mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-13664 Cache external link metadata when populating post metadata (#10128)

Просмотр исходного кода 
* MM-13664 Added LinkMetadata types

* MM-13664 Use LinkMetadata when populating post metadata

* Fix unused import

* Fix index name on SQLite

* Finish adding unit tests

* Address feedback

* Increase max length of URL column to 2048 characters
Этот коммит содержится в:
Harrison Healey
2019-01-28 10:25:08 -05:00
коммит произвёл Carlos Tadeu Panato Junior
родитель 5c76e90a83
Коммит 26684716aa
22 изменённых файлов: 1483 добавлений и 50 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

91
app/post_metadata.go
Просмотреть файл

@@ -41,7 +41,7 @@ func (a *App) PreparePostListForClient(originalList *model.PostList) *model.Post
}
for id, originalPost := range originalList.Posts {
post := a.PreparePostForClient(originalPost)
post := a.PreparePostForClient(originalPost, false)
list.Posts[id] = post
}
@@ -49,7 +49,7 @@ func (a *App) PreparePostListForClient(originalList *model.PostList) *model.Post
return list
}
func (a *App) PreparePostForClient(originalPost *model.Post) *model.Post {
func (a *App) PreparePostForClient(originalPost *model.Post, isNewPost bool) *model.Post {
post := originalPost.Clone()
// Proxy image links before constructing metadata so that requests go through the proxy
@@ -79,7 +79,7 @@ func (a *App) PreparePostForClient(originalPost *model.Post) *model.Post {
// Embeds and image dimensions
firstLink, images := getFirstLinkAndImages(post.Message)
if embed, err := a.getEmbedForPost(post, firstLink); err != nil {
if embed, err := a.getEmbedForPost(post, firstLink, isNewPost); err != nil {
mlog.Warn("Failed to get embedded content for a post", mlog.String("post_id", post.Id), mlog.Any("err", err))
} else if embed == nil {
post.Metadata.Embeds = []*model.PostEmbed{}
@@ -87,7 +87,7 @@ func (a *App) PreparePostForClient(originalPost *model.Post) *model.Post {
post.Metadata.Embeds = []*model.PostEmbed{embed}
}
post.Metadata.Images = a.getImagesForPost(post, images)
post.Metadata.Images = a.getImagesForPost(post, images, isNewPost)
return post
}
@@ -118,7 +118,7 @@ func (a *App) getEmojisAndReactionsForPost(post *model.Post) ([]*model.Emoji, []
return emojis, reactions, nil
}
func (a *App) getEmbedForPost(post *model.Post, firstLink string) (*model.PostEmbed, error) {
func (a *App) getEmbedForPost(post *model.Post, firstLink string, isNewPost bool) (*model.PostEmbed, error) {
if _, ok := post.Props["attachments"]; ok {
return &model.PostEmbed{
Type: model.POST_EMBED_MESSAGE_ATTACHMENT,
@@ -129,7 +129,7 @@ func (a *App) getEmbedForPost(post *model.Post, firstLink string) (*model.PostEm
return nil, nil
}
og, image, err := a.getLinkMetadata(firstLink, true)
og, image, err := a.getLinkMetadata(firstLink, post.CreateAt, isNewPost)
if err != nil {
return nil, err
}
@@ -153,7 +153,7 @@ func (a *App) getEmbedForPost(post *model.Post, firstLink string) (*model.PostEm
return nil, nil
}
func (a *App) getImagesForPost(post *model.Post, imageURLs []string) map[string]*model.PostImage {
func (a *App) getImagesForPost(post *model.Post, imageURLs []string, isNewPost bool) map[string]*model.PostImage {
images := map[string]*model.PostImage{}
for _, embed := range post.Metadata.Embeds {
@@ -187,7 +187,7 @@ func (a *App) getImagesForPost(post *model.Post, imageURLs []string) map[string]
}
for _, imageURL := range imageURLs {
if _, image, err := a.getLinkMetadata(imageURL, true); err != nil {
if _, image, err := a.getLinkMetadata(imageURL, post.CreateAt, isNewPost); err != nil {
mlog.Warn("Failed to get dimensions of an image in a post",
mlog.String("post_id", post.Id), mlog.String("image_url", imageURL), mlog.Any("err", err))
} else {
@@ -317,14 +317,23 @@ func getImagesInMessageAttachments(post *model.Post) []string {
return images
}
func (a *App) getLinkMetadata(requestURL string, useCache bool) (*opengraph.OpenGraph, *model.PostImage, error) {
func (a *App) getLinkMetadata(requestURL string, timestamp int64, isNewPost bool) (*opengraph.OpenGraph, *model.PostImage, error) {
requestURL = resolveMetadataURL(requestURL, a.GetSiteURL())
// Check cache
if useCache {
og, image, ok := getLinkMetadataFromCache(requestURL)
timestamp = model.FloorToNearestHour(timestamp)
// Check cache
og, image, ok := getLinkMetadataFromCache(requestURL, timestamp)
if ok {
return og, image, nil
}
// Check the database if this isn't a new post. If it is a new post and the data is cached, it should be in memory.
if !isNewPost {
og, image, ok := a.getLinkMetadataFromDatabase(requestURL, timestamp)
if ok {
cacheLinkMetadata(requestURL, timestamp, og, image)
return og, image, nil
}
}
@@ -345,12 +354,12 @@ func (a *App) getLinkMetadata(requestURL string, useCache bool) (*opengraph.Open
defer res.Body.Close()
// Parse the data
og, image, err := a.parseLinkMetadata(requestURL, res.Body, res.Header.Get("Content-Type"))
og, image, err = a.parseLinkMetadata(requestURL, res.Body, res.Header.Get("Content-Type"))
// Write back to cache
if useCache {
cacheLinkMetadata(requestURL, og, image)
}
// Write back to cache and database
cacheLinkMetadata(requestURL, timestamp, og, image)
a.saveLinkMetadataToDatabase(requestURL, timestamp, og, image)
return og, image, err
}
@@ -370,8 +379,8 @@ func resolveMetadataURL(requestURL string, siteURL string) string {
return resolved.String()
}
func getLinkMetadataFromCache(requestURL string) (*opengraph.OpenGraph, *model.PostImage, bool) {
cached, ok := linkCache.Get(requestURL)
func getLinkMetadataFromCache(requestURL string, timestamp int64) (*opengraph.OpenGraph, *model.PostImage, bool) {
cached, ok := linkCache.Get(model.GenerateLinkMetadataHash(requestURL, timestamp))
if !ok {
return nil, nil, false
}
@@ -386,7 +395,47 @@ func getLinkMetadataFromCache(requestURL string) (*opengraph.OpenGraph, *model.P
}
}
func cacheLinkMetadata(requestURL string, og *opengraph.OpenGraph, image *model.PostImage) {
func (a *App) getLinkMetadataFromDatabase(requestURL string, timestamp int64) (*opengraph.OpenGraph, *model.PostImage, bool) {
result := <-a.Srv.Store.LinkMetadata().Get(requestURL, timestamp)
if result.Err != nil {
return nil, nil, false
}
data := result.Data.(*model.LinkMetadata).Data
switch v := data.(type) {
case *opengraph.OpenGraph:
return v, nil, true
case *model.PostImage:
return nil, v, true
default:
return nil, nil, true
}
}
func (a *App) saveLinkMetadataToDatabase(requestURL string, timestamp int64, og *opengraph.OpenGraph, image *model.PostImage) {
metadata := &model.LinkMetadata{
URL: requestURL,
Timestamp: timestamp,
}
if og != nil {
metadata.Type = model.LINK_METADATA_TYPE_OPENGRAPH
metadata.Data = og
} else if image != nil {
metadata.Type = model.LINK_METADATA_TYPE_IMAGE
metadata.Data = image
} else {
metadata.Type = model.LINK_METADATA_TYPE_NONE
}
result := <-a.Srv.Store.LinkMetadata().Save(metadata)
if result.Err != nil {
mlog.Warn("Failed to write link metadata", mlog.String("request_url", requestURL), mlog.Err(result.Err))
}
}
func cacheLinkMetadata(requestURL string, timestamp int64, og *opengraph.OpenGraph, image *model.PostImage) {
var val interface{}
if og != nil {
val = og
@@ -394,7 +443,7 @@ func cacheLinkMetadata(requestURL string, og *opengraph.OpenGraph, image *model.
val = image
}
linkCache.AddWithExpiresInSecs(requestURL, val, LINK_CACHE_DURATION)
linkCache.AddWithExpiresInSecs(model.GenerateLinkMetadataHash(requestURL, timestamp), val, LINK_CACHE_DURATION)
}
func (a *App) parseLinkMetadata(requestURL string, body io.Reader, contentType string) (*opengraph.OpenGraph, *model.PostImage, error) {