mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-11434 Performance improvements for post metadata (#9849)

Просмотр исходного кода 
* Remove unused error return value from PreparePostForClient

* Remove unused error return value from PreparePostListForClient

* MM-11434 Parallelize PreparePostListForClient

* MM-11434 Skip looking reactions and files on post whenever possible

* Add note about the use of deprecated fields
Этот коммит содержится в:
Harrison Healey
2018-11-19 13:26:40 -05:00
коммит произвёл GitHub
родитель 7aef759fd8
Коммит 23c8950312
8 изменённых файлов: 100 добавлений и 128 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

7
app/notification.go
Просмотреть файл

@@ -317,13 +317,8 @@ func (a *App) SendNotifications(post *model.Post, team *model.Team, channel *mod
}
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare new post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POSTED, "", post.ChannelId, "", nil)
message.Add("post", clientPost.ToJson())
message.Add("post", a.PreparePostForClient(post).ToJson())
message.Add("channel_type", channel.Type)
message.Add("channel_display_name", notification.GetChannelName(model.SHOW_USERNAME, ""))
message.Add("channel_name", channel.Name)

21
app/post.go
Просмотреть файл

@@ -297,13 +297,8 @@ func (a *App) SendEphemeralPost(userId string, post *model.Post) *model.Post {
post.Props = model.StringInterface{}
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare ephemeral post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_EPHEMERAL_MESSAGE, "", post.ChannelId, userId, nil)
message.Add("post", clientPost.ToJson())
message.Add("post", a.PreparePostForClient(post).ToJson())
a.Publish(message)
return post
@@ -424,13 +419,8 @@ func (a *App) PatchPost(postId string, patch *model.PostPatch) (*model.Post, *mo
}
func (a *App) sendUpdatedPostEvent(post *model.Post) {
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare updated post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil)
message.Add("post", clientPost.ToJson())
message.Add("post", a.PreparePostForClient(post).ToJson())
a.Publish(message)
}
@@ -569,13 +559,8 @@ func (a *App) DeletePost(postId, deleteByID string) (*model.Post, *model.AppErro
return nil, result.Err
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare deleted post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_DELETED, "", post.ChannelId, "", nil)
message.Add("post", clientPost.ToJson())
message.Add("post", a.PreparePostForClient(post).ToJson())
a.Publish(message)
a.Srv.Go(func() {

52
app/post_metadata.go
Просмотреть файл

@@ -31,25 +31,33 @@ func (a *App) InitPostMetadata() {
})
}
func (a *App) PreparePostListForClient(originalList *model.PostList) (*model.PostList, *model.AppError) {
func (a *App) PreparePostListForClient(originalList *model.PostList) *model.PostList {
numPosts := len(originalList.Posts)
list := &model.PostList{
Posts: make(map[string]*model.Post),
Order: originalList.Order,
Posts: make(map[string]*model.Post, numPosts),
Order: originalList.Order, // Note that this uses the original Order array, so it isn't a deep copy
}
for id, originalPost := range originalList.Posts {
post, err := a.PreparePostForClient(originalPost)
if err != nil {
return originalList, err
}
posts := make(chan *model.Post, numPosts)
list.Posts[id] = post
for _, originalPost := range originalList.Posts {
go func(originalPost *model.Post) {
posts <- a.PreparePostForClient(originalPost)
}(originalPost)
}
return list, nil
for i := 0; i < numPosts; i++ {
post := <-posts
list.Posts[post.Id] = post
}
close(posts)
return list
}
func (a *App) PreparePostForClient(originalPost *model.Post) (*model.Post, *model.AppError) {
func (a *App) PreparePostForClient(originalPost *model.Post) *model.Post {
post := originalPost.Clone()
// Proxy image links before constructing metadata so that requests go through the proxy
@@ -67,7 +75,7 @@ func (a *App) PreparePostForClient(originalPost *model.Post) (*model.Post, *mode
}
// Files
if fileInfos, err := a.GetFileInfosForPost(post.Id, false); err != nil {
if fileInfos, err := a.getFileMetadataForPost(post); err != nil {
mlog.Warn("Failed to get files for a post", mlog.String("post_id", post.Id), mlog.Any("err", err))
} else {
post.Metadata.Files = fileInfos
@@ -87,13 +95,25 @@ func (a *App) PreparePostForClient(originalPost *model.Post) (*model.Post, *mode
post.Metadata.Images = a.getImagesForPost(post, images)
}
return post, nil
return post
}
func (a *App) getFileMetadataForPost(post *model.Post) ([]*model.FileInfo, *model.AppError) {
if len(post.FileIds) == 0 { // This field is deprecated, but still use it for now to avoid unnecessary database hits
return nil, nil
}
return a.GetFileInfosForPost(post.Id, false)
}
func (a *App) getEmojisAndReactionsForPost(post *model.Post) ([]*model.Emoji, []*model.Reaction, *model.AppError) {
reactions, err := a.GetReactionsForPost(post.Id)
if err != nil {
return nil, nil, err
var reactions []*model.Reaction
if post.HasReactions { // This field is deprecated, but still use it for now to avoid unnecessary database hits
var err *model.AppError
reactions, err = a.GetReactionsForPost(post.Id)
if err != nil {
return nil, nil, err
}
}
emojis, err := a.getCustomEmojisForPost(post, reactions)

75
app/post_metadata_test.go
Просмотреть файл

@@ -18,6 +18,37 @@ import (
"github.com/stretchr/testify/require"
)
func TestPreparePostListForClient(t *testing.T) {
// Most of this logic is covered by TestPreparePostForClient, so this just tests handling of multiple posts
th := Setup().InitBasic()
defer th.TearDown()
postList := model.NewPostList()
for i := 0; i < 5; i++ {
postList.AddPost(th.CreatePost(th.BasicChannel))
}
clientPostList := th.App.PreparePostListForClient(postList)
t.Run("doesn't mutate provided post list", func(t *testing.T) {
assert.NotEqual(t, clientPostList, postList, "should've returned a new post list")
assert.NotEqual(t, clientPostList.Posts, postList.Posts, "should've returned a new PostList.Posts")
assert.Equal(t, clientPostList.Order, postList.Order, "should've returned the existing PostList.Order")
for id, originalPost := range postList.Posts {
assert.NotEqual(t, clientPostList.Posts[id], originalPost, "should've returned new post objects")
assert.Equal(t, clientPostList.Posts[id].Id, originalPost.Id, "should've returned the same posts")
}
})
t.Run("adds metadata to each post", func(t *testing.T) {
for _, clientPost := range clientPostList.Posts {
assert.NotNil(t, clientPost.Metadata, "should've populated metadata for each post")
}
})
}
func TestPreparePostForClient(t *testing.T) {
setup := func() *TestHelper {
th := Setup().InitBasic()
@@ -38,8 +69,7 @@ func TestPreparePostForClient(t *testing.T) {
post := th.CreatePost(th.BasicChannel)
message := post.Message
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
t.Run("doesn't mutate provided post", func(t *testing.T) {
assert.NotEqual(t, clientPost, post, "should've returned a new post")
@@ -63,11 +93,9 @@ func TestPreparePostForClient(t *testing.T) {
th := setup()
defer th.TearDown()
post, err := th.App.PreparePostForClient(th.CreatePost(th.BasicChannel))
require.Nil(t, err)
post := th.App.PreparePostForClient(th.CreatePost(th.BasicChannel))
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
assert.False(t, clientPost == post, "should've returned a new post")
assert.Equal(t, clientPost, post, "shouldn't have changed any metadata")
@@ -81,9 +109,9 @@ func TestPreparePostForClient(t *testing.T) {
reaction1 := th.AddReactionToPost(post, th.BasicUser, "smile")
reaction2 := th.AddReactionToPost(post, th.BasicUser2, "smile")
reaction3 := th.AddReactionToPost(post, th.BasicUser2, "ice_cream")
post.HasReactions = true
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
assert.Len(t, clientPost.Metadata.Reactions, 3, "should've populated Reactions")
assert.Equal(t, reaction1, clientPost.Metadata.Reactions[0], "first reaction is incorrect")
@@ -107,8 +135,7 @@ func TestPreparePostForClient(t *testing.T) {
fileInfo.PostId = post.Id
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
assert.Equal(t, []*model.FileInfo{fileInfo}, clientPost.Metadata.Files, "should've populated Files")
})
@@ -140,9 +167,9 @@ func TestPreparePostForClient(t *testing.T) {
th.AddReactionToPost(post, th.BasicUser, "smile")
th.AddReactionToPost(post, th.BasicUser, "angry")
th.AddReactionToPost(post, th.BasicUser2, "angry")
post.HasReactions = true
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
t.Run("populates emojis", func(t *testing.T) {
assert.ElementsMatch(t, []*model.Emoji{}, clientPost.Metadata.Emojis, "should've populated empty Emojis")
@@ -185,9 +212,9 @@ func TestPreparePostForClient(t *testing.T) {
th.AddReactionToPost(post, th.BasicUser, emoji2.Name)
th.AddReactionToPost(post, th.BasicUser2, emoji2.Name)
th.AddReactionToPost(post, th.BasicUser2, "angry")
post.HasReactions = true
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
t.Run("pupulates emojis", func(t *testing.T) {
assert.ElementsMatch(t, []*model.Emoji{emoji1, emoji2, emoji3, emoji4}, clientPost.Metadata.Emojis, "should've populated post.Emojis")
@@ -210,8 +237,7 @@ func TestPreparePostForClient(t *testing.T) {
}, th.BasicChannel, false)
require.Nil(t, err)
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
t.Run("populates image dimensions", func(t *testing.T) {
imageDimensions := clientPost.Metadata.Images
@@ -253,8 +279,7 @@ func TestPreparePostForClient(t *testing.T) {
}, th.BasicChannel, false)
require.Nil(t, err)
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
// Reminder that only the first link gets an embed and dimensions
@@ -288,8 +313,7 @@ func TestPreparePostForClient(t *testing.T) {
}, th.BasicChannel, false)
require.Nil(t, err)
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
t.Run("populates embeds", func(t *testing.T) {
assert.ElementsMatch(t, []*model.PostEmbed{
@@ -339,8 +363,7 @@ func TestPreparePostForClient(t *testing.T) {
}, th.BasicChannel, false)
require.Nil(t, err)
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
t.Run("populates embeds", func(t *testing.T) {
assert.ElementsMatch(t, []*model.PostEmbed{
@@ -405,10 +428,7 @@ func testProxyLinkedImage(t *testing.T, th *TestHelper, shouldProxy bool) {
post, err = th.App.CreatePost(post, th.BasicChannel, false)
require.Nil(t, err)
clientPost, err := th.App.PreparePostForClient(post)
if err != nil && err.Id != "app.post.metadata.link.app_error" {
t.Fatal(err)
}
clientPost := th.App.PreparePostForClient(post)
if shouldProxy {
assert.Equal(t, post.Message, fmt.Sprintf(postTemplate, imageURL), "should not have mutated original post")
@@ -426,8 +446,7 @@ func testProxyOpenGraphImage(t *testing.T, th *TestHelper, shouldProxy bool) {
}, th.BasicChannel, false)
require.Nil(t, err)
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
clientPost := th.App.PreparePostForClient(post)
image := &opengraph.Image{}
if shouldProxy {

10
app/reaction.go
Просмотреть файл

@@ -6,7 +6,6 @@ package app
import (
"net/http"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
@@ -112,13 +111,10 @@ func (a *App) sendReactionEvent(event string, reaction *model.Reaction, post *mo
message.Add("reaction", reaction.ToJson())
a.Publish(message)
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare new post for client after reaction", mlog.Any("err", err))
}
post.HasReactions = hasReactions
post.UpdateAt = model.GetMillis()
clientPost.HasReactions = hasReactions
clientPost.UpdateAt = model.GetMillis()
clientPost := a.PreparePostForClient(post)
umessage := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil)
umessage.Add("post", clientPost.ToJson())