MM-11272 Added initial post metadata (#9175)

* MM-11272 Added app.PreparePostForClient

* MM-11272 Added app.PreparePostListForClient

* MM-11272 Added EmojiStore.GetMultipleByName

* MM-11272 Added emojis to PreparePostForClient

* MM-11272 Added unit tests for getting reaction counts

* MM-11272 Added unit tests for TestPreparePostForClient

* MM-11272 Added emojis from reactions to Post.Emojis

* MM-11272 Always update post.UpdateAt when reactions change to bust cache

* Fixed merge conflicts

* Moved post metadata-related code into its own file

* Update store mocks

* Fixed typo

* Add missing license headers

* Updated post metadata tests when custom emojis are disabled

* Fix unreliable unit tests

* Fix inconsistent casing in SQL statements

* Fix blank line

* Invalidate store cache after making changes

* Clear post cache synchronously with reactions
Этот коммит содержится в:
Harrison Healey
2018-08-07 16:24:56 -04:00
родитель 2e945e287d
Коммит 48f16b6401
27 изменённых файлов: 902 добавлений и 87 удалений

Просмотреть файл

@@ -390,6 +390,39 @@ func (me *TestHelper) CreateScheme() (*model.Scheme, []*model.Role) {
return scheme, roles
}
func (me *TestHelper) CreateEmoji() *model.Emoji {
utils.DisableDebugLogForTest()
result := <-me.App.Srv.Store.Emoji().Save(&model.Emoji{
CreatorId: me.BasicUser.Id,
Name: model.NewRandomString(10),
})
if result.Err != nil {
panic(result.Err)
}
utils.EnableDebugLogForTest()
return result.Data.(*model.Emoji)
}
func (me *TestHelper) AddReactionToPost(post *model.Post, user *model.User, emojiName string) *model.Reaction {
utils.DisableDebugLogForTest()
reaction, err := me.App.SaveReactionForPost(&model.Reaction{
UserId: user.Id,
PostId: post.Id,
EmojiName: emojiName,
})
if err != nil {
panic(err)
}
utils.EnableDebugLogForTest()
return reaction
}
func (me *TestHelper) TearDown() {
me.App.Shutdown()
os.Remove(me.tempConfigPath)

Просмотреть файл

@@ -185,6 +185,18 @@ func (a *App) GetEmojiByName(emojiName string) (*model.Emoji, *model.AppError) {
return result.Data.(*model.Emoji), nil
}
func (a *App) GetMultipleEmojiByName(names []string) ([]*model.Emoji, *model.AppError) {
if !*a.Config().ServiceSettings.EnableCustomEmoji {
return nil, model.NewAppError("GetMultipleEmojiByName", "api.emoji.disabled.app_error", nil, "", http.StatusNotImplemented)
}
if result := <-a.Srv.Store.Emoji().GetMultipleByName(names); result.Err != nil {
return nil, result.Err
} else {
return result.Data.([]*model.Emoji), nil
}
}
func (a *App) GetEmojiImage(emojiId string) ([]byte, string, *model.AppError) {
result := <-a.Srv.Store.Emoji().Get(emojiId, true)
if result.Err != nil {

Просмотреть файл

@@ -317,8 +317,13 @@ func (a *App) SendNotifications(post *model.Post, team *model.Team, channel *mod
}
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare new post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POSTED, "", post.ChannelId, "", nil)
message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
message.Add("post", clientPost.ToJson())
message.Add("channel_type", channel.Type)
message.Add("channel_display_name", notification.GetChannelName(model.SHOW_USERNAME, ""))
message.Add("channel_name", channel.Name)

Просмотреть файл

@@ -301,8 +301,13 @@ func (a *App) SendEphemeralPost(userId string, post *model.Post) *model.Post {
post.Props = model.StringInterface{}
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare ephemeral post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_EPHEMERAL_MESSAGE, "", post.ChannelId, userId, nil)
message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
message.Add("post", clientPost.ToJson())
a.Publish(message)
return post
@@ -423,8 +428,13 @@ func (a *App) PatchPost(postId string, patch *model.PostPatch) (*model.Post, *mo
}
func (a *App) sendUpdatedPostEvent(post *model.Post) {
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare updated post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil)
message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
message.Add("post", clientPost.ToJson())
a.Publish(message)
}
@@ -563,8 +573,13 @@ func (a *App) DeletePost(postId, deleteByID string) (*model.Post, *model.AppErro
return nil, result.Err
}
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare deleted post for client", mlog.Any("err", err))
}
message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_DELETED, "", post.ChannelId, "", nil)
message.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
message.Add("post", clientPost.ToJson())
a.Publish(message)
a.Go(func() {
@@ -967,13 +982,6 @@ func (a *App) DoPostAction(postId, actionId, userId, selectedOption string) *mod
return nil
}
func (a *App) PostListWithProxyAddedToImageURLs(list *model.PostList) *model.PostList {
if f := a.ImageProxyAdder(); f != nil {
return list.WithRewrittenImageURLs(f)
}
return list
}
func (a *App) PostWithProxyAddedToImageURLs(post *model.Post) *model.Post {
if f := a.ImageProxyAdder(); f != nil {
return post.WithRewrittenImageURLs(f)

111
app/post_metadata.go Обычный файл
Просмотреть файл

@@ -0,0 +1,111 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package app
import (
"strings"
"github.com/dyatlov/go-opengraph/opengraph"
"github.com/mattermost/mattermost-server/model"
)
func (a *App) PreparePostListForClient(originalList *model.PostList) (*model.PostList, *model.AppError) {
list := &model.PostList{
Posts: make(map[string]*model.Post),
Order: originalList.Order,
}
for id, originalPost := range originalList.Posts {
post, err := a.PreparePostForClient(originalPost)
if err != nil {
return originalList, err
}
list.Posts[id] = post
}
return list, nil
}
func (a *App) PreparePostForClient(originalPost *model.Post) (*model.Post, *model.AppError) {
post := originalPost.Clone()
var err *model.AppError
needReactionCounts := post.ReactionCounts == nil
needEmojis := post.Emojis == nil
needImageDimensions := post.ImageDimensions == nil
needOpenGraphData := post.OpenGraphData == nil
var reactions []*model.Reaction
if needReactionCounts || needEmojis {
reactions, err = a.GetReactionsForPost(post.Id)
if err != nil {
return post, err
}
}
if needReactionCounts {
post.ReactionCounts = model.CountReactions(reactions)
}
if post.FileInfos == nil {
fileInfos, err := a.GetFileInfosForPost(post.Id, false)
if err != nil {
return post, err
}
post.FileInfos = fileInfos
}
if needEmojis {
emojis, err := a.getCustomEmojisForPost(post.Message, reactions)
if err != nil {
return post, err
}
post.Emojis = emojis
}
post = a.PostWithProxyAddedToImageURLs(post)
if needImageDimensions || needOpenGraphData {
if needImageDimensions {
post.ImageDimensions = []*model.PostImageDimensions{}
}
if needOpenGraphData {
post.OpenGraphData = []*opengraph.OpenGraph{}
}
// TODO
}
return post, nil
}
func (a *App) getCustomEmojisForPost(message string, reactions []*model.Reaction) ([]*model.Emoji, *model.AppError) {
if !*a.Config().ServiceSettings.EnableCustomEmoji {
// Only custom emoji are returned
return []*model.Emoji{}, nil
}
names := model.EMOJI_PATTERN.FindAllString(message, -1)
for _, reaction := range reactions {
names = append(names, reaction.EmojiName)
}
if len(names) == 0 {
return []*model.Emoji{}, nil
}
names = model.RemoveDuplicateStrings(names)
for i, name := range names {
names[i] = strings.Trim(name, ":")
}
return a.GetMultipleEmojiByName(names)
}

356
app/post_metadata_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,356 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package app
import (
"fmt"
"testing"
"time"
"github.com/mattermost/mattermost-server/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestPreparePostForClient(t *testing.T) {
setup := func() *TestHelper {
th := Setup().InitBasic()
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.ServiceSettings.ImageProxyType = ""
*cfg.ServiceSettings.ImageProxyURL = ""
*cfg.ServiceSettings.ImageProxyOptions = ""
})
return th
}
t.Run("no metadata needed", func(t *testing.T) {
th := setup()
defer th.TearDown()
post := th.CreatePost(th.BasicChannel)
message := post.Message
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
assert.NotEqual(t, clientPost, post, "should've returned a new post")
assert.Equal(t, message, post.Message, "shouldn't have mutated post.Message")
assert.NotEqual(t, nil, post.ReactionCounts, "shouldn't have mutated post.ReactionCounts")
assert.NotEqual(t, nil, post.FileInfos, "shouldn't have mutated post.FileInfos")
assert.NotEqual(t, nil, post.Emojis, "shouldn't have mutated post.Emojis")
assert.NotEqual(t, nil, post.ImageDimensions, "shouldn't have mutated post.ImageDimensions")
assert.NotEqual(t, nil, post.OpenGraphData, "shouldn't have mutated post.OpenGraphData")
assert.Equal(t, clientPost.Message, post.Message, "shouldn't have changed Message")
assert.Len(t, post.ReactionCounts, 0, "should've populated ReactionCounts")
assert.Len(t, post.FileInfos, 0, "should've populated FileInfos")
assert.Len(t, post.Emojis, 0, "should've populated Emojis")
assert.Len(t, post.ImageDimensions, 0, "should've populated ImageDimensions")
assert.Len(t, post.OpenGraphData, 0, "should've populated OpenGraphData")
})
t.Run("metadata already set", func(t *testing.T) {
th := setup()
defer th.TearDown()
post, err := th.App.PreparePostForClient(th.CreatePost(th.BasicChannel))
require.Nil(t, err)
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
assert.False(t, clientPost == post, "should've returned a new post")
assert.Equal(t, clientPost, post, "shouldn't have changed any metadata")
})
t.Run("reaction counts", func(t *testing.T) {
th := setup()
defer th.TearDown()
post := th.CreatePost(th.BasicChannel)
th.AddReactionToPost(post, th.BasicUser, "smile")
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
assert.Equal(t, model.ReactionCounts{
"smile": 1,
}, clientPost.ReactionCounts, "should've populated post.ReactionCounts")
})
t.Run("file infos", func(t *testing.T) {
th := setup()
defer th.TearDown()
fileInfo, err := th.App.DoUploadFile(time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test"))
require.Nil(t, err)
post, err := th.App.CreatePost(&model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
FileIds: []string{fileInfo.Id},
}, th.BasicChannel, false)
require.Nil(t, err)
fileInfo.PostId = post.Id
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
assert.Equal(t, []*model.FileInfo{fileInfo}, clientPost.FileInfos, "should've populated post.FileInfos")
})
t.Run("emojis without custom emojis enabled", func(t *testing.T) {
th := setup()
defer th.TearDown()
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.ServiceSettings.EnableCustomEmoji = false
})
emoji := th.CreateEmoji()
post, err := th.App.CreatePost(&model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
Message: ":" + emoji.Name + ": :taco:",
}, th.BasicChannel, false)
require.Nil(t, err)
th.AddReactionToPost(post, th.BasicUser, "smile")
th.AddReactionToPost(post, th.BasicUser, "angry")
th.AddReactionToPost(post, th.BasicUser2, "angry")
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
assert.Len(t, clientPost.ReactionCounts, 2, "should've populated post.ReactionCounts")
assert.Equal(t, 1, clientPost.ReactionCounts["smile"], "should've populated post.ReactionCounts for smile")
assert.Equal(t, 2, clientPost.ReactionCounts["angry"], "should've populated post.ReactionCounts for angry")
assert.ElementsMatch(t, []*model.Emoji{}, clientPost.Emojis, "should've populated empty post.Emojis")
})
t.Run("emojis with custom emojis enabled", func(t *testing.T) {
th := setup()
defer th.TearDown()
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.ServiceSettings.EnableCustomEmoji = true
})
emoji1 := th.CreateEmoji()
emoji2 := th.CreateEmoji()
emoji3 := th.CreateEmoji()
post, err := th.App.CreatePost(&model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
Message: ":" + emoji3.Name + ": :taco:",
}, th.BasicChannel, false)
require.Nil(t, err)
th.AddReactionToPost(post, th.BasicUser, emoji1.Name)
th.AddReactionToPost(post, th.BasicUser, emoji2.Name)
th.AddReactionToPost(post, th.BasicUser2, emoji2.Name)
th.AddReactionToPost(post, th.BasicUser2, "angry")
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
assert.Len(t, clientPost.ReactionCounts, 3, "should've populated post.ReactionCounts")
assert.Equal(t, 1, clientPost.ReactionCounts[emoji1.Name], "should've populated post.ReactionCounts for emoji1")
assert.Equal(t, 2, clientPost.ReactionCounts[emoji2.Name], "should've populated post.ReactionCounts for emoji2")
assert.Equal(t, 1, clientPost.ReactionCounts["angry"], "should've populated post.ReactionCounts for angry")
assert.ElementsMatch(t, []*model.Emoji{emoji1, emoji2, emoji3}, clientPost.Emojis, "should've populated post.Emojis")
})
t.Run("linked image dimensions", func(t *testing.T) {
// TODO
})
t.Run("proxy linked images", func(t *testing.T) {
th := setup()
defer th.TearDown()
testProxyLinkedImage(t, th, false)
})
t.Run("opengraph", func(t *testing.T) {
// TODO
})
t.Run("opengraph image dimensions", func(t *testing.T) {
// TODO
})
t.Run("proxy opengraph images", func(t *testing.T) {
// TODO
})
}
func TestPreparePostForClientWithImageProxy(t *testing.T) {
setup := func() *TestHelper {
th := Setup().InitBasic()
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.ServiceSettings.SiteURL = "http://mymattermost.com"
*cfg.ServiceSettings.ImageProxyType = "atmos/camo"
*cfg.ServiceSettings.ImageProxyURL = "https://127.0.0.1"
*cfg.ServiceSettings.ImageProxyOptions = "foo"
})
return th
}
t.Run("proxy linked images", func(t *testing.T) {
th := setup()
defer th.TearDown()
testProxyLinkedImage(t, th, true)
})
t.Run("proxy opengraph images", func(t *testing.T) {
// TODO
})
}
func testProxyLinkedImage(t *testing.T, th *TestHelper, shouldProxy bool) {
postTemplate := "![foo](%v)"
imageURL := "http://mydomain.com/myimage"
proxiedImageURL := "https://127.0.0.1/f8dace906d23689e8d5b12c3cefbedbf7b9b72f5/687474703a2f2f6d79646f6d61696e2e636f6d2f6d79696d616765"
post := &model.Post{
UserId: th.BasicUser.Id,
ChannelId: th.BasicChannel.Id,
Message: fmt.Sprintf(postTemplate, imageURL),
}
var err *model.AppError
post, err = th.App.CreatePost(post, th.BasicChannel, false)
require.Nil(t, err)
clientPost, err := th.App.PreparePostForClient(post)
require.Nil(t, err)
if shouldProxy {
assert.Equal(t, post.Message, fmt.Sprintf(postTemplate, imageURL), "should not have mutated original post")
assert.Equal(t, clientPost.Message, fmt.Sprintf(postTemplate, proxiedImageURL), "should've replaced linked image URLs")
} else {
assert.Equal(t, clientPost.Message, fmt.Sprintf(postTemplate, imageURL), "shouldn't have replaced linked image URLs")
}
}
func TestGetCustomEmojisForPost_Message(t *testing.T) {
th := Setup().InitBasic()
defer th.TearDown()
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.ServiceSettings.EnableCustomEmoji = true
})
emoji1 := th.CreateEmoji()
emoji2 := th.CreateEmoji()
emoji3 := th.CreateEmoji()
testCases := []struct {
Description string
Input string
Expected []*model.Emoji
SkipExpectations bool
}{
{
Description: "no emojis",
Input: "this is a string",
Expected: []*model.Emoji{},
SkipExpectations: true,
},
{
Description: "one emoji",
Input: "this is an :" + emoji1.Name + ": string",
Expected: []*model.Emoji{
emoji1,
},
},
{
Description: "two emojis",
Input: "this is a :" + emoji3.Name + ": :" + emoji2.Name + ": string",
Expected: []*model.Emoji{
emoji3,
emoji2,
},
},
{
Description: "punctuation around emojis",
Input: ":" + emoji3.Name + ":/:" + emoji1.Name + ": (:" + emoji2.Name + ":)",
Expected: []*model.Emoji{
emoji3,
emoji1,
emoji2,
},
},
{
Description: "adjacent emojis",
Input: ":" + emoji3.Name + "::" + emoji1.Name + ":",
Expected: []*model.Emoji{
emoji3,
emoji1,
},
},
{
Description: "duplicate emojis",
Input: "" + emoji1.Name + ": :" + emoji1.Name + ": :" + emoji1.Name + ": :" + emoji2.Name + ": :" + emoji2.Name + ": :" + emoji1.Name + ":",
Expected: []*model.Emoji{
emoji1,
emoji2,
},
},
{
Description: "fake emojis",
Input: "these don't exist :tomato: :potato: :rotato:",
Expected: []*model.Emoji{},
},
{
Description: "fake and real emojis",
Input: ":tomato::" + emoji1.Name + ": :potato: :" + emoji2.Name + ":",
Expected: []*model.Emoji{
emoji1,
emoji2,
},
},
}
for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.Description, func(t *testing.T) {
emojis, err := th.App.getCustomEmojisForPost(testCase.Input, nil)
assert.Nil(t, err, "failed to get emojis in message")
assert.ElementsMatch(t, emojis, testCase.Expected, "received incorrect emojis")
})
}
}
func TestGetCustomEmojisForPost(t *testing.T) {
th := Setup().InitBasic()
defer th.TearDown()
th.App.UpdateConfig(func(cfg *model.Config) {
*cfg.ServiceSettings.EnableCustomEmoji = true
})
emoji1 := th.CreateEmoji()
emoji2 := th.CreateEmoji()
reactions := []*model.Reaction{
{
UserId: th.BasicUser.Id,
EmojiName: emoji1.Name,
},
}
emojis, err := th.App.getCustomEmojisForPost(":"+emoji2.Name+":", reactions)
assert.Nil(t, err, "failed to get emojis for post")
assert.ElementsMatch(t, emojis, []*model.Emoji{emoji1, emoji2}, "received incorrect emojis")
}

Просмотреть файл

@@ -466,7 +466,6 @@ func TestImageProxy(t *testing.T) {
list := model.NewPostList()
list.Posts[post.Id] = post
assert.Equal(t, "![foo]("+tc.ProxiedImageURL+")", th.App.PostListWithProxyAddedToImageURLs(list).Posts[post.Id].Message)
assert.Equal(t, "![foo]("+tc.ProxiedImageURL+")", th.App.PostWithProxyAddedToImageURLs(post).Message)
assert.Equal(t, "![foo]("+tc.ImageURL+")", th.App.PostWithProxyRemovedFromImageURLs(post).Message)

Просмотреть файл

@@ -6,6 +6,7 @@ package app
import (
"net/http"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
@@ -42,6 +43,9 @@ func (a *App) SaveReactionForPost(reaction *model.Reaction) (*model.Reaction, *m
reaction = result.Data.(*model.Reaction)
// The post is always modified since the UpdateAt always changes
a.InvalidateCacheForChannelPosts(post.ChannelId)
a.Go(func() {
a.sendReactionEvent(model.WEBSOCKET_EVENT_REACTION_ADDED, reaction, post, true)
})
@@ -92,6 +96,9 @@ func (a *App) DeleteReactionForPost(reaction *model.Reaction) *model.AppError {
return result.Err
}
// The post is always modified since the UpdateAt always changes
a.InvalidateCacheForChannelPosts(post.ChannelId)
a.Go(func() {
a.sendReactionEvent(model.WEBSOCKET_EVENT_REACTION_REMOVED, reaction, post, hasReactions)
})
@@ -105,11 +112,15 @@ func (a *App) sendReactionEvent(event string, reaction *model.Reaction, post *mo
message.Add("reaction", reaction.ToJson())
a.Publish(message)
// The post is always modified since the UpdateAt always changes
a.InvalidateCacheForChannelPosts(post.ChannelId)
post.HasReactions = hasReactions
post.UpdateAt = model.GetMillis()
clientPost, err := a.PreparePostForClient(post)
if err != nil {
mlog.Error("Failed to prepare new post for client after reaction", mlog.Any("err", err))
}
clientPost.HasReactions = hasReactions
clientPost.UpdateAt = model.GetMillis()
umessage := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_POST_EDITED, "", post.ChannelId, "", nil)
umessage.Add("post", a.PostWithProxyAddedToImageURLs(post).ToJson())
umessage.Add("post", clientPost.ToJson())
a.Publish(umessage)
}