From 21639f9c8730b2f49ee52322acd3acdbf64ac6d9 Mon Sep 17 00:00:00 2001 From: Martin Kraft Date: Mon, 9 Aug 2021 11:33:21 -0400 Subject: [PATCH] MM-22845: Added support for permalink previews. (#17796) * MM-22845: Added support for permalink previews. * MM-22845: Adds license to new file. * MM-22845: Adds endpoint to retrieve multiple posts by id. * MM-22845: Fix for deleted post. * MM-22845: Adds config setting for permalink previews. * MM-22845: Adds API test for new endpoint. * MM-22845: Fix typo. * MM-22845: Tests that post create or updated via App get the previewed_post prop. * MM-22845: Tests for matching permalinks. * MM-22845: Adds PreparePostForClient test for permalink previews. * MM-22845: Embeds entire post in permalink metadata. * MM-22845: Filter WS message payload of created and edited post based on permissions. * MM-22845: Runs app layer generator. * MM-22845: Lint check fix. * MM-22845: Adds feature flag. * MM-22845: Clones WS message. * MM-22845: Removes knowledge of permalink from LinkMetadata table. Removes knowledge of user id from post embedding methods in favour of a 'sanitize' method/step. * MM-22845: Handle nil post metadata. * MM-22845: Switch to cloning post. * MM-22845: Removes unused code. * MM-22845: Refactor. * MM-22845: Reverts whitespace change. * MM-22845: Removes unnecessary code. * MM-22845: Removes unnecessary function. * MM-22845: Warn but don't error if permalinked referenced post or channel is not found. * MM-22845: Fix for clone method. * MM-22845: Fix for clone method. * MM-22845: Updates translations. Co-authored-by: Mattermod --- api4/channel.go | 5 + api4/post.go | 38 +++- app/admin.go | 1 + app/app_iface.go | 3 + app/authorization.go | 4 + app/notification.go | 20 +- app/opentracing/opentracing_layer.go | 61 ++++++ app/post.go | 93 ++++++++- app/post_metadata.go | 190 +++++++++++++----- app/post_metadata_test.go | 154 +++++++++----- app/post_test.go | 76 ++++++- app/ratelimit.go | 2 +- app/role_test.go | 2 +- app/shared_channel.go | 8 +- app/syncables_test.go | 2 +- app/user_test.go | 8 +- app/web_conn.go | 2 +- config/client.go | 1 + model/config.go | 5 + model/feature_flags.go | 3 + model/message_export.go | 14 ++ model/permalink.go | 34 ++++ model/post.go | 13 ++ model/post_embed.go | 1 + model/post_list.go | 17 ++ services/telemetry/telemetry.go | 1 + ...globalrelay_compliance_export_message.html | 1 + tests/test-config.json | 1 + 28 files changed, 645 insertions(+), 115 deletions(-) create mode 100644 model/permalink.go diff --git a/api4/channel.go b/api4/channel.go index cfefcc8c2c..09326dd896 100644 --- a/api4/channel.go +++ b/api4/channel.go @@ -722,6 +722,11 @@ func getPinnedPosts(c *Context, w http.ResponseWriter, r *http.Request) { } clientPostList := c.App.PreparePostListForClient(posts) + clientPostList, err = c.App.SanitizePostListMetadataForUser(clientPostList, c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } w.Header().Set(model.HeaderEtagServer, clientPostList.Etag()) if err := json.NewEncoder(w).Encode(clientPostList); err != nil { diff --git a/api4/post.go b/api4/post.go index 7e2dfa3c2f..5e75a76713 100644 --- a/api4/post.go +++ b/api4/post.go @@ -128,6 +128,11 @@ func createEphemeralPost(c *Context, w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) rp = model.AddPostActionCookies(rp, c.App.PostActionCookieSecret()) rp = c.App.PreparePostForClient(rp, true, false) + rp, err := c.App.SanitizePostMetadataForUser(rp, c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } if err := json.NewEncoder(w).Encode(rp); err != nil { mlog.Warn("Error while writing response", mlog.Err(err)) } @@ -216,6 +221,11 @@ func getPostsForChannel(c *Context, w http.ResponseWriter, r *http.Request) { c.App.AddCursorIdsForPostList(list, afterPost, beforePost, since, page, perPage, collapsedThreads) clientPostList := c.App.PreparePostListForClient(list) + clientPostList, err = c.App.SanitizePostListMetadataForUser(clientPostList, c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } if err := json.NewEncoder(w).Encode(clientPostList); err != nil { mlog.Warn("Error while writing response", mlog.Err(err)) @@ -274,6 +284,11 @@ func getPostsForChannelAroundLastUnread(c *Context, w http.ResponseWriter, r *ht postList.PrevPostId = c.App.GetPrevPostIdFromPostList(postList, collapsedThreads) clientPostList := c.App.PreparePostListForClient(postList) + clientPostList, err = c.App.SanitizePostListMetadataForUser(clientPostList, c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } if etag != "" { w.Header().Set(model.HeaderEtagServer, etag) @@ -337,7 +352,13 @@ func getFlaggedPostsForUser(c *Context, w http.ResponseWriter, r *http.Request) } pl.SortByCreateAt() - if err := json.NewEncoder(w).Encode(c.App.PreparePostListForClient(pl)); err != nil { + clientPostList := c.App.PreparePostListForClient(pl) + clientPostList, err = c.App.SanitizePostListMetadataForUser(clientPostList, c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } + if err := json.NewEncoder(w).Encode(clientPostList); err != nil { mlog.Warn("Error while writing response", mlog.Err(err)) } } @@ -355,6 +376,11 @@ func getPost(c *Context, w http.ResponseWriter, r *http.Request) { } post = c.App.PreparePostForClient(post, false, false) + post, err = c.App.SanitizePostMetadataForUser(post, c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } if c.HandleEtag(post.Etag(), "Get Post", w, r) { return @@ -434,6 +460,11 @@ func getPostThread(c *Context, w http.ResponseWriter, r *http.Request) { } clientPostList := c.App.PreparePostListForClient(list) + clientPostList, err = c.App.SanitizePostListMetadataForUser(clientPostList, c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } w.Header().Set(model.HeaderEtagServer, clientPostList.Etag()) @@ -507,6 +538,11 @@ func searchPosts(c *Context, w http.ResponseWriter, r *http.Request) { } clientPostList := c.App.PreparePostListForClient(results.PostList) + clientPostList, err = c.App.SanitizePostListMetadataForUser(clientPostList, c.AppContext.Session().UserId) + if err != nil { + c.Err = err + return + } results = model.MakePostSearchResults(clientPostList, results.Matches) diff --git a/app/admin.go b/app/admin.go index e0dc72ab71..bee56f5dc4 100644 --- a/app/admin.go +++ b/app/admin.go @@ -176,6 +176,7 @@ func (s *Server) InvalidateAllCachesSkipSend() { s.Store.Post().ClearCaches() s.Store.FileInfo().ClearCaches() s.Store.Webhook().ClearCaches() + linkCache.Purge() s.LoadLicense() } diff --git a/app/app_iface.go b/app/app_iface.go index 6e1db22c41..a291fdc2f6 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -815,6 +815,7 @@ type AppIface interface { HasPermissionTo(askingUserId string, permission *model.Permission) bool HasPermissionToChannel(askingUserId string, channelID string, permission *model.Permission) bool HasPermissionToChannelByPost(askingUserId string, postID string, permission *model.Permission) bool + HasPermissionToReadChannel(userID string, channel *model.Channel) bool HasPermissionToTeam(askingUserId string, teamID string, permission *model.Permission) bool HasPermissionToUser(askingUserId string, userID string) bool HasSharedChannel(channelID string) (bool, error) @@ -941,6 +942,8 @@ type AppIface interface { RevokeUserAccessToken(token *model.UserAccessToken) *model.AppError RolesGrantPermission(roleNames []string, permissionId string) bool Saml() einterfaces.SamlInterface + SanitizePostListMetadataForUser(postList *model.PostList, userID string) (*model.PostList, *model.AppError) + SanitizePostMetadataForUser(post *model.Post, userID string) (*model.Post, *model.AppError) SanitizeProfile(user *model.User, asAdmin bool) SanitizeTeam(session model.Session, team *model.Team) *model.Team SanitizeTeams(session model.Session, teams []*model.Team) []*model.Team diff --git a/app/authorization.go b/app/authorization.go index 93e022590f..6edc586be3 100644 --- a/app/authorization.go +++ b/app/authorization.go @@ -278,3 +278,7 @@ func (a *App) SessionHasPermissionToManageBot(session model.Session, botUserId s return nil } + +func (a *App) HasPermissionToReadChannel(userID string, channel *model.Channel) bool { + return a.HasPermissionToChannel(userID, channel.Id, model.PermissionReadChannel) || (channel.Type == model.ChannelTypeOpen && a.HasPermissionToTeam(userID, channel.TeamId, model.PermissionReadPublicChannel)) +} diff --git a/app/notification.go b/app/notification.go index 46dcb4e5c3..72e7e9cf17 100644 --- a/app/notification.go +++ b/app/notification.go @@ -469,7 +469,13 @@ func (a *App) SendNotifications(post *model.Post, team *model.Team, channel *mod message.Add("mentions", model.ArrayToJson(mentionedUsersList)) } - a.Publish(message) + published, err := a.publishWebsocketEventForPermalinkPost(post, message) + if err != nil { + return nil, err + } + if !published { + a.Publish(message) + } // If this is a reply in a thread, notify participants if a.Config().FeatureFlags.CollapsedThreads && *a.Config().ServiceSettings.CollapsedThreads != model.CollapsedThreadsDisabled && post.RootId != "" { @@ -502,6 +508,18 @@ func (a *App) SendNotifications(post *model.Post, team *model.Team, channel *mod if userThread != nil { a.sanitizeProfiles(userThread.Participants, false) userThread.Post.SanitizeProps() + + previewPost := post.GetPreviewPost() + if previewPost != nil { + previewedChannel, err := a.GetChannel(previewPost.Post.ChannelId) + if err != nil { + return nil, err + } + if previewedChannel != nil && !a.HasPermissionToReadChannel(uid, previewedChannel) { + userThread.Post.Metadata.Embeds[0].Data = nil + } + } + message.Add("thread", userThread.ToJson()) a.Publish(message) } diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 6baa336a56..0ee47e45f8 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -10431,6 +10431,23 @@ func (a *OpenTracingAppLayer) HasPermissionToChannelByPost(askingUserId string, return resultVar0 } +func (a *OpenTracingAppLayer) HasPermissionToReadChannel(userID string, channel *model.Channel) bool { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.HasPermissionToReadChannel") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0 := a.app.HasPermissionToReadChannel(userID, channel) + + return resultVar0 +} + func (a *OpenTracingAppLayer) HasPermissionToTeam(askingUserId string, teamID string, permission *model.Permission) bool { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.HasPermissionToTeam") @@ -13432,6 +13449,50 @@ func (a *OpenTracingAppLayer) RolesGrantPermission(roleNames []string, permissio return resultVar0 } +func (a *OpenTracingAppLayer) SanitizePostListMetadataForUser(postList *model.PostList, userID string) (*model.PostList, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SanitizePostListMetadataForUser") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.SanitizePostListMetadataForUser(postList, userID) + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + +func (a *OpenTracingAppLayer) SanitizePostMetadataForUser(post *model.Post, userID string) (*model.Post, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SanitizePostMetadataForUser") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.SanitizePostMetadataForUser(post, userID) + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) SanitizeProfile(user *model.User, asAdmin bool) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SanitizeProfile") diff --git a/app/post.go b/app/post.go index aefc15eb6a..70c4fb7195 100644 --- a/app/post.go +++ b/app/post.go @@ -359,6 +359,11 @@ func (a *App) CreatePost(c *request.Context, post *model.Post, channel *model.Ch // to be done when we send the post over the websocket in handlePostEvents rpost = a.PreparePostForClient(rpost, true, false) + rpost, nErr = a.addPostPreviewProp(rpost) + if nErr != nil { + return nil, model.NewAppError("CreatePost", "app.post.save.app_error", nil, nErr.Error(), http.StatusInternalServerError) + } + // Make sure poster is following the thread if *a.Config().ServiceSettings.ThreadAutoFollow && rpost.RootId != "" { _, err := a.Srv().Store.Thread().MaintainMembership(user.Id, rpost.RootId, store.ThreadMembershipOpts{ @@ -379,9 +384,25 @@ func (a *App) CreatePost(c *request.Context, post *model.Post, channel *model.Ch a.SendEphemeralPost(post.UserId, ephemeralPost) } + rpost, err = a.SanitizePostMetadataForUser(rpost, c.Session().UserId) + if err != nil { + return nil, err + } + return rpost, nil } +func (a *App) addPostPreviewProp(post *model.Post) (*model.Post, error) { + previewPost := post.GetPreviewPost() + if previewPost != nil { + updatedPost := post.Clone() + updatedPost.AddProp(model.PostPropsPreviewedPost, previewPost.PostID) + updatedPost, err := a.Srv().Store.Post().Update(updatedPost, post) + return updatedPost, err + } + return post, nil +} + func (a *App) attachFilesToPost(post *model.Post) *model.AppError { var attachedIds []string for _, fileID := range post.FileIds { @@ -671,15 +692,79 @@ func (a *App) UpdatePost(c *request.Context, post *model.Post, safeUpdate bool) // individually. rpost.IsFollowing = nil + rpost, nErr = a.addPostPreviewProp(rpost) + if nErr != nil { + return nil, model.NewAppError("UpdatePost", "app.post.update.app_error", nil, nErr.Error(), http.StatusInternalServerError) + } + message := model.NewWebSocketEvent(model.WebsocketEventPostEdited, "", rpost.ChannelId, "", nil) message.Add("post", rpost.ToJson()) - a.Publish(message) + + published, err := a.publishWebsocketEventForPermalinkPost(rpost, message) + if err != nil { + return nil, err + } + if !published { + a.Publish(message) + } a.invalidateCacheForChannelPosts(rpost.ChannelId) return rpost, nil } +func (a *App) publishWebsocketEventForPermalinkPost(post *model.Post, message *model.WebSocketEvent) (published bool, err *model.AppError) { + var previewedPostID string + if val, ok := post.GetProp(model.PostPropsPreviewedPost).(string); ok { + previewedPostID = val + } else { + return false, nil + } + + if !model.IsValidId(previewedPostID) { + mlog.Warn("invalid post prop value", mlog.String("prop_key", model.PostPropsPreviewedPost), mlog.String("prop_value", previewedPostID)) + return false, nil + } + + previewedPost, err := a.GetSinglePost(previewedPostID) + if err != nil { + if err.StatusCode == http.StatusNotFound { + mlog.Warn("permalinked post not found", mlog.String("referenced_post_id", previewedPostID)) + return false, nil + } + return false, err + } + + previewedChannel, err := a.GetChannel(previewedPost.ChannelId) + if err != nil { + if err.StatusCode == http.StatusNotFound { + mlog.Warn("channel containing permalinked post not found", mlog.String("referenced_channel_id", previewedPost.ChannelId)) + return false, nil + } + return false, err + } + + channelMembers, err := a.GetChannelMembersPage(post.ChannelId, 0, 10000000) + if err != nil { + return false, err + } + + for _, cm := range *channelMembers { + postForUser := post.Clone() + if !a.HasPermissionToReadChannel(cm.UserId, previewedChannel) { + postForUser.Metadata.Embeds[0].Data = nil + } + messageCopy := message.Copy() + broadcastCopy := messageCopy.GetBroadcast() + broadcastCopy.UserId = cm.UserId + messageCopy.SetBroadcast(broadcastCopy) + messageCopy.Add("post", postForUser.ToJson()) + a.Publish(messageCopy) + } + + return true, nil +} + func (a *App) PatchPost(c *request.Context, postID string, patch *model.PostPatch) (*model.Post, *model.AppError) { post, err := a.GetSinglePost(postID) if err != nil { @@ -1063,15 +1148,13 @@ func (a *App) DeletePost(postID, deleteByID string) (*model.Post, *model.AppErro } } - postData := a.PreparePostForClient(post, false, false).ToJson() - userMessage := model.NewWebSocketEvent(model.WebsocketEventPostDeleted, "", post.ChannelId, "", nil) - userMessage.Add("post", postData) + userMessage.Add("post", post) userMessage.GetBroadcast().ContainsSanitizedData = true a.Publish(userMessage) adminMessage := model.NewWebSocketEvent(model.WebsocketEventPostDeleted, "", post.ChannelId, "", nil) - adminMessage.Add("post", postData) + adminMessage.Add("post", post) adminMessage.Add("delete_by", deleteByID) adminMessage.GetBroadcast().ContainsSensitiveData = true a.Publish(adminMessage) diff --git a/app/post_metadata.go b/app/post_metadata.go index 43ffbe144d..66cd719110 100644 --- a/app/post_metadata.go +++ b/app/post_metadata.go @@ -5,11 +5,13 @@ package app import ( "bytes" + "fmt" "image" "io" "io/ioutil" "net/http" "net/url" + "regexp" "strconv" "strings" "time" @@ -26,6 +28,7 @@ import ( type linkMetadataCache struct { OpenGraph *opengraph.OpenGraph PostImage *model.PostImage + Permalink *model.Permalink } const LinkCacheSize = 10000 @@ -138,6 +141,41 @@ func (a *App) PreparePostForClient(originalPost *model.Post, isNewPost bool, isE return post } +func (a *App) SanitizePostMetadataForUser(post *model.Post, userID string) (*model.Post, *model.AppError) { + if post.Metadata == nil || len(post.Metadata.Embeds) == 0 { + return post, nil + } + + previewPost := post.GetPreviewPost() + if previewPost == nil { + return post, nil + } + + previewedChannel, err := a.GetChannel(previewPost.Post.ChannelId) + if err != nil { + return nil, err + } + + if previewedChannel != nil && !a.HasPermissionToReadChannel(userID, previewedChannel) { + post = post.Clone() + post.Metadata.Embeds[0].Data = nil + } + + return post, nil +} + +func (a *App) SanitizePostListMetadataForUser(postList *model.PostList, userID string) (*model.PostList, *model.AppError) { + clonedPostList := postList.Clone() + for postID, post := range clonedPostList.Posts { + sanitizedPost, err := a.SanitizePostMetadataForUser(post, userID) + if err != nil { + return nil, err + } + clonedPostList.Posts[postID] = sanitizedPost + } + return clonedPostList, nil +} + func (a *App) getFileMetadataForPost(post *model.Post, fromMaster bool) ([]*model.FileInfo, *model.AppError) { if len(post.FileIds) == 0 { return nil, nil @@ -171,15 +209,24 @@ func (a *App) getEmbedForPost(post *model.Post, firstLink string, isNewPost bool }, nil } - if firstLink == "" || !*a.Config().ServiceSettings.EnableLinkPreviews { + if firstLink == "" { return nil, nil } - og, image, err := a.getLinkMetadata(firstLink, post.CreateAt, isNewPost) + // Permalink previews are not toggled via the ServiceSettings.EnableLinkPreviews config setting. + if !*a.Config().ServiceSettings.EnableLinkPreviews && !looksLikeAPermalink(firstLink, *a.Config().ServiceSettings.SiteURL) { + return nil, nil + } + + og, image, permalink, err := a.getLinkMetadata(firstLink, post.CreateAt, isNewPost) if err != nil { return nil, err } + if !*a.Config().ServiceSettings.EnablePermalinkPreviews || !a.Config().FeatureFlags.PermalinkPreviews { + permalink = nil + } + if og != nil { return &model.PostEmbed{ Type: model.PostEmbedOpengraph, @@ -196,6 +243,10 @@ func (a *App) getEmbedForPost(post *model.Post, firstLink string, isNewPost bool }, nil } + if permalink != nil { + return &model.PostEmbed{Type: model.PostEmbedPermalink, Data: permalink.PreviewPost}, nil + } + return &model.PostEmbed{ Type: model.PostEmbedLink, URL: firstLink, @@ -238,7 +289,7 @@ func (a *App) getImagesForPost(post *model.Post, imageURLs []string, isNewPost b } for _, imageURL := range imageURLs { - if _, image, err := a.getLinkMetadata(imageURL, post.CreateAt, isNewPost); err != nil { + if _, image, _, err := a.getLinkMetadata(imageURL, post.CreateAt, isNewPost); err != nil { mlog.Debug("Failed to get dimensions of an image in a post", mlog.String("post_id", post.Id), mlog.String("image_url", imageURL), mlog.Err(err)) } else if image != nil { @@ -393,74 +444,114 @@ func (a *App) getImagesInMessageAttachments(post *model.Post) []string { return images } -func (a *App) getLinkMetadata(requestURL string, timestamp int64, isNewPost bool) (*opengraph.OpenGraph, *model.PostImage, error) { +func looksLikeAPermalink(url, siteURL string) bool { + expression := fmt.Sprintf(`^(%s).*(/pl/)[a-z0-9]{26}$`, siteURL) + matched, err := regexp.MatchString(expression, strings.TrimSpace(url)) + if err != nil { + mlog.Warn("error matching regex", mlog.Err(err)) + } + return matched +} + +func (a *App) getLinkMetadata(requestURL string, timestamp int64, isNewPost bool) (*opengraph.OpenGraph, *model.PostImage, *model.Permalink, error) { requestURL = resolveMetadataURL(requestURL, a.GetSiteURL()) timestamp = model.FloorToNearestHour(timestamp) // Check cache - og, image, ok := getLinkMetadataFromCache(requestURL, timestamp) + og, image, permalink, ok := getLinkMetadataFromCache(requestURL, timestamp) + if !*a.Config().ServiceSettings.EnablePermalinkPreviews || !a.Config().FeatureFlags.PermalinkPreviews { + permalink = nil + } + if ok { - return og, image, nil + return og, image, permalink, nil } // Check the database if this isn't a new post. If it is a new post and the data is cached, it should be in memory. if !isNewPost { og, image, ok = a.getLinkMetadataFromDatabase(requestURL, timestamp) if ok { - cacheLinkMetadata(requestURL, timestamp, og, image) - - return og, image, nil + cacheLinkMetadata(requestURL, timestamp, og, image, nil) + return og, image, nil, nil } } - // Make request for a web page or an image - request, err := http.NewRequest("GET", requestURL, nil) - if err != nil { - return nil, nil, err - } + var err error - var body io.ReadCloser - var contentType string + if looksLikeAPermalink(requestURL, a.GetSiteURL()) && *a.Config().ServiceSettings.EnablePermalinkPreviews && a.Config().FeatureFlags.PermalinkPreviews { + referencedPostID := requestURL[len(requestURL)-26:] - if (request.URL.Scheme+"://"+request.URL.Host) == a.GetSiteURL() && request.URL.Path == "/api/v4/image" { - // /api/v4/image requires authentication, so bypass the API by hitting the proxy directly - body, contentType, err = a.ImageProxy().GetImageDirect(a.ImageProxy().GetUnproxiedImageURL(request.URL.String())) + referencedPost, appErr := a.GetSinglePost(referencedPostID) + // Ignore 'not found' errors; post could have been deleted via retention policy so we don't want to permanently log a warning. + // + // TODO: Look into saving a value in the LinkMetadat.Data field to prevent perpetually re-querying for the deleted post. + if appErr != nil && appErr.StatusCode != http.StatusNotFound { + return nil, nil, nil, appErr + } + + referencedChannel, appErr := a.GetChannel(referencedPost.ChannelId) + if appErr != nil { + return nil, nil, nil, appErr + } + + referencedTeam, appErr := a.GetTeam(referencedChannel.TeamId) + if appErr != nil { + return nil, nil, nil, appErr + } + + permalink = &model.Permalink{PreviewPost: model.NewPreviewPost(referencedPost, referencedTeam, referencedChannel)} } else { - request.Header.Add("Accept", "image/*") - request.Header.Add("Accept", "text/html;q=0.8") - client := a.HTTPService().MakeClient(false) - client.Timeout = time.Duration(*a.Config().ExperimentalSettings.LinkMetadataTimeoutMilliseconds) * time.Millisecond - - var res *http.Response - res, err = client.Do(request) - - if res != nil { - body = res.Body - contentType = res.Header.Get("Content-Type") + var request *http.Request + // Make request for a web page or an image + request, err = http.NewRequest("GET", requestURL, nil) + if err != nil { + return nil, nil, nil, err } - } - if body != nil { - defer func() { - io.Copy(ioutil.Discard, body) - body.Close() - }() - } + var body io.ReadCloser + var contentType string - if err == nil { - // Parse the data - og, image, err = a.parseLinkMetadata(requestURL, body, contentType) + if (request.URL.Scheme+"://"+request.URL.Host) == a.GetSiteURL() && request.URL.Path == "/api/v4/image" { + // /api/v4/image requires authentication, so bypass the API by hitting the proxy directly + body, contentType, err = a.ImageProxy().GetImageDirect(a.ImageProxy().GetUnproxiedImageURL(request.URL.String())) + } else { + request.Header.Add("Accept", "image/*") + request.Header.Add("Accept", "text/html;q=0.8") + + client := a.HTTPService().MakeClient(false) + client.Timeout = time.Duration(*a.Config().ExperimentalSettings.LinkMetadataTimeoutMilliseconds) * time.Millisecond + + var res *http.Response + res, err = client.Do(request) + + if res != nil { + body = res.Body + contentType = res.Header.Get("Content-Type") + } + } + + if body != nil { + defer func() { + io.Copy(ioutil.Discard, body) + body.Close() + }() + } + + if err == nil { + // Parse the data + og, image, err = a.parseLinkMetadata(requestURL, body, contentType) + } + og = model.TruncateOpenGraph(og) // remove unwanted length of texts + + a.saveLinkMetadataToDatabase(requestURL, timestamp, og, image) } - og = model.TruncateOpenGraph(og) // remove unwanted length of texts // Write back to cache and database, even if there was an error and the results are nil - cacheLinkMetadata(requestURL, timestamp, og, image) + cacheLinkMetadata(requestURL, timestamp, og, image, permalink) - a.saveLinkMetadataToDatabase(requestURL, timestamp, og, image) - - return og, image, err + return og, image, permalink, err } // resolveMetadataURL resolves a given URL relative to the server's site URL. @@ -478,14 +569,14 @@ func resolveMetadataURL(requestURL string, siteURL string) string { return resolved.String() } -func getLinkMetadataFromCache(requestURL string, timestamp int64) (*opengraph.OpenGraph, *model.PostImage, bool) { +func getLinkMetadataFromCache(requestURL string, timestamp int64) (*opengraph.OpenGraph, *model.PostImage, *model.Permalink, bool) { var cached linkMetadataCache err := linkCache.Get(strconv.FormatInt(model.GenerateLinkMetadataHash(requestURL, timestamp), 16), &cached) if err != nil { - return nil, nil, false + return nil, nil, nil, false } - return cached.OpenGraph, cached.PostImage, true + return cached.OpenGraph, cached.PostImage, cached.Permalink, true } func (a *App) getLinkMetadataFromDatabase(requestURL string, timestamp int64) (*opengraph.OpenGraph, *model.PostImage, bool) { @@ -528,10 +619,11 @@ func (a *App) saveLinkMetadataToDatabase(requestURL string, timestamp int64, og } } -func cacheLinkMetadata(requestURL string, timestamp int64, og *opengraph.OpenGraph, image *model.PostImage) { +func cacheLinkMetadata(requestURL string, timestamp int64, og *opengraph.OpenGraph, image *model.PostImage, permalink *model.Permalink) { metadata := linkMetadataCache{ OpenGraph: og, PostImage: image, + Permalink: permalink, } linkCache.SetWithExpiry(strconv.FormatInt(model.GenerateLinkMetadataHash(requestURL, timestamp), 16), metadata, LinkCacheDuration) diff --git a/app/post_metadata_test.go b/app/post_metadata_test.go index 4b7ff395e8..3c468bf5db 100644 --- a/app/post_metadata_test.go +++ b/app/post_metadata_test.go @@ -550,6 +550,38 @@ func TestPreparePostForClient(t *testing.T) { assert.Nil(t, clientPost.Metadata.Reactions, "should not have populated Reactions") assert.Nil(t, clientPost.Metadata.Files, "should not have populated Files") }) + + t.Run("permalink preview", func(t *testing.T) { + th := setup(t) + defer th.TearDown() + + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ServiceSettings.SiteURL = "http://mymattermost.com" + }) + + th.Context.Session().UserId = th.BasicUser.Id + + referencedPost, err := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: th.BasicChannel.Id, + Message: "hello world", + }, th.BasicChannel, false, true) + require.Nil(t, err) + + link := fmt.Sprintf("%s/%s/pl/%s", *th.App.Config().ServiceSettings.SiteURL, th.BasicTeam.Name, referencedPost.Id) + + previewPost, err := th.App.CreatePost(th.Context, &model.Post{ + UserId: th.BasicUser.Id, + ChannelId: th.BasicChannel.Id, + Message: link, + }, th.BasicChannel, false, true) + require.Nil(t, err) + + clientPost := th.App.PreparePostForClient(previewPost, false, false) + firstEmbed := clientPost.Metadata.Embeds[0] + preview := firstEmbed.Data.(*model.PreviewPost) + require.Equal(t, referencedPost.Id, preview.PostID) + }) } func TestPreparePostForClientWithImageProxy(t *testing.T) { @@ -1688,16 +1720,16 @@ func TestGetLinkMetadata(t *testing.T) { timestamp := int64(1547510400000) title := "from cache" - cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: title}, nil) + cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: title}, nil, nil) t.Run("should use cache if cached entry exists", func(t *testing.T) { - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.True(t, ok, "data should already exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) require.NotNil(t, og) assert.Nil(t, img) @@ -1706,13 +1738,13 @@ func TestGetLinkMetadata(t *testing.T) { }) t.Run("should use cache if cached entry exists near time", func(t *testing.T) { - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.True(t, ok, "data should already exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false) require.NotNil(t, og) assert.Nil(t, img) @@ -1723,13 +1755,13 @@ func TestGetLinkMetadata(t *testing.T) { t.Run("should not use cache if URL is different", func(t *testing.T) { differentURL := server.URL + "/other" - _, _, ok := getLinkMetadataFromCache(differentURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(differentURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(differentURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(differentURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(differentURL, timestamp, false) assert.Nil(t, og) assert.Nil(t, img) @@ -1739,13 +1771,13 @@ func TestGetLinkMetadata(t *testing.T) { t.Run("should not use cache if timestamp is different", func(t *testing.T) { differentTimestamp := timestamp + 60*60*1000 - _, _, ok := getLinkMetadataFromCache(requestURL, differentTimestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, differentTimestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, differentTimestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false) assert.Nil(t, og) assert.Nil(t, img) @@ -1766,13 +1798,13 @@ func TestGetLinkMetadata(t *testing.T) { t.Run("should use database if saved entry exists", func(t *testing.T) { linkCache.Purge() - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.True(t, ok, "data should already exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) require.NotNil(t, og) assert.Nil(t, img) @@ -1783,13 +1815,13 @@ func TestGetLinkMetadata(t *testing.T) { t.Run("should use database if saved entry exists near time", func(t *testing.T) { linkCache.Purge() - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.True(t, ok, "data should already exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false) require.NotNil(t, og) assert.Nil(t, img) @@ -1802,13 +1834,13 @@ func TestGetLinkMetadata(t *testing.T) { differentURL := requestURL + "/other" - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(differentURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(differentURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(differentURL, timestamp, false) assert.Nil(t, og) assert.Nil(t, img) @@ -1820,13 +1852,13 @@ func TestGetLinkMetadata(t *testing.T) { differentTimestamp := timestamp + 60*60*1000 - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, differentTimestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false) assert.Nil(t, og) assert.Nil(t, img) @@ -1841,13 +1873,13 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/opengraph?title=Remote&name=" + t.Name() timestamp := int64(1547510400000) - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.NotNil(t, og) assert.Nil(t, img) @@ -1861,19 +1893,19 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/opengraph?title=Remote&name=" + t.Name() timestamp := int64(1547510400000) - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.NotNil(t, og) assert.Nil(t, img) assert.NoError(t, err) - fromCache, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + fromCache, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) assert.True(t, ok) assert.Exactly(t, og, fromCache) @@ -1889,19 +1921,19 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/image?height=300&width=400&name=" + t.Name() timestamp := int64(1547510400000) - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.Nil(t, og) assert.NotNil(t, img) assert.NoError(t, err) - _, fromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, fromCache, _, ok := getLinkMetadataFromCache(requestURL, timestamp) assert.True(t, ok) assert.Exactly(t, img, fromCache) @@ -1917,19 +1949,19 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/error" timestamp := int64(1547510400000) - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.Nil(t, og) assert.Nil(t, img) assert.NoError(t, err) - ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) + ogFromCache, imgFromCache, _, ok := getLinkMetadataFromCache(requestURL, timestamp) assert.True(t, ok) assert.Nil(t, ogFromCache) assert.Nil(t, imgFromCache) @@ -1947,19 +1979,19 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := "http://notarealdomainthatactuallyexists.ca/?name=" + t.Name() timestamp := int64(1547510400000) - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.Nil(t, og) assert.Nil(t, img) assert.IsType(t, &url.Error{}, err) - ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) + ogFromCache, imgFromCache, _, ok := getLinkMetadataFromCache(requestURL, timestamp) assert.True(t, ok) assert.Nil(t, ogFromCache) assert.Nil(t, imgFromCache) @@ -1981,20 +2013,20 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/timeout?name=" + t.Name() timestamp := int64(1547510400000) - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.Nil(t, og) assert.Nil(t, img) assert.Error(t, err) assert.True(t, os.IsTimeout(err)) - ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) + ogFromCache, imgFromCache, _, ok := getLinkMetadataFromCache(requestURL, timestamp) assert.True(t, ok) assert.Nil(t, ogFromCache) assert.Nil(t, imgFromCache) @@ -2012,20 +2044,20 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/image?height=300&width=400&name=" + t.Name() timestamp := int64(1547510400000) - _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should not exist in in-memory cache") _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) require.False(t, ok, "data should not exist in database") - _, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + _, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) require.NoError(t, err) - _, _, ok = getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok = getLinkMetadataFromCache(requestURL, timestamp) require.True(t, ok, "data should now exist in in-memory cache") linkCache.Purge() - _, _, ok = getLinkMetadataFromCache(requestURL, timestamp) + _, _, _, ok = getLinkMetadataFromCache(requestURL, timestamp) require.False(t, ok, "data should no longer exist in in-memory cache") _, fromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) @@ -2040,7 +2072,7 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/json?name=" + t.Name() timestamp := int64(1547510400000) - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.Nil(t, og) assert.Nil(t, img) assert.NoError(t, err) @@ -2053,9 +2085,9 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/error?name=" + t.Name() timestamp := int64(1547510400000) - cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil) + cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil, nil) - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, true) assert.NotNil(t, og) assert.Nil(t, img) assert.NoError(t, err) @@ -2070,7 +2102,7 @@ func TestGetLinkMetadata(t *testing.T) { th.App.saveLinkMetadataToDatabase(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil) - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, true) assert.Nil(t, og) assert.Nil(t, img) assert.NoError(t, err) @@ -2093,7 +2125,7 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := "/image?height=200&width=300&name=" + t.Name() timestamp := int64(1547510400000) - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.Nil(t, og) assert.NotNil(t, img) assert.NoError(t, err) @@ -2121,7 +2153,7 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/image?height=200&width=300&name=" + t.Name() timestamp := int64(1547510400000) - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, false) assert.Nil(t, og) assert.Nil(t, img) assert.Error(t, err) @@ -2131,7 +2163,7 @@ func TestGetLinkMetadata(t *testing.T) { requestURL = th.App.GetSiteURL() + "/api/v4/image?url=" + url.QueryEscape(requestURL) // Note that this request still fails while testing because the request made by the image proxy is blocked - og, img, err = th.App.getLinkMetadata(requestURL, timestamp, false) + og, img, _, err = th.App.getLinkMetadata(requestURL, timestamp, false) assert.Nil(t, og) assert.Nil(t, img) assert.Error(t, err) @@ -2145,7 +2177,7 @@ func TestGetLinkMetadata(t *testing.T) { requestURL := server.URL + "/mixed?name=" + t.Name() timestamp := int64(1547510400000) - og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true) + og, img, _, err := th.App.getLinkMetadata(requestURL, timestamp, true) assert.Nil(t, og) assert.NotNil(t, img) assert.NoError(t, err) @@ -2323,3 +2355,33 @@ func TestParseImages(t *testing.T) { }) } } + +func TestLooksLikeAPermalink(t *testing.T) { + const siteURLWithSubpath = "http://localhost:8065/foo" + const siteURLWithTrailingSlash = "http://test.com/" + const siteURL = "http://test.com" + tests := map[string]struct { + input string + siteURL string + expect bool + }{ + "happy path": {input: fmt.Sprintf("%s/private-core/pl/dppezk51jp8afbhwxf1jpag66r", siteURLWithSubpath), siteURL: siteURLWithSubpath, expect: true}, + "looks nothing like a permalink": {input: "foobar", siteURL: siteURLWithSubpath, expect: false}, + "link has no subpath": {input: fmt.Sprintf("%s/private-core/pl/dppezk51jp8afbhwxf1jpag66r", "http://localhost:8065"), siteURL: siteURLWithSubpath, expect: false}, + "without port": {input: fmt.Sprintf("%s/private-core/pl/dppezk51jp8afbhwxf1jpag66r", "http://localhost/foo"), siteURL: siteURLWithSubpath, expect: false}, + "wrong port": {input: fmt.Sprintf("%s/private-core/pl/dppezk51jp8afbhwxf1jpag66r", "http://localhost:8066"), siteURL: siteURLWithSubpath, expect: false}, + "invalid post ID length": {input: fmt.Sprintf("%s/private-core/pl/dppezk51jp8afbhwxf1jpag66", siteURLWithSubpath), siteURL: siteURLWithSubpath, expect: false}, + "invalid post ID character": {input: fmt.Sprintf("%s/private-core/pl/dppezk51jp8$fbhwxf1jpag66r", siteURLWithSubpath), siteURL: siteURLWithSubpath, expect: false}, + "leading whitespace": {input: fmt.Sprintf(" %s/private-core/pl/dppezk51jp8afbhwxf1jpag66r", siteURLWithSubpath), siteURL: siteURLWithSubpath, expect: true}, + "trailing whitespace": {input: fmt.Sprintf("%s/private-core/pl/dppezk51jp8afbhwxf1jpag66r ", siteURLWithSubpath), siteURL: siteURLWithSubpath, expect: true}, + "siteURL without a subpath": {input: fmt.Sprintf("%sprivate-core/pl/dppezk51jp8afbhwxf1jpag66r", siteURLWithTrailingSlash), siteURL: siteURLWithTrailingSlash, expect: true}, + "siteURL without a trailing slash": {input: fmt.Sprintf("%s/private-core/pl/dppezk51jp8afbhwxf1jpag66r", siteURL), siteURL: siteURL, expect: true}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + actual := looksLikeAPermalink(tc.input, tc.siteURL) + assert.Equal(t, tc.expect, actual) + }) + } +} diff --git a/app/post_test.go b/app/post_test.go index c4c38ca6d8..a695cf1af6 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -651,7 +651,7 @@ func TestDeletePostWithFileAttachments(t *testing.T) { assert.Nil(t, err) // Delete the post. - post, err = th.App.DeletePost(post.Id, userID) + _, err = th.App.DeletePost(post.Id, userID) assert.Nil(t, err) // Wait for the cleanup routine to finish. @@ -755,6 +755,41 @@ func TestCreatePost(t *testing.T) { th.AddPermissionToRole(model.PermissionUseChannelMentions.Id, model.ChannelAdminRoleId) }) }) + + t.Run("Sets PostPropsPreviewedPost when a permalink is the first link", func(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + th.AddUserToChannel(th.BasicUser, th.BasicChannel) + + referencedPost := &model.Post{ + ChannelId: th.BasicChannel.Id, + Message: "hello world", + UserId: th.BasicUser.Id, + } + + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ServiceSettings.SiteURL = "http://mymattermost.com" + }) + + th.Context.Session().UserId = th.BasicUser.Id + + referencedPost, err := th.App.CreatePost(th.Context, referencedPost, th.BasicChannel, false, false) + require.Nil(t, err) + + permalink := fmt.Sprintf("%s/%s/pl/%s", *th.App.Config().ServiceSettings.SiteURL, th.BasicTeam.Name, referencedPost.Id) + + channelForPreview := th.CreateChannel(th.BasicTeam) + previewPost := &model.Post{ + ChannelId: channelForPreview.Id, + Message: permalink, + UserId: th.BasicUser.Id, + } + + previewPost, err = th.App.CreatePost(th.Context, previewPost, channelForPreview, false, false) + require.Nil(t, err) + + assert.Equal(t, previewPost.GetProps(), model.StringInterface{"previewed_post": referencedPost.Id}) + }) } func TestPatchPost(t *testing.T) { @@ -1085,6 +1120,45 @@ func TestUpdatePost(t *testing.T) { require.Nil(t, err) assert.Equal(t, "![image]("+proxiedImageURL+")", rpost.Message) }) + + t.Run("Sets PostPropsPreviewedPost when a post is updated to have a permalink as the first link", func(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + th.AddUserToChannel(th.BasicUser, th.BasicChannel) + + referencedPost := &model.Post{ + ChannelId: th.BasicChannel.Id, + Message: "hello world", + UserId: th.BasicUser.Id, + } + + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ServiceSettings.SiteURL = "http://mymattermost.com" + }) + + th.Context.Session().UserId = th.BasicUser.Id + + referencedPost, err := th.App.CreatePost(th.Context, referencedPost, th.BasicChannel, false, false) + require.Nil(t, err) + + permalink := fmt.Sprintf("%s/%s/pl/%s", *th.App.Config().ServiceSettings.SiteURL, th.BasicTeam.Name, referencedPost.Id) + + channelForTestPost := th.CreateChannel(th.BasicTeam) + testPost := &model.Post{ + ChannelId: channelForTestPost.Id, + Message: "hello world", + UserId: th.BasicUser.Id, + } + + testPost, err = th.App.CreatePost(th.Context, testPost, channelForTestPost, false, false) + require.Nil(t, err) + assert.Equal(t, testPost.GetProps(), model.StringInterface{}) + + testPost.Message = permalink + testPost, err = th.App.UpdatePost(th.Context, testPost, false) + require.Nil(t, err) + assert.Equal(t, testPost.GetProps(), model.StringInterface{"previewed_post": referencedPost.Id}) + }) } func TestSearchPostsInTeamForUser(t *testing.T) { diff --git a/app/ratelimit.go b/app/ratelimit.go index b05e082e25..cf6799ffc6 100644 --- a/app/ratelimit.go +++ b/app/ratelimit.go @@ -86,7 +86,7 @@ func (rl *RateLimiter) RateLimitWriter(key string, w http.ResponseWriter) bool { if limited { mlog.Debug("Denied due to throttling settings code=429", mlog.String("key", key)) - http.Error(w, "limit exceeded", 429) + http.Error(w, "limit exceeded", http.StatusTooManyRequests) } return limited diff --git a/app/role_test.go b/app/role_test.go index f59cd0da4f..0a184be28e 100644 --- a/app/role_test.go +++ b/app/role_test.go @@ -221,7 +221,7 @@ func testPermissionInheritance(t *testing.T, testCallback func(t *testing.T, th // assign the scheme to the team team.SchemeId = &teamScheme.Id - team, err = th.App.UpdateTeamScheme(team) + _, err = th.App.UpdateTeamScheme(team) require.Nil(t, err) // test 24 combinations where the higher-scoped scheme is a TEAM scheme diff --git a/app/shared_channel.go b/app/shared_channel.go index 1101eb8d2c..8478ad9b9c 100644 --- a/app/shared_channel.go +++ b/app/shared_channel.go @@ -22,7 +22,7 @@ func (a *App) checkChannelNotShared(channelId string) error { if _, err := a.GetSharedChannel(channelId); err == nil { var errNotFound *store.ErrNotFound if errors.As(err, &errNotFound) { - return errors.New("channel is already shared.") + return errors.New("channel is already shared") } return fmt.Errorf("cannot find channel: %w", err) } @@ -33,7 +33,7 @@ func (a *App) checkChannelIsShared(channelId string) error { if _, err := a.GetSharedChannel(channelId); err != nil { var errNotFound *store.ErrNotFound if errors.As(err, &errNotFound) { - return errors.New("channel is not shared.") + return errors.New("channel is not shared") } return fmt.Errorf("cannot find channel: %w", err) } @@ -45,13 +45,13 @@ func (a *App) CheckCanInviteToSharedChannel(channelId string) error { if err != nil { var errNotFound *store.ErrNotFound if errors.As(err, &errNotFound) { - return errors.New("channel is not shared.") + return errors.New("channel is not shared") } return fmt.Errorf("cannot find channel: %w", err) } if !sc.Home { - return errors.New("channel is homed on a remote cluster.") + return errors.New("channel is homed on a remote cluster") } return nil } diff --git a/app/syncables_test.go b/app/syncables_test.go index 416b227eff..376b0e7a9a 100644 --- a/app/syncables_test.go +++ b/app/syncables_test.go @@ -169,7 +169,7 @@ func TestCreateDefaultMemberships(t *testing.T) { // update AutoAdd to true scienceTeamGroupSyncable.AutoAdd = true - scienceTeamGroupSyncable, err = th.App.UpdateGroupSyncable(scienceTeamGroupSyncable) + _, err = th.App.UpdateGroupSyncable(scienceTeamGroupSyncable) if err != nil { t.Errorf("error updating group syncable: %s", err.Error()) } diff --git a/app/user_test.go b/app/user_test.go index 5874afbf44..d839ba628e 100644 --- a/app/user_test.go +++ b/app/user_test.go @@ -1162,7 +1162,7 @@ func TestPromoteGuestToUser(t *testing.T) { assert.Nil(t, err) assert.False(t, teamMember.SchemeGuest) assert.True(t, teamMember.SchemeUser) - channelMember, err = th.App.GetChannelMember(context.Background(), th.BasicChannel.Id, guest.Id) + _, err = th.App.GetChannelMember(context.Background(), th.BasicChannel.Id, guest.Id) assert.Nil(t, err) assert.False(t, teamMember.SchemeGuest) assert.True(t, teamMember.SchemeUser) @@ -1194,7 +1194,7 @@ func TestPromoteGuestToUser(t *testing.T) { assert.Nil(t, err) assert.False(t, teamMember.SchemeGuest) assert.True(t, teamMember.SchemeUser) - channelMember, err = th.App.GetChannelMember(context.Background(), th.BasicChannel.Id, guest.Id) + _, err = th.App.GetChannelMember(context.Background(), th.BasicChannel.Id, guest.Id) assert.Nil(t, err) assert.False(t, teamMember.SchemeGuest) assert.True(t, teamMember.SchemeUser) @@ -1325,7 +1325,7 @@ func TestDemoteUserToGuest(t *testing.T) { assert.Nil(t, err) assert.False(t, teamMember.SchemeUser) assert.True(t, teamMember.SchemeGuest) - channelMember, err = th.App.GetChannelMember(context.Background(), th.BasicChannel.Id, user.Id) + _, err = th.App.GetChannelMember(context.Background(), th.BasicChannel.Id, user.Id) assert.Nil(t, err) assert.False(t, teamMember.SchemeUser) assert.True(t, teamMember.SchemeGuest) @@ -1357,7 +1357,7 @@ func TestDemoteUserToGuest(t *testing.T) { assert.Nil(t, err) assert.False(t, teamMember.SchemeUser) assert.True(t, teamMember.SchemeGuest) - channelMember, err = th.App.GetChannelMember(context.Background(), th.BasicChannel.Id, user.Id) + _, err = th.App.GetChannelMember(context.Background(), th.BasicChannel.Id, user.Id) assert.Nil(t, err) assert.False(t, teamMember.SchemeUser) assert.True(t, teamMember.SchemeGuest) diff --git a/app/web_conn.go b/app/web_conn.go index 2489a11f43..c1d4b918d1 100644 --- a/app/web_conn.go +++ b/app/web_conn.go @@ -126,7 +126,7 @@ func (a *App) PopulateWebConnConfig(s *model.Session, cfg *WebConnConfig, seqVal if seqVal == "" { // Sequence_number must be sent with connection id. // A client must be either non-compliant or fully compliant. - return nil, errors.New("Sequence number not present in websocket request") + return nil, errors.New("sequence number not present in websocket request") } var err error cfg.sequence, err = strconv.Atoi(seqVal) diff --git a/config/client.go b/config/client.go index 51f990f763..d0690ff43f 100644 --- a/config/client.go +++ b/config/client.go @@ -35,6 +35,7 @@ func GenerateClientConfig(c *model.Config, telemetryID string, license *model.Li props["EnablePostIconOverride"] = strconv.FormatBool(*c.ServiceSettings.EnablePostIconOverride) props["EnableUserAccessTokens"] = strconv.FormatBool(*c.ServiceSettings.EnableUserAccessTokens) props["EnableLinkPreviews"] = strconv.FormatBool(*c.ServiceSettings.EnableLinkPreviews) + props["EnablePermalinkPreviews"] = strconv.FormatBool(*c.ServiceSettings.EnablePermalinkPreviews) props["EnableTesting"] = strconv.FormatBool(*c.ServiceSettings.EnableTesting) props["EnableDeveloper"] = strconv.FormatBool(*c.ServiceSettings.EnableDeveloper) props["PostEditTimeLimit"] = fmt.Sprintf("%v", *c.ServiceSettings.PostEditTimeLimit) diff --git a/model/config.go b/model/config.go index a0da8364c0..ae9d268732 100644 --- a/model/config.go +++ b/model/config.go @@ -303,6 +303,7 @@ type ServiceSettings struct { GoogleDeveloperKey *string `access:"site_posts,write_restrictable,cloud_restrictable"` DEPRECATED_DO_NOT_USE_EnableOnlyAdminIntegrations *bool `json:"EnableOnlyAdminIntegrations" mapstructure:"EnableOnlyAdminIntegrations"` // Deprecated: do not use EnableLinkPreviews *bool `access:"site_posts"` + EnablePermalinkPreviews *bool `access:"site_posts"` RestrictLinkPreviews *string `access:"site_posts"` EnableTesting *bool `access:"environment_developer,write_restrictable,cloud_restrictable"` EnableDeveloper *bool `access:"environment_developer,write_restrictable,cloud_restrictable"` @@ -413,6 +414,10 @@ func (s *ServiceSettings) SetDefaults(isUpdate bool) { s.EnableLinkPreviews = NewBool(true) } + if s.EnablePermalinkPreviews == nil { + s.EnablePermalinkPreviews = NewBool(true) + } + if s.RestrictLinkPreviews == nil { s.RestrictLinkPreviews = NewString("") } diff --git a/model/feature_flags.go b/model/feature_flags.go index d729af545e..1fab30302e 100644 --- a/model/feature_flags.go +++ b/model/feature_flags.go @@ -36,6 +36,8 @@ type FeatureFlags struct { // Enable timed dnd support for user status TimedDND bool + PermalinkPreviews bool + // Enable the Global Header GlobalHeader bool @@ -54,6 +56,7 @@ func (f *FeatureFlags) SetDefaults() { f.PluginApps = "" f.PluginFocalboard = "" f.TimedDND = false + f.PermalinkPreviews = true f.GlobalHeader = false f.InviteMembersButton = "none" } diff --git a/model/message_export.go b/model/message_export.go index 92eca9d66e..cc8f882b39 100644 --- a/model/message_export.go +++ b/model/message_export.go @@ -3,6 +3,8 @@ package model +import "encoding/json" + type MessageExport struct { TeamId *string TeamName *string @@ -34,3 +36,15 @@ type MessageExportCursor struct { LastPostUpdateAt int64 LastPostId string } + +// PreviewID returns the value of the post's previewed_post prop, if present, or an empty string. +func (m *MessageExport) PreviewID() string { + var previewID string + props := map[string]interface{}{} + if m.PostProps != nil && json.Unmarshal([]byte(*m.PostProps), &props) == nil { + if val, ok := props[PostPropsPreviewedPost]; ok { + previewID = val.(string) + } + } + return previewID +} diff --git a/model/permalink.go b/model/permalink.go new file mode 100644 index 0000000000..b6d3284ed3 --- /dev/null +++ b/model/permalink.go @@ -0,0 +1,34 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package model + +import "encoding/json" + +type Permalink struct { + PreviewPost *PreviewPost `json:"preview_post"` +} + +type PreviewPost struct { + PostID string `json:"post_id"` + Post *Post `json:"post"` + TeamName string `json:"team_name"` + ChannelDisplayName string `json:"channel_display_name"` +} + +func NewPreviewPost(post *Post, team *Team, channel *Channel) *PreviewPost { + if post == nil { + return nil + } + return &PreviewPost{ + PostID: post.Id, + Post: post, + TeamName: team.Name, + ChannelDisplayName: channel.DisplayName, + } +} + +func (o *Permalink) ToJson() string { + b, _ := json.Marshal(o) + return string(b) +} diff --git a/model/post.go b/model/post.go index b711e58b29..59f46522e0 100644 --- a/model/post.go +++ b/model/post.go @@ -67,6 +67,8 @@ const ( PostPropsMentionHighlightDisabled = "mentionHighlightDisabled" PostPropsGroupHighlightDisabled = "disable_group_highlight" + + PostPropsPreviewedPost = "previewed_post" ) type Post struct { @@ -749,3 +751,14 @@ func (o *Post) ToNilIfInvalid() *Post { } return o } + +func (o *Post) GetPreviewPost() *PreviewPost { + for _, embed := range o.Metadata.Embeds { + if embed.Type == PostEmbedPermalink { + if previewPost, ok := embed.Data.(*PreviewPost); ok { + return previewPost + } + } + } + return nil +} diff --git a/model/post_embed.go b/model/post_embed.go index 8df923d256..ea3e2c5bc8 100644 --- a/model/post_embed.go +++ b/model/post_embed.go @@ -8,6 +8,7 @@ const ( PostEmbedMessageAttachment PostEmbedType = "message_attachment" PostEmbedOpengraph PostEmbedType = "opengraph" PostEmbedLink PostEmbedType = "link" + PostEmbedPermalink PostEmbedType = "permalink" ) type PostEmbedType string diff --git a/model/post_list.go b/model/post_list.go index ba84f7493d..5236dd88d9 100644 --- a/model/post_list.go +++ b/model/post_list.go @@ -25,6 +25,23 @@ func NewPostList() *PostList { } } +func (o *PostList) Clone() *PostList { + orderCopy := make([]string, len(o.Order)) + postsCopy := make(map[string]*Post) + for i, v := range o.Order { + orderCopy[i] = v + } + for k, v := range o.Posts { + postsCopy[k] = v.Clone() + } + return &PostList{ + Order: orderCopy, + Posts: postsCopy, + NextPostId: o.NextPostId, + PrevPostId: o.PrevPostId, + } +} + func (o *PostList) ToSlice() []*Post { var posts []*Post diff --git a/services/telemetry/telemetry.go b/services/telemetry/telemetry.go index 377cae60b4..f1cd6a376f 100644 --- a/services/telemetry/telemetry.go +++ b/services/telemetry/telemetry.go @@ -442,6 +442,7 @@ func (ts *TelemetryService) trackConfig() { "enable_legacy_sidebar": *cfg.ServiceSettings.EnableLegacySidebar, "thread_auto_follow": *cfg.ServiceSettings.ThreadAutoFollow, "enable_link_previews": *cfg.ServiceSettings.EnableLinkPreviews, + "enable_permalink_previews": *cfg.ServiceSettings.EnablePermalinkPreviews, "enable_file_search": *cfg.ServiceSettings.EnableFileSearch, "restrict_link_previews": isDefault(*cfg.ServiceSettings.RestrictLinkPreviews, ""), }) diff --git a/templates/globalrelay_compliance_export_message.html b/templates/globalrelay_compliance_export_message.html index 27de9cf80d..74d07bc89d 100644 --- a/templates/globalrelay_compliance_export_message.html +++ b/templates/globalrelay_compliance_export_message.html @@ -6,5 +6,6 @@ {{.Props.UserType}} {{.Props.Message}} + {{.Props.PreviewsPost}} {{end}} diff --git a/tests/test-config.json b/tests/test-config.json index 3926ddf864..96665d18fb 100644 --- a/tests/test-config.json +++ b/tests/test-config.json @@ -23,6 +23,7 @@ "EnablePostUsernameOverride": false, "EnablePostIconOverride": false, "EnableLinkPreviews": false, + "EnablePermalinkPreviews": false, "EnableTesting": false, "EnableDeveloper": false, "EnableSecurityFixAlert": true,