diff --git a/api4/post.go b/api4/post.go index 4e63b1b087..99187afa6f 100644 --- a/api4/post.go +++ b/api4/post.go @@ -381,6 +381,7 @@ func getFlaggedPostsForUser(c *Context, w http.ResponseWriter, r *http.Request) } } +// getPost also sets a header to indicate, if post is inaccessible due to the cloud plan's limit. func getPost(c *Context, w http.ResponseWriter, r *http.Request) { c.RequirePostId() if c.Err != nil { @@ -396,6 +397,12 @@ func getPost(c *Context, w http.ResponseWriter, r *http.Request) { post, err := c.App.GetPostIfAuthorized(c.Params.PostId, c.AppContext.Session(), includeDeleted) if err != nil { c.Err = err + + // Post is inaccessible due to cloud plan's limit. + if err.Id == "app.post.cloud.get.app_error" { + w.Header().Set(model.HeaderHasInaccessiblePosts, "true") + } + return } @@ -416,6 +423,7 @@ func getPost(c *Context, w http.ResponseWriter, r *http.Request) { } } +// getPostsByIds also sets a header to indicate, if posts were truncated as per the cloud plan's limit. func getPostsByIds(c *Context, w http.ResponseWriter, r *http.Request) { postIDs := model.ArrayFromJSON(r.Body) @@ -429,7 +437,7 @@ func getPostsByIds(c *Context, w http.ResponseWriter, r *http.Request) { return } - postsList, err := c.App.GetPostsByIds(postIDs) + postsList, hasInaccessiblePosts, err := c.App.GetPostsByIds(postIDs) if err != nil { c.Err = err return @@ -462,6 +470,8 @@ func getPostsByIds(c *Context, w http.ResponseWriter, r *http.Request) { posts = append(posts, post) } + w.Header().Set(model.HeaderHasInaccessiblePosts, strconv.FormatBool(hasInaccessiblePosts)) + if err := json.NewEncoder(w).Encode(posts); err != nil { mlog.Warn("Error while writing response", mlog.Err(err)) } diff --git a/app/app_iface.go b/app/app_iface.go index 0e708bf8c3..375c945203 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -76,6 +76,9 @@ type AppIface interface { CheckProviderAttributes(user *model.User, patch *model.UserPatch) string // ClientConfigWithComputed gets the configuration in a format suitable for sending to the client. ClientConfigWithComputed() map[string]string + // ComputeLastAccessiblePostTime updates cache with CreateAt time of the last accessible post as per the cloud plan's limit. + // Use GetLastAccessiblePostTime() to access the result. + ComputeLastAccessiblePostTime() *model.AppError // ConvertBotToUser converts a bot to user. ConvertBotToUser(bot *model.Bot, userPatch *model.UserPatch, sysadmin bool) (*model.User, *model.AppError) // ConvertUserToBot converts a user to bot. @@ -181,6 +184,8 @@ type AppIface interface { // relationship with a user. That means any user sharing any channel, including // direct and group channels. GetKnownUsers(userID string) ([]string, *model.AppError) + // GetLastAccessiblePostTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit + GetLastAccessiblePostTime() (int64, *model.AppError) // GetLdapGroup retrieves a single LDAP group by the given LDAP group id. GetLdapGroup(ldapGroupID string) (*model.Group, *model.AppError) // GetMarketplacePlugins returns a list of plugins from the marketplace-server, @@ -196,6 +201,8 @@ type AppIface interface { // To get the plugins environment when the plugins are disabled, manually acquire the plugins // lock instead. GetPluginsEnvironment() *plugin.Environment + // GetPostsByIds response bool value indicates, if the post is inaccessible due to cloud plan's limit. + GetPostsByIds(postIDs []string) ([]*model.Post, bool, *model.AppError) // GetPostsUsage returns the total posts count rounded down to the most // significant digit GetPostsUsage() (int64, *model.AppError) @@ -697,7 +704,6 @@ type AppIface interface { GetPostsAfterPost(options model.GetPostsOptions) (*model.PostList, *model.AppError) GetPostsAroundPost(before bool, options model.GetPostsOptions) (*model.PostList, *model.AppError) GetPostsBeforePost(options model.GetPostsOptions) (*model.PostList, *model.AppError) - GetPostsByIds(postIDs []string) ([]*model.Post, *model.AppError) GetPostsEtag(channelID string, collapsedThreads bool) string GetPostsForChannelAroundLastUnread(channelID, userID string, limitBefore, limitAfter int, skipFetchThreads bool, collapsedThreads, collapsedThreadsExtended bool) (*model.PostList, *model.AppError) GetPostsPage(options model.GetPostsOptions) (*model.PostList, *model.AppError) diff --git a/app/channel.go b/app/channel.go index 54ab0be2f0..3825a90a58 100644 --- a/app/channel.go +++ b/app/channel.go @@ -3172,6 +3172,10 @@ func (a *App) GetPinnedPosts(channelID string) (*model.PostList, *model.AppError return nil, model.NewAppError("GetPinnedPosts", "app.channel.pinned_posts.app_error", nil, err.Error(), http.StatusInternalServerError) } + if appErr := a.filterInaccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil { + return nil, appErr + } + return posts, nil } diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 3e8784a8c0..2ddc75c5cc 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -1737,6 +1737,28 @@ func (a *OpenTracingAppLayer) CompleteSwitchWithOAuth(service string, userData i return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) ComputeLastAccessiblePostTime() *model.AppError { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ComputeLastAccessiblePostTime") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0 := a.app.ComputeLastAccessiblePostTime() + + if resultVar0 != nil { + span.LogFields(spanlog.Error(resultVar0)) + ext.Error.Set(span, true) + } + + return resultVar0 +} + func (a *OpenTracingAppLayer) Config() *model.Config { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.Config") @@ -6779,6 +6801,28 @@ func (a *OpenTracingAppLayer) GetKnownUsers(userID string) ([]string, *model.App return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) GetLastAccessiblePostTime() (int64, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLastAccessiblePostTime") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.GetLastAccessiblePostTime() + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) GetLatestTermsOfService() (*model.TermsOfService, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLatestTermsOfService") @@ -7837,7 +7881,7 @@ func (a *OpenTracingAppLayer) GetPostsBeforePost(options model.GetPostsOptions) return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) GetPostsByIds(postIDs []string) ([]*model.Post, *model.AppError) { +func (a *OpenTracingAppLayer) GetPostsByIds(postIDs []string) ([]*model.Post, bool, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetPostsByIds") @@ -7849,14 +7893,14 @@ func (a *OpenTracingAppLayer) GetPostsByIds(postIDs []string) ([]*model.Post, *m }() defer span.Finish() - resultVar0, resultVar1 := a.app.GetPostsByIds(postIDs) + resultVar0, resultVar1, resultVar2 := a.app.GetPostsByIds(postIDs) - if resultVar1 != nil { - span.LogFields(spanlog.Error(resultVar1)) + if resultVar2 != nil { + span.LogFields(spanlog.Error(resultVar2)) ext.Error.Set(span, true) } - return resultVar0, resultVar1 + return resultVar0, resultVar1, resultVar2 } func (a *OpenTracingAppLayer) GetPostsEtag(channelID string, collapsedThreads bool) string { diff --git a/app/post.go b/app/post.go index c174bb99f8..36e6632ea9 100644 --- a/app/post.go +++ b/app/post.go @@ -10,6 +10,7 @@ import ( "fmt" "net/http" "regexp" + "strconv" "strings" "sync" "time" @@ -809,6 +810,11 @@ func (a *App) GetPostsPage(options model.GetPostsOptions) (*model.PostList, *mod } } + // The postList is sorted as only rootPosts Order is included + if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -824,6 +830,10 @@ func (a *App) GetPosts(channelID string, offset int, limit int) (*model.PostList } } + if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{}); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -837,6 +847,10 @@ func (a *App) GetPostsSince(options model.GetPostsSinceOptions) (*model.PostList return nil, model.NewAppError("GetPostsSince", "app.post.get_posts_since.app_error", nil, err.Error(), http.StatusInternalServerError) } + if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -852,6 +866,14 @@ func (a *App) GetSinglePost(postID string, includeDeleted bool) (*model.Post, *m } } + isInaccessible, appErr := a.isInaccessiblePost(post) + if appErr != nil { + return nil, appErr + } + if isInaccessible { + return nil, model.NewAppError("GetSinglePost", "app.post.cloud.get.app_error", nil, "", http.StatusForbidden) + } + return post, nil } @@ -870,6 +892,18 @@ func (a *App) GetPostThread(postID string, opts model.GetPostsOptions, userID st } } + // Get inserts the requested post first in the list, then adds the sorted threadPosts. + // So, the whole postList.Order is not sorted. + // The fully sorted list comes only when the CollapsedThreads is true and the Directions is not empty. + filterOptions := filterPostOptions{} + if opts.CollapsedThreads && opts.Direction != "" { + filterOptions.assumeSortedCreatedAt = true + } + + if appErr := a.filterInaccessiblePosts(posts, filterOptions); appErr != nil { + return nil, appErr + } + return posts, nil } @@ -879,6 +913,10 @@ func (a *App) GetFlaggedPosts(userID string, offset int, limit int) (*model.Post return nil, model.NewAppError("GetFlaggedPosts", "app.post.get_flagged_posts.app_error", nil, err.Error(), http.StatusInternalServerError) } + if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -888,6 +926,10 @@ func (a *App) GetFlaggedPostsForTeam(userID, teamID string, offset int, limit in return nil, model.NewAppError("GetFlaggedPostsForTeam", "app.post.get_flagged_posts.app_error", nil, err.Error(), http.StatusInternalServerError) } + if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -897,6 +939,10 @@ func (a *App) GetFlaggedPostsForChannel(userID, channelID string, offset int, li return nil, model.NewAppError("GetFlaggedPostsForChannel", "app.post.get_flagged_posts.app_error", nil, err.Error(), http.StatusInternalServerError) } + if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -929,6 +975,10 @@ func (a *App) GetPermalinkPost(c *request.Context, postID string, userID string) return nil, err } + if appErr := a.filterInaccessiblePosts(list, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil { + return nil, appErr + } + return list, nil } @@ -944,6 +994,19 @@ func (a *App) GetPostsBeforePost(options model.GetPostsOptions) (*model.PostList } } + // GetPostsBefore orders by channel id and deleted at, + // before sorting based on created at. + // but the deleted at is only ever where deleted at = 0, + // and channel id may or may not be empty (all channels) or defined (single channel), + // so we can still optimize if the search is for a single channel + filterOptions := filterPostOptions{} + if options.ChannelId != "" { + filterOptions.assumeSortedCreatedAt = true + } + if appErr := a.filterInaccessiblePosts(postList, filterOptions); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -959,6 +1022,19 @@ func (a *App) GetPostsAfterPost(options model.GetPostsOptions) (*model.PostList, } } + // GetPostsAfter orders by channel id and deleted at, + // before sorting based on created at. + // but the deleted at is only ever where deleted at = 0, + // and channel id may or may not be empty (all channels) or defined (single channel), + // so we can still optimize if the search is for a single channel + filterOptions := filterPostOptions{} + if options.ChannelId != "" { + filterOptions.assumeSortedCreatedAt = true + } + if appErr := a.filterInaccessiblePosts(postList, filterOptions); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -982,6 +1058,19 @@ func (a *App) GetPostsAroundPost(before bool, options model.GetPostsOptions) (*m } } + // GetPostsBefore and GetPostsAfter order by channel id and deleted at, + // before sorting based on created at. + // but the deleted at is only ever where deleted at = 0, + // and channel id may or may not be empty (all channels) or defined (single channel), + // so we can still optimize if the search is for a single channel + filterOptions := filterPostOptions{} + if options.ChannelId != "" { + filterOptions.assumeSortedCreatedAt = true + } + if appErr := a.filterInaccessiblePosts(postList, filterOptions); appErr != nil { + return nil, appErr + } + return postList, nil } @@ -1270,6 +1359,9 @@ func (a *App) searchPostsInTeam(teamID string, userID string, paramsList []*mode } posts.SortByCreateAt() + + a.filterInaccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true}) + return posts, nil } @@ -1297,10 +1389,85 @@ func (a *App) convertUserNameToUserIds(usernames []string) []string { return usernames } +// GetLastAccessiblePostTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit +func (a *App) GetLastAccessiblePostTime() (int64, *model.AppError) { + license := a.Srv().License() + if license == nil || !*license.Features.Cloud { + return 0, nil + } + + system, err := a.Srv().Store.System().GetByName(model.SystemLastAccessiblePostTime) + if err != nil { + var nfErr *store.ErrNotFound + switch { + case errors.As(err, &nfErr): + // All posts are accessible + return 0, nil + default: + return 0, model.NewAppError("GetLastAccessiblePostTime", "app.system.get_by_name.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + + lastAccessiblePostTime, err := strconv.ParseInt(system.Value, 10, 64) + if err != nil { + return 0, model.NewAppError("GetLastAccessiblePostTime", "common.parse_error_int64", map[string]interface{}{"Value": system.Value}, err.Error(), http.StatusInternalServerError) + } + + return lastAccessiblePostTime, nil +} + +// ComputeLastAccessiblePostTime updates cache with CreateAt time of the last accessible post as per the cloud plan's limit. +// Use GetLastAccessiblePostTime() to access the result. +func (a *App) ComputeLastAccessiblePostTime() *model.AppError { + limit, appErr := a.getCloudMessagesHistoryLimit() + if appErr != nil { + return appErr + } + + createdAt, err := a.Srv().GetStore().Post().GetNthRecentPostTime(limit) + if err != nil { + var nfErr *store.ErrNotFound + if !errors.As(err, &nfErr) { + return model.NewAppError("ComputeLastAccessiblePostTime", "app.last_accessible_post.app_error", nil, err.Error(), http.StatusInternalServerError) + } + } + + // Update Cache + err = a.Srv().Store.System().SaveOrUpdate(&model.System{ + Name: model.SystemLastAccessiblePostTime, + Value: strconv.FormatInt(createdAt, 10), + }) + if err != nil { + return model.NewAppError("ComputeLastAccessiblePostTime", "app.system.save.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return nil +} + +func (a *App) getCloudMessagesHistoryLimit() (int64, *model.AppError) { + license := a.Srv().License() + if license == nil || !*license.Features.Cloud { + return 0, nil + } + + limits, err := a.Cloud().GetCloudLimits("") + if err != nil { + return 0, model.NewAppError("getCloudMessagesHistoryLimit", "api.cloud.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + if limits == nil || limits.Messages == nil || limits.Messages.History == nil { + // Cloud limit is not applicable + return 0, nil + } + + return int64(*limits.Messages.History), nil +} + func (a *App) SearchPostsInTeam(teamID string, paramsList []*model.SearchParams) (*model.PostList, *model.AppError) { if !*a.Config().ServiceSettings.EnablePostSearch { return nil, model.NewAppError("SearchPostsInTeam", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v", teamID), http.StatusNotImplemented) } + return a.searchPostsInTeam(teamID, "", paramsList, func(params *model.SearchParams) { params.SearchWithoutUserId = true }) @@ -1354,6 +1521,10 @@ func (a *App) SearchPostsForUser(c *request.Context, terms string, userID string } } + if appErr := a.filterInaccessiblePosts(postSearchResults.PostList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil { + return nil, appErr + } + return postSearchResults, nil } @@ -1703,19 +1874,25 @@ func (a *App) GetPostIfAuthorized(postID string, session *model.Session, include return post, nil } -func (a *App) GetPostsByIds(postIDs []string) ([]*model.Post, *model.AppError) { +// GetPostsByIds response bool value indicates, if the post is inaccessible due to cloud plan's limit. +func (a *App) GetPostsByIds(postIDs []string) ([]*model.Post, bool, *model.AppError) { posts, err := a.Srv().Store.Post().GetPostsByIds(postIDs) if err != nil { var nfErr *store.ErrNotFound switch { case errors.As(err, &nfErr): - return nil, model.NewAppError("GetPostsByIds", "app.post.get.app_error", nil, nfErr.Error(), http.StatusNotFound) + return nil, false, model.NewAppError("GetPostsByIds", "app.post.get.app_error", nil, nfErr.Error(), http.StatusNotFound) default: - return nil, model.NewAppError("GetPostsByIds", "app.post.get.app_error", nil, err.Error(), http.StatusInternalServerError) + return nil, false, model.NewAppError("GetPostsByIds", "app.post.get.app_error", nil, err.Error(), http.StatusInternalServerError) } } - return posts, nil + posts, hasInaccessiblePosts, appErr := a.getFilteredAccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true}) + if appErr != nil { + return nil, false, appErr + } + + return posts, hasInaccessiblePosts, nil } func (a *App) GetTopThreadsForTeamSince(teamID, userID string, opts *model.InsightsOpts) (*model.TopThreadList, *model.AppError) { diff --git a/app/post_helpers.go b/app/post_helpers.go new file mode 100644 index 0000000000..88f9487829 --- /dev/null +++ b/app/post_helpers.go @@ -0,0 +1,231 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package app + +import ( + "net/http" + "sort" + + "github.com/mattermost/mattermost-server/v6/model" +) + +type filterPostOptions struct { + assumeSortedCreatedAt bool +} + +type accessibleBounds struct { + start int + end int +} + +func (b accessibleBounds) allAccessible(lenPosts int) bool { + return b.start == allAccessibleBounds(lenPosts).start && b.end == allAccessibleBounds(lenPosts).end +} + +func (b accessibleBounds) noAccessible() bool { + return b.start == noAccessibleBounds.start && b.end == noAccessibleBounds.end +} + +var noAccessibleBounds = accessibleBounds{start: -1, end: -1} +var allAccessibleBounds = func(lenPosts int) accessibleBounds { return accessibleBounds{start: 0, end: lenPosts - 1} } + +// getTimeSortedPostAccessibleBounds returns what the boundaries are for accessible posts. +// It assumes that CreateAt time for posts is monotonically increasing or decreasing. +// It could be either because posts can be returned in ascending or descending time order. +// Special values (which can be checked with methods `allAccessible` and `allInaccessible`) +// denote if all or none of the posts are accessible. +func getTimeSortedPostAccessibleBounds(earliestAccessibleTime int64, lenPosts int, getCreateAt func(int) int64) accessibleBounds { + if lenPosts == 0 { + return allAccessibleBounds(lenPosts) + } + if lenPosts == 1 { + if getCreateAt(0) >= earliestAccessibleTime { + return allAccessibleBounds(lenPosts) + } + return noAccessibleBounds + } + + ascending := getCreateAt(0) < getCreateAt(lenPosts-1) + + idx := sort.Search(lenPosts, func(i int) bool { + if ascending { + // Ascending order automatically picks the left most post(at idx), + // in case multiple posts at idx, idx+1, idx+2... have the same time. + return getCreateAt(i) >= earliestAccessibleTime + } + // Special case(subtracting 1) for descending order to include the right most post(at idx+k), + // in case multiple posts at idx, idx+1, idx+2...idx+k have the same time. + return getCreateAt(i) <= earliestAccessibleTime-1 + }) + + if ascending { + if idx == lenPosts { + return noAccessibleBounds + } + return accessibleBounds{start: idx, end: lenPosts - 1} + } + + if idx == 0 { + return noAccessibleBounds + } + return accessibleBounds{start: 0, end: idx - 1} +} + +// linearFilterPostList make no assumptions about ordering, go through posts one by one +// this is the slower fallback that is still safe if we can not +// assume posts are ordered by CreatedAt +func linearFilterPostList(postList *model.PostList, earliestAccessibleTime int64) { + // filter Posts + posts := postList.Posts + order := postList.Order + + n := 0 + for i, postId := range order { + if posts[postId].CreateAt >= earliestAccessibleTime { + order[n] = order[i] + n++ + } else { + postList.HasInaccessiblePosts = true + delete(posts, postId) + } + } + postList.Order = order[:n] + + // it can happen that some post list results don't have all posts in the Order field. + // for example GetPosts in the CollapsedThreads = false path, parents are not added + // to Order + for postId := range posts { + if posts[postId].CreateAt < earliestAccessibleTime { + postList.HasInaccessiblePosts = true + delete(posts, postId) + } + } +} + +// linearFilterPostsSlice make no assumptions about ordering, go through posts one by one +// this is the slower fallback that is still safe if we can not +// assume posts are ordered by CreatedAt +func linearFilterPostsSlice(posts []*model.Post, earliestAccessibleTime int64) ([]*model.Post, bool) { + hasInaccessiblePosts := false + n := 0 + for i := range posts { + if posts[i].CreateAt >= earliestAccessibleTime { + posts[n] = posts[i] + n++ + } else { + hasInaccessiblePosts = true + } + } + return posts[:n], hasInaccessiblePosts +} + +// filterInaccessiblePosts filters out the posts, past the cloud limit +func (a *App) filterInaccessiblePosts(postList *model.PostList, options filterPostOptions) *model.AppError { + if postList == nil || postList.Posts == nil || len(postList.Posts) == 0 { + return nil + } + + lastAccessiblePostTime, appErr := a.GetLastAccessiblePostTime() + if appErr != nil { + return model.NewAppError("filterInaccessiblePosts", "app.last_accessible_post.app_error", nil, appErr.Error(), http.StatusInternalServerError) + } + if lastAccessiblePostTime == 0 { + // No need to filter, all posts are accessible + return nil + } + + if len(postList.Posts) == len(postList.Order) && options.assumeSortedCreatedAt { + lenPosts := len(postList.Posts) + getCreateAt := func(i int) int64 { return postList.Posts[postList.Order[i]].CreateAt } + + bounds := getTimeSortedPostAccessibleBounds(lastAccessiblePostTime, lenPosts, getCreateAt) + + if bounds.allAccessible(lenPosts) { + return nil + } + if bounds.noAccessible() { + if lenPosts > 0 { + postList.HasInaccessiblePosts = true + } + postList.Posts = map[string]*model.Post{} + postList.Order = []string{} + return nil + } + postList.HasInaccessiblePosts = true + + posts := postList.Posts + order := postList.Order + accessibleCount := bounds.end - bounds.start + 1 + inaccessibleCount := lenPosts - accessibleCount + // Linearly cover shorter route to traverse posts map + if inaccessibleCount < accessibleCount { + for i := 0; i < bounds.start; i++ { + delete(posts, order[i]) + } + for i := bounds.end + 1; i < lenPosts; i++ { + delete(posts, order[i]) + } + } else { + accessiblePosts := make(map[string]*model.Post, accessibleCount) + for i := bounds.start; i <= bounds.end; i++ { + accessiblePosts[order[i]] = posts[order[i]] + } + postList.Posts = accessiblePosts + } + + postList.Order = postList.Order[bounds.start : bounds.end+1] + } else { + linearFilterPostList(postList, lastAccessiblePostTime) + } + + return nil +} + +// isInaccessiblePost indicates if the post is past the cloud plan's limit. +func (a *App) isInaccessiblePost(post *model.Post) (bool, *model.AppError) { + if post == nil { + return false, nil + } + + pl := &model.PostList{ + Order: []string{post.Id}, + Posts: map[string]*model.Post{post.Id: post}, + } + + return pl.HasInaccessiblePosts, a.filterInaccessiblePosts(pl, filterPostOptions{assumeSortedCreatedAt: true}) +} + +// getFilteredAccessiblePosts returns accessible posts filtered as per the cloud plan's limit and also indicates if there were any inaccessible posts +func (a *App) getFilteredAccessiblePosts(posts []*model.Post, options filterPostOptions) ([]*model.Post, bool, *model.AppError) { + if len(posts) == 0 { + return posts, false, nil + } + + filteredPosts := []*model.Post{} + lastAccessiblePostTime, appErr := a.GetLastAccessiblePostTime() + if appErr != nil { + return filteredPosts, false, model.NewAppError("getFilteredAccessiblePosts", "app.last_accessible_post.app_error", nil, appErr.Error(), http.StatusInternalServerError) + } else if lastAccessiblePostTime == 0 { + // No need to filter, all posts are accessible + return posts, false, nil + } + + if options.assumeSortedCreatedAt { + lenPosts := len(posts) + getCreateAt := func(i int) int64 { return posts[i].CreateAt } + bounds := getTimeSortedPostAccessibleBounds(lastAccessiblePostTime, lenPosts, getCreateAt) + if bounds.allAccessible(lenPosts) { + return posts, false, nil + } + if bounds.noAccessible() { + return filteredPosts, lenPosts > 0, nil + } + + filteredPosts = posts[bounds.start : bounds.end+1] + return filteredPosts, true, nil + } + + filteredPosts, hasInaccessiblePosts := linearFilterPostsSlice(posts, lastAccessiblePostTime) + return filteredPosts, hasInaccessiblePosts, nil +} diff --git a/app/post_helpers_test.go b/app/post_helpers_test.go new file mode 100644 index 0000000000..ebde2f2e1a --- /dev/null +++ b/app/post_helpers_test.go @@ -0,0 +1,377 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package app + +import ( + "testing" + + "github.com/mattermost/mattermost-server/v6/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetTimeSortedPostAccessibleBounds(t *testing.T) { + var postFromCreateAt = func(at int64) *model.Post { + return &model.Post{CreateAt: at} + } + + getPostListCreateAtFunc := func(pl *model.PostList) func(i int) int64 { + return func(i int) int64 { + return pl.Posts[pl.Order[i]].CreateAt + } + } + + t.Run("empty posts returns all accessible posts", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{}, + Order: []string{}, + } + bounds := getTimeSortedPostAccessibleBounds(0, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.True(t, bounds.allAccessible(len(pl.Posts))) + }) + + t.Run("one accessible post returns all accessible posts", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(1), + }, + Order: []string{"post_a"}, + } + bounds := getTimeSortedPostAccessibleBounds(0, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.True(t, bounds.allAccessible(len(pl.Posts))) + }) + + t.Run("one inaccessible post returns no accessible posts", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + }, + Order: []string{"post_a"}, + } + bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.True(t, bounds.noAccessible()) + }) + + t.Run("all accessible posts returns all accessible posts", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(1), + "post_b": postFromCreateAt(2), + "post_c": postFromCreateAt(3), + "post_d": postFromCreateAt(4), + "post_e": postFromCreateAt(5), + "post_f": postFromCreateAt(6), + }, + Order: []string{"post_a", "post_b", "post_c", "post_d", "post_e", "post_f"}, + } + bounds := getTimeSortedPostAccessibleBounds(0, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.True(t, bounds.allAccessible(len(pl.Posts))) + }) + + t.Run("all inaccessible posts returns all inaccessible posts", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(1), + "post_b": postFromCreateAt(2), + "post_c": postFromCreateAt(3), + "post_d": postFromCreateAt(4), + "post_e": postFromCreateAt(5), + "post_f": postFromCreateAt(6), + }, + Order: []string{"post_a", "post_b", "post_c", "post_d", "post_e", "post_f"}, + } + bounds := getTimeSortedPostAccessibleBounds(7, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.True(t, bounds.noAccessible()) + }) + + t.Run("all accessible posts returns all accessible posts, descending ordered", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(1), + "post_b": postFromCreateAt(2), + "post_c": postFromCreateAt(3), + "post_d": postFromCreateAt(4), + "post_e": postFromCreateAt(5), + "post_f": postFromCreateAt(6), + }, + Order: []string{"post_f", "post_e", "post_d", "post_c", "post_b", "post_a"}, + } + bounds := getTimeSortedPostAccessibleBounds(0, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.True(t, bounds.allAccessible(len(pl.Posts))) + }) + + t.Run("all inaccessible posts returns all inaccessible posts, descending ordered", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(1), + "post_b": postFromCreateAt(2), + "post_c": postFromCreateAt(3), + "post_d": postFromCreateAt(4), + "post_e": postFromCreateAt(5), + "post_f": postFromCreateAt(6), + }, + Order: []string{"post_f", "post_e", "post_d", "post_c", "post_b", "post_a"}, + } + bounds := getTimeSortedPostAccessibleBounds(7, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.True(t, bounds.noAccessible()) + }) + + t.Run("two posts, first accessible", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(1), + "post_b": postFromCreateAt(0), + }, + Order: []string{"post_a", "post_b"}, + } + bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.Equal(t, accessibleBounds{start: 0, end: 0}, bounds) + }) + + t.Run("two posts, second accessible", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + "post_b": postFromCreateAt(1), + }, + Order: []string{"post_a", "post_b"}, + } + bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.Equal(t, accessibleBounds{start: 1, end: 1}, bounds) + }) + + t.Run("picks the left most post for boundaries when there are time ties", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + "post_b": postFromCreateAt(1), + "post_c": postFromCreateAt(1), + "post_d": postFromCreateAt(2), + }, + Order: []string{"post_a", "post_b", "post_c", "post_d"}, + } + bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.Equal(t, accessibleBounds{start: 1, end: len(pl.Posts) - 1}, bounds) + }) + + t.Run("picks the right most post for boundaries when there are time ties, descending ordered", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + "post_b": postFromCreateAt(1), + "post_c": postFromCreateAt(1), + "post_d": postFromCreateAt(2), + }, + Order: []string{"post_d", "post_c", "post_b", "post_a"}, + } + bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.Equal(t, accessibleBounds{start: 0, end: 2}, bounds) + }) + + t.Run("odd number of posts and reverse time selects right boundaries", func(t *testing.T) { + pl := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + "post_b": postFromCreateAt(1), + "post_c": postFromCreateAt(2), + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, + Order: []string{"post_e", "post_d", "post_c", "post_b", "post_a"}, + } + bounds := getTimeSortedPostAccessibleBounds(2, len(pl.Posts), getPostListCreateAtFunc(pl)) + require.Equal(t, accessibleBounds{start: 0, end: 2}, bounds) + }) + + t.Run("posts-slice: odd number of posts and reverse time selects right boundaries", func(t *testing.T) { + posts := []*model.Post{postFromCreateAt(4), postFromCreateAt(3), postFromCreateAt(2), postFromCreateAt(1), postFromCreateAt(0)} + bounds := getTimeSortedPostAccessibleBounds(2, len(posts), func(i int) int64 { return posts[i].CreateAt }) + require.Equal(t, accessibleBounds{start: 0, end: 2}, bounds) + }) +} + +func TestFilterInaccessiblePosts(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessiblePostTime, + Value: "2", + }) + + defer th.TearDown() + + var postFromCreateAt = func(at int64) *model.Post { + return &model.Post{CreateAt: at} + } + + t.Run("ascending order returns correct posts", func(t *testing.T) { + postList := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + "post_b": postFromCreateAt(1), + "post_c": postFromCreateAt(2), + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, + Order: []string{"post_a", "post_b", "post_c", "post_d", "post_e"}, + } + appErr := th.App.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.Post{ + "post_c": postFromCreateAt(2), + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, postList.Posts) + + assert.Equal(t, []string{ + "post_c", + "post_d", + "post_e", + }, postList.Order) + }) + + t.Run("descending order returns correct posts", func(t *testing.T) { + postList := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + "post_b": postFromCreateAt(1), + "post_c": postFromCreateAt(2), + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, + Order: []string{"post_e", "post_d", "post_c", "post_b", "post_a"}, + } + appErr := th.App.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.Post{ + "post_c": postFromCreateAt(2), + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, postList.Posts) + + assert.Equal(t, []string{ + "post_e", + "post_d", + "post_c", + }, postList.Order) + }) + + t.Run("handles mixed create at ordering correctly if correct options given", func(t *testing.T) { + postList := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + "post_b": postFromCreateAt(1), + "post_c": postFromCreateAt(2), + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, + Order: []string{"post_e", "post_b", "post_a", "post_d", "post_c"}, + } + appErr := th.App.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: false}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.Post{ + "post_c": postFromCreateAt(2), + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, postList.Posts) + + assert.Equal(t, []string{ + "post_e", + "post_d", + "post_c", + }, postList.Order) + }) + + t.Run("handles posts missing from order when doing linear search", func(t *testing.T) { + postList := &model.PostList{ + Posts: map[string]*model.Post{ + "post_a": postFromCreateAt(0), + "post_b": postFromCreateAt(1), + "post_c": postFromCreateAt(1), + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, + Order: []string{"post_e", "post_a", "post_d", "post_b"}, + } + appErr := th.App.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: false}) + + require.Nil(t, appErr) + + assert.Equal(t, map[string]*model.Post{ + "post_d": postFromCreateAt(3), + "post_e": postFromCreateAt(4), + }, postList.Posts) + + assert.Equal(t, []string{ + "post_e", + "post_d", + }, postList.Order) + }) +} + +func TestGetFilteredAccessiblePosts(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessiblePostTime, + Value: "2", + }) + + defer th.TearDown() + + var postFromCreateAt = func(at int64) *model.Post { + return &model.Post{CreateAt: at} + } + + t.Run("ascending order returns correct posts", func(t *testing.T) { + posts := []*model.Post{postFromCreateAt(0), postFromCreateAt(1), postFromCreateAt(2), postFromCreateAt(3), postFromCreateAt(4)} + filteredPosts, _, appErr := th.App.getFilteredAccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true}) + + assert.Nil(t, appErr) + assert.Equal(t, []*model.Post{postFromCreateAt(2), postFromCreateAt(3), postFromCreateAt(4)}, filteredPosts) + }) + + t.Run("descending order returns correct posts", func(t *testing.T) { + posts := []*model.Post{postFromCreateAt(4), postFromCreateAt(3), postFromCreateAt(2), postFromCreateAt(1), postFromCreateAt(0)} + filteredPosts, _, appErr := th.App.getFilteredAccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true}) + + assert.Nil(t, appErr) + assert.Equal(t, []*model.Post{postFromCreateAt(4), postFromCreateAt(3), postFromCreateAt(2)}, filteredPosts) + }) + + t.Run("handles mixed create at ordering correctly if correct options given", func(t *testing.T) { + posts := []*model.Post{postFromCreateAt(4), postFromCreateAt(1), postFromCreateAt(0), postFromCreateAt(3), postFromCreateAt(2)} + filteredPosts, _, appErr := th.App.getFilteredAccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: false}) + + assert.Nil(t, appErr) + assert.Equal(t, []*model.Post{postFromCreateAt(4), postFromCreateAt(3), postFromCreateAt(2)}, filteredPosts) + }) +} + +func TestIsInaccessiblePost(t *testing.T) { + th := Setup(t) + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + th.App.Srv().Store.System().Save(&model.System{ + Name: model.SystemLastAccessiblePostTime, + Value: "2", + }) + + defer th.TearDown() + + post := &model.Post{CreateAt: 3} + r, appErr := th.App.isInaccessiblePost(post) + assert.Nil(t, appErr) + assert.Equal(t, false, r) + + post = &model.Post{CreateAt: 1} + r, appErr = th.App.isInaccessiblePost(post) + assert.Nil(t, appErr) + assert.Equal(t, true, r) +} diff --git a/app/post_metadata.go b/app/post_metadata.go index 75329b3f23..b7cde0355c 100644 --- a/app/post_metadata.go +++ b/app/post_metadata.go @@ -53,11 +53,12 @@ func (s *Server) initPostMetadata() { func (a *App) PreparePostListForClient(originalList *model.PostList) *model.PostList { list := &model.PostList{ - Posts: make(map[string]*model.Post, len(originalList.Posts)), - Order: originalList.Order, - NextPostId: originalList.NextPostId, - PrevPostId: originalList.PrevPostId, - HasNext: originalList.HasNext, + Posts: make(map[string]*model.Post, len(originalList.Posts)), + Order: originalList.Order, + NextPostId: originalList.NextPostId, + PrevPostId: originalList.PrevPostId, + HasNext: originalList.HasNext, + HasInaccessiblePosts: originalList.HasInaccessiblePosts, } for id, originalPost := range originalList.Posts { diff --git a/app/post_test.go b/app/post_test.go index 86032957a6..cefa0f5ea9 100644 --- a/app/post_test.go +++ b/app/post_test.go @@ -5,6 +5,7 @@ package app import ( "context" + "errors" "fmt" "net/http" "os" @@ -15,6 +16,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + eMocks "github.com/mattermost/mattermost-server/v6/einterfaces/mocks" "github.com/mattermost/mattermost-server/v6/model" "github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock" "github.com/mattermost/mattermost-server/v6/services/imageproxy" @@ -2818,6 +2820,68 @@ func TestShouldNotRefollowOnOthersReply(t *testing.T) { require.True(t, m.Following) } +func TestGetLastAccessiblePostTime(t *testing.T) { + th := SetupWithStoreMock(t) + defer th.TearDown() + + r, err := th.App.GetLastAccessiblePostTime() + assert.Nil(t, err) + assert.Equal(t, int64(0), r) + + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + + mockStore := th.App.Srv().Store.(*storemocks.Store) + + mockSystemStore := storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(nil, store.NewErrNotFound("", "")) + r, err = th.App.GetLastAccessiblePostTime() + assert.Nil(t, err) + assert.Equal(t, int64(0), r) + + mockSystemStore = storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(nil, errors.New("test")) + _, err = th.App.GetLastAccessiblePostTime() + assert.NotNil(t, err) + + mockSystemStore = storemocks.SystemStore{} + mockStore.On("System").Return(&mockSystemStore) + mockSystemStore.On("GetByName", mock.Anything).Return(&model.System{Name: model.SystemLastAccessiblePostTime, Value: "10"}, nil) + r, err = th.App.GetLastAccessiblePostTime() + assert.Nil(t, err) + assert.Equal(t, int64(10), r) +} + +func TestComputeLastAccessiblePostTime(t *testing.T) { + th := SetupWithStoreMock(t) + defer th.TearDown() + + th.App.Srv().SetLicense(model.NewTestLicense("cloud")) + + cloud := &eMocks.CloudInterface{} + th.App.Srv().Cloud = cloud + + cloud.Mock.On("GetCloudLimits", mock.Anything).Return(&model.ProductLimits{ + Messages: &model.MessagesLimits{ + History: model.NewInt(1), + }, + }, nil) + + mockStore := th.App.Srv().Store.(*storemocks.Store) + mockPostStore := storemocks.PostStore{} + mockPostStore.On("GetNthRecentPostTime", mock.Anything).Return(int64(1), nil) + mockSystemStore := storemocks.SystemStore{} + mockSystemStore.On("SaveOrUpdate", mock.Anything).Return(nil) + mockStore.On("Post").Return(&mockPostStore) + mockStore.On("System").Return(&mockSystemStore) + + err := th.App.ComputeLastAccessiblePostTime() + assert.Nil(t, err) + + mockSystemStore.AssertCalled(t, "SaveOrUpdate", mock.Anything) +} + func TestGetTopThreadsForTeamSince(t *testing.T) { th := Setup(t).InitBasic() defer th.TearDown() diff --git a/app/server.go b/app/server.go index 9c9f2ae6e0..a71856b02c 100644 --- a/app/server.go +++ b/app/server.go @@ -49,6 +49,7 @@ import ( "github.com/mattermost/mattermost-server/v6/jobs/extract_content" "github.com/mattermost/mattermost-server/v6/jobs/import_delete" "github.com/mattermost/mattermost-server/v6/jobs/import_process" + "github.com/mattermost/mattermost-server/v6/jobs/last_accessible_post" "github.com/mattermost/mattermost-server/v6/jobs/migrations" "github.com/mattermost/mattermost-server/v6/jobs/product_notices" "github.com/mattermost/mattermost-server/v6/jobs/resend_invitation_email" @@ -2046,6 +2047,12 @@ func (s *Server) initJobs() { extract_content.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())), s.Store), nil, ) + + s.Jobs.RegisterJobType( + model.JobTypeLastAccessiblePost, + last_accessible_post.MakeWorker(s.Jobs, s.License(), New(ServerConnector(s.Channels()))), + last_accessible_post.MakeScheduler(s.Jobs, s.License()), + ) } func (s *Server) TelemetryId() string { diff --git a/i18n/en.json b/i18n/en.json index 9185c4ebe3..5e25b80e9e 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -5475,6 +5475,10 @@ "id": "app.job.update.app_error", "translation": "Unable to update the job." }, + { + "id": "app.last_accessible_post.app_error", + "translation": "Error fetching last accessible post" + }, { "id": "app.license.generate_renewal_token.app_error", "translation": "Failed to generate a new renewal token." @@ -5811,6 +5815,10 @@ "id": "app.post.analytics_user_counts_posts_by_day.app_error", "translation": "Unable to get user counts with posts." }, + { + "id": "app.post.cloud.get.app_error", + "translation": "Can not fetch the post as it is past the cloud's plan limit." + }, { "id": "app.post.delete.app_error", "translation": "Unable to delete the post." @@ -6895,6 +6903,10 @@ "id": "brand.save_brand_image.save_image.app_error", "translation": "Unable to write the image file to your file storage. Please check your connection and try again." }, + { + "id": "common.parse_error_int64", + "translation": "Failed to parse the value:{{.Value}} to int64" + }, { "id": "ent.account_migration.get_all_failed", "translation": "Unable to get users." diff --git a/jobs/last_accessible_post/scheduler.go b/jobs/last_accessible_post/scheduler.go new file mode 100644 index 0000000000..96db5f5ce8 --- /dev/null +++ b/jobs/last_accessible_post/scheduler.go @@ -0,0 +1,24 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package last_accessible_post + +import ( + "strconv" + "time" + + "github.com/mattermost/mattermost-server/v6/jobs" + "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/shared/mlog" +) + +const schedFreq = 30 * time.Minute + +func MakeScheduler(jobServer *jobs.JobServer, license *model.License) model.Scheduler { + isEnabled := func(cfg *model.Config) bool { + enabled := license != nil && *license.Features.Cloud + mlog.Debug("Scheduler: isEnabled: "+strconv.FormatBool(enabled), mlog.String("scheduler", model.JobTypeLastAccessiblePost)) + return enabled + } + return jobs.NewPeriodicScheduler(jobServer, model.JobTypeLastAccessiblePost, schedFreq, isEnabled) +} diff --git a/jobs/last_accessible_post/worker.go b/jobs/last_accessible_post/worker.go new file mode 100644 index 0000000000..5093f25ab9 --- /dev/null +++ b/jobs/last_accessible_post/worker.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package last_accessible_post + +import ( + "github.com/mattermost/mattermost-server/v6/jobs" + "github.com/mattermost/mattermost-server/v6/model" +) + +const ( + JobName = "LastAccessiblePost" +) + +type AppIface interface { + ComputeLastAccessiblePostTime() *model.AppError +} + +func MakeWorker(jobServer *jobs.JobServer, license *model.License, app AppIface) model.Worker { + isEnabled := func(_ *model.Config) bool { + return license != nil && *license.Features.Cloud + } + execute := func(_ *model.Job) error { + return app.ComputeLastAccessiblePostTime() + } + worker := jobs.NewSimpleWorker(JobName, jobServer, execute, isEnabled) + return worker +} diff --git a/model/client4.go b/model/client4.go index 9020a6ec53..37cb49b3f4 100644 --- a/model/client4.go +++ b/model/client4.go @@ -18,29 +18,30 @@ import ( ) const ( - HeaderRequestId = "X-Request-ID" - HeaderVersionId = "X-Version-ID" - HeaderClusterId = "X-Cluster-ID" - HeaderEtagServer = "ETag" - HeaderEtagClient = "If-None-Match" - HeaderForwarded = "X-Forwarded-For" - HeaderRealIP = "X-Real-IP" - HeaderForwardedProto = "X-Forwarded-Proto" - HeaderToken = "token" - HeaderCsrfToken = "X-CSRF-Token" - HeaderBearer = "BEARER" - HeaderAuth = "Authorization" - HeaderCloudToken = "X-Cloud-Token" - HeaderRemoteclusterToken = "X-RemoteCluster-Token" - HeaderRemoteclusterId = "X-RemoteCluster-Id" - HeaderRequestedWith = "X-Requested-With" - HeaderRequestedWithXML = "XMLHttpRequest" - HeaderRange = "Range" - STATUS = "status" - StatusOk = "OK" - StatusFail = "FAIL" - StatusUnhealthy = "UNHEALTHY" - StatusRemove = "REMOVE" + HeaderRequestId = "X-Request-ID" + HeaderVersionId = "X-Version-ID" + HeaderClusterId = "X-Cluster-ID" + HeaderEtagServer = "ETag" + HeaderEtagClient = "If-None-Match" + HeaderForwarded = "X-Forwarded-For" + HeaderRealIP = "X-Real-IP" + HeaderForwardedProto = "X-Forwarded-Proto" + HeaderToken = "token" + HeaderCsrfToken = "X-CSRF-Token" + HeaderBearer = "BEARER" + HeaderAuth = "Authorization" + HeaderCloudToken = "X-Cloud-Token" + HeaderRemoteclusterToken = "X-RemoteCluster-Token" + HeaderRemoteclusterId = "X-RemoteCluster-Id" + HeaderRequestedWith = "X-Requested-With" + HeaderRequestedWithXML = "XMLHttpRequest" + HeaderHasInaccessiblePosts = "Has-Inaccessible-Posts" + HeaderRange = "Range" + STATUS = "status" + StatusOk = "OK" + StatusFail = "FAIL" + StatusUnhealthy = "UNHEALTHY" + StatusRemove = "REMOVE" ClientDir = "client" diff --git a/model/job.go b/model/job.go index 8b6272e8af..d12c2245b7 100644 --- a/model/job.go +++ b/model/job.go @@ -27,6 +27,7 @@ const ( JobTypeCloud = "cloud" JobTypeResendInvitationEmail = "resend_invitation_email" JobTypeExtractContent = "extract_content" + JobTypeLastAccessiblePost = "last_accessible_post" JobStatusPending = "pending" JobStatusInProgress = "in_progress" @@ -55,6 +56,7 @@ var AllJobTypes = [...]string{ JobTypeExportDelete, JobTypeCloud, JobTypeExtractContent, + JobTypeLastAccessiblePost, } type Job struct { diff --git a/model/post_list.go b/model/post_list.go index 67b90ac7f6..0801b5c8e1 100644 --- a/model/post_list.go +++ b/model/post_list.go @@ -16,6 +16,8 @@ type PostList struct { PrevPostId string `json:"prev_post_id"` // HasNext indicates whether there are more items to be fetched or not. HasNext bool `json:"has_next"` + // HasInaccessiblePosts tells if there are inaccessible posts, past the cloud limit. + HasInaccessiblePosts bool `json:"has_inaccessible_posts"` } func NewPostList() *PostList { @@ -35,11 +37,12 @@ func (o *PostList) Clone() *PostList { postsCopy[k] = v.Clone() } return &PostList{ - Order: orderCopy, - Posts: postsCopy, - NextPostId: o.NextPostId, - PrevPostId: o.PrevPostId, - HasNext: o.HasNext, + Order: orderCopy, + Posts: postsCopy, + NextPostId: o.NextPostId, + PrevPostId: o.PrevPostId, + HasNext: o.HasNext, + HasInaccessiblePosts: o.HasInaccessiblePosts, } } diff --git a/model/system.go b/model/system.go index af0e944094..10370d0e08 100644 --- a/model/system.go +++ b/model/system.go @@ -31,6 +31,7 @@ const ( SystemWarnMetricLastRunTimestampKey = "LastWarnMetricRunTimestamp" SystemFirstAdminVisitMarketplace = "FirstAdminVisitMarketplace" SystemFirstAdminSetupComplete = "FirstAdminSetupComplete" + SystemLastAccessiblePostTime = "LastAccessiblePostTime" AwsMeteringReportInterval = 1 AwsMeteringDimensionUsageHrs = "UsageHrs" ) diff --git a/store/opentracinglayer/opentracinglayer.go b/store/opentracinglayer/opentracinglayer.go index 06e0b9b999..85c4829f58 100644 --- a/store/opentracinglayer/opentracinglayer.go +++ b/store/opentracinglayer/opentracinglayer.go @@ -5683,6 +5683,24 @@ func (s *OpenTracingLayerPostStore) GetMaxPostSize() int { return result } +func (s *OpenTracingLayerPostStore) GetNthRecentPostTime(n int64) (int64, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.GetNthRecentPostTime") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.PostStore.GetNthRecentPostTime(n) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerPostStore) GetOldest() (*model.Post, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.GetOldest") diff --git a/store/retrylayer/retrylayer.go b/store/retrylayer/retrylayer.go index 2accc6c080..5094c41fcb 100644 --- a/store/retrylayer/retrylayer.go +++ b/store/retrylayer/retrylayer.go @@ -6428,6 +6428,27 @@ func (s *RetryLayerPostStore) GetMaxPostSize() int { } +func (s *RetryLayerPostStore) GetNthRecentPostTime(n int64) (int64, error) { + + tries := 0 + for { + result, err := s.PostStore.GetNthRecentPostTime(n) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerPostStore) GetOldest() (*model.Post, error) { tries := 0 diff --git a/store/searchtest/helper.go b/store/searchtest/helper.go index 8dd70f8505..5ad88c0924 100644 --- a/store/searchtest/helper.go +++ b/store/searchtest/helper.go @@ -193,6 +193,13 @@ func (th *SearchTestHelper) deleteUser(user *model.User) error { return th.Store.User().PermanentDelete(user.Id) } +func (th *SearchTestHelper) deleteBotUser(botID string) error { + if err := th.deleteBot(botID); err != nil { + return err + } + return th.Store.User().PermanentDelete(botID) +} + func (th *SearchTestHelper) cleanAllUsers() error { users, err := th.Store.User().GetAll() if err != nil { diff --git a/store/searchtest/post_layer.go b/store/searchtest/post_layer.go index 5fb65219dc..1d03edf7f7 100644 --- a/store/searchtest/post_layer.go +++ b/store/searchtest/post_layer.go @@ -1649,7 +1649,7 @@ func testSearchTermsWithUnderscores(t *testing.T, th *SearchTestHelper) { func testSearchBotAccountsPosts(t *testing.T, th *SearchTestHelper) { bot, err := th.createBot("testbot", "Test Bot", th.User.Id) require.NoError(t, err) - defer th.deleteBot(bot.UserId) + defer th.deleteBotUser(bot.UserId) err = th.addUserToTeams(model.UserFromBot(bot), []string{th.Team.Id}) require.NoError(t, err) p1, err := th.createPost(bot.UserId, th.ChannelBasic.Id, "bot test message", "", model.PostTypeDefault, 0, false) diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index bee37bf0b9..af7b11ca01 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -1735,6 +1735,41 @@ var specialSearchChar = []string{ ":", } +// GetNthRecentPostTime returns the CreateAt time of the nth most recent post. +func (s *SqlPostStore) GetNthRecentPostTime(n int64) (int64, error) { + if n <= 0 { + return 0, errors.New("n can't be less than 1") + } + + builder := s.getQueryBuilder(). + Select("CreateAt"). + From("Posts p"). + // Consider users posts only for cloud limit + Where(sq.And{ + sq.Eq{"p.Type": ""}, + sq.Expr("p.UserId NOT IN (SELECT UserId FROM Bots)"), + }). + OrderBy("p.CreateAt DESC"). + Limit(1). + Offset(uint64(n - 1)) + + query, queryArgs, err := builder.ToSql() + if err != nil { + return 0, errors.Wrap(err, "GetNthRecentPostTime_tosql") + } + + var createAt int64 + if err := s.GetMasterX().Get(&createAt, query, queryArgs...); err != nil { + if err == sql.ErrNoRows { + return 0, store.NewErrNotFound("Post", "none") + } + + return 0, errors.Wrapf(err, "failed to get the Nth Post=%d", n) + } + + return createAt, nil +} + func (s *SqlPostStore) buildCreateDateFilterClause(params *model.SearchParams, builder sq.SelectBuilder) sq.SelectBuilder { // handle after: before: on: filters if params.OnDate != "" { diff --git a/store/store.go b/store/store.go index 9feffccf27..d5f67ccdbb 100644 --- a/store/store.go +++ b/store/store.go @@ -385,6 +385,8 @@ type PostStore interface { GetOldestEntityCreationTime() (int64, error) HasAutoResponsePostByUserSince(options model.GetPostsSinceOptions, userId string) (bool, error) GetPostsSinceForSync(options model.GetPostsSinceForSyncOptions, cursor model.GetPostsSinceForSyncCursor, limit int) ([]*model.Post, model.GetPostsSinceForSyncCursor, error) + // GetNthRecentPostTime returns the CreateAt time of the nth most recent post. + GetNthRecentPostTime(n int64) (int64, error) } type UserStore interface { diff --git a/store/storetest/mocks/PostStore.go b/store/storetest/mocks/PostStore.go index a55cf77949..0f24d48db0 100644 --- a/store/storetest/mocks/PostStore.go +++ b/store/storetest/mocks/PostStore.go @@ -287,6 +287,27 @@ func (_m *PostStore) GetMaxPostSize() int { return r0 } +// GetNthRecentPostTime provides a mock function with given fields: n +func (_m *PostStore) GetNthRecentPostTime(n int64) (int64, error) { + ret := _m.Called(n) + + var r0 int64 + if rf, ok := ret.Get(0).(func(int64) int64); ok { + r0 = rf(n) + } else { + r0 = ret.Get(0).(int64) + } + + var r1 error + if rf, ok := ret.Get(1).(func(int64) error); ok { + r1 = rf(n) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetOldest provides a mock function with given fields: func (_m *PostStore) GetOldest() (*model.Post, error) { ret := _m.Called() diff --git a/store/storetest/post_store.go b/store/storetest/post_store.go index 1e7bb7b386..15147cbfb6 100644 --- a/store/storetest/post_store.go +++ b/store/storetest/post_store.go @@ -57,6 +57,7 @@ func TestPostStore(t *testing.T, ss store.Store, s SqlStore) { t.Run("GetForThread", func(t *testing.T) { testPostStoreGetForThread(t, ss) }) t.Run("HasAutoResponsePostByUserSince", func(t *testing.T) { testHasAutoResponsePostByUserSince(t, ss) }) t.Run("GetPostsSinceForSync", func(t *testing.T) { testGetPostsSinceForSync(t, ss, s) }) + t.Run("GetNthRecentPostTime", func(t *testing.T) { testGetNthRecentPostTime(t, ss) }) } func testPostStoreSave(t *testing.T, ss store.Store) { @@ -3765,3 +3766,98 @@ func getPostIds(posts []*model.Post, morePosts ...*model.Post) []string { } return ids } + +func testGetNthRecentPostTime(t *testing.T, ss store.Store) { + _, err := ss.Post().GetNthRecentPostTime(0) + assert.Error(t, err) + _, err = ss.Post().GetNthRecentPostTime(-1) + assert.Error(t, err) + + diff := int64(10000) + now := utils.MillisFromTime(time.Now()) + diff + + p1 := &model.Post{} + p1.ChannelId = model.NewId() + p1.UserId = model.NewId() + p1.Message = "test" + p1.CreateAt = now + p1, err = ss.Post().Save(p1) + require.NoError(t, err) + + p2 := &model.Post{} + p2.ChannelId = p1.ChannelId + p2.UserId = p1.UserId + p2.Message = p1.Message + now = now + diff + p2.CreateAt = now + p2, err = ss.Post().Save(p2) + require.NoError(t, err) + + bot1 := &model.Bot{ + Username: "username", + Description: "a bot", + OwnerId: model.NewId(), + UserId: model.NewId(), + } + _, err = ss.Bot().Save(bot1) + require.NoError(t, err) + + b1 := &model.Post{} + b1.Message = "bot test" + b1.ChannelId = p1.ChannelId + b1.UserId = bot1.UserId + now = now + diff + b1.CreateAt = now + _, err = ss.Post().Save(b1) + require.NoError(t, err) + + p3 := &model.Post{} + p3.ChannelId = p1.ChannelId + p3.UserId = p1.UserId + p3.Message = p1.Message + now = now + diff + p3.CreateAt = now + p3, err = ss.Post().Save(p3) + require.NoError(t, err) + + s1 := &model.Post{} + s1.Type = model.PostTypeJoinChannel + s1.ChannelId = p1.ChannelId + s1.UserId = model.NewId() + s1.Message = "system_join_channel message" + now = now + diff + s1.CreateAt = now + _, err = ss.Post().Save(s1) + require.NoError(t, err) + + p4 := &model.Post{} + p4.ChannelId = p1.ChannelId + p4.UserId = p1.UserId + p4.Message = p1.Message + now = now + diff + p4.CreateAt = now + p4, err = ss.Post().Save(p4) + require.NoError(t, err) + + r, err := ss.Post().GetNthRecentPostTime(1) + assert.NoError(t, err) + assert.Equal(t, p4.CreateAt, r) + + // Skip system post + r, err = ss.Post().GetNthRecentPostTime(2) + assert.NoError(t, err) + assert.Equal(t, p3.CreateAt, r) + + // Skip system & bot post + r, err = ss.Post().GetNthRecentPostTime(3) + assert.NoError(t, err) + assert.Equal(t, p2.CreateAt, r) + + r, err = ss.Post().GetNthRecentPostTime(4) + assert.NoError(t, err) + assert.Equal(t, p1.CreateAt, r) + + _, err = ss.Post().GetNthRecentPostTime(10000) + assert.Error(t, err) + assert.IsType(t, &store.ErrNotFound{}, err) +} diff --git a/store/timerlayer/timerlayer.go b/store/timerlayer/timerlayer.go index 209f6c528e..eb920da051 100644 --- a/store/timerlayer/timerlayer.go +++ b/store/timerlayer/timerlayer.go @@ -5146,6 +5146,22 @@ func (s *TimerLayerPostStore) GetMaxPostSize() int { return result } +func (s *TimerLayerPostStore) GetNthRecentPostTime(n int64) (int64, error) { + start := time.Now() + + result, err := s.PostStore.GetNthRecentPostTime(n) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("PostStore.GetNthRecentPostTime", success, elapsed) + } + return result, err +} + func (s *TimerLayerPostStore) GetOldest() (*model.Post, error) { start := time.Now()