[MM-44488] Cloud limits: enforcing messages (#20362)

* Add new Job to keep updating the last_accessible_post time

* Filter out posts for funcs returning PostList model

* Separate methods to get and compute cache

* filter pinned posts

* For posts with sorted CreateAt order, support a faster form of filtering.

* Add inaccessible header for getPost and getPostsByIDs APIs

* replace manual binary search with the std. library

* in-place filter posts

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Nathaniel Allred <neallred@protonmail.com>
Этот коммит содержится в:
Vishal
2022-07-06 11:56:39 +05:30
коммит произвёл GitHub
родитель c62d4bee5f
Коммит e5ee5eecd8
26 изменённых файлов: 1253 добавлений и 45 удалений

Просмотреть файл

@@ -76,6 +76,9 @@ type AppIface interface {
CheckProviderAttributes(user *model.User, patch *model.UserPatch) string
// ClientConfigWithComputed gets the configuration in a format suitable for sending to the client.
ClientConfigWithComputed() map[string]string
// ComputeLastAccessiblePostTime updates cache with CreateAt time of the last accessible post as per the cloud plan's limit.
// Use GetLastAccessiblePostTime() to access the result.
ComputeLastAccessiblePostTime() *model.AppError
// ConvertBotToUser converts a bot to user.
ConvertBotToUser(bot *model.Bot, userPatch *model.UserPatch, sysadmin bool) (*model.User, *model.AppError)
// ConvertUserToBot converts a user to bot.
@@ -181,6 +184,8 @@ type AppIface interface {
// relationship with a user. That means any user sharing any channel, including
// direct and group channels.
GetKnownUsers(userID string) ([]string, *model.AppError)
// GetLastAccessiblePostTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit
GetLastAccessiblePostTime() (int64, *model.AppError)
// GetLdapGroup retrieves a single LDAP group by the given LDAP group id.
GetLdapGroup(ldapGroupID string) (*model.Group, *model.AppError)
// GetMarketplacePlugins returns a list of plugins from the marketplace-server,
@@ -196,6 +201,8 @@ type AppIface interface {
// To get the plugins environment when the plugins are disabled, manually acquire the plugins
// lock instead.
GetPluginsEnvironment() *plugin.Environment
// GetPostsByIds response bool value indicates, if the post is inaccessible due to cloud plan's limit.
GetPostsByIds(postIDs []string) ([]*model.Post, bool, *model.AppError)
// GetPostsUsage returns the total posts count rounded down to the most
// significant digit
GetPostsUsage() (int64, *model.AppError)
@@ -697,7 +704,6 @@ type AppIface interface {
GetPostsAfterPost(options model.GetPostsOptions) (*model.PostList, *model.AppError)
GetPostsAroundPost(before bool, options model.GetPostsOptions) (*model.PostList, *model.AppError)
GetPostsBeforePost(options model.GetPostsOptions) (*model.PostList, *model.AppError)
GetPostsByIds(postIDs []string) ([]*model.Post, *model.AppError)
GetPostsEtag(channelID string, collapsedThreads bool) string
GetPostsForChannelAroundLastUnread(channelID, userID string, limitBefore, limitAfter int, skipFetchThreads bool, collapsedThreads, collapsedThreadsExtended bool) (*model.PostList, *model.AppError)
GetPostsPage(options model.GetPostsOptions) (*model.PostList, *model.AppError)

Просмотреть файл

@@ -3172,6 +3172,10 @@ func (a *App) GetPinnedPosts(channelID string) (*model.PostList, *model.AppError
return nil, model.NewAppError("GetPinnedPosts", "app.channel.pinned_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if appErr := a.filterInaccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil {
return nil, appErr
}
return posts, nil
}

Просмотреть файл

@@ -1737,6 +1737,28 @@ func (a *OpenTracingAppLayer) CompleteSwitchWithOAuth(service string, userData i
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) ComputeLastAccessiblePostTime() *model.AppError {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ComputeLastAccessiblePostTime")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0 := a.app.ComputeLastAccessiblePostTime()
if resultVar0 != nil {
span.LogFields(spanlog.Error(resultVar0))
ext.Error.Set(span, true)
}
return resultVar0
}
func (a *OpenTracingAppLayer) Config() *model.Config {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.Config")
@@ -6779,6 +6801,28 @@ func (a *OpenTracingAppLayer) GetKnownUsers(userID string) ([]string, *model.App
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) GetLastAccessiblePostTime() (int64, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLastAccessiblePostTime")
a.ctx = newCtx
a.app.Srv().Store.SetContext(newCtx)
defer func() {
a.app.Srv().Store.SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.GetLastAccessiblePostTime()
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) GetLatestTermsOfService() (*model.TermsOfService, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetLatestTermsOfService")
@@ -7837,7 +7881,7 @@ func (a *OpenTracingAppLayer) GetPostsBeforePost(options model.GetPostsOptions)
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) GetPostsByIds(postIDs []string) ([]*model.Post, *model.AppError) {
func (a *OpenTracingAppLayer) GetPostsByIds(postIDs []string) ([]*model.Post, bool, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetPostsByIds")
@@ -7849,14 +7893,14 @@ func (a *OpenTracingAppLayer) GetPostsByIds(postIDs []string) ([]*model.Post, *m
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.GetPostsByIds(postIDs)
resultVar0, resultVar1, resultVar2 := a.app.GetPostsByIds(postIDs)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
if resultVar2 != nil {
span.LogFields(spanlog.Error(resultVar2))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
return resultVar0, resultVar1, resultVar2
}
func (a *OpenTracingAppLayer) GetPostsEtag(channelID string, collapsedThreads bool) string {

Просмотреть файл

@@ -10,6 +10,7 @@ import (
"fmt"
"net/http"
"regexp"
"strconv"
"strings"
"sync"
"time"
@@ -809,6 +810,11 @@ func (a *App) GetPostsPage(options model.GetPostsOptions) (*model.PostList, *mod
}
}
// The postList is sorted as only rootPosts Order is included
if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -824,6 +830,10 @@ func (a *App) GetPosts(channelID string, offset int, limit int) (*model.PostList
}
}
if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{}); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -837,6 +847,10 @@ func (a *App) GetPostsSince(options model.GetPostsSinceOptions) (*model.PostList
return nil, model.NewAppError("GetPostsSince", "app.post.get_posts_since.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -852,6 +866,14 @@ func (a *App) GetSinglePost(postID string, includeDeleted bool) (*model.Post, *m
}
}
isInaccessible, appErr := a.isInaccessiblePost(post)
if appErr != nil {
return nil, appErr
}
if isInaccessible {
return nil, model.NewAppError("GetSinglePost", "app.post.cloud.get.app_error", nil, "", http.StatusForbidden)
}
return post, nil
}
@@ -870,6 +892,18 @@ func (a *App) GetPostThread(postID string, opts model.GetPostsOptions, userID st
}
}
// Get inserts the requested post first in the list, then adds the sorted threadPosts.
// So, the whole postList.Order is not sorted.
// The fully sorted list comes only when the CollapsedThreads is true and the Directions is not empty.
filterOptions := filterPostOptions{}
if opts.CollapsedThreads && opts.Direction != "" {
filterOptions.assumeSortedCreatedAt = true
}
if appErr := a.filterInaccessiblePosts(posts, filterOptions); appErr != nil {
return nil, appErr
}
return posts, nil
}
@@ -879,6 +913,10 @@ func (a *App) GetFlaggedPosts(userID string, offset int, limit int) (*model.Post
return nil, model.NewAppError("GetFlaggedPosts", "app.post.get_flagged_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -888,6 +926,10 @@ func (a *App) GetFlaggedPostsForTeam(userID, teamID string, offset int, limit in
return nil, model.NewAppError("GetFlaggedPostsForTeam", "app.post.get_flagged_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -897,6 +939,10 @@ func (a *App) GetFlaggedPostsForChannel(userID, channelID string, offset int, li
return nil, model.NewAppError("GetFlaggedPostsForChannel", "app.post.get_flagged_posts.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if appErr := a.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -929,6 +975,10 @@ func (a *App) GetPermalinkPost(c *request.Context, postID string, userID string)
return nil, err
}
if appErr := a.filterInaccessiblePosts(list, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil {
return nil, appErr
}
return list, nil
}
@@ -944,6 +994,19 @@ func (a *App) GetPostsBeforePost(options model.GetPostsOptions) (*model.PostList
}
}
// GetPostsBefore orders by channel id and deleted at,
// before sorting based on created at.
// but the deleted at is only ever where deleted at = 0,
// and channel id may or may not be empty (all channels) or defined (single channel),
// so we can still optimize if the search is for a single channel
filterOptions := filterPostOptions{}
if options.ChannelId != "" {
filterOptions.assumeSortedCreatedAt = true
}
if appErr := a.filterInaccessiblePosts(postList, filterOptions); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -959,6 +1022,19 @@ func (a *App) GetPostsAfterPost(options model.GetPostsOptions) (*model.PostList,
}
}
// GetPostsAfter orders by channel id and deleted at,
// before sorting based on created at.
// but the deleted at is only ever where deleted at = 0,
// and channel id may or may not be empty (all channels) or defined (single channel),
// so we can still optimize if the search is for a single channel
filterOptions := filterPostOptions{}
if options.ChannelId != "" {
filterOptions.assumeSortedCreatedAt = true
}
if appErr := a.filterInaccessiblePosts(postList, filterOptions); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -982,6 +1058,19 @@ func (a *App) GetPostsAroundPost(before bool, options model.GetPostsOptions) (*m
}
}
// GetPostsBefore and GetPostsAfter order by channel id and deleted at,
// before sorting based on created at.
// but the deleted at is only ever where deleted at = 0,
// and channel id may or may not be empty (all channels) or defined (single channel),
// so we can still optimize if the search is for a single channel
filterOptions := filterPostOptions{}
if options.ChannelId != "" {
filterOptions.assumeSortedCreatedAt = true
}
if appErr := a.filterInaccessiblePosts(postList, filterOptions); appErr != nil {
return nil, appErr
}
return postList, nil
}
@@ -1270,6 +1359,9 @@ func (a *App) searchPostsInTeam(teamID string, userID string, paramsList []*mode
}
posts.SortByCreateAt()
a.filterInaccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true})
return posts, nil
}
@@ -1297,10 +1389,85 @@ func (a *App) convertUserNameToUserIds(usernames []string) []string {
return usernames
}
// GetLastAccessiblePostTime returns CreateAt time(from cache) of the last accessible post as per the cloud limit
func (a *App) GetLastAccessiblePostTime() (int64, *model.AppError) {
license := a.Srv().License()
if license == nil || !*license.Features.Cloud {
return 0, nil
}
system, err := a.Srv().Store.System().GetByName(model.SystemLastAccessiblePostTime)
if err != nil {
var nfErr *store.ErrNotFound
switch {
case errors.As(err, &nfErr):
// All posts are accessible
return 0, nil
default:
return 0, model.NewAppError("GetLastAccessiblePostTime", "app.system.get_by_name.app_error", nil, err.Error(), http.StatusInternalServerError)
}
}
lastAccessiblePostTime, err := strconv.ParseInt(system.Value, 10, 64)
if err != nil {
return 0, model.NewAppError("GetLastAccessiblePostTime", "common.parse_error_int64", map[string]interface{}{"Value": system.Value}, err.Error(), http.StatusInternalServerError)
}
return lastAccessiblePostTime, nil
}
// ComputeLastAccessiblePostTime updates cache with CreateAt time of the last accessible post as per the cloud plan's limit.
// Use GetLastAccessiblePostTime() to access the result.
func (a *App) ComputeLastAccessiblePostTime() *model.AppError {
limit, appErr := a.getCloudMessagesHistoryLimit()
if appErr != nil {
return appErr
}
createdAt, err := a.Srv().GetStore().Post().GetNthRecentPostTime(limit)
if err != nil {
var nfErr *store.ErrNotFound
if !errors.As(err, &nfErr) {
return model.NewAppError("ComputeLastAccessiblePostTime", "app.last_accessible_post.app_error", nil, err.Error(), http.StatusInternalServerError)
}
}
// Update Cache
err = a.Srv().Store.System().SaveOrUpdate(&model.System{
Name: model.SystemLastAccessiblePostTime,
Value: strconv.FormatInt(createdAt, 10),
})
if err != nil {
return model.NewAppError("ComputeLastAccessiblePostTime", "app.system.save.app_error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (a *App) getCloudMessagesHistoryLimit() (int64, *model.AppError) {
license := a.Srv().License()
if license == nil || !*license.Features.Cloud {
return 0, nil
}
limits, err := a.Cloud().GetCloudLimits("")
if err != nil {
return 0, model.NewAppError("getCloudMessagesHistoryLimit", "api.cloud.app_error", nil, err.Error(), http.StatusInternalServerError)
}
if limits == nil || limits.Messages == nil || limits.Messages.History == nil {
// Cloud limit is not applicable
return 0, nil
}
return int64(*limits.Messages.History), nil
}
func (a *App) SearchPostsInTeam(teamID string, paramsList []*model.SearchParams) (*model.PostList, *model.AppError) {
if !*a.Config().ServiceSettings.EnablePostSearch {
return nil, model.NewAppError("SearchPostsInTeam", "store.sql_post.search.disabled", nil, fmt.Sprintf("teamId=%v", teamID), http.StatusNotImplemented)
}
return a.searchPostsInTeam(teamID, "", paramsList, func(params *model.SearchParams) {
params.SearchWithoutUserId = true
})
@@ -1354,6 +1521,10 @@ func (a *App) SearchPostsForUser(c *request.Context, terms string, userID string
}
}
if appErr := a.filterInaccessiblePosts(postSearchResults.PostList, filterPostOptions{assumeSortedCreatedAt: true}); appErr != nil {
return nil, appErr
}
return postSearchResults, nil
}
@@ -1703,19 +1874,25 @@ func (a *App) GetPostIfAuthorized(postID string, session *model.Session, include
return post, nil
}
func (a *App) GetPostsByIds(postIDs []string) ([]*model.Post, *model.AppError) {
// GetPostsByIds response bool value indicates, if the post is inaccessible due to cloud plan's limit.
func (a *App) GetPostsByIds(postIDs []string) ([]*model.Post, bool, *model.AppError) {
posts, err := a.Srv().Store.Post().GetPostsByIds(postIDs)
if err != nil {
var nfErr *store.ErrNotFound
switch {
case errors.As(err, &nfErr):
return nil, model.NewAppError("GetPostsByIds", "app.post.get.app_error", nil, nfErr.Error(), http.StatusNotFound)
return nil, false, model.NewAppError("GetPostsByIds", "app.post.get.app_error", nil, nfErr.Error(), http.StatusNotFound)
default:
return nil, model.NewAppError("GetPostsByIds", "app.post.get.app_error", nil, err.Error(), http.StatusInternalServerError)
return nil, false, model.NewAppError("GetPostsByIds", "app.post.get.app_error", nil, err.Error(), http.StatusInternalServerError)
}
}
return posts, nil
posts, hasInaccessiblePosts, appErr := a.getFilteredAccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true})
if appErr != nil {
return nil, false, appErr
}
return posts, hasInaccessiblePosts, nil
}
func (a *App) GetTopThreadsForTeamSince(teamID, userID string, opts *model.InsightsOpts) (*model.TopThreadList, *model.AppError) {

231
app/post_helpers.go Обычный файл
Просмотреть файл

@@ -0,0 +1,231 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package app
import (
"net/http"
"sort"
"github.com/mattermost/mattermost-server/v6/model"
)
type filterPostOptions struct {
assumeSortedCreatedAt bool
}
type accessibleBounds struct {
start int
end int
}
func (b accessibleBounds) allAccessible(lenPosts int) bool {
return b.start == allAccessibleBounds(lenPosts).start && b.end == allAccessibleBounds(lenPosts).end
}
func (b accessibleBounds) noAccessible() bool {
return b.start == noAccessibleBounds.start && b.end == noAccessibleBounds.end
}
var noAccessibleBounds = accessibleBounds{start: -1, end: -1}
var allAccessibleBounds = func(lenPosts int) accessibleBounds { return accessibleBounds{start: 0, end: lenPosts - 1} }
// getTimeSortedPostAccessibleBounds returns what the boundaries are for accessible posts.
// It assumes that CreateAt time for posts is monotonically increasing or decreasing.
// It could be either because posts can be returned in ascending or descending time order.
// Special values (which can be checked with methods `allAccessible` and `allInaccessible`)
// denote if all or none of the posts are accessible.
func getTimeSortedPostAccessibleBounds(earliestAccessibleTime int64, lenPosts int, getCreateAt func(int) int64) accessibleBounds {
if lenPosts == 0 {
return allAccessibleBounds(lenPosts)
}
if lenPosts == 1 {
if getCreateAt(0) >= earliestAccessibleTime {
return allAccessibleBounds(lenPosts)
}
return noAccessibleBounds
}
ascending := getCreateAt(0) < getCreateAt(lenPosts-1)
idx := sort.Search(lenPosts, func(i int) bool {
if ascending {
// Ascending order automatically picks the left most post(at idx),
// in case multiple posts at idx, idx+1, idx+2... have the same time.
return getCreateAt(i) >= earliestAccessibleTime
}
// Special case(subtracting 1) for descending order to include the right most post(at idx+k),
// in case multiple posts at idx, idx+1, idx+2...idx+k have the same time.
return getCreateAt(i) <= earliestAccessibleTime-1
})
if ascending {
if idx == lenPosts {
return noAccessibleBounds
}
return accessibleBounds{start: idx, end: lenPosts - 1}
}
if idx == 0 {
return noAccessibleBounds
}
return accessibleBounds{start: 0, end: idx - 1}
}
// linearFilterPostList make no assumptions about ordering, go through posts one by one
// this is the slower fallback that is still safe if we can not
// assume posts are ordered by CreatedAt
func linearFilterPostList(postList *model.PostList, earliestAccessibleTime int64) {
// filter Posts
posts := postList.Posts
order := postList.Order
n := 0
for i, postId := range order {
if posts[postId].CreateAt >= earliestAccessibleTime {
order[n] = order[i]
n++
} else {
postList.HasInaccessiblePosts = true
delete(posts, postId)
}
}
postList.Order = order[:n]
// it can happen that some post list results don't have all posts in the Order field.
// for example GetPosts in the CollapsedThreads = false path, parents are not added
// to Order
for postId := range posts {
if posts[postId].CreateAt < earliestAccessibleTime {
postList.HasInaccessiblePosts = true
delete(posts, postId)
}
}
}
// linearFilterPostsSlice make no assumptions about ordering, go through posts one by one
// this is the slower fallback that is still safe if we can not
// assume posts are ordered by CreatedAt
func linearFilterPostsSlice(posts []*model.Post, earliestAccessibleTime int64) ([]*model.Post, bool) {
hasInaccessiblePosts := false
n := 0
for i := range posts {
if posts[i].CreateAt >= earliestAccessibleTime {
posts[n] = posts[i]
n++
} else {
hasInaccessiblePosts = true
}
}
return posts[:n], hasInaccessiblePosts
}
// filterInaccessiblePosts filters out the posts, past the cloud limit
func (a *App) filterInaccessiblePosts(postList *model.PostList, options filterPostOptions) *model.AppError {
if postList == nil || postList.Posts == nil || len(postList.Posts) == 0 {
return nil
}
lastAccessiblePostTime, appErr := a.GetLastAccessiblePostTime()
if appErr != nil {
return model.NewAppError("filterInaccessiblePosts", "app.last_accessible_post.app_error", nil, appErr.Error(), http.StatusInternalServerError)
}
if lastAccessiblePostTime == 0 {
// No need to filter, all posts are accessible
return nil
}
if len(postList.Posts) == len(postList.Order) && options.assumeSortedCreatedAt {
lenPosts := len(postList.Posts)
getCreateAt := func(i int) int64 { return postList.Posts[postList.Order[i]].CreateAt }
bounds := getTimeSortedPostAccessibleBounds(lastAccessiblePostTime, lenPosts, getCreateAt)
if bounds.allAccessible(lenPosts) {
return nil
}
if bounds.noAccessible() {
if lenPosts > 0 {
postList.HasInaccessiblePosts = true
}
postList.Posts = map[string]*model.Post{}
postList.Order = []string{}
return nil
}
postList.HasInaccessiblePosts = true
posts := postList.Posts
order := postList.Order
accessibleCount := bounds.end - bounds.start + 1
inaccessibleCount := lenPosts - accessibleCount
// Linearly cover shorter route to traverse posts map
if inaccessibleCount < accessibleCount {
for i := 0; i < bounds.start; i++ {
delete(posts, order[i])
}
for i := bounds.end + 1; i < lenPosts; i++ {
delete(posts, order[i])
}
} else {
accessiblePosts := make(map[string]*model.Post, accessibleCount)
for i := bounds.start; i <= bounds.end; i++ {
accessiblePosts[order[i]] = posts[order[i]]
}
postList.Posts = accessiblePosts
}
postList.Order = postList.Order[bounds.start : bounds.end+1]
} else {
linearFilterPostList(postList, lastAccessiblePostTime)
}
return nil
}
// isInaccessiblePost indicates if the post is past the cloud plan's limit.
func (a *App) isInaccessiblePost(post *model.Post) (bool, *model.AppError) {
if post == nil {
return false, nil
}
pl := &model.PostList{
Order: []string{post.Id},
Posts: map[string]*model.Post{post.Id: post},
}
return pl.HasInaccessiblePosts, a.filterInaccessiblePosts(pl, filterPostOptions{assumeSortedCreatedAt: true})
}
// getFilteredAccessiblePosts returns accessible posts filtered as per the cloud plan's limit and also indicates if there were any inaccessible posts
func (a *App) getFilteredAccessiblePosts(posts []*model.Post, options filterPostOptions) ([]*model.Post, bool, *model.AppError) {
if len(posts) == 0 {
return posts, false, nil
}
filteredPosts := []*model.Post{}
lastAccessiblePostTime, appErr := a.GetLastAccessiblePostTime()
if appErr != nil {
return filteredPosts, false, model.NewAppError("getFilteredAccessiblePosts", "app.last_accessible_post.app_error", nil, appErr.Error(), http.StatusInternalServerError)
} else if lastAccessiblePostTime == 0 {
// No need to filter, all posts are accessible
return posts, false, nil
}
if options.assumeSortedCreatedAt {
lenPosts := len(posts)
getCreateAt := func(i int) int64 { return posts[i].CreateAt }
bounds := getTimeSortedPostAccessibleBounds(lastAccessiblePostTime, lenPosts, getCreateAt)
if bounds.allAccessible(lenPosts) {
return posts, false, nil
}
if bounds.noAccessible() {
return filteredPosts, lenPosts > 0, nil
}
filteredPosts = posts[bounds.start : bounds.end+1]
return filteredPosts, true, nil
}
filteredPosts, hasInaccessiblePosts := linearFilterPostsSlice(posts, lastAccessiblePostTime)
return filteredPosts, hasInaccessiblePosts, nil
}

377
app/post_helpers_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,377 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package app
import (
"testing"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetTimeSortedPostAccessibleBounds(t *testing.T) {
var postFromCreateAt = func(at int64) *model.Post {
return &model.Post{CreateAt: at}
}
getPostListCreateAtFunc := func(pl *model.PostList) func(i int) int64 {
return func(i int) int64 {
return pl.Posts[pl.Order[i]].CreateAt
}
}
t.Run("empty posts returns all accessible posts", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{},
Order: []string{},
}
bounds := getTimeSortedPostAccessibleBounds(0, len(pl.Posts), getPostListCreateAtFunc(pl))
require.True(t, bounds.allAccessible(len(pl.Posts)))
})
t.Run("one accessible post returns all accessible posts", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(1),
},
Order: []string{"post_a"},
}
bounds := getTimeSortedPostAccessibleBounds(0, len(pl.Posts), getPostListCreateAtFunc(pl))
require.True(t, bounds.allAccessible(len(pl.Posts)))
})
t.Run("one inaccessible post returns no accessible posts", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
},
Order: []string{"post_a"},
}
bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl))
require.True(t, bounds.noAccessible())
})
t.Run("all accessible posts returns all accessible posts", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(1),
"post_b": postFromCreateAt(2),
"post_c": postFromCreateAt(3),
"post_d": postFromCreateAt(4),
"post_e": postFromCreateAt(5),
"post_f": postFromCreateAt(6),
},
Order: []string{"post_a", "post_b", "post_c", "post_d", "post_e", "post_f"},
}
bounds := getTimeSortedPostAccessibleBounds(0, len(pl.Posts), getPostListCreateAtFunc(pl))
require.True(t, bounds.allAccessible(len(pl.Posts)))
})
t.Run("all inaccessible posts returns all inaccessible posts", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(1),
"post_b": postFromCreateAt(2),
"post_c": postFromCreateAt(3),
"post_d": postFromCreateAt(4),
"post_e": postFromCreateAt(5),
"post_f": postFromCreateAt(6),
},
Order: []string{"post_a", "post_b", "post_c", "post_d", "post_e", "post_f"},
}
bounds := getTimeSortedPostAccessibleBounds(7, len(pl.Posts), getPostListCreateAtFunc(pl))
require.True(t, bounds.noAccessible())
})
t.Run("all accessible posts returns all accessible posts, descending ordered", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(1),
"post_b": postFromCreateAt(2),
"post_c": postFromCreateAt(3),
"post_d": postFromCreateAt(4),
"post_e": postFromCreateAt(5),
"post_f": postFromCreateAt(6),
},
Order: []string{"post_f", "post_e", "post_d", "post_c", "post_b", "post_a"},
}
bounds := getTimeSortedPostAccessibleBounds(0, len(pl.Posts), getPostListCreateAtFunc(pl))
require.True(t, bounds.allAccessible(len(pl.Posts)))
})
t.Run("all inaccessible posts returns all inaccessible posts, descending ordered", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(1),
"post_b": postFromCreateAt(2),
"post_c": postFromCreateAt(3),
"post_d": postFromCreateAt(4),
"post_e": postFromCreateAt(5),
"post_f": postFromCreateAt(6),
},
Order: []string{"post_f", "post_e", "post_d", "post_c", "post_b", "post_a"},
}
bounds := getTimeSortedPostAccessibleBounds(7, len(pl.Posts), getPostListCreateAtFunc(pl))
require.True(t, bounds.noAccessible())
})
t.Run("two posts, first accessible", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(1),
"post_b": postFromCreateAt(0),
},
Order: []string{"post_a", "post_b"},
}
bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl))
require.Equal(t, accessibleBounds{start: 0, end: 0}, bounds)
})
t.Run("two posts, second accessible", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
"post_b": postFromCreateAt(1),
},
Order: []string{"post_a", "post_b"},
}
bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl))
require.Equal(t, accessibleBounds{start: 1, end: 1}, bounds)
})
t.Run("picks the left most post for boundaries when there are time ties", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
"post_b": postFromCreateAt(1),
"post_c": postFromCreateAt(1),
"post_d": postFromCreateAt(2),
},
Order: []string{"post_a", "post_b", "post_c", "post_d"},
}
bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl))
require.Equal(t, accessibleBounds{start: 1, end: len(pl.Posts) - 1}, bounds)
})
t.Run("picks the right most post for boundaries when there are time ties, descending ordered", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
"post_b": postFromCreateAt(1),
"post_c": postFromCreateAt(1),
"post_d": postFromCreateAt(2),
},
Order: []string{"post_d", "post_c", "post_b", "post_a"},
}
bounds := getTimeSortedPostAccessibleBounds(1, len(pl.Posts), getPostListCreateAtFunc(pl))
require.Equal(t, accessibleBounds{start: 0, end: 2}, bounds)
})
t.Run("odd number of posts and reverse time selects right boundaries", func(t *testing.T) {
pl := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
"post_b": postFromCreateAt(1),
"post_c": postFromCreateAt(2),
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
},
Order: []string{"post_e", "post_d", "post_c", "post_b", "post_a"},
}
bounds := getTimeSortedPostAccessibleBounds(2, len(pl.Posts), getPostListCreateAtFunc(pl))
require.Equal(t, accessibleBounds{start: 0, end: 2}, bounds)
})
t.Run("posts-slice: odd number of posts and reverse time selects right boundaries", func(t *testing.T) {
posts := []*model.Post{postFromCreateAt(4), postFromCreateAt(3), postFromCreateAt(2), postFromCreateAt(1), postFromCreateAt(0)}
bounds := getTimeSortedPostAccessibleBounds(2, len(posts), func(i int) int64 { return posts[i].CreateAt })
require.Equal(t, accessibleBounds{start: 0, end: 2}, bounds)
})
}
func TestFilterInaccessiblePosts(t *testing.T) {
th := Setup(t)
th.App.Srv().SetLicense(model.NewTestLicense("cloud"))
th.App.Srv().Store.System().Save(&model.System{
Name: model.SystemLastAccessiblePostTime,
Value: "2",
})
defer th.TearDown()
var postFromCreateAt = func(at int64) *model.Post {
return &model.Post{CreateAt: at}
}
t.Run("ascending order returns correct posts", func(t *testing.T) {
postList := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
"post_b": postFromCreateAt(1),
"post_c": postFromCreateAt(2),
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
},
Order: []string{"post_a", "post_b", "post_c", "post_d", "post_e"},
}
appErr := th.App.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true})
require.Nil(t, appErr)
assert.Equal(t, map[string]*model.Post{
"post_c": postFromCreateAt(2),
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
}, postList.Posts)
assert.Equal(t, []string{
"post_c",
"post_d",
"post_e",
}, postList.Order)
})
t.Run("descending order returns correct posts", func(t *testing.T) {
postList := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
"post_b": postFromCreateAt(1),
"post_c": postFromCreateAt(2),
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
},
Order: []string{"post_e", "post_d", "post_c", "post_b", "post_a"},
}
appErr := th.App.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: true})
require.Nil(t, appErr)
assert.Equal(t, map[string]*model.Post{
"post_c": postFromCreateAt(2),
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
}, postList.Posts)
assert.Equal(t, []string{
"post_e",
"post_d",
"post_c",
}, postList.Order)
})
t.Run("handles mixed create at ordering correctly if correct options given", func(t *testing.T) {
postList := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
"post_b": postFromCreateAt(1),
"post_c": postFromCreateAt(2),
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
},
Order: []string{"post_e", "post_b", "post_a", "post_d", "post_c"},
}
appErr := th.App.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: false})
require.Nil(t, appErr)
assert.Equal(t, map[string]*model.Post{
"post_c": postFromCreateAt(2),
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
}, postList.Posts)
assert.Equal(t, []string{
"post_e",
"post_d",
"post_c",
}, postList.Order)
})
t.Run("handles posts missing from order when doing linear search", func(t *testing.T) {
postList := &model.PostList{
Posts: map[string]*model.Post{
"post_a": postFromCreateAt(0),
"post_b": postFromCreateAt(1),
"post_c": postFromCreateAt(1),
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
},
Order: []string{"post_e", "post_a", "post_d", "post_b"},
}
appErr := th.App.filterInaccessiblePosts(postList, filterPostOptions{assumeSortedCreatedAt: false})
require.Nil(t, appErr)
assert.Equal(t, map[string]*model.Post{
"post_d": postFromCreateAt(3),
"post_e": postFromCreateAt(4),
}, postList.Posts)
assert.Equal(t, []string{
"post_e",
"post_d",
}, postList.Order)
})
}
func TestGetFilteredAccessiblePosts(t *testing.T) {
th := Setup(t)
th.App.Srv().SetLicense(model.NewTestLicense("cloud"))
th.App.Srv().Store.System().Save(&model.System{
Name: model.SystemLastAccessiblePostTime,
Value: "2",
})
defer th.TearDown()
var postFromCreateAt = func(at int64) *model.Post {
return &model.Post{CreateAt: at}
}
t.Run("ascending order returns correct posts", func(t *testing.T) {
posts := []*model.Post{postFromCreateAt(0), postFromCreateAt(1), postFromCreateAt(2), postFromCreateAt(3), postFromCreateAt(4)}
filteredPosts, _, appErr := th.App.getFilteredAccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true})
assert.Nil(t, appErr)
assert.Equal(t, []*model.Post{postFromCreateAt(2), postFromCreateAt(3), postFromCreateAt(4)}, filteredPosts)
})
t.Run("descending order returns correct posts", func(t *testing.T) {
posts := []*model.Post{postFromCreateAt(4), postFromCreateAt(3), postFromCreateAt(2), postFromCreateAt(1), postFromCreateAt(0)}
filteredPosts, _, appErr := th.App.getFilteredAccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: true})
assert.Nil(t, appErr)
assert.Equal(t, []*model.Post{postFromCreateAt(4), postFromCreateAt(3), postFromCreateAt(2)}, filteredPosts)
})
t.Run("handles mixed create at ordering correctly if correct options given", func(t *testing.T) {
posts := []*model.Post{postFromCreateAt(4), postFromCreateAt(1), postFromCreateAt(0), postFromCreateAt(3), postFromCreateAt(2)}
filteredPosts, _, appErr := th.App.getFilteredAccessiblePosts(posts, filterPostOptions{assumeSortedCreatedAt: false})
assert.Nil(t, appErr)
assert.Equal(t, []*model.Post{postFromCreateAt(4), postFromCreateAt(3), postFromCreateAt(2)}, filteredPosts)
})
}
func TestIsInaccessiblePost(t *testing.T) {
th := Setup(t)
th.App.Srv().SetLicense(model.NewTestLicense("cloud"))
th.App.Srv().Store.System().Save(&model.System{
Name: model.SystemLastAccessiblePostTime,
Value: "2",
})
defer th.TearDown()
post := &model.Post{CreateAt: 3}
r, appErr := th.App.isInaccessiblePost(post)
assert.Nil(t, appErr)
assert.Equal(t, false, r)
post = &model.Post{CreateAt: 1}
r, appErr = th.App.isInaccessiblePost(post)
assert.Nil(t, appErr)
assert.Equal(t, true, r)
}

Просмотреть файл

@@ -53,11 +53,12 @@ func (s *Server) initPostMetadata() {
func (a *App) PreparePostListForClient(originalList *model.PostList) *model.PostList {
list := &model.PostList{
Posts: make(map[string]*model.Post, len(originalList.Posts)),
Order: originalList.Order,
NextPostId: originalList.NextPostId,
PrevPostId: originalList.PrevPostId,
HasNext: originalList.HasNext,
Posts: make(map[string]*model.Post, len(originalList.Posts)),
Order: originalList.Order,
NextPostId: originalList.NextPostId,
PrevPostId: originalList.PrevPostId,
HasNext: originalList.HasNext,
HasInaccessiblePosts: originalList.HasInaccessiblePosts,
}
for id, originalPost := range originalList.Posts {

Просмотреть файл

@@ -5,6 +5,7 @@ package app
import (
"context"
"errors"
"fmt"
"net/http"
"os"
@@ -15,6 +16,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
eMocks "github.com/mattermost/mattermost-server/v6/einterfaces/mocks"
"github.com/mattermost/mattermost-server/v6/model"
"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
"github.com/mattermost/mattermost-server/v6/services/imageproxy"
@@ -2818,6 +2820,68 @@ func TestShouldNotRefollowOnOthersReply(t *testing.T) {
require.True(t, m.Following)
}
func TestGetLastAccessiblePostTime(t *testing.T) {
th := SetupWithStoreMock(t)
defer th.TearDown()
r, err := th.App.GetLastAccessiblePostTime()
assert.Nil(t, err)
assert.Equal(t, int64(0), r)
th.App.Srv().SetLicense(model.NewTestLicense("cloud"))
mockStore := th.App.Srv().Store.(*storemocks.Store)
mockSystemStore := storemocks.SystemStore{}
mockStore.On("System").Return(&mockSystemStore)
mockSystemStore.On("GetByName", mock.Anything).Return(nil, store.NewErrNotFound("", ""))
r, err = th.App.GetLastAccessiblePostTime()
assert.Nil(t, err)
assert.Equal(t, int64(0), r)
mockSystemStore = storemocks.SystemStore{}
mockStore.On("System").Return(&mockSystemStore)
mockSystemStore.On("GetByName", mock.Anything).Return(nil, errors.New("test"))
_, err = th.App.GetLastAccessiblePostTime()
assert.NotNil(t, err)
mockSystemStore = storemocks.SystemStore{}
mockStore.On("System").Return(&mockSystemStore)
mockSystemStore.On("GetByName", mock.Anything).Return(&model.System{Name: model.SystemLastAccessiblePostTime, Value: "10"}, nil)
r, err = th.App.GetLastAccessiblePostTime()
assert.Nil(t, err)
assert.Equal(t, int64(10), r)
}
func TestComputeLastAccessiblePostTime(t *testing.T) {
th := SetupWithStoreMock(t)
defer th.TearDown()
th.App.Srv().SetLicense(model.NewTestLicense("cloud"))
cloud := &eMocks.CloudInterface{}
th.App.Srv().Cloud = cloud
cloud.Mock.On("GetCloudLimits", mock.Anything).Return(&model.ProductLimits{
Messages: &model.MessagesLimits{
History: model.NewInt(1),
},
}, nil)
mockStore := th.App.Srv().Store.(*storemocks.Store)
mockPostStore := storemocks.PostStore{}
mockPostStore.On("GetNthRecentPostTime", mock.Anything).Return(int64(1), nil)
mockSystemStore := storemocks.SystemStore{}
mockSystemStore.On("SaveOrUpdate", mock.Anything).Return(nil)
mockStore.On("Post").Return(&mockPostStore)
mockStore.On("System").Return(&mockSystemStore)
err := th.App.ComputeLastAccessiblePostTime()
assert.Nil(t, err)
mockSystemStore.AssertCalled(t, "SaveOrUpdate", mock.Anything)
}
func TestGetTopThreadsForTeamSince(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()

Просмотреть файл

@@ -49,6 +49,7 @@ import (
"github.com/mattermost/mattermost-server/v6/jobs/extract_content"
"github.com/mattermost/mattermost-server/v6/jobs/import_delete"
"github.com/mattermost/mattermost-server/v6/jobs/import_process"
"github.com/mattermost/mattermost-server/v6/jobs/last_accessible_post"
"github.com/mattermost/mattermost-server/v6/jobs/migrations"
"github.com/mattermost/mattermost-server/v6/jobs/product_notices"
"github.com/mattermost/mattermost-server/v6/jobs/resend_invitation_email"
@@ -2046,6 +2047,12 @@ func (s *Server) initJobs() {
extract_content.MakeWorker(s.Jobs, New(ServerConnector(s.Channels())), s.Store),
nil,
)
s.Jobs.RegisterJobType(
model.JobTypeLastAccessiblePost,
last_accessible_post.MakeWorker(s.Jobs, s.License(), New(ServerConnector(s.Channels()))),
last_accessible_post.MakeScheduler(s.Jobs, s.License()),
)
}
func (s *Server) TelemetryId() string {