From d87fe35bdf02c57dd513429a9481e1a9410c2d20 Mon Sep 17 00:00:00 2001 From: Vishal Date: Tue, 31 May 2022 16:25:46 +0530 Subject: [PATCH] [MM-44263] Cache Posts Usage (#20254) * Fetch users only posts * Cache Posts Usage * Explicitly validate posts count before testing usage * Add DocString * Revert cache changes * Revert "Revert cache changes" This reverts commit ba693f8d5028b62c19f6a730e8dca73a224d0761. --- app/usage.go | 2 +- model/cluster_message.go | 1 + model/post.go | 2 ++ store/localcachelayer/layer.go | 16 ++++++++++++-- store/localcachelayer/post_layer.go | 33 +++++++++++++++++++++++++++++ 5 files changed, 51 insertions(+), 3 deletions(-) diff --git a/app/usage.go b/app/usage.go index 262c419365..71a75d2193 100644 --- a/app/usage.go +++ b/app/usage.go @@ -47,7 +47,7 @@ func (ch *Channels) getIntegrationsUsage() (*model.IntegrationsUsage, *model.App // GetPostsUsage returns "rounded off" total posts count like returns 900 instead of 987 func (a *App) GetPostsUsage() (int64, *model.AppError) { - count, err := a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeDeleted: true, UsersPostsOnly: true}) + count, err := a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeDeleted: true, UsersPostsOnly: true, AllowFromCache: true}) if err != nil { return 0, model.NewAppError("GetPostsUsage", "app.post.analytics_posts_count.app_error", nil, err.Error(), http.StatusInternalServerError) } diff --git a/model/cluster_message.go b/model/cluster_message.go index 90999ab03a..35bae0371e 100644 --- a/model/cluster_message.go +++ b/model/cluster_message.go @@ -31,6 +31,7 @@ const ( ClusterEventInvalidateCacheForChannelMemberCounts ClusterEvent = "inv_channel_member_counts" ClusterEventInvalidateCacheForLastPosts ClusterEvent = "inv_last_posts" ClusterEventInvalidateCacheForLastPostTime ClusterEvent = "inv_last_post_time" + ClusterEventInvalidateCacheForPostsUsage ClusterEvent = "inv_posts_usage" ClusterEventInvalidateCacheForTeams ClusterEvent = "inv_teams" ClusterEventClearSessionCacheForAllUsers ClusterEvent = "inv_all_user_sessions" ClusterEventInstallPlugin ClusterEvent = "install_plugin" diff --git a/model/post.go b/model/post.go index 740969e283..f5ae88c759 100644 --- a/model/post.go +++ b/model/post.go @@ -281,6 +281,8 @@ type PostCountOptions struct { MustHaveHashtag bool ExcludeDeleted bool UsersPostsOnly bool + // AllowFromCache looks up cache only when ExcludeDeleted and UsersPostsOnly are true and rest are falsy. + AllowFromCache bool } func (o *Post) Etag() string { diff --git a/store/localcachelayer/layer.go b/store/localcachelayer/layer.go index f715c7685e..4ebb47180e 100644 --- a/store/localcachelayer/layer.go +++ b/store/localcachelayer/layer.go @@ -41,8 +41,10 @@ const ( ChannelMembersCountsCacheSize = model.ChannelCacheSize ChannelMembersCountsCacheSec = 30 * 60 - LastPostsCacheSize = 20000 - LastPostsCacheSec = 30 * 60 + LastPostsCacheSize = 20000 + LastPostsCacheSec = 30 * 60 + PostsUsageCacheSize = 1 + PostsUsageCacheSec = 30 * 60 TermsOfServiceCacheSize = 20000 TermsOfServiceCacheSec = 30 * 60 @@ -97,6 +99,7 @@ type LocalCacheStore struct { post LocalCachePostStore postLastPostsCache cache.Cache lastPostTimeCache cache.Cache + postsUsageCache cache.Cache user *LocalCacheUserStore userProfileByIdsCache cache.Cache @@ -256,6 +259,14 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf }); err != nil { return } + if localCacheStore.postsUsageCache, err = cacheProvider.NewCache(&cache.CacheOptions{ + Size: PostsUsageCacheSize, + Name: "PostsUsage", + DefaultExpiry: PostsUsageCacheSec * time.Second, + InvalidateClusterEvent: model.ClusterEventInvalidateCacheForPostsUsage, + }); err != nil { + return + } localCacheStore.post = LocalCachePostStore{PostStore: baseStore.Post(), rootStore: &localCacheStore} // TOS @@ -312,6 +323,7 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForSchemes, localCacheStore.scheme.handleClusterInvalidateScheme) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForFileInfos, localCacheStore.fileInfo.handleClusterInvalidateFileInfo) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForLastPostTime, localCacheStore.post.handleClusterInvalidateLastPostTime) + cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForPostsUsage, localCacheStore.post.handleClusterInvalidatePostsUsage) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForWebhooks, localCacheStore.webhook.handleClusterInvalidateWebhook) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForEmojisById, localCacheStore.emoji.handleClusterInvalidateEmojiById) cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForEmojisIdByName, localCacheStore.emoji.handleClusterInvalidateEmojiIdByName) diff --git a/store/localcachelayer/post_layer.go b/store/localcachelayer/post_layer.go index 80bc09bdcb..331367cd0e 100644 --- a/store/localcachelayer/post_layer.go +++ b/store/localcachelayer/post_layer.go @@ -34,14 +34,24 @@ func (s *LocalCachePostStore) handleClusterInvalidateLastPosts(msg *model.Cluste } } +func (s *LocalCachePostStore) handleClusterInvalidatePostsUsage(msg *model.ClusterMessage) { + if bytes.Equal(msg.Data, clearCacheMessageData) { + s.rootStore.postsUsageCache.Purge() + } else { + s.rootStore.postsUsageCache.Remove(string(msg.Data)) + } +} + func (s LocalCachePostStore) ClearCaches() { s.rootStore.doClearCacheCluster(s.rootStore.lastPostTimeCache) s.rootStore.doClearCacheCluster(s.rootStore.postLastPostsCache) + s.rootStore.doClearCacheCluster(s.rootStore.postsUsageCache) s.PostStore.ClearCaches() if s.rootStore.metrics != nil { s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Last Post Time - Purge") s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Last Posts Cache - Purge") + s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Posts Usage Cache - Purge") } } @@ -131,3 +141,26 @@ func (s LocalCachePostStore) GetPosts(options model.GetPostsOptions, allowFromCa return list, err } + +// AnalyticsPostCount looks up cache only when ExcludeDeleted and UsersPostsOnly are true and rest are falsy. +func (s LocalCachePostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) { + if !options.AllowFromCache || options.MustHaveFile || options.MustHaveHashtag || !options.UsersPostsOnly || !options.ExcludeDeleted || options.TeamId != "" { + return s.PostStore.AnalyticsPostCount(options) + } + + // Currently cache only for app > usage > GetPostsUsage() + // Other filter combinations can be cached if required + cacheKey := "posts_usage" + var count int64 + if err := s.rootStore.doStandardReadCache(s.rootStore.postsUsageCache, cacheKey, &count); err == nil { + return count, nil + } + + count, err := s.PostStore.AnalyticsPostCount(options) + if err != nil { + return 0, err + } + + s.rootStore.doStandardAddToCache(s.rootStore.postsUsageCache, cacheKey, count) + return count, nil +}