[MM-44263] Cache Posts Usage (#20254)

* Fetch users only posts

* Cache Posts Usage

* Explicitly validate posts count before testing usage

* Add DocString

* Revert cache changes

* Revert "Revert cache changes"

This reverts commit ba693f8d5028b62c19f6a730e8dca73a224d0761.

app/usage.go

@@ -47,7 +47,7 @@ func (ch *Channels) getIntegrationsUsage() (*model.IntegrationsUsage, *model.App
 
 // GetPostsUsage returns "rounded off" total posts count like returns 900 instead of 987
 func (a *App) GetPostsUsage() (int64, *model.AppError) {
-	count, err := a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeDeleted: true, UsersPostsOnly: true})
+	count, err := a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeDeleted: true, UsersPostsOnly: true, AllowFromCache: true})
 	if err != nil {
 		return 0, model.NewAppError("GetPostsUsage", "app.post.analytics_posts_count.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}

model/cluster_message.go

@@ -31,6 +31,7 @@ const (
 	ClusterEventInvalidateCacheForChannelMemberCounts       ClusterEvent = "inv_channel_member_counts"
 	ClusterEventInvalidateCacheForLastPosts                 ClusterEvent = "inv_last_posts"
 	ClusterEventInvalidateCacheForLastPostTime              ClusterEvent = "inv_last_post_time"
+	ClusterEventInvalidateCacheForPostsUsage                ClusterEvent = "inv_posts_usage"
 	ClusterEventInvalidateCacheForTeams                     ClusterEvent = "inv_teams"
 	ClusterEventClearSessionCacheForAllUsers                ClusterEvent = "inv_all_user_sessions"
 	ClusterEventInstallPlugin                               ClusterEvent = "install_plugin"

model/post.go

@@ -281,6 +281,8 @@ type PostCountOptions struct {
 	MustHaveHashtag bool
 	ExcludeDeleted  bool
 	UsersPostsOnly  bool
+	// AllowFromCache looks up cache only when ExcludeDeleted and UsersPostsOnly are true and rest are falsy.
+	AllowFromCache bool
 }
 
 func (o *Post) Etag() string {

store/localcachelayer/layer.go

@@ -41,8 +41,10 @@ const (
 	ChannelMembersCountsCacheSize = model.ChannelCacheSize
 	ChannelMembersCountsCacheSec  = 30 * 60
 
-	LastPostsCacheSize = 20000
+	LastPostsCacheSize  = 20000
-	LastPostsCacheSec  = 30 * 60
+	LastPostsCacheSec   = 30 * 60
+	PostsUsageCacheSize = 1
+	PostsUsageCacheSec  = 30 * 60
 
 	TermsOfServiceCacheSize = 20000
 	TermsOfServiceCacheSec  = 30 * 60
@@ -97,6 +99,7 @@ type LocalCacheStore struct {
 	post               LocalCachePostStore
 	postLastPostsCache cache.Cache
 	lastPostTimeCache  cache.Cache
+	postsUsageCache    cache.Cache
 
 	user                   *LocalCacheUserStore
 	userProfileByIdsCache  cache.Cache
@@ -256,6 +259,14 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf
 	}); err != nil {
 		return
 	}
+	if localCacheStore.postsUsageCache, err = cacheProvider.NewCache(&cache.CacheOptions{
+		Size:                   PostsUsageCacheSize,
+		Name:                   "PostsUsage",
+		DefaultExpiry:          PostsUsageCacheSec * time.Second,
+		InvalidateClusterEvent: model.ClusterEventInvalidateCacheForPostsUsage,
+	}); err != nil {
+		return
+	}
 	localCacheStore.post = LocalCachePostStore{PostStore: baseStore.Post(), rootStore: &localCacheStore}
 
 	// TOS
@@ -312,6 +323,7 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf
 		cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForSchemes, localCacheStore.scheme.handleClusterInvalidateScheme)
 		cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForFileInfos, localCacheStore.fileInfo.handleClusterInvalidateFileInfo)
 		cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForLastPostTime, localCacheStore.post.handleClusterInvalidateLastPostTime)
+		cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForPostsUsage, localCacheStore.post.handleClusterInvalidatePostsUsage)
 		cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForWebhooks, localCacheStore.webhook.handleClusterInvalidateWebhook)
 		cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForEmojisById, localCacheStore.emoji.handleClusterInvalidateEmojiById)
 		cluster.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForEmojisIdByName, localCacheStore.emoji.handleClusterInvalidateEmojiIdByName)

store/localcachelayer/post_layer.go

@@ -34,14 +34,24 @@ func (s *LocalCachePostStore) handleClusterInvalidateLastPosts(msg *model.Cluste
 	}
 }
 
+func (s *LocalCachePostStore) handleClusterInvalidatePostsUsage(msg *model.ClusterMessage) {
+	if bytes.Equal(msg.Data, clearCacheMessageData) {
+		s.rootStore.postsUsageCache.Purge()
+	} else {
+		s.rootStore.postsUsageCache.Remove(string(msg.Data))
+	}
+}
+
 func (s LocalCachePostStore) ClearCaches() {
 	s.rootStore.doClearCacheCluster(s.rootStore.lastPostTimeCache)
 	s.rootStore.doClearCacheCluster(s.rootStore.postLastPostsCache)
+	s.rootStore.doClearCacheCluster(s.rootStore.postsUsageCache)
 	s.PostStore.ClearCaches()
 
 	if s.rootStore.metrics != nil {
 		s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Last Post Time - Purge")
 		s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Last Posts Cache - Purge")
+		s.rootStore.metrics.IncrementMemCacheInvalidationCounter("Posts Usage Cache - Purge")
 	}
 }
 
@@ -131,3 +141,26 @@ func (s LocalCachePostStore) GetPosts(options model.GetPostsOptions, allowFromCa
 
 	return list, err
 }
+
+// AnalyticsPostCount looks up cache only when ExcludeDeleted and UsersPostsOnly are true and rest are falsy.
+func (s LocalCachePostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) {
+	if !options.AllowFromCache || options.MustHaveFile || options.MustHaveHashtag || !options.UsersPostsOnly || !options.ExcludeDeleted || options.TeamId != "" {
+		return s.PostStore.AnalyticsPostCount(options)
+	}
+
+	// Currently cache only for app > usage > GetPostsUsage()
+	// Other filter combinations can be cached if required
+	cacheKey := "posts_usage"
+	var count int64
+	if err := s.rootStore.doStandardReadCache(s.rootStore.postsUsageCache, cacheKey, &count); err == nil {
+		return count, nil
+	}
+
+	count, err := s.PostStore.AnalyticsPostCount(options)
+	if err != nil {
+		return 0, err
+	}
+
+	s.rootStore.doStandardAddToCache(s.rootStore.postsUsageCache, cacheKey, count)
+	return count, nil
+}