MM-9664 Add invalidation metrics for store caches (#8340)

* Add invalidation metrics for store caches

* Increment session invalidation metric

* Fix tests

api/channel_test.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/store"
-	"github.com/mattermost/mattermost-server/store/sqlstore"
 )
 
 func TestCreateChannel(t *testing.T) {
@@ -374,7 +373,7 @@ func TestUpdateChannel(t *testing.T) {
 	})
 	th.MakeUserChannelUser(th.BasicUser, channel2)
 	th.MakeUserChannelUser(th.BasicUser, channel3)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	if _, err := Client.UpdateChannel(channel2); err == nil {
 		t.Fatal("should have errored not channel admin")
@@ -396,7 +395,7 @@ func TestUpdateChannel(t *testing.T) {
 
 	th.MakeUserChannelAdmin(th.BasicUser, channel2)
 	th.MakeUserChannelAdmin(th.BasicUser, channel3)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	if _, err := Client.UpdateChannel(channel2); err != nil {
 		t.Fatal(err)
@@ -596,7 +595,7 @@ func TestUpdateChannelHeader(t *testing.T) {
 	})
 	th.MakeUserChannelUser(th.BasicUser, channel2)
 	th.MakeUserChannelUser(th.BasicUser, channel3)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	if _, err := Client.UpdateChannelHeader(data2); err == nil {
 		t.Fatal("should have errored not channel admin")
@@ -607,7 +606,7 @@ func TestUpdateChannelHeader(t *testing.T) {
 
 	th.MakeUserChannelAdmin(th.BasicUser, channel2)
 	th.MakeUserChannelAdmin(th.BasicUser, channel3)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	if _, err := Client.UpdateChannelHeader(data2); err != nil {
 		t.Fatal(err)
@@ -767,7 +766,7 @@ func TestUpdateChannelPurpose(t *testing.T) {
 	})
 	th.MakeUserChannelUser(th.BasicUser, channel2)
 	th.MakeUserChannelUser(th.BasicUser, channel3)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	if _, err := Client.UpdateChannelPurpose(data2); err == nil {
 		t.Fatal("should have errored not channel admin")
@@ -778,7 +777,7 @@ func TestUpdateChannelPurpose(t *testing.T) {
 
 	th.MakeUserChannelAdmin(th.BasicUser, channel2)
 	th.MakeUserChannelAdmin(th.BasicUser, channel3)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	if _, err := Client.UpdateChannelPurpose(data2); err != nil {
 		t.Fatal(err)
@@ -1344,7 +1343,7 @@ func TestDeleteChannel(t *testing.T) {
 
 	th.MakeUserChannelAdmin(th.BasicUser, channel2)
 	th.MakeUserChannelAdmin(th.BasicUser, channel3)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	if _, err := Client.DeleteChannel(channel2.Id); err != nil {
 		t.Fatal(err)

api4/channel_test.go

@@ -13,7 +13,6 @@ import (
 	"testing"
 
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/store/sqlstore"
 )
 
 func TestCreateChannel(t *testing.T) {
@@ -909,7 +908,7 @@ func TestDeleteChannel(t *testing.T) {
 	// successful delete by channel admin
 	th.MakeUserChannelAdmin(user, publicChannel6)
 	th.MakeUserChannelAdmin(user, privateChannel7)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	_, resp = Client.DeleteChannel(publicChannel6.Id)
 	CheckNoError(t, resp)
@@ -960,7 +959,7 @@ func TestDeleteChannel(t *testing.T) {
 	// // cannot delete by channel admin
 	th.MakeUserChannelAdmin(user, publicChannel6)
 	th.MakeUserChannelAdmin(user, privateChannel7)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	_, resp = Client.DeleteChannel(publicChannel6.Id)
 	CheckForbiddenStatus(t, resp)
@@ -1001,7 +1000,7 @@ func TestDeleteChannel(t *testing.T) {
 	// cannot delete by channel admin
 	th.MakeUserChannelAdmin(user, publicChannel6)
 	th.MakeUserChannelAdmin(user, privateChannel7)
-	sqlstore.ClearChannelCaches()
+	th.App.Srv.Store.Channel().ClearCaches()
 
 	_, resp = Client.DeleteChannel(publicChannel6.Id)
 	CheckForbiddenStatus(t, resp)

app/admin.go

@@ -15,7 +15,6 @@ import (
 
 	l4g "github.com/alecthomas/log4go"
 	"github.com/mattermost/mattermost-server/model"
-	"github.com/mattermost/mattermost-server/store/sqlstore"
 	"github.com/mattermost/mattermost-server/utils"
 )
 
@@ -141,10 +140,11 @@ func (a *App) InvalidateAllCachesSkipSend() {
 	l4g.Info(utils.T("api.context.invalidate_all_caches"))
 	a.sessionCache.Purge()
 	ClearStatusCache()
-	sqlstore.ClearChannelCaches()
+	a.Srv.Store.Channel().ClearCaches()
-	sqlstore.ClearUserCaches()
+	a.Srv.Store.User().ClearCaches()
-	sqlstore.ClearPostCaches()
+	a.Srv.Store.Post().ClearCaches()
-	sqlstore.ClearWebhookCaches()
+	a.Srv.Store.FileInfo().ClearCaches()
+	a.Srv.Store.Webhook().ClearCaches()
 	a.LoadLicense()
 }
 

app/session.go

@@ -138,6 +138,9 @@ func (a *App) ClearSessionCacheForUserSkipClusterSend(userId string) {
 			session := ts.(*model.Session)
 			if session.UserId == userId {
 				a.sessionCache.Remove(key)
+				if a.Metrics != nil {
+					a.Metrics.IncrementMemCacheInvalidationCounterSession()
+				}
 			}
 		}
 	}

einterfaces/metrics.go

@@ -29,8 +29,10 @@ type MetricsInterface interface {
 
 	IncrementMemCacheHitCounter(cacheName string)
 	IncrementMemCacheMissCounter(cacheName string)
+	IncrementMemCacheInvalidationCounter(cacheName string)
 	IncrementMemCacheMissCounterSession()
 	IncrementMemCacheHitCounterSession()
+	IncrementMemCacheInvalidationCounterSession()
 
 	IncrementWebsocketEvent(eventType string)
 	IncrementWebSocketBroadcast(eventType string)

store/sqlstore/channel_store.go

@@ -43,12 +43,20 @@ var allChannelMembersNotifyPropsForChannelCache = utils.NewLru(ALL_CHANNEL_MEMBE
 var channelCache = utils.NewLru(model.CHANNEL_CACHE_SIZE)
 var channelByNameCache = utils.NewLru(model.CHANNEL_CACHE_SIZE)
 
-func ClearChannelCaches() {
+func (s SqlChannelStore) ClearCaches() {
 	channelMemberCountsCache.Purge()
 	allChannelMembersForUserCache.Purge()
 	allChannelMembersNotifyPropsForChannelCache.Purge()
 	channelCache.Purge()
 	channelByNameCache.Purge()
+
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Channel Member Counts - Purge")
+		s.metrics.IncrementMemCacheInvalidationCounter("All Channel Members for User - Purge")
+		s.metrics.IncrementMemCacheInvalidationCounter("All Channel Members Notify Props for Channel - Purge")
+		s.metrics.IncrementMemCacheInvalidationCounter("Channel - Purge")
+		s.metrics.IncrementMemCacheInvalidationCounter("Channel By Name - Purge")
+	}
 }
 
 func NewSqlChannelStore(sqlStore SqlStore, metrics einterfaces.MetricsInterface) store.ChannelStore {
@@ -308,12 +316,18 @@ func (s SqlChannelStore) GetChannelUnread(channelId, userId string) store.StoreC
 	})
 }
 
-func (us SqlChannelStore) InvalidateChannel(id string) {
+func (s SqlChannelStore) InvalidateChannel(id string) {
 	channelCache.Remove(id)
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Channel - Remove by ChannelId")
+	}
 }
 
-func (us SqlChannelStore) InvalidateChannelByName(teamId, name string) {
+func (s SqlChannelStore) InvalidateChannelByName(teamId, name string) {
 	channelByNameCache.Remove(teamId + name)
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Channel by Name - Remove by TeamId and Name")
+	}
 }
 
 func (s SqlChannelStore) Get(id string, allowFromCache bool) store.StoreChannel {
@@ -814,14 +828,17 @@ func (s SqlChannelStore) GetMember(channelId string, userId string) store.StoreC
 	})
 }
 
-func (us SqlChannelStore) InvalidateAllChannelMembersForUser(userId string) {
+func (s SqlChannelStore) InvalidateAllChannelMembersForUser(userId string) {
 	allChannelMembersForUserCache.Remove(userId)
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("All Channel Members for User - Remove by UserId")
+	}
 }
 
-func (us SqlChannelStore) IsUserInChannelUseCache(userId string, channelId string) bool {
+func (s SqlChannelStore) IsUserInChannelUseCache(userId string, channelId string) bool {
 	if cacheItem, ok := allChannelMembersForUserCache.Get(userId); ok {
-		if us.metrics != nil {
+		if s.metrics != nil {
-			us.metrics.IncrementMemCacheHitCounter("All Channel Members for User")
+			s.metrics.IncrementMemCacheHitCounter("All Channel Members for User")
 		}
 		ids := cacheItem.(map[string]string)
 		if _, ok := ids[channelId]; ok {
@@ -830,12 +847,12 @@ func (us SqlChannelStore) IsUserInChannelUseCache(userId string, channelId strin
 			return false
 		}
 	} else {
-		if us.metrics != nil {
+		if s.metrics != nil {
-			us.metrics.IncrementMemCacheMissCounter("All Channel Members for User")
+			s.metrics.IncrementMemCacheMissCounter("All Channel Members for User")
 		}
 	}
 
-	if result := <-us.GetAllChannelMembersForUser(userId, true); result.Err != nil {
+	if result := <-s.GetAllChannelMembersForUser(userId, true); result.Err != nil {
 		l4g.Error("SqlChannelStore.IsUserInChannelUseCache: " + result.Err.Error())
 		return false
 	} else {
@@ -915,8 +932,11 @@ func (s SqlChannelStore) GetAllChannelMembersForUser(userId string, allowFromCac
 	})
 }
 
-func (us SqlChannelStore) InvalidateCacheForChannelMembersNotifyProps(channelId string) {
+func (s SqlChannelStore) InvalidateCacheForChannelMembersNotifyProps(channelId string) {
 	allChannelMembersNotifyPropsForChannelCache.Remove(channelId)
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("All Channel Members Notify Props for Channel - Remove by ChannelId")
+	}
 }
 
 type allChannelMemberNotifyProps struct {
@@ -966,8 +986,11 @@ func (s SqlChannelStore) GetAllChannelMembersNotifyPropsForChannel(channelId str
 	})
 }
 
-func (us SqlChannelStore) InvalidateMemberCount(channelId string) {
+func (s SqlChannelStore) InvalidateMemberCount(channelId string) {
 	channelMemberCountsCache.Remove(channelId)
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Channel Member Counts - Remove by ChannelId")
+	}
 }
 
 func (s SqlChannelStore) GetMemberCountFromCache(channelId string) int64 {

store/sqlstore/file_info_store.go

@@ -25,8 +25,11 @@ const (
 
 var fileInfoCache *utils.Cache = utils.NewLru(FILE_INFO_CACHE_SIZE)
 
-func ClearFileCaches() {
+func (fs SqlFileInfoStore) ClearCaches() {
 	fileInfoCache.Purge()
+	if fs.metrics != nil {
+		fs.metrics.IncrementMemCacheInvalidationCounter("File Info Cache - Purge")
+	}
 }
 
 func NewSqlFileInfoStore(sqlStore SqlStore, metrics einterfaces.MetricsInterface) store.FileInfoStore {
@@ -118,6 +121,9 @@ func (fs SqlFileInfoStore) GetByPath(path string) store.StoreChannel {
 
 func (fs SqlFileInfoStore) InvalidateFileInfosForPostCache(postId string) {
 	fileInfoCache.Remove(postId)
+	if fs.metrics != nil {
+		fs.metrics.IncrementMemCacheInvalidationCounter("File Info Cache - Remove by PostId")
+	}
 }
 
 func (fs SqlFileInfoStore) GetForPost(postId string, readFromMaster bool, allowFromCache bool) store.StoreChannel {

store/sqlstore/post_store.go

@@ -35,9 +35,14 @@ const (
 var lastPostTimeCache = utils.NewLru(LAST_POST_TIME_CACHE_SIZE)
 var lastPostsCache = utils.NewLru(LAST_POSTS_CACHE_SIZE)
 
-func ClearPostCaches() {
+func (s SqlPostStore) ClearCaches() {
 	lastPostTimeCache.Purge()
 	lastPostsCache.Purge()
+
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Last Post Time - Purge")
+		s.metrics.IncrementMemCacheInvalidationCounter("Last Posts Cache - Purge")
+	}
 }
 
 func NewSqlPostStore(sqlStore SqlStore, metrics einterfaces.MetricsInterface) store.PostStore {
@@ -326,6 +331,11 @@ func (s SqlPostStore) InvalidateLastPostTimeCache(channelId string) {
 	// Keys are "{channelid}{limit}" and caching only occurs on limits of 30 and 60
 	lastPostsCache.Remove(channelId + "30")
 	lastPostsCache.Remove(channelId + "60")
+
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Last Post Time - Remove by Channel Id")
+		s.metrics.IncrementMemCacheInvalidationCounter("Last Posts Cache - Remove by Channel Id")
+	}
 }
 
 func (s SqlPostStore) GetEtag(channelId string, allowFromCache bool) store.StoreChannel {

store/sqlstore/user_store.go

@@ -38,13 +38,22 @@ type SqlUserStore struct {
 var profilesInChannelCache *utils.Cache = utils.NewLru(PROFILES_IN_CHANNEL_CACHE_SIZE)
 var profileByIdsCache *utils.Cache = utils.NewLru(PROFILE_BY_IDS_CACHE_SIZE)
 
-func ClearUserCaches() {
+func (us SqlUserStore) ClearCaches() {
 	profilesInChannelCache.Purge()
 	profileByIdsCache.Purge()
+
+	if us.metrics != nil {
+		us.metrics.IncrementMemCacheInvalidationCounter("Profiles in Channel - Purge")
+		us.metrics.IncrementMemCacheInvalidationCounter("Profile By Ids - Purge")
+	}
 }
 
 func (us SqlUserStore) InvalidatProfileCacheForUser(userId string) {
 	profileByIdsCache.Remove(userId)
+
+	if us.metrics != nil {
+		us.metrics.IncrementMemCacheInvalidationCounter("Profile By Ids - Remove")
+	}
 }
 
 func NewSqlUserStore(sqlStore SqlStore, metrics einterfaces.MetricsInterface) store.UserStore {
@@ -384,6 +393,9 @@ func (us SqlUserStore) InvalidateProfilesInChannelCacheByUser(userId string) {
 			userMap := cacheItem.(map[string]*model.User)
 			if _, userInCache := userMap[userId]; userInCache {
 				profilesInChannelCache.Remove(key)
+				if us.metrics != nil {
+					us.metrics.IncrementMemCacheInvalidationCounter("Profiles in Channel - Remove by User")
+				}
 			}
 		}
 	}
@@ -391,6 +403,9 @@ func (us SqlUserStore) InvalidateProfilesInChannelCacheByUser(userId string) {
 
 func (us SqlUserStore) InvalidateProfilesInChannelCache(channelId string) {
 	profilesInChannelCache.Remove(channelId)
+	if us.metrics != nil {
+		us.metrics.IncrementMemCacheInvalidationCounter("Profiles in Channel - Remove by Channel")
+	}
 }
 
 func (us SqlUserStore) GetProfilesInChannel(channelId string, offset int, limit int) store.StoreChannel {

store/sqlstore/webhook_store.go

@@ -26,8 +26,12 @@ const (
 
 var webhookCache = utils.NewLru(WEBHOOK_CACHE_SIZE)
 
-func ClearWebhookCaches() {
+func (s SqlWebhookStore) ClearCaches() {
 	webhookCache.Purge()
+
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Webhook - Purge")
+	}
 }
 
 func NewSqlWebhookStore(sqlStore SqlStore, metrics einterfaces.MetricsInterface) store.WebhookStore {
@@ -78,6 +82,9 @@ func (s SqlWebhookStore) CreateIndexesIfNotExists() {
 
 func (s SqlWebhookStore) InvalidateWebhookCache(webhookId string) {
 	webhookCache.Remove(webhookId)
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Webhook - Remove by WebhookId")
+	}
 }
 
 func (s SqlWebhookStore) SaveIncoming(webhook *model.IncomingWebhook) store.StoreChannel {
@@ -164,7 +171,7 @@ func (s SqlWebhookStore) PermanentDeleteIncomingByUser(userId string) store.Stor
 			result.Err = model.NewAppError("SqlWebhookStore.DeleteIncomingByUser", "store.sql_webhooks.permanent_delete_incoming_by_user.app_error", nil, "id="+userId+", err="+err.Error(), http.StatusInternalServerError)
 		}
 
-		ClearWebhookCaches()
+		s.ClearCaches()
 	})
 }
 
@@ -175,7 +182,7 @@ func (s SqlWebhookStore) PermanentDeleteIncomingByChannel(channelId string) stor
 			result.Err = model.NewAppError("SqlWebhookStore.DeleteIncomingByChannel", "store.sql_webhooks.permanent_delete_incoming_by_channel.app_error", nil, "id="+channelId+", err="+err.Error(), http.StatusInternalServerError)
 		}
 
-		ClearWebhookCaches()
+		s.ClearCaches()
 	})
 }
 
@@ -322,7 +329,7 @@ func (s SqlWebhookStore) PermanentDeleteOutgoingByChannel(channelId string) stor
 			result.Err = model.NewAppError("SqlWebhookStore.DeleteOutgoingByChannel", "store.sql_webhooks.permanent_delete_outgoing_by_channel.app_error", nil, "id="+channelId+", err="+err.Error(), http.StatusInternalServerError)
 		}
 
-		ClearWebhookCaches()
+		s.ClearCaches()
 	})
 }
 

store/store.go

@@ -161,6 +161,7 @@ type ChannelStore interface {
 	GetMembersByIds(channelId string, userIds []string) StoreChannel
 	AnalyticsDeletedTypeCount(teamId string, channelType string) StoreChannel
 	GetChannelUnread(channelId, userId string) StoreChannel
+	ClearCaches()
 }
 
 type ChannelMemberHistoryStore interface {
@@ -190,6 +191,7 @@ type PostStore interface {
 	AnalyticsUserCountsWithPostsByDay(teamId string) StoreChannel
 	AnalyticsPostCountsByDay(teamId string) StoreChannel
 	AnalyticsPostCount(teamId string, mustHaveFile bool, mustHaveHashtag bool) StoreChannel
+	ClearCaches()
 	InvalidateLastPostTimeCache(channelId string)
 	GetPostsCreatedAt(channelId string, time int64) StoreChannel
 	Overwrite(post *model.Post) StoreChannel
@@ -210,6 +212,7 @@ type UserStore interface {
 	UpdateMfaActive(userId string, active bool) StoreChannel
 	Get(id string) StoreChannel
 	GetAll() StoreChannel
+	ClearCaches()
 	InvalidateProfilesInChannelCacheByUser(userId string)
 	InvalidateProfilesInChannelCache(channelId string)
 	GetProfilesInChannel(channelId string, offset int, limit int) StoreChannel
@@ -343,6 +346,7 @@ type WebhookStore interface {
 	AnalyticsIncomingCount(teamId string) StoreChannel
 	AnalyticsOutgoingCount(teamId string) StoreChannel
 	InvalidateWebhookCache(webhook string)
+	ClearCaches()
 }
 
 type CommandStore interface {
@@ -420,6 +424,7 @@ type FileInfoStore interface {
 	DeleteForPost(postId string) StoreChannel
 	PermanentDelete(fileId string) StoreChannel
 	PermanentDeleteBatch(endTime int64, limit int64) StoreChannel
+	ClearCaches()
 }
 
 type ReactionStore interface {

store/storetest/mocks/ChannelStore.go

@@ -61,6 +61,11 @@ func (_m *ChannelStore) AutocompleteInTeam(teamId string, term string) store.Sto
 	return r0
 }
 
+// ClearCaches provides a mock function with given fields:
+func (_m *ChannelStore) ClearCaches() {
+	_m.Called()
+}
+
 // CreateDirectChannel provides a mock function with given fields: userId, otherUserId
 func (_m *ChannelStore) CreateDirectChannel(userId string, otherUserId string) store.StoreChannel {
 	ret := _m.Called(userId, otherUserId)

store/storetest/mocks/FileInfoStore.go

@@ -29,6 +29,11 @@ func (_m *FileInfoStore) AttachToPost(fileId string, postId string) store.StoreC
 	return r0
 }
 
+// ClearCaches provides a mock function with given fields:
+func (_m *FileInfoStore) ClearCaches() {
+	_m.Called()
+}
+
 // DeleteForPost provides a mock function with given fields: postId
 func (_m *FileInfoStore) DeleteForPost(postId string) store.StoreChannel {
 	ret := _m.Called(postId)

store/storetest/mocks/PostStore.go

@@ -61,6 +61,11 @@ func (_m *PostStore) AnalyticsUserCountsWithPostsByDay(teamId string) store.Stor
 	return r0
 }
 
+// ClearCaches provides a mock function with given fields:
+func (_m *PostStore) ClearCaches() {
+	_m.Called()
+}
+
 // Delete provides a mock function with given fields: postId, time
 func (_m *PostStore) Delete(postId string, time int64) store.StoreChannel {
 	ret := _m.Called(postId, time)

store/storetest/mocks/UserStore.go

@@ -77,6 +77,11 @@ func (_m *UserStore) AnalyticsUniqueUserCount(teamId string) store.StoreChannel
 	return r0
 }
 
+// ClearCaches provides a mock function with given fields:
+func (_m *UserStore) ClearCaches() {
+	_m.Called()
+}
+
 // Get provides a mock function with given fields: id
 func (_m *UserStore) Get(id string) store.StoreChannel {
 	ret := _m.Called(id)

store/storetest/mocks/WebhookStore.go

@@ -45,6 +45,11 @@ func (_m *WebhookStore) AnalyticsOutgoingCount(teamId string) store.StoreChannel
 	return r0
 }
 
+// ClearCaches provides a mock function with given fields:
+func (_m *WebhookStore) ClearCaches() {
+	_m.Called()
+}
+
 // DeleteIncoming provides a mock function with given fields: webhookId, time
 func (_m *WebhookStore) DeleteIncoming(webhookId string, time int64) store.StoreChannel {
 	ret := _m.Called(webhookId, time)