Adding metrics for caching mechanisms (#4828)

api/admin.go

@@ -13,6 +13,8 @@ import (
 	"strings"
 	"time"
 
+	"runtime/debug"
+
 	l4g "github.com/alecthomas/log4go"
 	"github.com/gorilla/mux"
 	"github.com/mattermost/platform/einterfaces"
@@ -20,7 +22,6 @@ import (
 	"github.com/mattermost/platform/store"
 	"github.com/mattermost/platform/utils"
 	"github.com/mssola/user_agent"
-	"runtime/debug"
 )
 
 func InitAdmin() {
@@ -112,7 +113,7 @@ func getAllAudits(c *Context, w http.ResponseWriter, r *http.Request) {
 		audits := result.Data.(model.Audits)
 		etag := audits.Etag()
 
-		if HandleEtag(etag, w, r) {
+		if HandleEtag(etag, "Get All Audits", w, r) {
 			return
 		}
 

api/api.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/utils"
 
@@ -114,15 +115,23 @@ func InitApi() {
 	InitEmailBatching()
 }
 
-func HandleEtag(etag string, w http.ResponseWriter, r *http.Request) bool {
+func HandleEtag(etag string, routeName string, w http.ResponseWriter, r *http.Request) bool {
+	metrics := einterfaces.GetMetricsInterface()
 	if et := r.Header.Get(model.HEADER_ETAG_CLIENT); len(etag) > 0 {
 		if et == etag {
 			w.Header().Set(model.HEADER_ETAG_SERVER, etag)
 			w.WriteHeader(http.StatusNotModified)
+			if metrics != nil {
+				metrics.IncrementEtagHitCounter(routeName)
+			}
 			return true
 		}
 	}
 
+	if metrics != nil {
+		metrics.IncrementEtagMissCounter(routeName)
+	}
+
 	return false
 }
 

api/channel.go

@@ -414,7 +414,7 @@ func getChannels(c *Context, w http.ResponseWriter, r *http.Request) {
 		}
 		c.Err = result.Err
 		return
-	} else if HandleEtag(result.Data.(*model.ChannelList).Etag(), w, r) {
+	} else if HandleEtag(result.Data.(*model.ChannelList).Etag(), "Get Channels", w, r) {
 		return
 	} else {
 		data := result.Data.(*model.ChannelList)
@@ -460,7 +460,7 @@ func getChannelCounts(c *Context, w http.ResponseWriter, r *http.Request) {
 	if result := <-Srv.Store.Channel().GetChannelCounts(c.TeamId, c.Session.UserId); result.Err != nil {
 		c.Err = model.NewLocAppError("getChannelCounts", "api.channel.get_channel_counts.app_error", nil, result.Err.Message)
 		return
-	} else if HandleEtag(result.Data.(*model.ChannelCounts).Etag(), w, r) {
+	} else if HandleEtag(result.Data.(*model.ChannelCounts).Etag(), "Get Channel Counts", w, r) {
 		return
 	} else {
 		data := result.Data.(*model.ChannelCounts)
@@ -958,7 +958,7 @@ func getChannel(c *Context, w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		if HandleEtag(data.Etag(), w, r) {
+		if HandleEtag(data.Etag(), "Get Channel", w, r) {
 			return
 		} else {
 			w.Header().Set(model.HEADER_ETAG_SERVER, data.Etag())
@@ -990,7 +990,7 @@ func getChannelByName(c *Context, w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		if HandleEtag(data.Etag(), w, r) {
+		if HandleEtag(data.Etag(), "Get Channel By Name", w, r) {
 			return
 		} else {
 			w.Header().Set(model.HEADER_ETAG_SERVER, data.Etag())

api/context.go

@@ -508,9 +508,18 @@ func Handle404(w http.ResponseWriter, r *http.Request) {
 }
 
 func GetSession(token string) *model.Session {
+	metrics := einterfaces.GetMetricsInterface()
+
 	var session *model.Session
 	if ts, ok := sessionCache.Get(token); ok {
 		session = ts.(*model.Session)
+		if metrics != nil {
+			metrics.IncrementMemCacheHitCounter("Session")
+		}
+	} else {
+		if metrics != nil {
+			metrics.IncrementMemCacheMissCounter("Session")
+		}
 	}
 
 	if session == nil {

api/deprecated.go

@@ -29,7 +29,7 @@ func getMoreChannels(c *Context, w http.ResponseWriter, r *http.Request) {
 	if result := <-Srv.Store.Channel().GetMoreChannels(c.TeamId, c.Session.UserId, 0, 100000); result.Err != nil {
 		c.Err = result.Err
 		return
-	} else if HandleEtag(result.Data.(*model.ChannelList).Etag(), w, r) {
+	} else if HandleEtag(result.Data.(*model.ChannelList).Etag(), "Get More Channels (deprecated)", w, r) {
 		return
 	} else {
 		data := result.Data.(*model.ChannelList)

api/license.go

@@ -177,7 +177,7 @@ func getClientLicenceConfig(c *Context, w http.ResponseWriter, r *http.Request)
 	c.Err = nil
 
 	etag := utils.GetClientLicenseEtag(useSanitizedLicense)
-	if HandleEtag(etag, w, r) {
+	if HandleEtag(etag, "Get Client License Config", w, r) {
 		return
 	}
 

api/post.go

@@ -1291,7 +1291,7 @@ func getPosts(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	etag := (<-etagChan).Data.(string)
 
-	if HandleEtag(etag, w, r) {
+	if HandleEtag(etag, "Get Posts", w, r) {
 		return
 	}
 
@@ -1365,7 +1365,7 @@ func getPost(c *Context, w http.ResponseWriter, r *http.Request) {
 	if result := <-pchan; result.Err != nil {
 		c.Err = result.Err
 		return
-	} else if HandleEtag(result.Data.(*model.PostList).Etag(), w, r) {
+	} else if HandleEtag(result.Data.(*model.PostList).Etag(), "Get Post", w, r) {
 		return
 	} else {
 		list := result.Data.(*model.PostList)
@@ -1406,7 +1406,7 @@ func getPostById(c *Context, w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		if HandleEtag(list.Etag(), w, r) {
+		if HandleEtag(list.Etag(), "Get Post By Id", w, r) {
 			return
 		}
 
@@ -1447,7 +1447,7 @@ func getPermalinkTmp(c *Context, w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		if HandleEtag(list.Etag(), w, r) {
+		if HandleEtag(list.Etag(), "Get Permalink TMP", w, r) {
 			return
 		}
 
@@ -1581,7 +1581,7 @@ func getPostsBeforeOrAfter(c *Context, w http.ResponseWriter, r *http.Request, b
 	}
 
 	etag := (<-etagChan).Data.(string)
-	if HandleEtag(etag, w, r) {
+	if HandleEtag(etag, "Get Posts Before or After", w, r) {
 		return
 	}
 
@@ -1691,7 +1691,7 @@ func getFileInfosForPost(c *Context, w http.ResponseWriter, r *http.Request) {
 
 	etag := model.GetEtagForFileInfos(infos)
 
-	if HandleEtag(etag, w, r) {
+	if HandleEtag(etag, "Get File Infos For Post", w, r) {
 		return
 	} else {
 		w.Header().Set("Cache-Control", "max-age=2592000, public")

api/status.go

@@ -111,13 +111,20 @@ func getStatusesByIdsWebSocket(req *model.WebSocketRequest) (map[string]interfac
 
 func GetStatusesByIds(userIds []string) (map[string]interface{}, *model.AppError) {
 	statusMap := map[string]interface{}{}
+	metrics := einterfaces.GetMetricsInterface()
 
 	missingUserIds := []string{}
 	for _, userId := range userIds {
 		if result, ok := statusCache.Get(userId); ok {
 			statusMap[userId] = result.(*model.Status).Status
+			if metrics != nil {
+				metrics.IncrementMemCacheHitCounter("Status")
+			}
 		} else {
 			missingUserIds = append(missingUserIds, userId)
+			if metrics != nil {
+				metrics.IncrementMemCacheMissCounter("Status")
+			}
 		}
 	}
 

api/team.go

@@ -927,7 +927,7 @@ func getMyTeam(c *Context, w http.ResponseWriter, r *http.Request) {
 	if result := <-Srv.Store.Team().Get(c.TeamId); result.Err != nil {
 		c.Err = result.Err
 		return
-	} else if HandleEtag(result.Data.(*model.Team).Etag(), w, r) {
+	} else if HandleEtag(result.Data.(*model.Team).Etag(), "Get My Team", w, r) {
 		return
 	} else {
 		w.Header().Set(model.HEADER_ETAG_SERVER, result.Data.(*model.Team).Etag())

api/user.go

@@ -865,7 +865,7 @@ func getMe(c *Context, w http.ResponseWriter, r *http.Request) {
 		c.RemoveSessionCookie(w, r)
 		l4g.Error(utils.T("api.user.get_me.getting.error"), c.Session.UserId)
 		return
-	} else if HandleEtag(result.Data.(*model.User).Etag(utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress), w, r) {
+	} else if HandleEtag(result.Data.(*model.User).Etag(utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress), "Get Me", w, r) {
 		return
 	} else {
 		result.Data.(*model.User).Sanitize(map[string]bool{})
@@ -952,7 +952,7 @@ func getUser(c *Context, w http.ResponseWriter, r *http.Request) {
 	if result := <-Srv.Store.User().Get(id); result.Err != nil {
 		c.Err = result.Err
 		return
-	} else if HandleEtag(result.Data.(*model.User).Etag(utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress), w, r) {
+	} else if HandleEtag(result.Data.(*model.User).Etag(utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress), "Get User", w, r) {
 		return
 	} else {
 		user := sanitizeProfile(c, result.Data.(*model.User))
@@ -970,7 +970,7 @@ func getByUsername(c *Context, w http.ResponseWriter, r *http.Request) {
 	if result := <-Srv.Store.User().GetByUsername(username); result.Err != nil {
 		c.Err = result.Err
 		return
-	} else if HandleEtag(result.Data.(*model.User).Etag(utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress), w, r) {
+	} else if HandleEtag(result.Data.(*model.User).Etag(utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress), "Get By Username", w, r) {
 		return
 	} else {
 		user := sanitizeProfile(c, result.Data.(*model.User))
@@ -997,7 +997,7 @@ func getProfiles(c *Context, w http.ResponseWriter, r *http.Request) {
 	}
 
 	etag := (<-Srv.Store.User().GetEtagForAllProfiles()).Data.(string)
-	if HandleEtag(etag, w, r) {
+	if HandleEtag(etag, "Get Profiles", w, r) {
 		return
 	}
 
@@ -1039,7 +1039,7 @@ func getProfilesInTeam(c *Context, w http.ResponseWriter, r *http.Request) {
 	}
 
 	etag := (<-Srv.Store.User().GetEtagForProfiles(teamId)).Data.(string)
-	if HandleEtag(etag, w, r) {
+	if HandleEtag(etag, "Get Profiles In Team", w, r) {
 		return
 	}
 
@@ -1160,7 +1160,7 @@ func getAudits(c *Context, w http.ResponseWriter, r *http.Request) {
 		audits := result.Data.(model.Audits)
 		etag := audits.Etag()
 
-		if HandleEtag(etag, w, r) {
+		if HandleEtag(etag, "Get Audits", w, r) {
 			return
 		}
 

einterfaces/metrics.go

@@ -19,6 +19,12 @@ type MetricsInterface interface {
 
 	IncrementLogin()
 	IncrementLoginFail()
+
+	IncrementEtagHitCounter(route string)
+	IncrementEtagMissCounter(route string)
+
+	IncrementMemCacheHitCounter(cacheName string)
+	IncrementMemCacheMissCounter(cacheName string)
 }
 
 var theMetricsInterface MetricsInterface

store/sql_channel_store.go

@@ -10,6 +10,7 @@ import (
 
 	l4g "github.com/alecthomas/log4go"
 	"github.com/go-gorp/gorp"
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/utils"
 )
@@ -684,13 +685,21 @@ func (us SqlChannelStore) InvalidateAllChannelMembersForUser(userId string) {
 }
 
 func (us SqlChannelStore) IsUserInChannelUseCache(userId string, channelId string) bool {
+	metrics := einterfaces.GetMetricsInterface()
 	if cacheItem, ok := allChannelMembersForUserCache.Get(userId); ok {
+		if metrics != nil {
+			metrics.IncrementMemCacheHitCounter("All Channel Members for User")
+		}
 		ids := cacheItem.(map[string]string)
 		if _, ok := ids[channelId]; ok {
 			return true
 		} else {
 			return false
 		}
+	} else {
+		if metrics != nil {
+			metrics.IncrementMemCacheMissCounter("All Channel Members for User")
+		}
 	}
 
 	if result := <-us.GetAllChannelMembersForUser(userId, true); result.Err != nil {
@@ -746,13 +755,25 @@ func (s SqlChannelStore) GetAllChannelMembersForUser(userId string, allowFromCac
 
 	go func() {
 		result := StoreResult{}
+		metrics := einterfaces.GetMetricsInterface()
 
 		if allowFromCache {
 			if cacheItem, ok := allChannelMembersForUserCache.Get(userId); ok {
+				if metrics != nil {
+					metrics.IncrementMemCacheHitCounter("All Channel Members for User")
+				}
 				result.Data = cacheItem.(map[string]string)
 				storeChannel <- result
 				close(storeChannel)
 				return
+			} else {
+				if metrics != nil {
+					metrics.IncrementMemCacheMissCounter("All Channel Members for User")
+				}
+			}
+		} else {
+			if metrics != nil {
+				metrics.IncrementMemCacheMissCounter("All Channel Members for User")
 			}
 		}
 
@@ -788,16 +809,28 @@ func (us SqlChannelStore) InvalidateMemberCount(channelId string) {
 
 func (s SqlChannelStore) GetMemberCount(channelId string, allowFromCache bool) StoreChannel {
 	storeChannel := make(StoreChannel, 1)
+	metrics := einterfaces.GetMetricsInterface()
 
 	go func() {
 		result := StoreResult{}
 
 		if allowFromCache {
 			if cacheItem, ok := channelMemberCountsCache.Get(channelId); ok {
+				if metrics != nil {
+					metrics.IncrementMemCacheHitCounter("Channel Member Counts")
+				}
 				result.Data = cacheItem.(int64)
 				storeChannel <- result
 				close(storeChannel)
 				return
+			} else {
+				if metrics != nil {
+					metrics.IncrementMemCacheMissCounter("Channel Member Counts")
+				}
+			}
+		} else {
+			if metrics != nil {
+				metrics.IncrementMemCacheMissCounter("Channel Member Counts")
 			}
 		}
 

store/sql_post_store.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/utils"
 )
@@ -227,16 +228,28 @@ func (s SqlPostStore) InvalidatePostEtagCache(channelId string) {
 
 func (s SqlPostStore) GetEtag(channelId string, allowFromCache bool) StoreChannel {
 	storeChannel := make(StoreChannel, 1)
+	metrics := einterfaces.GetMetricsInterface()
 
 	go func() {
 		result := StoreResult{}
 
 		if allowFromCache {
 			if cacheItem, ok := postEtagCache.Get(channelId); ok {
+				if metrics != nil {
+					metrics.IncrementMemCacheHitCounter("Post Etag")
+				}
 				result.Data = cacheItem.(string)
 				storeChannel <- result
 				close(storeChannel)
 				return
+			} else {
+				if metrics != nil {
+					metrics.IncrementMemCacheMissCounter("Post Etag")
+				}
+			}
+		} else {
+			if metrics != nil {
+				metrics.IncrementMemCacheMissCounter("Post Etag")
 			}
 		}
 

store/sql_user_store.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/utils"
 )
@@ -601,13 +602,25 @@ func (us SqlUserStore) GetProfilesInChannel(channelId string, offset int, limit
 
 	go func() {
 		result := StoreResult{}
+		metrics := einterfaces.GetMetricsInterface()
 
 		if allowFromCache && offset == -1 && limit == -1 {
 			if cacheItem, ok := profilesInChannelCache.Get(channelId); ok {
+				if metrics != nil {
+					metrics.IncrementMemCacheHitCounter("Profiles in Channel")
+				}
 				result.Data = cacheItem.(map[string]*model.User)
 				storeChannel <- result
 				close(storeChannel)
 				return
+			} else {
+				if metrics != nil {
+					metrics.IncrementMemCacheMissCounter("Profiles in Channel")
+				}
+			}
+		} else {
+			if metrics != nil {
+				metrics.IncrementMemCacheMissCounter("Profiles in Channel")
 			}
 		}