Merge branch 'master' into mark-as-unread

store/sqlstore/channel_store.go

@@ -35,6 +35,9 @@ const (
 	CHANNEL_GUESTS_COUNTS_CACHE_SIZE = model.CHANNEL_CACHE_SIZE
 	CHANNEL_GUESTS_COUNTS_CACHE_SEC  = 1800 // 30 mins
 
+	CHANNEL_PINNEDPOSTS_COUNTS_CACHE_SIZE = model.CHANNEL_CACHE_SIZE
+	CHANNEL_PINNEDPOSTS_COUNTS_CACHE_SEC  = 1800 // 30 mins
+
 	CHANNEL_CACHE_SEC = 900 // 15 mins
 )
 
@@ -281,6 +284,7 @@ type publicChannel struct {
 }
 
 var channelMemberCountsCache = utils.NewLru(CHANNEL_MEMBERS_COUNTS_CACHE_SIZE)
+var channelPinnedPostCountsCache = utils.NewLru(CHANNEL_PINNEDPOSTS_COUNTS_CACHE_SIZE)
 var channelGuestCountsCache = utils.NewLru(CHANNEL_GUESTS_COUNTS_CACHE_SIZE)
 var allChannelMembersForUserCache = utils.NewLru(ALL_CHANNEL_MEMBERS_FOR_USER_CACHE_SIZE)
 var allChannelMembersNotifyPropsForChannelCache = utils.NewLru(ALL_CHANNEL_MEMBERS_NOTIFY_PROPS_FOR_CHANNEL_CACHE_SIZE)
@@ -289,6 +293,7 @@ var channelByNameCache = utils.NewLru(model.CHANNEL_CACHE_SIZE)
 
 func (s SqlChannelStore) ClearCaches() {
 	channelMemberCountsCache.Purge()
+	channelPinnedPostCountsCache.Purge()
 	channelGuestCountsCache.Purge()
 	allChannelMembersForUserCache.Purge()
 	allChannelMembersNotifyPropsForChannelCache.Purge()
@@ -297,6 +302,7 @@ func (s SqlChannelStore) ClearCaches() {
 
 	if s.metrics != nil {
 		s.metrics.IncrementMemCacheInvalidationCounter("Channel Member Counts - Purge")
+		s.metrics.IncrementMemCacheInvalidationCounter("Channel Pinned Post Counts - Purge")
 		s.metrics.IncrementMemCacheInvalidationCounter("All Channel Members for User - Purge")
 		s.metrics.IncrementMemCacheInvalidationCounter("All Channel Members Notify Props for Channel - Purge")
 		s.metrics.IncrementMemCacheInvalidationCounter("Channel - Purge")
@@ -1626,6 +1632,66 @@ func (s SqlChannelStore) GetMemberCount(channelId string, allowFromCache bool) (
 	return count, nil
 }
 
+func (s SqlChannelStore) InvalidatePinnedPostCount(channelId string) {
+	channelPinnedPostCountsCache.Remove(channelId)
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheInvalidationCounter("Channel Pinned Post Counts - Remove by ChannelId")
+	}
+}
+
+func (s SqlChannelStore) GetPinnedPostCountFromCache(channelId string) int64 {
+	if cacheItem, ok := channelPinnedPostCountsCache.Get(channelId); ok {
+		if s.metrics != nil {
+			s.metrics.IncrementMemCacheHitCounter("Channel Pinned Post Counts")
+		}
+		return cacheItem.(int64)
+	}
+
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheMissCounter("Channel Pinned Post Counts")
+	}
+
+	count, err := s.GetPinnedPostCount(channelId, true)
+	if err != nil {
+		return 0
+	}
+
+	return count
+}
+
+func (s SqlChannelStore) GetPinnedPostCount(channelId string, allowFromCache bool) (int64, *model.AppError) {
+	if allowFromCache {
+		if cacheItem, ok := channelPinnedPostCountsCache.Get(channelId); ok {
+			if s.metrics != nil {
+				s.metrics.IncrementMemCacheHitCounter("Channel Pinned Post Counts")
+			}
+			return cacheItem.(int64), nil
+		}
+	}
+
+	if s.metrics != nil {
+		s.metrics.IncrementMemCacheMissCounter("Channel Pinned Post Counts")
+	}
+
+	count, err := s.GetReplica().SelectInt(`
+		SELECT count(*)
+			FROM Posts
+		WHERE
+			IsPinned = true
+			AND ChannelId = :ChannelId 
+			AND DeleteAt = 0`, map[string]interface{}{"ChannelId": channelId})
+
+	if err != nil {
+		return 0, model.NewAppError("SqlChannelStore.GetPinnedPostCount", "store.sql_channel.get_pinnedpost_count.app_error", nil, "channel_id="+channelId+", "+err.Error(), http.StatusInternalServerError)
+	}
+
+	if allowFromCache {
+		channelPinnedPostCountsCache.AddWithExpiresInSecs(channelId, count, CHANNEL_PINNEDPOSTS_COUNTS_CACHE_SEC)
+	}
+
+	return count, nil
+}
+
 func (s SqlChannelStore) InvalidateGuestCount(channelId string) {
 	channelGuestCountsCache.Remove(channelId)
 	if s.metrics != nil {

store/sqlstore/integrity.go

@@ -0,0 +1,514 @@
+// Copyright (c) 2019-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package sqlstore
+
+import (
+	"github.com/mattermost/mattermost-server/mlog"
+	"github.com/mattermost/mattermost-server/store"
+
+	sq "github.com/Masterminds/squirrel"
+)
+
+type relationalCheckConfig struct {
+	parentName         string
+	parentIdAttr       string
+	childName          string
+	childIdAttr        string
+	canParentIdBeEmpty bool
+	sortRecords        bool
+}
+
+func getOrphanedRecords(ss *SqlSupplier, cfg relationalCheckConfig) ([]store.OrphanedRecord, error) {
+	var records []store.OrphanedRecord
+
+	sub := ss.getQueryBuilder().
+		Select("TRUE").
+		From(cfg.parentName).
+		Prefix("NOT EXISTS (").
+		Suffix(")").
+		Where(sq.Eq{"id": cfg.childName + "." + cfg.parentIdAttr})
+
+	main := ss.getQueryBuilder().
+		Select().
+		Column(cfg.parentIdAttr + " AS ParentId").
+		From(cfg.childName).
+		Where(sub)
+
+	if cfg.childIdAttr != "" {
+		main = main.Column(cfg.childIdAttr + " AS ChildId")
+	}
+
+	if cfg.canParentIdBeEmpty {
+		main = main.Where(sq.NotEq{cfg.parentIdAttr: ""})
+	}
+
+	if cfg.sortRecords {
+		main = main.OrderBy(cfg.parentIdAttr)
+	}
+
+	query, args, _ := main.ToSql()
+	_, err := ss.GetMaster().Select(&records, query, args...)
+
+	return records, err
+}
+
+func checkParentChildIntegrity(ss *SqlSupplier, config relationalCheckConfig) store.IntegrityCheckResult {
+	var result store.IntegrityCheckResult
+	var data store.RelationalIntegrityCheckData
+
+	config.sortRecords = true
+	data.Records, result.Err = getOrphanedRecords(ss, config)
+	if result.Err != nil {
+		mlog.Error(result.Err.Error())
+		return result
+	}
+	data.ParentName = config.parentName
+	data.ChildName = config.childName
+	data.ParentIdAttr = config.parentIdAttr
+	data.ChildIdAttr = config.childIdAttr
+	result.Data = data
+
+	return result
+}
+
+func checkChannelsCommandWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Channels",
+		parentIdAttr: "ChannelId",
+		childName:    "CommandWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkChannelsChannelMemberHistoryIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Channels",
+		parentIdAttr: "ChannelId",
+		childName:    "ChannelMemberHistory",
+		childIdAttr:  "",
+	})
+}
+
+func checkChannelsChannelMembersIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Channels",
+		parentIdAttr: "ChannelId",
+		childName:    "ChannelMembers",
+		childIdAttr:  "",
+	})
+}
+
+func checkChannelsIncomingWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Channels",
+		parentIdAttr: "ChannelId",
+		childName:    "IncomingWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkChannelsOutgoingWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Channels",
+		parentIdAttr: "ChannelId",
+		childName:    "OutgoingWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkChannelsPostsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Channels",
+		parentIdAttr: "ChannelId",
+		childName:    "Posts",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkCommandsCommandWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Commands",
+		parentIdAttr: "CommandId",
+		childName:    "CommandWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkPostsFileInfoIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Posts",
+		parentIdAttr: "PostId",
+		childName:    "FileInfo",
+		childIdAttr:  "",
+	})
+}
+
+func checkPostsPostsParentIdIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:         "Posts",
+		parentIdAttr:       "ParentId",
+		childName:          "Posts",
+		childIdAttr:        "Id",
+		canParentIdBeEmpty: true,
+	})
+}
+
+func checkPostsPostsRootIdIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:         "Posts",
+		parentIdAttr:       "RootId",
+		childName:          "Posts",
+		childIdAttr:        "Id",
+		canParentIdBeEmpty: true,
+	})
+}
+
+func checkPostsReactionsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Posts",
+		parentIdAttr: "PostId",
+		childName:    "Reactions",
+		childIdAttr:  "",
+	})
+}
+
+func checkSchemesChannelsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:         "Schemes",
+		parentIdAttr:       "SchemeId",
+		childName:          "Channels",
+		childIdAttr:        "Id",
+		canParentIdBeEmpty: true,
+	})
+}
+
+func checkSchemesTeamsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:         "Schemes",
+		parentIdAttr:       "SchemeId",
+		childName:          "Teams",
+		childIdAttr:        "Id",
+		canParentIdBeEmpty: true,
+	})
+}
+
+func checkSessionsAuditsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:         "Sessions",
+		parentIdAttr:       "SessionId",
+		childName:          "Audits",
+		childIdAttr:        "Id",
+		canParentIdBeEmpty: true,
+	})
+}
+
+func checkTeamsChannelsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Teams",
+		parentIdAttr: "TeamId",
+		childName:    "Channels",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkTeamsCommandsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Teams",
+		parentIdAttr: "TeamId",
+		childName:    "Commands",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkTeamsIncomingWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Teams",
+		parentIdAttr: "TeamId",
+		childName:    "IncomingWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkTeamsOutgoingWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Teams",
+		parentIdAttr: "TeamId",
+		childName:    "OutgoingWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkTeamsTeamMembersIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Teams",
+		parentIdAttr: "TeamId",
+		childName:    "TeamMembers",
+		childIdAttr:  "",
+	})
+}
+
+func checkUsersAuditsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:         "Users",
+		parentIdAttr:       "UserId",
+		childName:          "Audits",
+		childIdAttr:        "Id",
+		canParentIdBeEmpty: true,
+	})
+}
+
+func checkUsersCommandWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "CommandWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersChannelMemberHistoryIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "ChannelMemberHistory",
+		childIdAttr:  "",
+	})
+}
+
+func checkUsersChannelMembersIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "ChannelMembers",
+		childIdAttr:  "",
+	})
+}
+
+func checkUsersChannelsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:         "Users",
+		parentIdAttr:       "CreatorId",
+		childName:          "Channels",
+		childIdAttr:        "Id",
+		canParentIdBeEmpty: true,
+	})
+}
+
+func checkUsersCommandsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "CreatorId",
+		childName:    "Commands",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersCompliancesIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "Compliances",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersEmojiIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "CreatorId",
+		childName:    "Emoji",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersFileInfoIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Posts",
+		parentIdAttr: "CreatorId",
+		childName:    "FileInfo",
+		childIdAttr:  "",
+	})
+}
+
+func checkUsersIncomingWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "IncomingWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersOAuthAccessDataIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "OAuthAccessData",
+		childIdAttr:  "Token",
+	})
+}
+
+func checkUsersOAuthAppsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "CreatorId",
+		childName:    "OAuthApps",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersOAuthAuthDataIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "OAuthAuthData",
+		childIdAttr:  "Code",
+	})
+}
+
+func checkUsersOutgoingWebhooksIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "CreatorId",
+		childName:    "OutgoingWebhooks",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersPostsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "Posts",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersPreferencesIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "Preferences",
+		childIdAttr:  "",
+	})
+}
+
+func checkUsersReactionsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "Reactions",
+		childIdAttr:  "",
+	})
+}
+
+func checkUsersSessionsIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "Sessions",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkUsersStatusIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "Status",
+		childIdAttr:  "",
+	})
+}
+
+func checkUsersTeamMembersIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "TeamMembers",
+		childIdAttr:  "",
+	})
+}
+
+func checkUsersUserAccessTokensIntegrity(ss *SqlSupplier) store.IntegrityCheckResult {
+	return checkParentChildIntegrity(ss, relationalCheckConfig{
+		parentName:   "Users",
+		parentIdAttr: "UserId",
+		childName:    "UserAccessTokens",
+		childIdAttr:  "Id",
+	})
+}
+
+func checkChannelsIntegrity(ss *SqlSupplier, results chan<- store.IntegrityCheckResult) {
+	results <- checkChannelsCommandWebhooksIntegrity(ss)
+	results <- checkChannelsChannelMemberHistoryIntegrity(ss)
+	results <- checkChannelsChannelMembersIntegrity(ss)
+	results <- checkChannelsIncomingWebhooksIntegrity(ss)
+	results <- checkChannelsOutgoingWebhooksIntegrity(ss)
+	results <- checkChannelsPostsIntegrity(ss)
+}
+
+func checkCommandsIntegrity(ss *SqlSupplier, results chan<- store.IntegrityCheckResult) {
+	results <- checkCommandsCommandWebhooksIntegrity(ss)
+}
+
+func checkPostsIntegrity(ss *SqlSupplier, results chan<- store.IntegrityCheckResult) {
+	results <- checkPostsFileInfoIntegrity(ss)
+	results <- checkPostsPostsParentIdIntegrity(ss)
+	results <- checkPostsPostsRootIdIntegrity(ss)
+	results <- checkPostsReactionsIntegrity(ss)
+}
+
+func checkSchemesIntegrity(ss *SqlSupplier, results chan<- store.IntegrityCheckResult) {
+	results <- checkSchemesChannelsIntegrity(ss)
+	results <- checkSchemesTeamsIntegrity(ss)
+}
+
+func checkSessionsIntegrity(ss *SqlSupplier, results chan<- store.IntegrityCheckResult) {
+	results <- checkSessionsAuditsIntegrity(ss)
+}
+
+func checkTeamsIntegrity(ss *SqlSupplier, results chan<- store.IntegrityCheckResult) {
+	results <- checkTeamsChannelsIntegrity(ss)
+	results <- checkTeamsCommandsIntegrity(ss)
+	results <- checkTeamsIncomingWebhooksIntegrity(ss)
+	results <- checkTeamsOutgoingWebhooksIntegrity(ss)
+	results <- checkTeamsTeamMembersIntegrity(ss)
+}
+
+func checkUsersIntegrity(ss *SqlSupplier, results chan<- store.IntegrityCheckResult) {
+	results <- checkUsersAuditsIntegrity(ss)
+	results <- checkUsersCommandWebhooksIntegrity(ss)
+	results <- checkUsersChannelMemberHistoryIntegrity(ss)
+	results <- checkUsersChannelMembersIntegrity(ss)
+	results <- checkUsersChannelsIntegrity(ss)
+	results <- checkUsersCommandsIntegrity(ss)
+	results <- checkUsersCompliancesIntegrity(ss)
+	results <- checkUsersEmojiIntegrity(ss)
+	results <- checkUsersFileInfoIntegrity(ss)
+	results <- checkUsersIncomingWebhooksIntegrity(ss)
+	results <- checkUsersOAuthAccessDataIntegrity(ss)
+	results <- checkUsersOAuthAppsIntegrity(ss)
+	results <- checkUsersOAuthAuthDataIntegrity(ss)
+	results <- checkUsersOutgoingWebhooksIntegrity(ss)
+	results <- checkUsersPostsIntegrity(ss)
+	results <- checkUsersPreferencesIntegrity(ss)
+	results <- checkUsersReactionsIntegrity(ss)
+	results <- checkUsersSessionsIntegrity(ss)
+	results <- checkUsersStatusIntegrity(ss)
+	results <- checkUsersTeamMembersIntegrity(ss)
+	results <- checkUsersUserAccessTokensIntegrity(ss)
+}
+
+func CheckRelationalIntegrity(ss *SqlSupplier, results chan<- store.IntegrityCheckResult) {
+	mlog.Info("Starting relational integrity checks...")
+	checkChannelsIntegrity(ss, results)
+	checkCommandsIntegrity(ss, results)
+	checkPostsIntegrity(ss, results)
+	checkSchemesIntegrity(ss, results)
+	checkSessionsIntegrity(ss, results)
+	checkTeamsIntegrity(ss, results)
+	checkUsersIntegrity(ss, results)
+	mlog.Info("Done with relational integrity checks")
+	close(results)
+}

store/sqlstore/session_store.go

@@ -4,7 +4,6 @@
 package sqlstore
 
 import (
-	"fmt"
 	"net/http"
 	"time"
 
@@ -251,13 +250,13 @@ func (me SqlSessionStore) Cleanup(expiryTime int64, batchSize int64) {
 
 	for rowsAffected > 0 {
 		if sqlResult, err := me.GetMaster().Exec(query, map[string]interface{}{"ExpiresAt": expiryTime, "Limit": batchSize}); err != nil {
-			mlog.Error(fmt.Sprintf("Unable to cleanup session store. err=%v", err.Error()))
+			mlog.Error("Unable to cleanup session store.", mlog.Err(err))
 			return
 		} else {
 			var rowErr error
 			rowsAffected, rowErr = sqlResult.RowsAffected()
 			if rowErr != nil {
-				mlog.Error(fmt.Sprintf("Unable to cleanup session store. err=%v", err.Error()))
+				mlog.Error("Unable to cleanup session store.", mlog.Err(err))
 				return
 			}
 		}

store/sqlstore/supplier.go

@@ -159,14 +159,14 @@ func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInter
 
 	err := supplier.GetMaster().CreateTablesIfNotExists()
 	if err != nil {
-		mlog.Critical(fmt.Sprintf("Error creating database tables: %v", err))
+		mlog.Critical("Error creating database tables.", mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_CREATE_TABLE)
 	}
 
 	err = UpgradeDatabase(supplier, model.CurrentVersion)
 	if err != nil {
-		mlog.Critical("Failed to upgrade database", mlog.Err(err))
+		mlog.Critical("Failed to upgrade database.", mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_GENERIC_FAILURE)
 	}
@@ -214,13 +214,13 @@ func (s *SqlSupplier) Next() store.LayeredStoreSupplier {
 func setupConnection(con_type string, dataSource string, settings *model.SqlSettings) *gorp.DbMap {
 	db, err := dbsql.Open(*settings.DriverName, dataSource)
 	if err != nil {
-		mlog.Critical(fmt.Sprintf("Failed to open SQL connection to err:%v", err.Error()))
+		mlog.Critical("Failed to open SQL connection to err.", mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_DB_OPEN)
 	}
 
 	for i := 0; i < DB_PING_ATTEMPTS; i++ {
-		mlog.Info(fmt.Sprintf("Pinging SQL %v database", con_type))
+		mlog.Info("Pinging SQL", mlog.String("database", con_type))
 		ctx, cancel := context.WithTimeout(context.Background(), DB_PING_TIMEOUT_SECS*time.Second)
 		defer cancel()
 		err = db.PingContext(ctx)
@@ -228,11 +228,11 @@ func setupConnection(con_type string, dataSource string, settings *model.SqlSett
 			break
 		} else {
 			if i == DB_PING_ATTEMPTS-1 {
-				mlog.Critical(fmt.Sprintf("Failed to ping DB, server will exit err=%v", err))
+				mlog.Critical("Failed to ping DB, server will exit.", mlog.Err(err))
 				time.Sleep(time.Second)
 				os.Exit(EXIT_PING)
 			} else {
-				mlog.Error(fmt.Sprintf("Failed to ping DB retrying in %v seconds err=%v", DB_PING_TIMEOUT_SECS, err))
+				mlog.Error("Failed to ping DB", mlog.Err(err), mlog.Int("retrying in seconds", DB_PING_TIMEOUT_SECS))
 				time.Sleep(DB_PING_TIMEOUT_SECS * time.Second)
 			}
 		}
@@ -365,7 +365,7 @@ func (ss *SqlSupplier) DoesTableExist(tableName string) bool {
 		)
 
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check if table exists %v", err))
+			mlog.Critical("Failed to check if table exists", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_TABLE_EXISTS)
 		}
@@ -387,7 +387,7 @@ func (ss *SqlSupplier) DoesTableExist(tableName string) bool {
 		)
 
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check if table exists %v", err))
+			mlog.Critical("Failed to check if table exists", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_TABLE_EXISTS_MYSQL)
 		}
@@ -401,7 +401,7 @@ func (ss *SqlSupplier) DoesTableExist(tableName string) bool {
 		)
 
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check if table exists %v", err))
+			mlog.Critical("Failed to check if table exists", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_TABLE_EXISTS_SQLITE)
 		}
@@ -433,7 +433,7 @@ func (ss *SqlSupplier) DoesColumnExist(tableName string, columnName string) bool
 				return false
 			}
 
-			mlog.Critical(fmt.Sprintf("Failed to check if column exists %v", err))
+			mlog.Critical("Failed to check if column exists", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_DOES_COLUMN_EXISTS_POSTGRES)
 		}
@@ -456,7 +456,7 @@ func (ss *SqlSupplier) DoesColumnExist(tableName string, columnName string) bool
 		)
 
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check if column exists %v", err))
+			mlog.Critical("Failed to check if column exists", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_DOES_COLUMN_EXISTS_MYSQL)
 		}
@@ -471,7 +471,7 @@ func (ss *SqlSupplier) DoesColumnExist(tableName string, columnName string) bool
 		)
 
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check if column exists %v", err))
+			mlog.Critical("Failed to check if column exists", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_DOES_COLUMN_EXISTS_SQLITE)
 		}
@@ -498,7 +498,7 @@ func (ss *SqlSupplier) DoesTriggerExist(triggerName string) bool {
 		`, triggerName)
 
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check if trigger exists %v", err))
+			mlog.Critical("Failed to check if trigger exists", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_GENERIC_FAILURE)
 		}
@@ -517,7 +517,7 @@ func (ss *SqlSupplier) DoesTriggerExist(triggerName string) bool {
 		`, triggerName)
 
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check if trigger exists %v", err))
+			mlog.Critical("Failed to check if trigger exists", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_GENERIC_FAILURE)
 		}
@@ -541,7 +541,7 @@ func (ss *SqlSupplier) CreateColumnIfNotExists(tableName string, columnName stri
 	if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
 		_, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " ADD " + columnName + " " + postgresColType + " DEFAULT '" + defaultValue + "'")
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to create column %v", err))
+			mlog.Critical("Failed to create column", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_CREATE_COLUMN_POSTGRES)
 		}
@@ -551,7 +551,7 @@ func (ss *SqlSupplier) CreateColumnIfNotExists(tableName string, columnName stri
 	} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
 		_, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " ADD " + columnName + " " + mySqlColType + " DEFAULT '" + defaultValue + "'")
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to create column %v", err))
+			mlog.Critical("Failed to create column", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_CREATE_COLUMN_MYSQL)
 		}
@@ -575,7 +575,7 @@ func (ss *SqlSupplier) CreateColumnIfNotExistsNoDefault(tableName string, column
 	if ss.DriverName() == model.DATABASE_DRIVER_POSTGRES {
 		_, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " ADD " + columnName + " " + postgresColType)
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to create column %v", err))
+			mlog.Critical("Failed to create column", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_CREATE_COLUMN_POSTGRES)
 		}
@@ -585,7 +585,7 @@ func (ss *SqlSupplier) CreateColumnIfNotExistsNoDefault(tableName string, column
 	} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
 		_, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " ADD " + columnName + " " + mySqlColType)
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to create column %v", err))
+			mlog.Critical("Failed to create column", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_CREATE_COLUMN_MYSQL)
 		}
@@ -608,7 +608,7 @@ func (ss *SqlSupplier) RemoveColumnIfExists(tableName string, columnName string)
 
 	_, err := ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " DROP COLUMN " + columnName)
 	if err != nil {
-		mlog.Critical(fmt.Sprintf("Failed to drop column %v", err))
+		mlog.Critical("Failed to drop column", mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_REMOVE_COLUMN)
 	}
@@ -623,7 +623,7 @@ func (ss *SqlSupplier) RemoveTableIfExists(tableName string) bool {
 
 	_, err := ss.GetMaster().ExecNoTimeout("DROP TABLE " + tableName)
 	if err != nil {
-		mlog.Critical(fmt.Sprintf("Failed to drop table %v", err))
+		mlog.Critical("Failed to drop table", mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_REMOVE_TABLE)
 	}
@@ -644,7 +644,7 @@ func (ss *SqlSupplier) RenameColumnIfExists(tableName string, oldColumnName stri
 	}
 
 	if err != nil {
-		mlog.Critical(fmt.Sprintf("Failed to rename column %v", err))
+		mlog.Critical("Failed to rename column", mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_RENAME_COLUMN)
 	}
@@ -666,7 +666,7 @@ func (ss *SqlSupplier) GetMaxLengthOfColumnIfExists(tableName string, columnName
 	}
 
 	if err != nil {
-		mlog.Critical(fmt.Sprintf("Failed to get max length of column %v", err))
+		mlog.Critical("Failed to get max length of column", mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_MAX_COLUMN)
 	}
@@ -687,7 +687,7 @@ func (ss *SqlSupplier) AlterColumnTypeIfExists(tableName string, columnName stri
 	}
 
 	if err != nil {
-		mlog.Critical(fmt.Sprintf("Failed to alter column type %v", err))
+		mlog.Critical("Failed to alter column type", mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_ALTER_COLUMN)
 	}
@@ -736,7 +736,7 @@ func (ss *SqlSupplier) AlterColumnDefaultIfExists(tableName string, columnName s
 	}
 
 	if err != nil {
-		mlog.Critical(fmt.Sprintf("Failed to alter column %s.%s default %s: %v", tableName, columnName, defaultValue, err))
+		mlog.Critical("Failed to alter column", mlog.String("table", tableName), mlog.String("column", columnName), mlog.String("default value", defaultValue), mlog.Err(err))
 		time.Sleep(time.Second)
 		os.Exit(EXIT_GENERIC_FAILURE)
 		return false
@@ -790,7 +790,7 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string
 
 		_, err := ss.GetMaster().ExecNoTimeout(query)
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to create index %v, %v", errExists, err))
+			mlog.Critical("Failed to create index", mlog.Err(errExists), mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_CREATE_INDEX_POSTGRES)
 		}
@@ -798,7 +798,7 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string
 
 		count, err := ss.GetMaster().SelectInt("SELECT COUNT(0) AS index_exists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() and table_name = ? AND index_name = ?", tableName, indexName)
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check index %v", err))
+			mlog.Critical("Failed to check index", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_CREATE_INDEX_MYSQL)
 		}
@@ -814,14 +814,14 @@ func (ss *SqlSupplier) createIndexIfNotExists(indexName string, tableName string
 
 		_, err = ss.GetMaster().ExecNoTimeout("CREATE  " + uniqueStr + fullTextIndex + " INDEX " + indexName + " ON " + tableName + " (" + strings.Join(columnNames, ", ") + ")")
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to create index %v", err))
+			mlog.Critical("Failed to create index", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_CREATE_INDEX_FULL_MYSQL)
 		}
 	} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
 		_, err := ss.GetMaster().ExecNoTimeout("CREATE INDEX IF NOT EXISTS " + indexName + " ON " + tableName + " (" + strings.Join(columnNames, ", ") + ")")
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to create index %v", err))
+			mlog.Critical("Failed to create index", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_CREATE_INDEX_SQLITE)
 		}
@@ -845,7 +845,7 @@ func (ss *SqlSupplier) RemoveIndexIfExists(indexName string, tableName string) b
 
 		_, err = ss.GetMaster().ExecNoTimeout("DROP INDEX " + indexName)
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to remove index %v", err))
+			mlog.Critical("Failed to remove index", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_REMOVE_INDEX_POSTGRES)
 		}
@@ -855,7 +855,7 @@ func (ss *SqlSupplier) RemoveIndexIfExists(indexName string, tableName string) b
 
 		count, err := ss.GetMaster().SelectInt("SELECT COUNT(0) AS index_exists FROM information_schema.statistics WHERE TABLE_SCHEMA = DATABASE() and table_name = ? AND index_name = ?", tableName, indexName)
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to check index %v", err))
+			mlog.Critical("Failed to check index", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_REMOVE_INDEX_MYSQL)
 		}
@@ -866,14 +866,14 @@ func (ss *SqlSupplier) RemoveIndexIfExists(indexName string, tableName string) b
 
 		_, err = ss.GetMaster().ExecNoTimeout("DROP INDEX " + indexName + " ON " + tableName)
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to remove index %v", err))
+			mlog.Critical("Failed to remove index", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_REMOVE_INDEX_MYSQL)
 		}
 	} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
 		_, err := ss.GetMaster().ExecNoTimeout("DROP INDEX IF EXISTS " + indexName)
 		if err != nil {
-			mlog.Critical(fmt.Sprintf("Failed to remove index %v", err))
+			mlog.Critical("Failed to remove index", mlog.Err(err))
 			time.Sleep(time.Second)
 			os.Exit(EXIT_REMOVE_INDEX_SQLITE)
 		}
@@ -1066,6 +1066,12 @@ func (ss *SqlSupplier) getQueryBuilder() sq.StatementBuilderType {
 	return builder
 }
 
+func (ss *SqlSupplier) CheckIntegrity() <-chan store.IntegrityCheckResult {
+	results := make(chan store.IntegrityCheckResult)
+	go CheckRelationalIntegrity(ss, results)
+	return results
+}
+
 type mattermConverter struct{}
 
 func (me mattermConverter) ToDb(val interface{}) (interface{}, error) {

store/sqlstore/supplier_reactions.go

@@ -4,7 +4,6 @@
 package sqlstore
 
 import (
-	"fmt"
 	"net/http"
 
 	"github.com/mattermost/gorp"
@@ -141,7 +140,7 @@ func (s *SqlReactionStore) DeleteAllWithEmojiName(emojiName string) *model.AppEr
 	for _, reaction := range reactions {
 		if _, err := s.GetMaster().Exec(UPDATE_POST_HAS_REACTIONS_ON_DELETE_QUERY,
 			map[string]interface{}{"PostId": reaction.PostId, "UpdateAt": model.GetMillis()}); err != nil {
-			mlog.Warn(fmt.Sprintf("Unable to update Post.HasReactions while removing reactions post_id=%v, error=%v", reaction.PostId, err.Error()))
+			mlog.Warn("Unable to update Post.HasReactions while removing reactions", mlog.String("post_id", reaction.PostId), mlog.Err(err))
 		}
 	}
 

store/sqlstore/team_store.go

@@ -381,6 +381,21 @@ func (s SqlTeamStore) GetAllPrivateTeamListing() ([]*model.Team, *model.AppError
 	return data, nil
 }
 
+func (s SqlTeamStore) GetAllPublicTeamPageListing(offset int, limit int) ([]*model.Team, *model.AppError) {
+	query := "SELECT * FROM Teams WHERE AllowOpenInvite = 1 ORDER BY DisplayName LIMIT :Limit OFFSET :Offset"
+
+	if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
+		query = "SELECT * FROM Teams WHERE AllowOpenInvite = true ORDER BY DisplayName LIMIT :Limit OFFSET :Offset"
+	}
+
+	var data []*model.Team
+	if _, err := s.GetReplica().Select(&data, query, map[string]interface{}{"Offset": offset, "Limit": limit}); err != nil {
+		return nil, model.NewAppError("SqlTeamStore.GetAllPrivateTeamListing", "store.sql_team.get_all_private_team_listing.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	return data, nil
+}
+
 func (s SqlTeamStore) GetAllPrivateTeamPageListing(offset int, limit int) ([]*model.Team, *model.AppError) {
 	query := "SELECT * FROM Teams WHERE AllowOpenInvite = 0 ORDER BY DisplayName LIMIT :Limit OFFSET :Offset"
 
@@ -433,6 +448,35 @@ func (s SqlTeamStore) PermanentDelete(teamId string) *model.AppError {
 	return nil
 }
 
+func (s SqlTeamStore) AnalyticsPublicTeamCount() (int64, *model.AppError) {
+
+	c, err := s.GetReplica().SelectInt("SELECT COUNT(*) FROM Teams WHERE DeleteAt = 0 AND AllowOpenInvite = 1", map[string]interface{}{})
+
+	if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
+		c, err = s.GetReplica().SelectInt("SELECT COUNT(*) FROM Teams WHERE DeleteAt = 0 AND AllowOpenInvite = true", map[string]interface{}{})
+	}
+
+	if err != nil {
+		return int64(0), model.NewAppError("SqlTeamStore.AnalyticsPublicTeamCount", "store.sql_team.analytics_public_team_count.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	return c, nil
+}
+
+func (s SqlTeamStore) AnalyticsPrivateTeamCount() (int64, *model.AppError) {
+	c, err := s.GetReplica().SelectInt("SELECT COUNT(*) FROM Teams WHERE DeleteAt = 0 AND AllowOpenInvite = 0", map[string]interface{}{})
+
+	if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
+		c, err = s.GetReplica().SelectInt("SELECT COUNT(*) FROM Teams WHERE DeleteAt = 0 AND AllowOpenInvite = false", map[string]interface{}{})
+	}
+
+	if err != nil {
+		return int64(0), model.NewAppError("SqlTeamStore.AnalyticsPrivateTeamCount", "store.sql_team.analytics_private_team_count.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+
+	return c, nil
+}
+
 func (s SqlTeamStore) AnalyticsTeamCount() (int64, *model.AppError) {
 	c, err := s.GetReplica().SelectInt("SELECT COUNT(*) FROM Teams WHERE DeleteAt = 0", map[string]interface{}{})
 

store/sqlstore/upgrade.go

@@ -729,11 +729,9 @@ func UpgradeDatabaseToVersion514(sqlStore SqlStore) {
 }
 
 func UpgradeDatabaseToVersion515(sqlStore SqlStore) {
-	// TODO: Uncomment following condition when version 5.15.0 is released
-	// if shouldPerformUpgrade(sqlStore, VERSION_5_14_0, VERSION_5_15_0) {
-
-	// 	saveSchemaVersion(sqlStore, VERSION_5_15_0)
-	// }
+	if shouldPerformUpgrade(sqlStore, VERSION_5_14_0, VERSION_5_15_0) {
+		saveSchemaVersion(sqlStore, VERSION_5_15_0)
+	}
 }
 
 func UpgradeDatabaseToVersion516(sqlStore SqlStore) {