[AI assisted]: Improve system console statistics performance (#29899)

```release-note NONE ``` Co-authored-by: Matthew Birtch <mattbirtch@gmail.com>
2025-02-04 21:54:01 +05:30
--- a/server/channels/store/sqlstore/channel_store.go
+++ b/server/channels/store/sqlstore/channel_store.go
@@ -2961,6 +2961,40 @@ func (s SqlChannelStore) AnalyticsDeletedTypeCount(teamId string, channelType mo
 	return v, nil
 }

+func (s SqlChannelStore) AnalyticsCountAll(teamId string) (map[model.ChannelType]int64, error) {
+	query := s.getQueryBuilder().
+		Select("Type, COUNT(*) AS Count").
+		From("Channels").
+		GroupBy("Type")
+
+	if teamId != "" {
+		query = query.Where(sq.Eq{"TeamId": teamId})
+	}
+
+	sql, args, err := query.ToSql()
+	if err != nil {
+		return nil, errors.Wrap(err, "AnalyticsCountAll_ToSql")
+	}
+
+	rows, err := s.GetReplica().Query(sql, args...)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to count Channels by type")
+	}
+	defer rows.Close()
+
+	counts := make(map[model.ChannelType]int64)
+	for rows.Next() {
+		var channelType model.ChannelType
+		var count int64
+		if err := rows.Scan(&channelType, &count); err != nil {
+			return nil, errors.Wrap(err, "unable to scan row")
+		}
+		counts[channelType] = count
+	}
+
+	return counts, nil
+}
+
 func (s SqlChannelStore) GetMembersForUser(teamID string, userID string) (model.ChannelMembers, error) {
 	sql, args, err := s.channelMembersForTeamWithSchemeSelectQuery.
 		Where(sq.And{
--- a/server/channels/store/sqlstore/file_info_store.go
+++ b/server/channels/store/sqlstore/file_info_store.go
@@ -712,18 +712,20 @@ func (fs SqlFileInfoStore) Search(rctx request.CTX, paramsList []*model.SearchPa
 }

 func (fs SqlFileInfoStore) CountAll() (int64, error) {
-	query := fs.getQueryBuilder().
-		Select("COUNT(*)").
-		From("FileInfo").
-		Where("DeleteAt = 0")
-
-	queryString, args, err := query.ToSql()
-	if err != nil {
-		return int64(0), errors.Wrap(err, "count_tosql")
+	var query sq.SelectBuilder
+	if fs.DriverName() == model.DatabaseDriverPostgres {
+		query = fs.getQueryBuilder().
+			Select("num").
+			From("file_stats")
+	} else {
+		query = fs.getQueryBuilder().
+			Select("COUNT(*)").
+			From("FileInfo").
+			Where("DeleteAt = 0")
 	}

 	var count int64
-	err = fs.GetReplica().Get(&count, queryString, args...)
+	err := fs.GetReplica().GetBuilder(&count, query)
 	if err != nil {
 		return int64(0), errors.Wrap(err, "failed to count Files")
 	}
@@ -758,13 +760,20 @@ func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID
 	return files, nil
 }

-func (fs SqlFileInfoStore) GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error) {
-	query := fs.getQueryBuilder().
-		Select("COALESCE(SUM(Size), 0)").
-		From("FileInfo")
+func (fs SqlFileInfoStore) GetStorageUsage(_, includeDeleted bool) (int64, error) {
+	var query sq.SelectBuilder
+	if fs.DriverName() == model.DatabaseDriverPostgres && !includeDeleted {
+		query = fs.getQueryBuilder().
+			Select("usage").
+			From("file_stats")
+	} else {
+		query = fs.getQueryBuilder().
+			Select("COALESCE(SUM(Size), 0)").
+			From("FileInfo")

-	if !includeDeleted {
-		query = query.Where("DeleteAt = 0")
+		if !includeDeleted {
+			query = query.Where("DeleteAt = 0")
+		}
 	}

 	var size int64
@@ -841,3 +850,18 @@ func (fs SqlFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string,

 	return nil
 }
+
+func (fs SqlFileInfoStore) RefreshFileStats() error {
+	if fs.DriverName() == model.DatabaseDriverPostgres {
+		// CONCURRENTLY is not used deliberately because as per Postgres docs,
+		// not using CONCURRENTLY takes less resources and completes faster
+		// at the expense of locking the mat view. Since viewing admin console
+		// is not a very frequent activity, we accept the tradeoff to let the
+		// refresh happen as fast as possible.
+		if _, err := fs.GetMaster().Exec("REFRESH MATERIALIZED VIEW file_stats"); err != nil {
+			return errors.Wrap(err, "error refreshing materialized view file_stats")
+		}
+	}
+
+	return nil
+}
--- a/server/channels/store/sqlstore/post_store.go
+++ b/server/channels/store/sqlstore/post_store.go
@@ -2292,8 +2292,79 @@ func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.A
 	return rows, nil
 }

+func (s *SqlPostStore) countBotPostsByDay(teamID, startDay, endDay string) (model.AnalyticsRows, error) {
+	var query sq.SelectBuilder
+	if teamID != "" {
+		query = s.getQueryBuilder().
+			Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, num as Value").
+			From("bot_posts_by_team_day").
+			Where(sq.Eq{"teamid": teamID})
+	} else {
+		query = s.getQueryBuilder().
+			Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, COALESCE(SUM(num), 0) as Value").
+			From("bot_posts_by_team_day").
+			GroupBy("Name")
+	}
+
+	query = query.
+		Where(sq.GtOrEq{"day": startDay}).
+		Where(sq.LtOrEq{"day": endDay}).
+		OrderBy("Name DESC").
+		Limit(30)
+
+	rows := model.AnalyticsRows{}
+	err := s.GetReplica().SelectBuilder(&rows, query)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find bot posts with teamId=%s", teamID)
+	}
+
+	return rows, nil
+}
+
+func (s *SqlPostStore) countPostsByDay(teamID, startDay, endDay string) (model.AnalyticsRows, error) {
+	var query sq.SelectBuilder
+	if teamID != "" {
+		query = s.getQueryBuilder().
+			Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, num as Value").
+			From("posts_by_team_day").
+			Where(sq.Eq{"teamid": teamID})
+	} else {
+		query = s.getQueryBuilder().
+			Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, COALESCE(SUM(num), 0) as Value").
+			From("posts_by_team_day").
+			GroupBy("Name")
+	}
+
+	query = query.
+		Where(sq.GtOrEq{"day": startDay}).
+		Where(sq.LtOrEq{"day": endDay}).
+		OrderBy("Name DESC").
+		Limit(30)
+
+	rows := model.AnalyticsRows{}
+	err := s.GetReplica().SelectBuilder(&rows, query)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to find posts with teamId=%s", teamID)
+	}
+
+	return rows, nil
+}
+
 // TODO: convert to squirrel HW
 func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
+	if s.DriverName() == model.DatabaseDriverPostgres {
+		endDay := utils.Yesterday().Format("2006-01-02")
+		startDay := utils.Yesterday().AddDate(0, 0, -31).Format("2006-01-02")
+		if options.YesterdayOnly {
+			startDay = utils.Yesterday().AddDate(0, 0, -1).Format("2006-01-02")
+		}
+		// Use materialized views
+		if options.BotsOnly {
+			return s.countBotPostsByDay(options.TeamId, startDay, endDay)
+		}
+		return s.countPostsByDay(options.TeamId, startDay, endDay)
+	}
+
 	var args []any
 	query :=
 		`SELECT
@@ -2318,30 +2389,6 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun
 		ORDER BY Name DESC
 		LIMIT 30`

-	if s.DriverName() == model.DatabaseDriverPostgres {
-		query =
-			`SELECT
-				TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, Count(Posts.Id) AS Value
-			FROM Posts`
-
-		if options.BotsOnly {
-			query += " INNER JOIN Bots ON Posts.UserId = Bots.Userid"
-		}
-
-		if options.TeamId != "" {
-			query += " INNER JOIN Channels ON Posts.ChannelId = Channels.Id  AND Channels.TeamId = ? AND"
-			args = []any{options.TeamId}
-		} else {
-			query += " WHERE"
-		}
-
-		query += ` Posts.CreateAt <= ?
-			            AND Posts.CreateAt >= ?
-			GROUP BY DATE(TO_TIMESTAMP(Posts.CreateAt / 1000))
-			ORDER BY Name DESC
-			LIMIT 30`
-	}
-
 	end := utils.MillisFromTime(utils.EndOfDay(utils.Yesterday()))
 	start := utils.MillisFromTime(utils.StartOfDay(utils.Yesterday().AddDate(0, 0, -31)))
 	if options.YesterdayOnly {
@@ -2350,16 +2397,39 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun
 	args = append(args, end, start)

 	rows := model.AnalyticsRows{}
-	err := s.GetReplica().Select(
-		&rows,
-		query,
-		args...)
+	err := s.GetReplica().Select(&rows, query, args...)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to find Posts with teamId=%s", options.TeamId)
 	}
 	return rows, nil
 }

+func (s *SqlPostStore) countByTeam(teamID string) (int64, error) {
+	query := s.getQueryBuilder().
+		Select("COALESCE(SUM(num), 0) AS total").
+		From("posts_by_team_day")
+
+	if teamID != "" {
+		query = query.Where(sq.Eq{"teamid": teamID})
+	}
+
+	var v int64
+	err := s.GetReplica().GetBuilder(&v, query)
+	if err != nil {
+		return 0, fmt.Errorf("failed to count Posts by team: %w, teamID: %s", err, teamID)
+	}
+
+	return v, nil
+}
+
+func (s *SqlPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
+	if s.DriverName() == model.DatabaseDriverPostgres {
+		return s.countByTeam(teamID)
+	}
+
+	return s.AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID})
+}
+
 func (s *SqlPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) {
 	query := s.getQueryBuilder().
 		Select("COUNT(*) AS Value").
@@ -2553,7 +2623,7 @@ func (s *SqlPostStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolic

 func (s *SqlPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
 	var query string
-	if s.DriverName() == "postgres" {
+	if s.DriverName() == model.DatabaseDriverPostgres {
 		query = "DELETE from Posts WHERE Id = any (array (SELECT Id FROM Posts WHERE CreateAt < ? LIMIT ?))"
 	} else {
 		query = "DELETE from Posts WHERE CreateAt < ? LIMIT ?"
@@ -3295,3 +3365,22 @@ func (s *SqlPostStore) GetPostReminderMetadata(postID string) (*store.PostRemind

 	return meta, nil
 }
+
+func (s *SqlPostStore) RefreshPostStats() error {
+	if s.DriverName() == model.DatabaseDriverPostgres {
+		// CONCURRENTLY is not used deliberately because as per Postgres docs,
+		// not using CONCURRENTLY takes less resources and completes faster
+		// at the expense of locking the mat view. Since viewing admin console
+		// is not a very frequent activity, we accept the tradeoff to let the
+		// refresh happen as fast as possible.
+		if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW posts_by_team_day"); err != nil {
+			return errors.Wrap(err, "error refreshing materialized view posts_by_team_day")
+		}
+
+		if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW bot_posts_by_team_day"); err != nil {
+			return errors.Wrap(err, "error refreshing materialized view bot_posts_by_team_day")
+		}
+	}
+
+	return nil
+}
--- a/server/channels/store/sqlstore/user_store.go
+++ b/server/channels/store/sqlstore/user_store.go
@@ -1696,7 +1696,6 @@ func (us SqlUserStore) performSearch(query sq.SelectBuilder, term string, option
 }

 func (us SqlUserStore) AnalyticsGetInactiveUsersCount() (int64, error) {
-	var count int64
 	query := us.getQueryBuilder().
 		Select("COUNT(Id)").
 		From("Users")
@@ -1712,11 +1711,9 @@ func (us SqlUserStore) AnalyticsGetInactiveUsersCount() (int64, error) {
 			sq.Gt{"Users.DeleteAt": 0},
 		})
 	}
-	queryStr, args, err := query.ToSql()
-	if err != nil {
-		return int64(0), errors.Wrap(err, "failed to create a SQL query to count inactive users")
-	}
-	err = us.GetReplica().Get(&count, queryStr, args...)
+
+	var count int64
+	err := us.GetReplica().GetBuilder(&count, query)
 	if err != nil {
 		return int64(0), errors.Wrap(err, "failed to count inactive Users")
 	}