mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[AI assisted]: Improve system console statistics performance (#29899)

Просмотр исходного кода 
```release-note
NONE
```

Co-authored-by: Matthew Birtch <mattbirtch@gmail.com>
Этот коммит содержится в:
Agniva De Sarker
2025-02-04 21:54:01 +05:30
коммит произвёл GitHub
родитель 6046a304b2
Коммит ae9e6174e5
45 изменённых файлов: 1052 добавлений и 407 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
server/channels/api4/user_test.go
Просмотреть файл

@@ -2649,6 +2649,7 @@ func TestPermanentDeleteAllUsers(t *testing.T) {
require.NoError(t, err)
require.Greater(t, len(users), 0)
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
postCount, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, err)
require.Greater(t, postCount, int64(0))
@@ -2662,6 +2663,7 @@ func TestPermanentDeleteAllUsers(t *testing.T) {
require.NoError(t, err)
require.Len(t, users, 0)
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
postCount, err = th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, err)
require.Equal(t, postCount, int64(0))

492
server/channels/app/analytics.go
Просмотреть файл

@@ -34,287 +34,273 @@ func (a *App) getAnalytics(rctx request.CTX, name string, teamID string, forSupp
skipIntensiveQueries := false
// When generating a Support Packet, always run intensive queries.
if !forSupportPacket {
if systemUserCount > int64(*a.Config().AnalyticsSettings.MaxUsersForStatistics) {
rctx.Logger().Debug("More than limit users are on the system, intensive queries skipped", mlog.Int("limit", *a.Config().AnalyticsSettings.MaxUsersForStatistics))
skipIntensiveQueries = true
}
if !forSupportPacket && systemUserCount > int64(*a.Config().AnalyticsSettings.MaxUsersForStatistics) {
rctx.Logger().Warn("Number of users in the system is higher than the configured limit. Skipping intensive SQL queries.", mlog.Int("limit", *a.Config().AnalyticsSettings.MaxUsersForStatistics))
skipIntensiveQueries = true
}
if name == "standard" {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 11)
rows[0] = &model.AnalyticsRow{Name: "channel_open_count", Value: 0}
rows[1] = &model.AnalyticsRow{Name: "channel_private_count", Value: 0}
rows[2] = &model.AnalyticsRow{Name: "post_count", Value: 0}
rows[3] = &model.AnalyticsRow{Name: "unique_user_count", Value: 0}
rows[4] = &model.AnalyticsRow{Name: "team_count", Value: 0}
rows[5] = &model.AnalyticsRow{Name: "total_websocket_connections", Value: 0}
rows[6] = &model.AnalyticsRow{Name: "total_master_db_connections", Value: 0}
rows[7] = &model.AnalyticsRow{Name: "total_read_db_connections", Value: 0}
rows[8] = &model.AnalyticsRow{Name: "daily_active_users", Value: 0}
rows[9] = &model.AnalyticsRow{Name: "monthly_active_users", Value: 0}
rows[10] = &model.AnalyticsRow{Name: "inactive_user_count", Value: 0}
switch name {
case "standard":
return a.getStandardAnalytics(rctx, teamID, systemUserCount)
case "bot_post_counts_day":
return a.getBotPostCountsAnalytics(rctx, teamID)
case "post_counts_day":
return a.getPostCountsAnalytics(rctx, teamID)
case "user_counts_with_posts_day":
return a.getUserCountsWithPostsAnalytics(rctx, teamID, skipIntensiveQueries)
case "extra_counts":
return a.getExtraCountsAnalytics(rctx, teamID)
default:
return nil, nil
}
}
var g errgroup.Group
var openChannelsCount int64
func (a *App) getStandardAnalytics(rctx request.CTX, teamID string, systemUserCount int64) (model.AnalyticsRows, *model.AppError) {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 11)
rows[0] = &model.AnalyticsRow{Name: "channel_open_count", Value: 0}
rows[1] = &model.AnalyticsRow{Name: "channel_private_count", Value: 0}
rows[2] = &model.AnalyticsRow{Name: "post_count", Value: 0}
rows[3] = &model.AnalyticsRow{Name: "unique_user_count", Value: 0}
rows[4] = &model.AnalyticsRow{Name: "team_count", Value: 0}
rows[5] = &model.AnalyticsRow{Name: "total_websocket_connections", Value: 0}
rows[6] = &model.AnalyticsRow{Name: "total_master_db_connections", Value: 0}
rows[7] = &model.AnalyticsRow{Name: "total_read_db_connections", Value: 0}
rows[8] = &model.AnalyticsRow{Name: "daily_active_users", Value: 0}
rows[9] = &model.AnalyticsRow{Name: "monthly_active_users", Value: 0}
rows[10] = &model.AnalyticsRow{Name: "inactive_user_count", Value: 0}
var g errgroup.Group
g.SetLimit(2)
var channelCounts map[model.ChannelType]int64
g.Go(func() error {
var err error
if channelCounts, err = a.Srv().Store().Channel().AnalyticsCountAll(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var usersCount int64
var inactiveUsersCount int64
if teamID == "" {
g.Go(func() error {
var err error
if openChannelsCount, err = a.Srv().Store().Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen); err != nil {
return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
if inactiveUsersCount, err = a.Srv().Store().User().AnalyticsGetInactiveUsersCount(); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_get_inactive_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var privateChannelsCount int64
} else {
g.Go(func() error {
var err error
if privateChannelsCount, err = a.Srv().Store().Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate); err != nil {
return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
if usersCount, err = a.Srv().Store().User().Count(model.UserCountOptions{TeamId: teamID}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.get_total_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
}
var usersCount int64
var inactiveUsersCount int64
if teamID == "" {
g.Go(func() error {
var err error
if inactiveUsersCount, err = a.Srv().Store().User().AnalyticsGetInactiveUsersCount(); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_get_inactive_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
} else {
g.Go(func() error {
var err error
if usersCount, err = a.Srv().Store().User().Count(model.UserCountOptions{TeamId: teamID}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.get_total_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var postsCount int64
g.Go(func() error {
var err error
if postsCount, err = a.Srv().Store().Post().AnalyticsPostCountByTeam(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var teamsCount int64
g.Go(func() error {
var err error
if teamsCount, err = a.Srv().Store().Team().AnalyticsTeamCount(nil); err != nil {
return model.NewAppError("GetAnalytics", "app.team.analytics_team_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var dailyActiveUsersCount int64
g.Go(func() error {
var err error
if dailyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(DayMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var monthlyActiveUsersCount int64
g.Go(func() error {
var err error
if monthlyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(MonthMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
if err := g.Wait(); err != nil {
return nil, err.(*model.AppError)
}
rows[0].Value = float64(channelCounts[model.ChannelTypeOpen])
rows[1].Value = float64(channelCounts[model.ChannelTypePrivate])
rows[2].Value = float64(postsCount)
if teamID == "" {
rows[3].Value = float64(systemUserCount)
rows[10].Value = float64(inactiveUsersCount)
} else {
rows[3].Value = float64(usersCount)
rows[10].Value = -1
}
rows[4].Value = float64(teamsCount)
// If in HA mode then aggregate all the stats
if a.Cluster() != nil && *a.Config().ClusterSettings.Enable {
stats, err2 := a.Cluster().GetClusterStats(rctx)
if err2 != nil {
return nil, err2
}
var postsCount int64
if !skipIntensiveQueries {
g.Go(func() error {
var err error
if postsCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID}); err != nil {
return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
totalSockets := a.TotalWebsocketConnections()
totalMasterDb := a.Srv().Store().TotalMasterDbConnections()
totalReadDb := a.Srv().Store().TotalReadDbConnections()
for _, stat := range stats {
totalSockets = totalSockets + stat.TotalWebsocketConnections
totalMasterDb = totalMasterDb + stat.TotalMasterDbConnections
totalReadDb = totalReadDb + stat.TotalReadDbConnections
}
var teamsCount int64
g.Go(func() error {
var err error
if teamsCount, err = a.Srv().Store().Team().AnalyticsTeamCount(nil); err != nil {
return model.NewAppError("GetAnalytics", "app.team.analytics_team_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
rows[5].Value = float64(totalSockets)
rows[6].Value = float64(totalMasterDb)
rows[7].Value = float64(totalReadDb)
} else {
rows[5].Value = float64(a.TotalWebsocketConnections())
rows[6].Value = float64(a.Srv().Store().TotalMasterDbConnections())
rows[7].Value = float64(a.Srv().Store().TotalReadDbConnections())
}
var dailyActiveUsersCount int64
g.Go(func() error {
var err error
if dailyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(DayMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
rows[8].Value = float64(dailyActiveUsersCount)
rows[9].Value = float64(monthlyActiveUsersCount)
var monthlyActiveUsersCount int64
g.Go(func() error {
var err error
if monthlyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(MonthMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
return rows, nil
}
if err := g.Wait(); err != nil {
return nil, err.(*model.AppError)
}
func (a *App) getBotPostCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) {
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
TeamId: teamID,
BotsOnly: true,
})
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
rows[0].Value = float64(openChannelsCount)
rows[1].Value = float64(privateChannelsCount)
return analyticsRows, nil
}
if skipIntensiveQueries {
rows[2].Value = -1
} else {
rows[2].Value = float64(postsCount)
}
func (a *App) getPostCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) {
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
TeamId: teamID,
BotsOnly: false,
})
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
if teamID == "" {
rows[3].Value = float64(systemUserCount)
rows[10].Value = float64(inactiveUsersCount)
} else {
rows[10].Value = -1
rows[3].Value = float64(usersCount)
}
rows[4].Value = float64(teamsCount)
// If in HA mode then aggregate all the stats
if a.Cluster() != nil && *a.Config().ClusterSettings.Enable {
stats, err2 := a.Cluster().GetClusterStats(rctx)
if err2 != nil {
return nil, err2
}
totalSockets := a.TotalWebsocketConnections()
totalMasterDb := a.Srv().Store().TotalMasterDbConnections()
totalReadDb := a.Srv().Store().TotalReadDbConnections()
for _, stat := range stats {
totalSockets = totalSockets + stat.TotalWebsocketConnections
totalMasterDb = totalMasterDb + stat.TotalMasterDbConnections
totalReadDb = totalReadDb + stat.TotalReadDbConnections
}
rows[5].Value = float64(totalSockets)
rows[6].Value = float64(totalMasterDb)
rows[7].Value = float64(totalReadDb)
} else {
rows[5].Value = float64(a.TotalWebsocketConnections())
rows[6].Value = float64(a.Srv().Store().TotalMasterDbConnections())
rows[7].Value = float64(a.Srv().Store().TotalReadDbConnections())
}
rows[8].Value = float64(dailyActiveUsersCount)
rows[9].Value = float64(monthlyActiveUsersCount)
return rows, nil
} else if name == "bot_post_counts_day" {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
return rows, nil
}
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
TeamId: teamID,
BotsOnly: true,
YesterdayOnly: false,
})
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return analyticsRows, nil
} else if name == "post_counts_day" {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
return rows, nil
}
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
TeamId: teamID,
BotsOnly: false,
YesterdayOnly: false,
})
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return analyticsRows, nil
} else if name == "user_counts_with_posts_day" {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
return rows, nil
}
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsUserCountsWithPostsByDay(teamID)
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_user_counts_posts_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return analyticsRows, nil
} else if name == "extra_counts" {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 6)
rows[0] = &model.AnalyticsRow{Name: "file_post_count", Value: 0}
rows[1] = &model.AnalyticsRow{Name: "hashtag_post_count", Value: 0}
rows[2] = &model.AnalyticsRow{Name: "incoming_webhook_count", Value: 0}
rows[3] = &model.AnalyticsRow{Name: "outgoing_webhook_count", Value: 0}
rows[4] = &model.AnalyticsRow{Name: "command_count", Value: 0}
rows[5] = &model.AnalyticsRow{Name: "session_count", Value: 0}
var g2 errgroup.Group
var incomingWebhookCount int64
g2.Go(func() error {
var err error
if incomingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsIncomingCount(teamID, ""); err != nil {
return model.NewAppError("GetAnalytics", "app.webhooks.analytics_incoming_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var outgoingWebhookCount int64
g2.Go(func() error {
var err error
if outgoingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsOutgoingCount(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.webhooks.analytics_outgoing_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var commandsCount int64
g2.Go(func() error {
var err error
if commandsCount, err = a.Srv().Store().Command().AnalyticsCommandCount(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.analytics.getanalytics.internal_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var sessionsCount int64
g2.Go(func() error {
var err error
if sessionsCount, err = a.Srv().Store().Session().AnalyticsSessionCount(); err != nil {
return model.NewAppError("GetAnalytics", "app.session.analytics_session_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var filesCount int64
var hashtagsCount int64
if !skipIntensiveQueries {
g2.Go(func() error {
var err error
if filesCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID, MustHaveFile: true}); err != nil {
return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
g2.Go(func() error {
var err error
if hashtagsCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID, MustHaveHashtag: true}); err != nil {
return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
}
if err := g2.Wait(); err != nil {
return nil, err.(*model.AppError)
}
if skipIntensiveQueries {
rows[0].Value = -1
rows[1].Value = -1
} else {
rows[0].Value = float64(filesCount)
rows[1].Value = float64(hashtagsCount)
}
rows[2].Value = float64(incomingWebhookCount)
rows[3].Value = float64(outgoingWebhookCount)
rows[4].Value = float64(commandsCount)
rows[5].Value = float64(sessionsCount)
return analyticsRows, nil
}
func (a *App) getUserCountsWithPostsAnalytics(rctx request.CTX, teamID string, skipIntensiveQueries bool) (model.AnalyticsRows, *model.AppError) {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
return rows, nil
}
return nil, nil
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsUserCountsWithPostsByDay(teamID)
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_user_counts_posts_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return analyticsRows, nil
}
func (a *App) getExtraCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 6)
rows[0] = &model.AnalyticsRow{Name: "incoming_webhook_count", Value: 0}
rows[1] = &model.AnalyticsRow{Name: "outgoing_webhook_count", Value: 0}
rows[2] = &model.AnalyticsRow{Name: "command_count", Value: 0}
rows[3] = &model.AnalyticsRow{Name: "session_count", Value: 0}
rows[4] = &model.AnalyticsRow{Name: "total_file_count", Value: 0}
rows[5] = &model.AnalyticsRow{Name: "total_file_size", Value: 0}
var incomingWebhookCount int64
var g errgroup.Group
g.SetLimit(2)
g.Go(func() error {
var err error
if incomingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsIncomingCount(teamID, ""); err != nil {
return model.NewAppError("GetAnalytics", "app.webhooks.analytics_incoming_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var outgoingWebhookCount int64
g.Go(func() error {
var err error
if outgoingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsOutgoingCount(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.webhooks.analytics_outgoing_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var commandsCount int64
g.Go(func() error {
var err error
if commandsCount, err = a.Srv().Store().Command().AnalyticsCommandCount(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.analytics.getanalytics.internal_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var sessionsCount int64
g.Go(func() error {
var err error
if sessionsCount, err = a.Srv().Store().Session().AnalyticsSessionCount(); err != nil {
return model.NewAppError("GetAnalytics", "app.session.analytics_session_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var fileCount int64
g.Go(func() error {
var err error
if fileCount, err = a.Srv().Store().FileInfo().CountAll(); err != nil {
return model.NewAppError("GetAnalytics", "app.file_info.get_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var fileSize int64
g.Go(func() error {
var err error
if fileSize, err = a.Srv().Store().FileInfo().GetStorageUsage(false, false); err != nil {
return model.NewAppError("GetAnalytics", "app.file_info.get_storage_usage.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
if err := g.Wait(); err != nil {
return nil, err.(*model.AppError)
}
rows[0].Value = float64(incomingWebhookCount)
rows[1].Value = float64(outgoingWebhookCount)
rows[2].Value = float64(commandsCount)
rows[3].Value = float64(sessionsCount)
rows[4].Value = float64(fileCount)
rows[5].Value = float64(fileSize)
return rows, nil
}
func (a *App) GetRecentlyActiveUsersForTeam(rctx request.CTX, teamID string) (map[string]*model.User, *model.AppError) {

5
server/channels/app/import_functions_test.go
Просмотреть файл

@@ -2182,6 +2182,7 @@ func TestImportimportMultiplePostLines(t *testing.T) {
require.Nil(t, err, "Failed to get user from database.")
// Count the number of posts in the testing team.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
initialPostCount, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team.Id})
require.NoError(t, nErr)
@@ -2859,6 +2860,7 @@ func TestImportimportMultiplePostLines(t *testing.T) {
require.Nil(t, err, "Failed to get channel from database.")
// Count the number of posts in the team2.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
initialPostCountForTeam2, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team2.Id})
require.NoError(t, nErr)
@@ -3239,6 +3241,7 @@ func TestImportImportPost(t *testing.T) {
require.Nil(t, appErr, "Failed to get user from database.")
// Count the number of posts in the testing team.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
initialPostCount, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team.Id})
require.NoError(t, nErr)
@@ -4194,6 +4197,7 @@ func TestImportImportDirectPost(t *testing.T) {
directChannel = channel
// Get the number of posts in the system.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
result, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, err)
initialPostCount := result
@@ -4664,6 +4668,7 @@ func TestImportImportDirectPost(t *testing.T) {
groupChannel = channel
// Get the number of posts in the system.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
result, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, nErr)
initialPostCount = result

2
server/channels/app/import_test.go
Просмотреть файл

@@ -53,6 +53,8 @@ func checkNoError(t *testing.T, err *model.AppError) {
}
func AssertAllPostsCount(t *testing.T, a *App, initialCount int64, change int64, teamName string) {
t.Helper()
require.NoError(t, a.Srv().Store().Post().RefreshPostStats())
result, err := a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamName})
require.NoError(t, err)
require.Equal(t, initialCount+change, result, "Did not find the expected number of posts.")

8
server/channels/app/server.go
Просмотреть файл

@@ -60,7 +60,7 @@ import (
"github.com/mattermost/mattermost/server/v8/channels/jobs/plugins"
"github.com/mattermost/mattermost/server/v8/channels/jobs/post_persistent_notifications"
"github.com/mattermost/mattermost/server/v8/channels/jobs/product_notices"
"github.com/mattermost/mattermost/server/v8/channels/jobs/refresh_post_stats"
"github.com/mattermost/mattermost/server/v8/channels/jobs/refresh_materialized_views"
"github.com/mattermost/mattermost/server/v8/channels/jobs/resend_invitation_email"
"github.com/mattermost/mattermost/server/v8/channels/jobs/s3_path_migration"
"github.com/mattermost/mattermost/server/v8/channels/store"
@@ -1643,9 +1643,9 @@ func (s *Server) initJobs() {
)
s.Jobs.RegisterJobType(
model.JobTypeRefreshPostStats,
refresh_post_stats.MakeWorker(s.Jobs, *s.platform.Config().SqlSettings.DriverName),
refresh_post_stats.MakeScheduler(s.Jobs, *s.platform.Config().SqlSettings.DriverName),
model.JobTypeRefreshMaterializedViews,
refresh_materialized_views.MakeWorker(s.Jobs, *s.platform.Config().SqlSettings.DriverName),
refresh_materialized_views.MakeScheduler(s.Jobs, *s.platform.Config().SqlSettings.DriverName),
)
s.Jobs.RegisterJobType(

2
server/channels/app/support_packet_test.go
Просмотреть файл

@@ -337,6 +337,8 @@ func TestGetSupportPacketStats(t *testing.T) {
generateStats := func(t *testing.T) *model.SupportPacketStats {
t.Helper()
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
fileData, err := th.App.getSupportPacketStats(th.Context)
require.NotNil(t, fileData)
assert.Equal(t, "stats.yaml", fileData.Filename)

4
server/channels/db/migrations/migrations.list
Просмотреть файл

@@ -255,6 +255,8 @@ channels/db/migrations/mysql/000128_create_scheduled_posts.down.sql
channels/db/migrations/mysql/000128_create_scheduled_posts.up.sql
channels/db/migrations/mysql/000129_add_property_system_architecture.down.sql
channels/db/migrations/mysql/000129_add_property_system_architecture.up.sql
channels/db/migrations/mysql/000130_system_console_stats.down.sql
channels/db/migrations/mysql/000130_system_console_stats.up.sql
channels/db/migrations/postgres/000001_create_teams.down.sql
channels/db/migrations/postgres/000001_create_teams.up.sql
channels/db/migrations/postgres/000002_create_team_members.down.sql
@@ -511,3 +513,5 @@ channels/db/migrations/postgres/000128_create_scheduled_posts.down.sql
channels/db/migrations/postgres/000128_create_scheduled_posts.up.sql
channels/db/migrations/postgres/000129_add_property_system_architecture.down.sql
channels/db/migrations/postgres/000129_add_property_system_architecture.up.sql
channels/db/migrations/postgres/000130_system_console_stats.down.sql
channels/db/migrations/postgres/000130_system_console_stats.up.sql

1
server/channels/db/migrations/mysql/000130_system_console_stats.down.sql Обычный файл
Просмотреть файл

@@ -0,0 +1 @@
-- Nothing to do for MySQL

1
server/channels/db/migrations/mysql/000130_system_console_stats.up.sql Обычный файл
Просмотреть файл

@@ -0,0 +1 @@
-- Nothing to do for MySQL

5
server/channels/db/migrations/postgres/000130_system_console_stats.down.sql Обычный файл
Просмотреть файл

@@ -0,0 +1,5 @@
DROP MATERIALIZED VIEW IF EXISTS posts_by_team_day;
DROP MATERIALIZED VIEW IF EXISTS bot_posts_by_team_day;
DROP MATERIALIZED VIEW IF EXISTS file_stats;

16
server/channels/db/migrations/postgres/000130_system_console_stats.up.sql Обычный файл
Просмотреть файл

@@ -0,0 +1,16 @@
CREATE MATERIALIZED VIEW IF NOT EXISTS posts_by_team_day as
SELECT to_timestamp(p.createat/1000)::date as day, COUNT(*) as num, teamid
FROM posts p JOIN channels c on p.channelid=c.id
GROUP BY day, c.teamid;
CREATE MATERIALIZED VIEW IF NOT EXISTS bot_posts_by_team_day as
SELECT to_timestamp(p.createat/1000)::date as day, COUNT(*) as num, teamid
FROM posts p
JOIN Bots b ON p.UserId = b.Userid
JOIN channels c on p.channelid=c.id
GROUP BY day, c.teamid;
CREATE MATERIALIZED VIEW IF NOT EXISTS file_stats as
SELECT COUNT(*) as num, COALESCE(SUM(Size), 0) as usage
FROM fileinfo
WHERE DeleteAt = 0;

4
server/channels/jobs/refresh_post_stats/scheduler.go → server/channels/jobs/refresh_materialized_views/scheduler.go
Просмотреть файл

@@ -1,7 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package refresh_post_stats
package refresh_materialized_views
import (
"time"
@@ -21,5 +21,5 @@ func MakeScheduler(jobServer *jobs.JobServer, sqlDriverName string) *jobs.DailyS
isEnabled := func(cfg *model.Config) bool {
return sqlDriverName == model.DatabaseDriverPostgres
}
return jobs.NewDailyScheduler(jobServer, model.JobTypeRefreshPostStats, startTime, isEnabled)
return jobs.NewDailyScheduler(jobServer, model.JobTypeRefreshMaterializedViews, startTime, isEnabled)
}

13
server/channels/jobs/refresh_post_stats/worker.go → server/channels/jobs/refresh_materialized_views/worker.go
Просмотреть файл

@@ -1,7 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package refresh_post_stats
package refresh_materialized_views
import (
"github.com/mattermost/mattermost/server/public/model"
@@ -9,7 +9,7 @@ import (
"github.com/mattermost/mattermost/server/v8/channels/jobs"
)
const jobName = "RefreshPostStats"
const jobName = "RefreshMaterializedViews"
func MakeWorker(jobServer *jobs.JobServer, sqlDriverName string) *jobs.SimpleWorker {
isEnabled := func(cfg *model.Config) bool {
@@ -18,8 +18,17 @@ func MakeWorker(jobServer *jobs.JobServer, sqlDriverName string) *jobs.SimpleWor
execute := func(logger mlog.LoggerIFace, job *model.Job) error {
defer jobServer.HandleJobPanic(logger, job)
if err := jobServer.Store.Post().RefreshPostStats(); err != nil {
return err
}
if err := jobServer.Store.FileInfo().RefreshFileStats(); err != nil {
return err
}
return jobServer.Store.User().RefreshPostStatsForUsers()
}
worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
return worker
}

84
server/channels/store/retrylayer/retrylayer.go
Просмотреть файл

@@ -741,6 +741,27 @@ func (s *RetryLayerBotStore) Update(bot *model.Bot) (*model.Bot, error) {
}
func (s *RetryLayerChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) {
tries := 0
for {
result, err := s.ChannelStore.AnalyticsCountAll(teamID)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) {
tries := 0
@@ -4746,6 +4767,27 @@ func (s *RetryLayerFileInfoStore) PermanentDeleteForPost(rctx request.CTX, postI
}
func (s *RetryLayerFileInfoStore) RefreshFileStats() error {
tries := 0
for {
err := s.FileInfoStore.RefreshFileStats()
if err == nil {
return nil
}
if !isRepeatableError(err) {
return err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error {
tries := 0
@@ -7308,6 +7350,27 @@ func (s *RetryLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions
}
func (s *RetryLayerPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
tries := 0
for {
result, err := s.PostStore.AnalyticsPostCountByTeam(teamID)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
tries := 0
@@ -8109,6 +8172,27 @@ func (s *RetryLayerPostStore) PermanentDeleteByUser(rctx request.CTX, userID str
}
func (s *RetryLayerPostStore) RefreshPostStats() error {
tries := 0
for {
err := s.PostStore.RefreshPostStats()
if err == nil {
return nil
}
if !isRepeatableError(err) {
return err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerPostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) {
tries := 0

34
server/channels/store/sqlstore/channel_store.go
Просмотреть файл

@@ -2961,6 +2961,40 @@ func (s SqlChannelStore) AnalyticsDeletedTypeCount(teamId string, channelType mo
return v, nil
}
func (s SqlChannelStore) AnalyticsCountAll(teamId string) (map[model.ChannelType]int64, error) {
query := s.getQueryBuilder().
Select("Type, COUNT(*) AS Count").
From("Channels").
GroupBy("Type")
if teamId != "" {
query = query.Where(sq.Eq{"TeamId": teamId})
}
sql, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "AnalyticsCountAll_ToSql")
}
rows, err := s.GetReplica().Query(sql, args...)
if err != nil {
return nil, errors.Wrap(err, "failed to count Channels by type")
}
defer rows.Close()
counts := make(map[model.ChannelType]int64)
for rows.Next() {
var channelType model.ChannelType
var count int64
if err := rows.Scan(&channelType, &count); err != nil {
return nil, errors.Wrap(err, "unable to scan row")
}
counts[channelType] = count
}
return counts, nil
}
func (s SqlChannelStore) GetMembersForUser(teamID string, userID string) (model.ChannelMembers, error) {
sql, args, err := s.channelMembersForTeamWithSchemeSelectQuery.
Where(sq.And{

54
server/channels/store/sqlstore/file_info_store.go
Просмотреть файл

@@ -712,18 +712,20 @@ func (fs SqlFileInfoStore) Search(rctx request.CTX, paramsList []*model.SearchPa
}
func (fs SqlFileInfoStore) CountAll() (int64, error) {
query := fs.getQueryBuilder().
Select("COUNT(*)").
From("FileInfo").
Where("DeleteAt = 0")
queryString, args, err := query.ToSql()
if err != nil {
return int64(0), errors.Wrap(err, "count_tosql")
var query sq.SelectBuilder
if fs.DriverName() == model.DatabaseDriverPostgres {
query = fs.getQueryBuilder().
Select("num").
From("file_stats")
} else {
query = fs.getQueryBuilder().
Select("COUNT(*)").
From("FileInfo").
Where("DeleteAt = 0")
}
var count int64
err = fs.GetReplica().Get(&count, queryString, args...)
err := fs.GetReplica().GetBuilder(&count, query)
if err != nil {
return int64(0), errors.Wrap(err, "failed to count Files")
}
@@ -758,13 +760,20 @@ func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID
return files, nil
}
func (fs SqlFileInfoStore) GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error) {
query := fs.getQueryBuilder().
Select("COALESCE(SUM(Size), 0)").
From("FileInfo")
func (fs SqlFileInfoStore) GetStorageUsage(_, includeDeleted bool) (int64, error) {
var query sq.SelectBuilder
if fs.DriverName() == model.DatabaseDriverPostgres && !includeDeleted {
query = fs.getQueryBuilder().
Select("usage").
From("file_stats")
} else {
query = fs.getQueryBuilder().
Select("COALESCE(SUM(Size), 0)").
From("FileInfo")
if !includeDeleted {
query = query.Where("DeleteAt = 0")
if !includeDeleted {
query = query.Where("DeleteAt = 0")
}
}
var size int64
@@ -841,3 +850,18 @@ func (fs SqlFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string,
return nil
}
func (fs SqlFileInfoStore) RefreshFileStats() error {
if fs.DriverName() == model.DatabaseDriverPostgres {
// CONCURRENTLY is not used deliberately because as per Postgres docs,
// not using CONCURRENTLY takes less resources and completes faster
// at the expense of locking the mat view. Since viewing admin console
// is not a very frequent activity, we accept the tradeoff to let the
// refresh happen as fast as possible.
if _, err := fs.GetMaster().Exec("REFRESH MATERIALIZED VIEW file_stats"); err != nil {
return errors.Wrap(err, "error refreshing materialized view file_stats")
}
}
return nil
}

147
server/channels/store/sqlstore/post_store.go
Просмотреть файл

@@ -2292,8 +2292,79 @@ func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.A
return rows, nil
}
func (s *SqlPostStore) countBotPostsByDay(teamID, startDay, endDay string) (model.AnalyticsRows, error) {
var query sq.SelectBuilder
if teamID != "" {
query = s.getQueryBuilder().
Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, num as Value").
From("bot_posts_by_team_day").
Where(sq.Eq{"teamid": teamID})
} else {
query = s.getQueryBuilder().
Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, COALESCE(SUM(num), 0) as Value").
From("bot_posts_by_team_day").
GroupBy("Name")
}
query = query.
Where(sq.GtOrEq{"day": startDay}).
Where(sq.LtOrEq{"day": endDay}).
OrderBy("Name DESC").
Limit(30)
rows := model.AnalyticsRows{}
err := s.GetReplica().SelectBuilder(&rows, query)
if err != nil {
return nil, errors.Wrapf(err, "failed to find bot posts with teamId=%s", teamID)
}
return rows, nil
}
func (s *SqlPostStore) countPostsByDay(teamID, startDay, endDay string) (model.AnalyticsRows, error) {
var query sq.SelectBuilder
if teamID != "" {
query = s.getQueryBuilder().
Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, num as Value").
From("posts_by_team_day").
Where(sq.Eq{"teamid": teamID})
} else {
query = s.getQueryBuilder().
Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, COALESCE(SUM(num), 0) as Value").
From("posts_by_team_day").
GroupBy("Name")
}
query = query.
Where(sq.GtOrEq{"day": startDay}).
Where(sq.LtOrEq{"day": endDay}).
OrderBy("Name DESC").
Limit(30)
rows := model.AnalyticsRows{}
err := s.GetReplica().SelectBuilder(&rows, query)
if err != nil {
return nil, errors.Wrapf(err, "failed to find posts with teamId=%s", teamID)
}
return rows, nil
}
// TODO: convert to squirrel HW
func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
if s.DriverName() == model.DatabaseDriverPostgres {
endDay := utils.Yesterday().Format("2006-01-02")
startDay := utils.Yesterday().AddDate(0, 0, -31).Format("2006-01-02")
if options.YesterdayOnly {
startDay = utils.Yesterday().AddDate(0, 0, -1).Format("2006-01-02")
}
// Use materialized views
if options.BotsOnly {
return s.countBotPostsByDay(options.TeamId, startDay, endDay)
}
return s.countPostsByDay(options.TeamId, startDay, endDay)
}
var args []any
query :=
`SELECT
@@ -2318,30 +2389,6 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun
ORDER BY Name DESC
LIMIT 30`
if s.DriverName() == model.DatabaseDriverPostgres {
query =
`SELECT
TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, Count(Posts.Id) AS Value
FROM Posts`
if options.BotsOnly {
query += " INNER JOIN Bots ON Posts.UserId = Bots.Userid"
}
if options.TeamId != "" {
query += " INNER JOIN Channels ON Posts.ChannelId = Channels.Id AND Channels.TeamId = ? AND"
args = []any{options.TeamId}
} else {
query += " WHERE"
}
query += ` Posts.CreateAt <= ?
AND Posts.CreateAt >= ?
GROUP BY DATE(TO_TIMESTAMP(Posts.CreateAt / 1000))
ORDER BY Name DESC
LIMIT 30`
}
end := utils.MillisFromTime(utils.EndOfDay(utils.Yesterday()))
start := utils.MillisFromTime(utils.StartOfDay(utils.Yesterday().AddDate(0, 0, -31)))
if options.YesterdayOnly {
@@ -2350,16 +2397,39 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun
args = append(args, end, start)
rows := model.AnalyticsRows{}
err := s.GetReplica().Select(
&rows,
query,
args...)
err := s.GetReplica().Select(&rows, query, args...)
if err != nil {
return nil, errors.Wrapf(err, "failed to find Posts with teamId=%s", options.TeamId)
}
return rows, nil
}
func (s *SqlPostStore) countByTeam(teamID string) (int64, error) {
query := s.getQueryBuilder().
Select("COALESCE(SUM(num), 0) AS total").
From("posts_by_team_day")
if teamID != "" {
query = query.Where(sq.Eq{"teamid": teamID})
}
var v int64
err := s.GetReplica().GetBuilder(&v, query)
if err != nil {
return 0, fmt.Errorf("failed to count Posts by team: %w, teamID: %s", err, teamID)
}
return v, nil
}
func (s *SqlPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
if s.DriverName() == model.DatabaseDriverPostgres {
return s.countByTeam(teamID)
}
return s.AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID})
}
func (s *SqlPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) {
query := s.getQueryBuilder().
Select("COUNT(*) AS Value").
@@ -2553,7 +2623,7 @@ func (s *SqlPostStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolic
func (s *SqlPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
var query string
if s.DriverName() == "postgres" {
if s.DriverName() == model.DatabaseDriverPostgres {
query = "DELETE from Posts WHERE Id = any (array (SELECT Id FROM Posts WHERE CreateAt < ? LIMIT ?))"
} else {
query = "DELETE from Posts WHERE CreateAt < ? LIMIT ?"
@@ -3295,3 +3365,22 @@ func (s *SqlPostStore) GetPostReminderMetadata(postID string) (*store.PostRemind
return meta, nil
}
func (s *SqlPostStore) RefreshPostStats() error {
if s.DriverName() == model.DatabaseDriverPostgres {
// CONCURRENTLY is not used deliberately because as per Postgres docs,
// not using CONCURRENTLY takes less resources and completes faster
// at the expense of locking the mat view. Since viewing admin console
// is not a very frequent activity, we accept the tradeoff to let the
// refresh happen as fast as possible.
if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW posts_by_team_day"); err != nil {
return errors.Wrap(err, "error refreshing materialized view posts_by_team_day")
}
if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW bot_posts_by_team_day"); err != nil {
return errors.Wrap(err, "error refreshing materialized view bot_posts_by_team_day")
}
}
return nil
}

9
server/channels/store/sqlstore/user_store.go
Просмотреть файл

@@ -1696,7 +1696,6 @@ func (us SqlUserStore) performSearch(query sq.SelectBuilder, term string, option
}
func (us SqlUserStore) AnalyticsGetInactiveUsersCount() (int64, error) {
var count int64
query := us.getQueryBuilder().
Select("COUNT(Id)").
From("Users")
@@ -1712,11 +1711,9 @@ func (us SqlUserStore) AnalyticsGetInactiveUsersCount() (int64, error) {
sq.Gt{"Users.DeleteAt": 0},
})
}
queryStr, args, err := query.ToSql()
if err != nil {
return int64(0), errors.Wrap(err, "failed to create a SQL query to count inactive users")
}
err = us.GetReplica().Get(&count, queryStr, args...)
var count int64
err := us.GetReplica().GetBuilder(&count, query)
if err != nil {
return int64(0), errors.Wrap(err, "failed to count inactive Users")
}

8
server/channels/store/store.go
Просмотреть файл

@@ -260,6 +260,8 @@ type ChannelStore interface {
CountUrgentPostsAfter(channelID string, timestamp int64, excludedUserID string) (int, error)
IncrementMentionCount(channelID string, userIDs []string, isRoot, isUrgent bool) error
AnalyticsTypeCount(teamID string, channelType model.ChannelType) (int64, error)
AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error)
AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error)
GetMembersForUser(teamID string, userID string) (model.ChannelMembers, error)
GetTeamMembersForChannel(channelID string) ([]string, error)
GetMembersForUserWithPagination(userID string, page, perPage int) (model.ChannelMembersWithTeamData, error)
@@ -275,7 +277,6 @@ type ChannelStore interface {
GetMembersByIds(channelID string, userIds []string) (model.ChannelMembers, error)
GetMembersByChannelIds(channelIds []string, userID string) (model.ChannelMembers, error)
GetMembersInfoByChannelIds(channelIDs []string) (map[string][]*model.User, error)
AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error)
GetChannelUnread(channelID, userID string) (*model.ChannelUnread, error)
GetChannelsWithUnreadsAndWithMentions(ctx context.Context, channelIDs []string, userID string, userNotifyProps model.StringMap) ([]string, []string, map[string]int64, error)
ClearCaches()
@@ -387,6 +388,7 @@ type PostStore interface {
AnalyticsUserCountsWithPostsByDay(teamID string) (model.AnalyticsRows, error)
AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error)
AnalyticsPostCount(options *model.PostCountOptions) (int64, error)
AnalyticsPostCountByTeam(teamID string) (int64, error)
ClearCaches()
InvalidateLastPostTimeCache(channelID string)
GetPostsCreatedAt(channelID string, timestamp int64) ([]*model.Post, error)
@@ -411,6 +413,8 @@ type PostStore interface {
GetPostReminderMetadata(postID string) (*PostReminderMetadata, error)
// GetNthRecentPostTime returns the CreateAt time of the nth most recent post.
GetNthRecentPostTime(n int64) (int64, error)
// RefreshPostStats refreshes the various materialized views for admin console post stats.
RefreshPostStats() error
}
type UserStore interface {
@@ -743,6 +747,8 @@ type FileInfoStore interface {
GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error)
// GetUptoNSizeFileTime returns the CreateAt time of the last accessible file with a running-total size upto n bytes.
GetUptoNSizeFileTime(n int64) (int64, error)
// RefreshFileStats recomputes the fileinfo materialized views.
RefreshFileStats() error
}
type UploadSessionStore interface {

24
server/channels/store/storetest/channel_store.go
Просмотреть файл

@@ -4148,18 +4148,34 @@ func testChannelStoreGetMoreChannels(t *testing.T, rctx request.CTX, ss store.St
count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen)
require.NoError(t, err)
require.EqualValues(t, 4, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
require.EqualValues(t, 4, counts[model.ChannelTypeOpen])
})
t.Run("verify analytics for private channels", func(t *testing.T) {
count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate)
require.NoError(t, err)
require.EqualValues(t, 2, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
require.EqualValues(t, 2, counts[model.ChannelTypePrivate])
})
t.Run("verify analytics for all channels", func(t *testing.T) {
count, err := ss.Channel().AnalyticsTypeCount(teamID, "")
require.NoError(t, err)
require.EqualValues(t, 6, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
total := int64(0)
for _, count := range counts {
total += count
}
require.EqualValues(t, 6, total)
})
}
@@ -4338,12 +4354,20 @@ func testChannelStoreGetPublicChannelsForTeam(t *testing.T, rctx request.CTX, ss
count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen)
require.NoError(t, err)
require.EqualValues(t, 3, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
require.EqualValues(t, 3, counts[model.ChannelTypeOpen])
})
t.Run("verify analytics for private channels", func(t *testing.T) {
count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate)
require.NoError(t, err)
require.EqualValues(t, 1, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
require.EqualValues(t, 1, counts[model.ChannelTypePrivate])
})
}

6
server/channels/store/storetest/file_info_store.go
Просмотреть файл

@@ -815,12 +815,15 @@ func testFileInfoStoreCountAll(t *testing.T, rctx request.CTX, ss store.Store) {
})
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
count, err := ss.FileInfo().CountAll()
require.NoError(t, err)
require.Equal(t, int64(3), count)
_, err = ss.FileInfo().DeleteForPost(rctx, f1.PostId)
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
count, err = ss.FileInfo().CountAll()
require.NoError(t, err)
require.Equal(t, int64(2), count)
@@ -830,6 +833,7 @@ func testFileInfoGetStorageUsage(t *testing.T, rctx request.CTX, ss store.Store)
_, err := ss.FileInfo().PermanentDeleteBatch(rctx, model.GetMillis(), 100000)
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
usage, err := ss.FileInfo().GetStorageUsage(false, false)
require.NoError(t, err)
require.Equal(t, int64(0), usage)
@@ -857,12 +861,14 @@ func testFileInfoGetStorageUsage(t *testing.T, rctx request.CTX, ss store.Store)
})
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
usage, err = ss.FileInfo().GetStorageUsage(false, false)
require.NoError(t, err)
require.Equal(t, int64(30), usage)
_, err = ss.FileInfo().DeleteForPost(rctx, f1.PostId)
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
usage, err = ss.FileInfo().GetStorageUsage(false, false)
require.NoError(t, err)
require.Equal(t, int64(20), usage)

30
server/channels/store/storetest/mocks/ChannelStore.go
Просмотреть файл

@@ -20,6 +20,36 @@ type ChannelStore struct {
mock.Mock
}
// AnalyticsCountAll provides a mock function with given fields: teamID
func (_m *ChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) {
ret := _m.Called(teamID)
if len(ret) == 0 {
panic("no return value specified for AnalyticsCountAll")
}
var r0 map[model.ChannelType]int64
var r1 error
if rf, ok := ret.Get(0).(func(string) (map[model.ChannelType]int64, error)); ok {
return rf(teamID)
}
if rf, ok := ret.Get(0).(func(string) map[model.ChannelType]int64); ok {
r0 = rf(teamID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(map[model.ChannelType]int64)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(teamID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// AnalyticsDeletedTypeCount provides a mock function with given fields: teamID, channelType
func (_m *ChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) {
ret := _m.Called(teamID, channelType)

18
server/channels/store/storetest/mocks/FileInfoStore.go
Просмотреть файл

@@ -505,6 +505,24 @@ func (_m *FileInfoStore) PermanentDeleteForPost(rctx request.CTX, postID string)
return r0
}
// RefreshFileStats provides a mock function with given fields:
func (_m *FileInfoStore) RefreshFileStats() error {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for RefreshFileStats")
}
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// RestoreForPostByIds provides a mock function with given fields: rctx, postId, fileIDs
func (_m *FileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error {
ret := _m.Called(rctx, postId, fileIDs)

46
server/channels/store/storetest/mocks/PostStore.go
Просмотреть файл

@@ -48,6 +48,34 @@ func (_m *PostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64,
return r0, r1
}
// AnalyticsPostCountByTeam provides a mock function with given fields: teamID
func (_m *PostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
ret := _m.Called(teamID)
if len(ret) == 0 {
panic("no return value specified for AnalyticsPostCountByTeam")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(string) (int64, error)); ok {
return rf(teamID)
}
if rf, ok := ret.Get(0).(func(string) int64); ok {
r0 = rf(teamID)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(teamID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// AnalyticsPostCountsByDay provides a mock function with given fields: options
func (_m *PostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
ret := _m.Called(options)
@@ -1163,6 +1191,24 @@ func (_m *PostStore) PermanentDeleteByUser(rctx request.CTX, userID string) erro
return r0
}
// RefreshPostStats provides a mock function with given fields:
func (_m *PostStore) RefreshPostStats() error {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for RefreshPostStats")
}
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// Save provides a mock function with given fields: rctx, post
func (_m *PostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) {
ret := _m.Called(rctx, post)

14
server/channels/store/storetest/post_store.go
Просмотреть файл

@@ -2895,6 +2895,8 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) {
_, nErr = ss.Post().Save(rctx, b1a)
require.NoError(t, nErr)
require.NoError(t, ss.Post().RefreshPostStats())
time.Sleep(1 * time.Second)
// summary of posts
@@ -2907,6 +2909,8 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) {
require.NoError(t, err)
assert.Equal(t, float64(3), r1[0].Value)
assert.Equal(t, float64(3), r1[1].Value)
assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name)
assert.Equal(t, utils.Yesterday().Add(-48*time.Hour).Format("2006-01-02"), r1[1].Name)
// last 31 days, bots only
postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: true, YesterdayOnly: false}
@@ -2914,18 +2918,22 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) {
require.NoError(t, err)
assert.Equal(t, float64(1), r1[0].Value)
assert.Equal(t, float64(1), r1[1].Value)
assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name)
assert.Equal(t, utils.Yesterday().Add(-48*time.Hour).Format("2006-01-02"), r1[1].Name)
// yesterday only, all users (including bots)
postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: false, YesterdayOnly: true}
r1, err = ss.Post().AnalyticsPostCountsByDay(postCountsOptions)
require.NoError(t, err)
assert.Equal(t, float64(3), r1[0].Value)
assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name)
// yesterday only, bots only
postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: true, YesterdayOnly: true}
r1, err = ss.Post().AnalyticsPostCountsByDay(postCountsOptions)
require.NoError(t, err)
assert.Equal(t, float64(1), r1[0].Value)
assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name)
}
func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) {
@@ -3029,6 +3037,8 @@ func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) {
_, nErr = ss.Post().Save(rctx, p7)
require.NoError(t, nErr)
require.NoError(t, ss.Post().RefreshPostStats())
// total across all teams
c, err := ss.Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, err)
@@ -3039,6 +3049,10 @@ func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) {
require.NoError(t, err)
assert.Equal(t, int64(7), c)
c, err = ss.Post().AnalyticsPostCountByTeam(t1.Id)
require.NoError(t, err)
assert.Equal(t, int64(7), c)
// with files
c, err = ss.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: t1.Id, MustHaveFile: true})
require.NoError(t, err)

64
server/channels/store/timerlayer/timerlayer.go
Просмотреть файл

@@ -671,6 +671,22 @@ func (s *TimerLayerBotStore) Update(bot *model.Bot) (*model.Bot, error) {
return result, err
}
func (s *TimerLayerChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) {
start := time.Now()
result, err := s.ChannelStore.AnalyticsCountAll(teamID)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("ChannelStore.AnalyticsCountAll", success, elapsed)
}
return result, err
}
func (s *TimerLayerChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) {
start := time.Now()
@@ -3859,6 +3875,22 @@ func (s *TimerLayerFileInfoStore) PermanentDeleteForPost(rctx request.CTX, postI
return err
}
func (s *TimerLayerFileInfoStore) RefreshFileStats() error {
start := time.Now()
err := s.FileInfoStore.RefreshFileStats()
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.RefreshFileStats", success, elapsed)
}
return err
}
func (s *TimerLayerFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error {
start := time.Now()
@@ -5811,6 +5843,22 @@ func (s *TimerLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions
return result, err
}
func (s *TimerLayerPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
start := time.Now()
result, err := s.PostStore.AnalyticsPostCountByTeam(teamID)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("PostStore.AnalyticsPostCountByTeam", success, elapsed)
}
return result, err
}
func (s *TimerLayerPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
start := time.Now()
@@ -6465,6 +6513,22 @@ func (s *TimerLayerPostStore) PermanentDeleteByUser(rctx request.CTX, userID str
return err
}
func (s *TimerLayerPostStore) RefreshPostStats() error {
start := time.Now()
err := s.PostStore.RefreshPostStats()
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("PostStore.RefreshPostStats", success, elapsed)
}
return err
}
func (s *TimerLayerPostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) {
start := time.Now()

8
server/i18n/en.json
Просмотреть файл

@@ -5218,10 +5218,18 @@
"id": "app.file_info.get_by_post_id.app_error",
"translation": "Failed to find files for post."
},
{
"id": "app.file_info.get_count.app_error",
"translation": "Failed to get count of all files."
},
{
"id": "app.file_info.get_for_post.app_error",
"translation": "Unable to get the file info for the post."
},
{
"id": "app.file_info.get_storage_usage.app_error",
"translation": "Failed to get storage usage of all files."
},
{
"id": "app.file_info.get_with_options.app_error",
"translation": "Unable to get the file info with options"

4
server/public/model/job.go
Просмотреть файл

@@ -39,7 +39,7 @@ const (
JobTypeS3PathMigration = "s3_path_migration"
JobTypeCleanupDesktopTokens = "cleanup_desktop_tokens"
JobTypeDeleteEmptyDraftsMigration = "delete_empty_drafts_migration"
JobTypeRefreshPostStats = "refresh_post_stats"
JobTypeRefreshMaterializedViews = "refresh_materialized_views"
JobTypeDeleteOrphanDraftsMigration = "delete_orphan_drafts_migration"
JobTypeExportUsersToCSV = "export_users_to_csv"
JobTypeDeleteDmsPreferencesMigration = "delete_dms_preferences_migration"
@@ -75,7 +75,7 @@ var AllJobTypes = [...]string{
JobTypeLastAccessiblePost,
JobTypeLastAccessibleFile,
JobTypeCleanupDesktopTokens,
JobTypeRefreshPostStats,
JobTypeRefreshMaterializedViews,
JobTypeMobileSessionMetadata,
}