From ae9e6174e5ad46e244ee141c8317565bc97d003e Mon Sep 17 00:00:00 2001 From: Agniva De Sarker Date: Tue, 4 Feb 2025 21:54:01 +0530 Subject: [PATCH] [AI assisted]: Improve system console statistics performance (#29899) ```release-note NONE ``` Co-authored-by: Matthew Birtch --- .../reporting/site_statistics_spec.js | 37 +- .../reporting/team_statistics_spec.js | 14 +- e2e-tests/cypress/tests/plugins/db_request.js | 27 + e2e-tests/cypress/tests/plugins/index.js | 2 + .../cypress/tests/support/db_commands.ts | 17 + server/channels/api4/user_test.go | 2 + server/channels/app/analytics.go | 492 +++++++++--------- server/channels/app/import_functions_test.go | 5 + server/channels/app/import_test.go | 2 + server/channels/app/server.go | 8 +- server/channels/app/support_packet_test.go | 2 + server/channels/db/migrations/migrations.list | 4 + .../000130_system_console_stats.down.sql | 1 + .../mysql/000130_system_console_stats.up.sql | 1 + .../000130_system_console_stats.down.sql | 5 + .../000130_system_console_stats.up.sql | 16 + .../scheduler.go | 4 +- .../worker.go | 13 +- .../channels/store/retrylayer/retrylayer.go | 84 +++ .../channels/store/sqlstore/channel_store.go | 34 ++ .../store/sqlstore/file_info_store.go | 54 +- server/channels/store/sqlstore/post_store.go | 147 ++++-- server/channels/store/sqlstore/user_store.go | 9 +- server/channels/store/store.go | 8 +- .../channels/store/storetest/channel_store.go | 24 + .../store/storetest/file_info_store.go | 6 + .../store/storetest/mocks/ChannelStore.go | 30 ++ .../store/storetest/mocks/FileInfoStore.go | 18 + .../store/storetest/mocks/PostStore.go | 46 ++ server/channels/store/storetest/post_store.go | 14 + .../channels/store/timerlayer/timerlayer.go | 64 +++ server/i18n/en.json | 8 + server/public/model/job.go | 4 +- .../src/components/analytics/format.tsx | 17 - .../analytics/statistic_count.test.tsx | 15 + .../components/analytics/statistic_count.tsx | 7 +- .../analytics/system_analytics/analytics.scss | 49 ++ .../system_analytics.test.tsx | 13 +- .../system_analytics/system_analytics.tsx | 123 +++-- webapp/channels/src/i18n/en.json | 6 +- .../mattermost-redux/src/constants/stats.ts | 4 +- .../src/reducers/entities/admin.ts | 12 +- .../mattermost-redux/src/utils/file_utils.ts | 3 +- webapp/channels/src/utils/constants.tsx | 4 +- webapp/platform/types/src/admin.ts | 4 +- 45 files changed, 1052 insertions(+), 407 deletions(-) create mode 100644 server/channels/db/migrations/mysql/000130_system_console_stats.down.sql create mode 100644 server/channels/db/migrations/mysql/000130_system_console_stats.up.sql create mode 100644 server/channels/db/migrations/postgres/000130_system_console_stats.down.sql create mode 100644 server/channels/db/migrations/postgres/000130_system_console_stats.up.sql rename server/channels/jobs/{refresh_post_stats => refresh_materialized_views}/scheduler.go (82%) rename server/channels/jobs/{refresh_post_stats => refresh_materialized_views}/worker.go (74%) create mode 100644 webapp/channels/src/components/analytics/system_analytics/analytics.scss diff --git a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/reporting/site_statistics_spec.js b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/reporting/site_statistics_spec.js index ae32a30ed6..a07a707669 100644 --- a/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/reporting/site_statistics_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/enterprise/system_console/reporting/site_statistics_spec.js @@ -128,25 +128,30 @@ describe('System Console > Site Statistics', () => { cy.visit('/admin_console'); cy.wait('@resources'); - // * Find site statistics and click it - cy.findByTestId('reporting.system_analytics', {timeout: TIMEOUTS.ONE_MIN}).click(); + cy.dbRefreshPostStats().then(() => { + // * Find site statistics and click it + cy.findByTestId('reporting.system_analytics', {timeout: TIMEOUTS.ONE_MIN}).click(); - let totalPostsDataSet; - let totalPostsFromBots; - let activeUsersWithPosts; + // * Expand the details + cy.findByTestId('details-expander', {timeout: TIMEOUTS.ONE_MIN}).click(); - // # Grab all data from the 3 charts from there data labels - cy.findByTestId('totalPostsLineChart').then((el) => { - totalPostsDataSet = el[0].dataset.labels; - cy.findByTestId('totalPostsFromBotsLineChart').then((el2) => { - totalPostsFromBots = el2[0].dataset.labels; - cy.findByTestId('activeUsersWithPostsLineChart').then((el3) => { - activeUsersWithPosts = el3[0].dataset.labels; + let totalPostsDataSet; + let totalPostsFromBots; + let activeUsersWithPosts; - // * Assert that all the dates are the same - expect(totalPostsDataSet).equal(totalPostsFromBots); - expect(totalPostsDataSet).equal(activeUsersWithPosts); - expect(totalPostsFromBots).equal(activeUsersWithPosts); + // # Grab all data from the 3 charts from there data labels + cy.findByTestId('totalPostsLineChart').then((el) => { + totalPostsDataSet = el[0].dataset.labels; + cy.findByTestId('totalPostsFromBotsLineChart').then((el2) => { + totalPostsFromBots = el2[0].dataset.labels; + cy.findByTestId('activeUsersWithPostsLineChart').then((el3) => { + activeUsersWithPosts = el3[0].dataset.labels; + + // * Assert that all the dates are the same + expect(totalPostsDataSet).equal(totalPostsFromBots); + expect(totalPostsDataSet).equal(activeUsersWithPosts); + expect(totalPostsFromBots).equal(activeUsersWithPosts); + }); }); }); }); diff --git a/e2e-tests/cypress/tests/integration/channels/system_console/reporting/team_statistics_spec.js b/e2e-tests/cypress/tests/integration/channels/system_console/reporting/team_statistics_spec.js index c33dba9ac5..57b05dd8a9 100644 --- a/e2e-tests/cypress/tests/integration/channels/system_console/reporting/team_statistics_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/system_console/reporting/team_statistics_spec.js @@ -19,14 +19,16 @@ describe('System Console > Team Statistics', () => { // # Create private channel. cy.apiCreateChannel(team.id, 'mmt906-ch', 'mmt906-ch', 'P'); - // # Visit team statistics page. - cy.visit('/admin_console/reporting/team_statistics'); + cy.dbRefreshPostStats().then(() => { + // # Visit team statistics page. + cy.visit('/admin_console/reporting/team_statistics'); - // # Select created team. - cy.get('select.team-statistics__team-filter__dropdown').select(team.id); + // # Select created team. + cy.get('select.team-statistics__team-filter__dropdown').select(team.id); - // # Explicit wait to allow stats to get loaded - cy.wait(TIMEOUTS.TWO_SEC); + // # Explicit wait to allow stats to get loaded + cy.wait(TIMEOUTS.TWO_SEC); + }); }); }); diff --git a/e2e-tests/cypress/tests/plugins/db_request.js b/e2e-tests/cypress/tests/plugins/db_request.js index 8dcc49d2d7..267171dd14 100644 --- a/e2e-tests/cypress/tests/plugins/db_request.js +++ b/e2e-tests/cypress/tests/plugins/db_request.js @@ -127,9 +127,36 @@ function toLowerCase(config, name) { return name.toLowerCase(); } +const dbRefreshPostStats = async ({dbConfig}) => { + if (!knexClient) { + knexClient = getKnexClient(dbConfig); + } + + // Only run for PostgreSQL + if (dbConfig.client !== 'postgres') { + return { + skipped: true, + message: 'Refresh post stats is only supported for PostgreSQL', + }; + } + + try { + await knexClient.raw('REFRESH MATERIALIZED VIEW posts_by_team_day;'); + await knexClient.raw('REFRESH MATERIALIZED VIEW bot_posts_by_team_day;'); + + return { + success: true, + }; + } catch (error) { + const errorMessage = 'Failed to refresh post statistics materialized views.'; + return {error, errorMessage}; + } +}; + module.exports = { dbGetActiveUserSessions, dbGetUser, dbGetUserSession, dbUpdateUserSession, + dbRefreshPostStats, }; diff --git a/e2e-tests/cypress/tests/plugins/index.js b/e2e-tests/cypress/tests/plugins/index.js index ca0b6e9dc1..5f8226ed6d 100644 --- a/e2e-tests/cypress/tests/plugins/index.js +++ b/e2e-tests/cypress/tests/plugins/index.js @@ -11,6 +11,7 @@ const { dbGetUser, dbGetUserSession, dbUpdateUserSession, + dbRefreshPostStats, } = require('./db_request'); const externalRequest = require('./external_request').default; const {fileExist, writeToFile} = require('./file_util'); @@ -42,6 +43,7 @@ module.exports = (on, config) => { dbGetUser, dbGetUserSession, dbUpdateUserSession, + dbRefreshPostStats, externalRequest, fileExist, writeToFile, diff --git a/e2e-tests/cypress/tests/support/db_commands.ts b/e2e-tests/cypress/tests/support/db_commands.ts index 472a45c920..5f5d7c5aeb 100644 --- a/e2e-tests/cypress/tests/support/db_commands.ts +++ b/e2e-tests/cypress/tests/support/db_commands.ts @@ -96,6 +96,14 @@ function dbUpdateUserSession(params: UpdateUserSessionParam): ChainableT { + return cy.task('dbRefreshPostStats', {dbConfig}).then(({success, skipped, message, errorMessage, error}) => { + verifyError(error, errorMessage); + return cy.wrap({success, skipped, message}); + }); +} +Cypress.Commands.add('dbRefreshPostStats', dbRefreshPostStats); + function verifyError(error, errorMessage) { if (errorMessage) { expect(errorMessage, `${errorMessage}\n\n${message}\n\n${JSON.stringify(error)}`).to.be.undefined; @@ -150,6 +158,15 @@ declare global { * @returns {Session} session */ dbUpdateUserSession: typeof dbUpdateUserSession; + + /** + * Refreshes PostgreSQL materialized views for post statistics + * @returns {Object} result + * @returns {boolean} result.success - true if refresh was successful + * @returns {boolean} result.skipped - true if operation was skipped (non-PostgreSQL) + * @returns {string} result.message - message when operation is skipped + */ + dbRefreshPostStats: typeof dbRefreshPostStats; } } } diff --git a/server/channels/api4/user_test.go b/server/channels/api4/user_test.go index 4a6a9f5cac..356d966d98 100644 --- a/server/channels/api4/user_test.go +++ b/server/channels/api4/user_test.go @@ -2649,6 +2649,7 @@ func TestPermanentDeleteAllUsers(t *testing.T) { require.NoError(t, err) require.Greater(t, len(users), 0) + require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats()) postCount, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, err) require.Greater(t, postCount, int64(0)) @@ -2662,6 +2663,7 @@ func TestPermanentDeleteAllUsers(t *testing.T) { require.NoError(t, err) require.Len(t, users, 0) + require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats()) postCount, err = th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, err) require.Equal(t, postCount, int64(0)) diff --git a/server/channels/app/analytics.go b/server/channels/app/analytics.go index c2ccb8409e..ae86b48363 100644 --- a/server/channels/app/analytics.go +++ b/server/channels/app/analytics.go @@ -34,287 +34,273 @@ func (a *App) getAnalytics(rctx request.CTX, name string, teamID string, forSupp skipIntensiveQueries := false // When generating a Support Packet, always run intensive queries. - if !forSupportPacket { - if systemUserCount > int64(*a.Config().AnalyticsSettings.MaxUsersForStatistics) { - rctx.Logger().Debug("More than limit users are on the system, intensive queries skipped", mlog.Int("limit", *a.Config().AnalyticsSettings.MaxUsersForStatistics)) - skipIntensiveQueries = true - } + if !forSupportPacket && systemUserCount > int64(*a.Config().AnalyticsSettings.MaxUsersForStatistics) { + rctx.Logger().Warn("Number of users in the system is higher than the configured limit. Skipping intensive SQL queries.", mlog.Int("limit", *a.Config().AnalyticsSettings.MaxUsersForStatistics)) + skipIntensiveQueries = true } - if name == "standard" { - var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 11) - rows[0] = &model.AnalyticsRow{Name: "channel_open_count", Value: 0} - rows[1] = &model.AnalyticsRow{Name: "channel_private_count", Value: 0} - rows[2] = &model.AnalyticsRow{Name: "post_count", Value: 0} - rows[3] = &model.AnalyticsRow{Name: "unique_user_count", Value: 0} - rows[4] = &model.AnalyticsRow{Name: "team_count", Value: 0} - rows[5] = &model.AnalyticsRow{Name: "total_websocket_connections", Value: 0} - rows[6] = &model.AnalyticsRow{Name: "total_master_db_connections", Value: 0} - rows[7] = &model.AnalyticsRow{Name: "total_read_db_connections", Value: 0} - rows[8] = &model.AnalyticsRow{Name: "daily_active_users", Value: 0} - rows[9] = &model.AnalyticsRow{Name: "monthly_active_users", Value: 0} - rows[10] = &model.AnalyticsRow{Name: "inactive_user_count", Value: 0} + switch name { + case "standard": + return a.getStandardAnalytics(rctx, teamID, systemUserCount) + case "bot_post_counts_day": + return a.getBotPostCountsAnalytics(rctx, teamID) + case "post_counts_day": + return a.getPostCountsAnalytics(rctx, teamID) + case "user_counts_with_posts_day": + return a.getUserCountsWithPostsAnalytics(rctx, teamID, skipIntensiveQueries) + case "extra_counts": + return a.getExtraCountsAnalytics(rctx, teamID) + default: + return nil, nil + } +} - var g errgroup.Group - var openChannelsCount int64 +func (a *App) getStandardAnalytics(rctx request.CTX, teamID string, systemUserCount int64) (model.AnalyticsRows, *model.AppError) { + var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 11) + rows[0] = &model.AnalyticsRow{Name: "channel_open_count", Value: 0} + rows[1] = &model.AnalyticsRow{Name: "channel_private_count", Value: 0} + rows[2] = &model.AnalyticsRow{Name: "post_count", Value: 0} + rows[3] = &model.AnalyticsRow{Name: "unique_user_count", Value: 0} + rows[4] = &model.AnalyticsRow{Name: "team_count", Value: 0} + rows[5] = &model.AnalyticsRow{Name: "total_websocket_connections", Value: 0} + rows[6] = &model.AnalyticsRow{Name: "total_master_db_connections", Value: 0} + rows[7] = &model.AnalyticsRow{Name: "total_read_db_connections", Value: 0} + rows[8] = &model.AnalyticsRow{Name: "daily_active_users", Value: 0} + rows[9] = &model.AnalyticsRow{Name: "monthly_active_users", Value: 0} + rows[10] = &model.AnalyticsRow{Name: "inactive_user_count", Value: 0} + + var g errgroup.Group + g.SetLimit(2) + var channelCounts map[model.ChannelType]int64 + g.Go(func() error { + var err error + if channelCounts, err = a.Srv().Store().Channel().AnalyticsCountAll(teamID); err != nil { + return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var usersCount int64 + var inactiveUsersCount int64 + if teamID == "" { g.Go(func() error { var err error - if openChannelsCount, err = a.Srv().Store().Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen); err != nil { - return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + if inactiveUsersCount, err = a.Srv().Store().User().AnalyticsGetInactiveUsersCount(); err != nil { + return model.NewAppError("GetAnalytics", "app.user.analytics_get_inactive_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } return nil }) - - var privateChannelsCount int64 + } else { g.Go(func() error { var err error - if privateChannelsCount, err = a.Srv().Store().Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate); err != nil { - return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + if usersCount, err = a.Srv().Store().User().Count(model.UserCountOptions{TeamId: teamID}); err != nil { + return model.NewAppError("GetAnalytics", "app.user.get_total_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) } return nil }) + } - var usersCount int64 - var inactiveUsersCount int64 - if teamID == "" { - g.Go(func() error { - var err error - if inactiveUsersCount, err = a.Srv().Store().User().AnalyticsGetInactiveUsersCount(); err != nil { - return model.NewAppError("GetAnalytics", "app.user.analytics_get_inactive_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) - } else { - g.Go(func() error { - var err error - if usersCount, err = a.Srv().Store().User().Count(model.UserCountOptions{TeamId: teamID}); err != nil { - return model.NewAppError("GetAnalytics", "app.user.get_total_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) + var postsCount int64 + g.Go(func() error { + var err error + if postsCount, err = a.Srv().Store().Post().AnalyticsPostCountByTeam(teamID); err != nil { + return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var teamsCount int64 + g.Go(func() error { + var err error + if teamsCount, err = a.Srv().Store().Team().AnalyticsTeamCount(nil); err != nil { + return model.NewAppError("GetAnalytics", "app.team.analytics_team_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var dailyActiveUsersCount int64 + g.Go(func() error { + var err error + if dailyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(DayMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil { + return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var monthlyActiveUsersCount int64 + g.Go(func() error { + var err error + if monthlyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(MonthMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil { + return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + if err := g.Wait(); err != nil { + return nil, err.(*model.AppError) + } + + rows[0].Value = float64(channelCounts[model.ChannelTypeOpen]) + rows[1].Value = float64(channelCounts[model.ChannelTypePrivate]) + rows[2].Value = float64(postsCount) + + if teamID == "" { + rows[3].Value = float64(systemUserCount) + rows[10].Value = float64(inactiveUsersCount) + } else { + rows[3].Value = float64(usersCount) + rows[10].Value = -1 + } + + rows[4].Value = float64(teamsCount) + + // If in HA mode then aggregate all the stats + if a.Cluster() != nil && *a.Config().ClusterSettings.Enable { + stats, err2 := a.Cluster().GetClusterStats(rctx) + if err2 != nil { + return nil, err2 } - var postsCount int64 - if !skipIntensiveQueries { - g.Go(func() error { - var err error - if postsCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID}); err != nil { - return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) + totalSockets := a.TotalWebsocketConnections() + totalMasterDb := a.Srv().Store().TotalMasterDbConnections() + totalReadDb := a.Srv().Store().TotalReadDbConnections() + + for _, stat := range stats { + totalSockets = totalSockets + stat.TotalWebsocketConnections + totalMasterDb = totalMasterDb + stat.TotalMasterDbConnections + totalReadDb = totalReadDb + stat.TotalReadDbConnections } - var teamsCount int64 - g.Go(func() error { - var err error - if teamsCount, err = a.Srv().Store().Team().AnalyticsTeamCount(nil); err != nil { - return model.NewAppError("GetAnalytics", "app.team.analytics_team_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) + rows[5].Value = float64(totalSockets) + rows[6].Value = float64(totalMasterDb) + rows[7].Value = float64(totalReadDb) + } else { + rows[5].Value = float64(a.TotalWebsocketConnections()) + rows[6].Value = float64(a.Srv().Store().TotalMasterDbConnections()) + rows[7].Value = float64(a.Srv().Store().TotalReadDbConnections()) + } - var dailyActiveUsersCount int64 - g.Go(func() error { - var err error - if dailyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(DayMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil { - return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) + rows[8].Value = float64(dailyActiveUsersCount) + rows[9].Value = float64(monthlyActiveUsersCount) - var monthlyActiveUsersCount int64 - g.Go(func() error { - var err error - if monthlyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(MonthMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil { - return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) + return rows, nil +} - if err := g.Wait(); err != nil { - return nil, err.(*model.AppError) - } +func (a *App) getBotPostCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) { + analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{ + TeamId: teamID, + BotsOnly: true, + }) + if nErr != nil { + return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr) + } - rows[0].Value = float64(openChannelsCount) - rows[1].Value = float64(privateChannelsCount) + return analyticsRows, nil +} - if skipIntensiveQueries { - rows[2].Value = -1 - } else { - rows[2].Value = float64(postsCount) - } +func (a *App) getPostCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) { + analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{ + TeamId: teamID, + BotsOnly: false, + }) + if nErr != nil { + return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr) + } - if teamID == "" { - rows[3].Value = float64(systemUserCount) - rows[10].Value = float64(inactiveUsersCount) - } else { - rows[10].Value = -1 - rows[3].Value = float64(usersCount) - } - - rows[4].Value = float64(teamsCount) - - // If in HA mode then aggregate all the stats - if a.Cluster() != nil && *a.Config().ClusterSettings.Enable { - stats, err2 := a.Cluster().GetClusterStats(rctx) - if err2 != nil { - return nil, err2 - } - - totalSockets := a.TotalWebsocketConnections() - totalMasterDb := a.Srv().Store().TotalMasterDbConnections() - totalReadDb := a.Srv().Store().TotalReadDbConnections() - - for _, stat := range stats { - totalSockets = totalSockets + stat.TotalWebsocketConnections - totalMasterDb = totalMasterDb + stat.TotalMasterDbConnections - totalReadDb = totalReadDb + stat.TotalReadDbConnections - } - - rows[5].Value = float64(totalSockets) - rows[6].Value = float64(totalMasterDb) - rows[7].Value = float64(totalReadDb) - } else { - rows[5].Value = float64(a.TotalWebsocketConnections()) - rows[6].Value = float64(a.Srv().Store().TotalMasterDbConnections()) - rows[7].Value = float64(a.Srv().Store().TotalReadDbConnections()) - } - - rows[8].Value = float64(dailyActiveUsersCount) - rows[9].Value = float64(monthlyActiveUsersCount) - - return rows, nil - } else if name == "bot_post_counts_day" { - if skipIntensiveQueries { - rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}} - return rows, nil - } - analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{ - TeamId: teamID, - BotsOnly: true, - YesterdayOnly: false, - }) - if nErr != nil { - return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr) - } - - return analyticsRows, nil - } else if name == "post_counts_day" { - if skipIntensiveQueries { - rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}} - return rows, nil - } - analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{ - TeamId: teamID, - BotsOnly: false, - YesterdayOnly: false, - }) - if nErr != nil { - return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr) - } - - return analyticsRows, nil - } else if name == "user_counts_with_posts_day" { - if skipIntensiveQueries { - rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}} - return rows, nil - } - - analyticsRows, nErr := a.Srv().Store().Post().AnalyticsUserCountsWithPostsByDay(teamID) - if nErr != nil { - return nil, model.NewAppError("GetAnalytics", "app.post.analytics_user_counts_posts_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr) - } - - return analyticsRows, nil - } else if name == "extra_counts" { - var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 6) - rows[0] = &model.AnalyticsRow{Name: "file_post_count", Value: 0} - rows[1] = &model.AnalyticsRow{Name: "hashtag_post_count", Value: 0} - rows[2] = &model.AnalyticsRow{Name: "incoming_webhook_count", Value: 0} - rows[3] = &model.AnalyticsRow{Name: "outgoing_webhook_count", Value: 0} - rows[4] = &model.AnalyticsRow{Name: "command_count", Value: 0} - rows[5] = &model.AnalyticsRow{Name: "session_count", Value: 0} - - var g2 errgroup.Group - - var incomingWebhookCount int64 - g2.Go(func() error { - var err error - if incomingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsIncomingCount(teamID, ""); err != nil { - return model.NewAppError("GetAnalytics", "app.webhooks.analytics_incoming_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) - - var outgoingWebhookCount int64 - g2.Go(func() error { - var err error - if outgoingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsOutgoingCount(teamID); err != nil { - return model.NewAppError("GetAnalytics", "app.webhooks.analytics_outgoing_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) - - var commandsCount int64 - g2.Go(func() error { - var err error - if commandsCount, err = a.Srv().Store().Command().AnalyticsCommandCount(teamID); err != nil { - return model.NewAppError("GetAnalytics", "app.analytics.getanalytics.internal_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) - - var sessionsCount int64 - g2.Go(func() error { - var err error - if sessionsCount, err = a.Srv().Store().Session().AnalyticsSessionCount(); err != nil { - return model.NewAppError("GetAnalytics", "app.session.analytics_session_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) - - var filesCount int64 - var hashtagsCount int64 - if !skipIntensiveQueries { - g2.Go(func() error { - var err error - if filesCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID, MustHaveFile: true}); err != nil { - return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) - - g2.Go(func() error { - var err error - if hashtagsCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID, MustHaveHashtag: true}); err != nil { - return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) - } - return nil - }) - } - - if err := g2.Wait(); err != nil { - return nil, err.(*model.AppError) - } - - if skipIntensiveQueries { - rows[0].Value = -1 - rows[1].Value = -1 - } else { - rows[0].Value = float64(filesCount) - rows[1].Value = float64(hashtagsCount) - } - - rows[2].Value = float64(incomingWebhookCount) - rows[3].Value = float64(outgoingWebhookCount) - rows[4].Value = float64(commandsCount) - rows[5].Value = float64(sessionsCount) + return analyticsRows, nil +} +func (a *App) getUserCountsWithPostsAnalytics(rctx request.CTX, teamID string, skipIntensiveQueries bool) (model.AnalyticsRows, *model.AppError) { + if skipIntensiveQueries { + rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}} return rows, nil } - return nil, nil + analyticsRows, nErr := a.Srv().Store().Post().AnalyticsUserCountsWithPostsByDay(teamID) + if nErr != nil { + return nil, model.NewAppError("GetAnalytics", "app.post.analytics_user_counts_posts_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr) + } + + return analyticsRows, nil +} + +func (a *App) getExtraCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) { + var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 6) + rows[0] = &model.AnalyticsRow{Name: "incoming_webhook_count", Value: 0} + rows[1] = &model.AnalyticsRow{Name: "outgoing_webhook_count", Value: 0} + rows[2] = &model.AnalyticsRow{Name: "command_count", Value: 0} + rows[3] = &model.AnalyticsRow{Name: "session_count", Value: 0} + rows[4] = &model.AnalyticsRow{Name: "total_file_count", Value: 0} + rows[5] = &model.AnalyticsRow{Name: "total_file_size", Value: 0} + + var incomingWebhookCount int64 + var g errgroup.Group + g.SetLimit(2) + g.Go(func() error { + var err error + if incomingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsIncomingCount(teamID, ""); err != nil { + return model.NewAppError("GetAnalytics", "app.webhooks.analytics_incoming_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var outgoingWebhookCount int64 + g.Go(func() error { + var err error + if outgoingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsOutgoingCount(teamID); err != nil { + return model.NewAppError("GetAnalytics", "app.webhooks.analytics_outgoing_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var commandsCount int64 + g.Go(func() error { + var err error + if commandsCount, err = a.Srv().Store().Command().AnalyticsCommandCount(teamID); err != nil { + return model.NewAppError("GetAnalytics", "app.analytics.getanalytics.internal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var sessionsCount int64 + g.Go(func() error { + var err error + if sessionsCount, err = a.Srv().Store().Session().AnalyticsSessionCount(); err != nil { + return model.NewAppError("GetAnalytics", "app.session.analytics_session_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var fileCount int64 + g.Go(func() error { + var err error + if fileCount, err = a.Srv().Store().FileInfo().CountAll(); err != nil { + return model.NewAppError("GetAnalytics", "app.file_info.get_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + var fileSize int64 + g.Go(func() error { + var err error + if fileSize, err = a.Srv().Store().FileInfo().GetStorageUsage(false, false); err != nil { + return model.NewAppError("GetAnalytics", "app.file_info.get_storage_usage.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil + }) + + if err := g.Wait(); err != nil { + return nil, err.(*model.AppError) + } + + rows[0].Value = float64(incomingWebhookCount) + rows[1].Value = float64(outgoingWebhookCount) + rows[2].Value = float64(commandsCount) + rows[3].Value = float64(sessionsCount) + rows[4].Value = float64(fileCount) + rows[5].Value = float64(fileSize) + + return rows, nil } func (a *App) GetRecentlyActiveUsersForTeam(rctx request.CTX, teamID string) (map[string]*model.User, *model.AppError) { diff --git a/server/channels/app/import_functions_test.go b/server/channels/app/import_functions_test.go index 790df9da7a..5a00967f2e 100644 --- a/server/channels/app/import_functions_test.go +++ b/server/channels/app/import_functions_test.go @@ -2182,6 +2182,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { require.Nil(t, err, "Failed to get user from database.") // Count the number of posts in the testing team. + require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats()) initialPostCount, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team.Id}) require.NoError(t, nErr) @@ -2859,6 +2860,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { require.Nil(t, err, "Failed to get channel from database.") // Count the number of posts in the team2. + require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats()) initialPostCountForTeam2, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team2.Id}) require.NoError(t, nErr) @@ -3239,6 +3241,7 @@ func TestImportImportPost(t *testing.T) { require.Nil(t, appErr, "Failed to get user from database.") // Count the number of posts in the testing team. + require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats()) initialPostCount, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team.Id}) require.NoError(t, nErr) @@ -4194,6 +4197,7 @@ func TestImportImportDirectPost(t *testing.T) { directChannel = channel // Get the number of posts in the system. + require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats()) result, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, err) initialPostCount := result @@ -4664,6 +4668,7 @@ func TestImportImportDirectPost(t *testing.T) { groupChannel = channel // Get the number of posts in the system. + require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats()) result, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, nErr) initialPostCount = result diff --git a/server/channels/app/import_test.go b/server/channels/app/import_test.go index 02e66d461e..60a386adef 100644 --- a/server/channels/app/import_test.go +++ b/server/channels/app/import_test.go @@ -53,6 +53,8 @@ func checkNoError(t *testing.T, err *model.AppError) { } func AssertAllPostsCount(t *testing.T, a *App, initialCount int64, change int64, teamName string) { + t.Helper() + require.NoError(t, a.Srv().Store().Post().RefreshPostStats()) result, err := a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamName}) require.NoError(t, err) require.Equal(t, initialCount+change, result, "Did not find the expected number of posts.") diff --git a/server/channels/app/server.go b/server/channels/app/server.go index e0b888c487..aef479a7b1 100644 --- a/server/channels/app/server.go +++ b/server/channels/app/server.go @@ -60,7 +60,7 @@ import ( "github.com/mattermost/mattermost/server/v8/channels/jobs/plugins" "github.com/mattermost/mattermost/server/v8/channels/jobs/post_persistent_notifications" "github.com/mattermost/mattermost/server/v8/channels/jobs/product_notices" - "github.com/mattermost/mattermost/server/v8/channels/jobs/refresh_post_stats" + "github.com/mattermost/mattermost/server/v8/channels/jobs/refresh_materialized_views" "github.com/mattermost/mattermost/server/v8/channels/jobs/resend_invitation_email" "github.com/mattermost/mattermost/server/v8/channels/jobs/s3_path_migration" "github.com/mattermost/mattermost/server/v8/channels/store" @@ -1643,9 +1643,9 @@ func (s *Server) initJobs() { ) s.Jobs.RegisterJobType( - model.JobTypeRefreshPostStats, - refresh_post_stats.MakeWorker(s.Jobs, *s.platform.Config().SqlSettings.DriverName), - refresh_post_stats.MakeScheduler(s.Jobs, *s.platform.Config().SqlSettings.DriverName), + model.JobTypeRefreshMaterializedViews, + refresh_materialized_views.MakeWorker(s.Jobs, *s.platform.Config().SqlSettings.DriverName), + refresh_materialized_views.MakeScheduler(s.Jobs, *s.platform.Config().SqlSettings.DriverName), ) s.Jobs.RegisterJobType( diff --git a/server/channels/app/support_packet_test.go b/server/channels/app/support_packet_test.go index d7ce7f8097..74f08867ad 100644 --- a/server/channels/app/support_packet_test.go +++ b/server/channels/app/support_packet_test.go @@ -337,6 +337,8 @@ func TestGetSupportPacketStats(t *testing.T) { generateStats := func(t *testing.T) *model.SupportPacketStats { t.Helper() + require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats()) + fileData, err := th.App.getSupportPacketStats(th.Context) require.NotNil(t, fileData) assert.Equal(t, "stats.yaml", fileData.Filename) diff --git a/server/channels/db/migrations/migrations.list b/server/channels/db/migrations/migrations.list index 827f3664ff..8bf3e7a518 100644 --- a/server/channels/db/migrations/migrations.list +++ b/server/channels/db/migrations/migrations.list @@ -255,6 +255,8 @@ channels/db/migrations/mysql/000128_create_scheduled_posts.down.sql channels/db/migrations/mysql/000128_create_scheduled_posts.up.sql channels/db/migrations/mysql/000129_add_property_system_architecture.down.sql channels/db/migrations/mysql/000129_add_property_system_architecture.up.sql +channels/db/migrations/mysql/000130_system_console_stats.down.sql +channels/db/migrations/mysql/000130_system_console_stats.up.sql channels/db/migrations/postgres/000001_create_teams.down.sql channels/db/migrations/postgres/000001_create_teams.up.sql channels/db/migrations/postgres/000002_create_team_members.down.sql @@ -511,3 +513,5 @@ channels/db/migrations/postgres/000128_create_scheduled_posts.down.sql channels/db/migrations/postgres/000128_create_scheduled_posts.up.sql channels/db/migrations/postgres/000129_add_property_system_architecture.down.sql channels/db/migrations/postgres/000129_add_property_system_architecture.up.sql +channels/db/migrations/postgres/000130_system_console_stats.down.sql +channels/db/migrations/postgres/000130_system_console_stats.up.sql diff --git a/server/channels/db/migrations/mysql/000130_system_console_stats.down.sql b/server/channels/db/migrations/mysql/000130_system_console_stats.down.sql new file mode 100644 index 0000000000..20933ebd79 --- /dev/null +++ b/server/channels/db/migrations/mysql/000130_system_console_stats.down.sql @@ -0,0 +1 @@ +-- Nothing to do for MySQL diff --git a/server/channels/db/migrations/mysql/000130_system_console_stats.up.sql b/server/channels/db/migrations/mysql/000130_system_console_stats.up.sql new file mode 100644 index 0000000000..20933ebd79 --- /dev/null +++ b/server/channels/db/migrations/mysql/000130_system_console_stats.up.sql @@ -0,0 +1 @@ +-- Nothing to do for MySQL diff --git a/server/channels/db/migrations/postgres/000130_system_console_stats.down.sql b/server/channels/db/migrations/postgres/000130_system_console_stats.down.sql new file mode 100644 index 0000000000..76ca8b9d51 --- /dev/null +++ b/server/channels/db/migrations/postgres/000130_system_console_stats.down.sql @@ -0,0 +1,5 @@ +DROP MATERIALIZED VIEW IF EXISTS posts_by_team_day; + +DROP MATERIALIZED VIEW IF EXISTS bot_posts_by_team_day; + +DROP MATERIALIZED VIEW IF EXISTS file_stats; diff --git a/server/channels/db/migrations/postgres/000130_system_console_stats.up.sql b/server/channels/db/migrations/postgres/000130_system_console_stats.up.sql new file mode 100644 index 0000000000..9d9486da4f --- /dev/null +++ b/server/channels/db/migrations/postgres/000130_system_console_stats.up.sql @@ -0,0 +1,16 @@ +CREATE MATERIALIZED VIEW IF NOT EXISTS posts_by_team_day as +SELECT to_timestamp(p.createat/1000)::date as day, COUNT(*) as num, teamid +FROM posts p JOIN channels c on p.channelid=c.id +GROUP BY day, c.teamid; + +CREATE MATERIALIZED VIEW IF NOT EXISTS bot_posts_by_team_day as +SELECT to_timestamp(p.createat/1000)::date as day, COUNT(*) as num, teamid +FROM posts p +JOIN Bots b ON p.UserId = b.Userid +JOIN channels c on p.channelid=c.id +GROUP BY day, c.teamid; + +CREATE MATERIALIZED VIEW IF NOT EXISTS file_stats as +SELECT COUNT(*) as num, COALESCE(SUM(Size), 0) as usage +FROM fileinfo +WHERE DeleteAt = 0; diff --git a/server/channels/jobs/refresh_post_stats/scheduler.go b/server/channels/jobs/refresh_materialized_views/scheduler.go similarity index 82% rename from server/channels/jobs/refresh_post_stats/scheduler.go rename to server/channels/jobs/refresh_materialized_views/scheduler.go index bfe68baa3b..3539351aec 100644 --- a/server/channels/jobs/refresh_post_stats/scheduler.go +++ b/server/channels/jobs/refresh_materialized_views/scheduler.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package refresh_post_stats +package refresh_materialized_views import ( "time" @@ -21,5 +21,5 @@ func MakeScheduler(jobServer *jobs.JobServer, sqlDriverName string) *jobs.DailyS isEnabled := func(cfg *model.Config) bool { return sqlDriverName == model.DatabaseDriverPostgres } - return jobs.NewDailyScheduler(jobServer, model.JobTypeRefreshPostStats, startTime, isEnabled) + return jobs.NewDailyScheduler(jobServer, model.JobTypeRefreshMaterializedViews, startTime, isEnabled) } diff --git a/server/channels/jobs/refresh_post_stats/worker.go b/server/channels/jobs/refresh_materialized_views/worker.go similarity index 74% rename from server/channels/jobs/refresh_post_stats/worker.go rename to server/channels/jobs/refresh_materialized_views/worker.go index cd5f285149..f6c5f15bc1 100644 --- a/server/channels/jobs/refresh_post_stats/worker.go +++ b/server/channels/jobs/refresh_materialized_views/worker.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package refresh_post_stats +package refresh_materialized_views import ( "github.com/mattermost/mattermost/server/public/model" @@ -9,7 +9,7 @@ import ( "github.com/mattermost/mattermost/server/v8/channels/jobs" ) -const jobName = "RefreshPostStats" +const jobName = "RefreshMaterializedViews" func MakeWorker(jobServer *jobs.JobServer, sqlDriverName string) *jobs.SimpleWorker { isEnabled := func(cfg *model.Config) bool { @@ -18,8 +18,17 @@ func MakeWorker(jobServer *jobs.JobServer, sqlDriverName string) *jobs.SimpleWor execute := func(logger mlog.LoggerIFace, job *model.Job) error { defer jobServer.HandleJobPanic(logger, job) + if err := jobServer.Store.Post().RefreshPostStats(); err != nil { + return err + } + + if err := jobServer.Store.FileInfo().RefreshFileStats(); err != nil { + return err + } + return jobServer.Store.User().RefreshPostStatsForUsers() } + worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled) return worker } diff --git a/server/channels/store/retrylayer/retrylayer.go b/server/channels/store/retrylayer/retrylayer.go index 97dc64c2f3..0e29f23f1b 100644 --- a/server/channels/store/retrylayer/retrylayer.go +++ b/server/channels/store/retrylayer/retrylayer.go @@ -741,6 +741,27 @@ func (s *RetryLayerBotStore) Update(bot *model.Bot) (*model.Bot, error) { } +func (s *RetryLayerChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) { + + tries := 0 + for { + result, err := s.ChannelStore.AnalyticsCountAll(teamID) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) { tries := 0 @@ -4746,6 +4767,27 @@ func (s *RetryLayerFileInfoStore) PermanentDeleteForPost(rctx request.CTX, postI } +func (s *RetryLayerFileInfoStore) RefreshFileStats() error { + + tries := 0 + for { + err := s.FileInfoStore.RefreshFileStats() + if err == nil { + return nil + } + if !isRepeatableError(err) { + return err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error { tries := 0 @@ -7308,6 +7350,27 @@ func (s *RetryLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions } +func (s *RetryLayerPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) { + + tries := 0 + for { + result, err := s.PostStore.AnalyticsPostCountByTeam(teamID) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) { tries := 0 @@ -8109,6 +8172,27 @@ func (s *RetryLayerPostStore) PermanentDeleteByUser(rctx request.CTX, userID str } +func (s *RetryLayerPostStore) RefreshPostStats() error { + + tries := 0 + for { + err := s.PostStore.RefreshPostStats() + if err == nil { + return nil + } + if !isRepeatableError(err) { + return err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return err + } + timepkg.Sleep(100 * timepkg.Millisecond) + } + +} + func (s *RetryLayerPostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) { tries := 0 diff --git a/server/channels/store/sqlstore/channel_store.go b/server/channels/store/sqlstore/channel_store.go index 5229ed024c..173160e8b6 100644 --- a/server/channels/store/sqlstore/channel_store.go +++ b/server/channels/store/sqlstore/channel_store.go @@ -2961,6 +2961,40 @@ func (s SqlChannelStore) AnalyticsDeletedTypeCount(teamId string, channelType mo return v, nil } +func (s SqlChannelStore) AnalyticsCountAll(teamId string) (map[model.ChannelType]int64, error) { + query := s.getQueryBuilder(). + Select("Type, COUNT(*) AS Count"). + From("Channels"). + GroupBy("Type") + + if teamId != "" { + query = query.Where(sq.Eq{"TeamId": teamId}) + } + + sql, args, err := query.ToSql() + if err != nil { + return nil, errors.Wrap(err, "AnalyticsCountAll_ToSql") + } + + rows, err := s.GetReplica().Query(sql, args...) + if err != nil { + return nil, errors.Wrap(err, "failed to count Channels by type") + } + defer rows.Close() + + counts := make(map[model.ChannelType]int64) + for rows.Next() { + var channelType model.ChannelType + var count int64 + if err := rows.Scan(&channelType, &count); err != nil { + return nil, errors.Wrap(err, "unable to scan row") + } + counts[channelType] = count + } + + return counts, nil +} + func (s SqlChannelStore) GetMembersForUser(teamID string, userID string) (model.ChannelMembers, error) { sql, args, err := s.channelMembersForTeamWithSchemeSelectQuery. Where(sq.And{ diff --git a/server/channels/store/sqlstore/file_info_store.go b/server/channels/store/sqlstore/file_info_store.go index a48bcd12c3..6a82513213 100644 --- a/server/channels/store/sqlstore/file_info_store.go +++ b/server/channels/store/sqlstore/file_info_store.go @@ -712,18 +712,20 @@ func (fs SqlFileInfoStore) Search(rctx request.CTX, paramsList []*model.SearchPa } func (fs SqlFileInfoStore) CountAll() (int64, error) { - query := fs.getQueryBuilder(). - Select("COUNT(*)"). - From("FileInfo"). - Where("DeleteAt = 0") - - queryString, args, err := query.ToSql() - if err != nil { - return int64(0), errors.Wrap(err, "count_tosql") + var query sq.SelectBuilder + if fs.DriverName() == model.DatabaseDriverPostgres { + query = fs.getQueryBuilder(). + Select("num"). + From("file_stats") + } else { + query = fs.getQueryBuilder(). + Select("COUNT(*)"). + From("FileInfo"). + Where("DeleteAt = 0") } var count int64 - err = fs.GetReplica().Get(&count, queryString, args...) + err := fs.GetReplica().GetBuilder(&count, query) if err != nil { return int64(0), errors.Wrap(err, "failed to count Files") } @@ -758,13 +760,20 @@ func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID return files, nil } -func (fs SqlFileInfoStore) GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error) { - query := fs.getQueryBuilder(). - Select("COALESCE(SUM(Size), 0)"). - From("FileInfo") +func (fs SqlFileInfoStore) GetStorageUsage(_, includeDeleted bool) (int64, error) { + var query sq.SelectBuilder + if fs.DriverName() == model.DatabaseDriverPostgres && !includeDeleted { + query = fs.getQueryBuilder(). + Select("usage"). + From("file_stats") + } else { + query = fs.getQueryBuilder(). + Select("COALESCE(SUM(Size), 0)"). + From("FileInfo") - if !includeDeleted { - query = query.Where("DeleteAt = 0") + if !includeDeleted { + query = query.Where("DeleteAt = 0") + } } var size int64 @@ -841,3 +850,18 @@ func (fs SqlFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, return nil } + +func (fs SqlFileInfoStore) RefreshFileStats() error { + if fs.DriverName() == model.DatabaseDriverPostgres { + // CONCURRENTLY is not used deliberately because as per Postgres docs, + // not using CONCURRENTLY takes less resources and completes faster + // at the expense of locking the mat view. Since viewing admin console + // is not a very frequent activity, we accept the tradeoff to let the + // refresh happen as fast as possible. + if _, err := fs.GetMaster().Exec("REFRESH MATERIALIZED VIEW file_stats"); err != nil { + return errors.Wrap(err, "error refreshing materialized view file_stats") + } + } + + return nil +} diff --git a/server/channels/store/sqlstore/post_store.go b/server/channels/store/sqlstore/post_store.go index d0cb61127d..8b2b136b39 100644 --- a/server/channels/store/sqlstore/post_store.go +++ b/server/channels/store/sqlstore/post_store.go @@ -2292,8 +2292,79 @@ func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.A return rows, nil } +func (s *SqlPostStore) countBotPostsByDay(teamID, startDay, endDay string) (model.AnalyticsRows, error) { + var query sq.SelectBuilder + if teamID != "" { + query = s.getQueryBuilder(). + Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, num as Value"). + From("bot_posts_by_team_day"). + Where(sq.Eq{"teamid": teamID}) + } else { + query = s.getQueryBuilder(). + Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, COALESCE(SUM(num), 0) as Value"). + From("bot_posts_by_team_day"). + GroupBy("Name") + } + + query = query. + Where(sq.GtOrEq{"day": startDay}). + Where(sq.LtOrEq{"day": endDay}). + OrderBy("Name DESC"). + Limit(30) + + rows := model.AnalyticsRows{} + err := s.GetReplica().SelectBuilder(&rows, query) + if err != nil { + return nil, errors.Wrapf(err, "failed to find bot posts with teamId=%s", teamID) + } + + return rows, nil +} + +func (s *SqlPostStore) countPostsByDay(teamID, startDay, endDay string) (model.AnalyticsRows, error) { + var query sq.SelectBuilder + if teamID != "" { + query = s.getQueryBuilder(). + Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, num as Value"). + From("posts_by_team_day"). + Where(sq.Eq{"teamid": teamID}) + } else { + query = s.getQueryBuilder(). + Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, COALESCE(SUM(num), 0) as Value"). + From("posts_by_team_day"). + GroupBy("Name") + } + + query = query. + Where(sq.GtOrEq{"day": startDay}). + Where(sq.LtOrEq{"day": endDay}). + OrderBy("Name DESC"). + Limit(30) + + rows := model.AnalyticsRows{} + err := s.GetReplica().SelectBuilder(&rows, query) + if err != nil { + return nil, errors.Wrapf(err, "failed to find posts with teamId=%s", teamID) + } + + return rows, nil +} + // TODO: convert to squirrel HW func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) { + if s.DriverName() == model.DatabaseDriverPostgres { + endDay := utils.Yesterday().Format("2006-01-02") + startDay := utils.Yesterday().AddDate(0, 0, -31).Format("2006-01-02") + if options.YesterdayOnly { + startDay = utils.Yesterday().AddDate(0, 0, -1).Format("2006-01-02") + } + // Use materialized views + if options.BotsOnly { + return s.countBotPostsByDay(options.TeamId, startDay, endDay) + } + return s.countPostsByDay(options.TeamId, startDay, endDay) + } + var args []any query := `SELECT @@ -2318,30 +2389,6 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun ORDER BY Name DESC LIMIT 30` - if s.DriverName() == model.DatabaseDriverPostgres { - query = - `SELECT - TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, Count(Posts.Id) AS Value - FROM Posts` - - if options.BotsOnly { - query += " INNER JOIN Bots ON Posts.UserId = Bots.Userid" - } - - if options.TeamId != "" { - query += " INNER JOIN Channels ON Posts.ChannelId = Channels.Id AND Channels.TeamId = ? AND" - args = []any{options.TeamId} - } else { - query += " WHERE" - } - - query += ` Posts.CreateAt <= ? - AND Posts.CreateAt >= ? - GROUP BY DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)) - ORDER BY Name DESC - LIMIT 30` - } - end := utils.MillisFromTime(utils.EndOfDay(utils.Yesterday())) start := utils.MillisFromTime(utils.StartOfDay(utils.Yesterday().AddDate(0, 0, -31))) if options.YesterdayOnly { @@ -2350,16 +2397,39 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun args = append(args, end, start) rows := model.AnalyticsRows{} - err := s.GetReplica().Select( - &rows, - query, - args...) + err := s.GetReplica().Select(&rows, query, args...) if err != nil { return nil, errors.Wrapf(err, "failed to find Posts with teamId=%s", options.TeamId) } return rows, nil } +func (s *SqlPostStore) countByTeam(teamID string) (int64, error) { + query := s.getQueryBuilder(). + Select("COALESCE(SUM(num), 0) AS total"). + From("posts_by_team_day") + + if teamID != "" { + query = query.Where(sq.Eq{"teamid": teamID}) + } + + var v int64 + err := s.GetReplica().GetBuilder(&v, query) + if err != nil { + return 0, fmt.Errorf("failed to count Posts by team: %w, teamID: %s", err, teamID) + } + + return v, nil +} + +func (s *SqlPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) { + if s.DriverName() == model.DatabaseDriverPostgres { + return s.countByTeam(teamID) + } + + return s.AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID}) +} + func (s *SqlPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) { query := s.getQueryBuilder(). Select("COUNT(*) AS Value"). @@ -2553,7 +2623,7 @@ func (s *SqlPostStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolic func (s *SqlPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) { var query string - if s.DriverName() == "postgres" { + if s.DriverName() == model.DatabaseDriverPostgres { query = "DELETE from Posts WHERE Id = any (array (SELECT Id FROM Posts WHERE CreateAt < ? LIMIT ?))" } else { query = "DELETE from Posts WHERE CreateAt < ? LIMIT ?" @@ -3295,3 +3365,22 @@ func (s *SqlPostStore) GetPostReminderMetadata(postID string) (*store.PostRemind return meta, nil } + +func (s *SqlPostStore) RefreshPostStats() error { + if s.DriverName() == model.DatabaseDriverPostgres { + // CONCURRENTLY is not used deliberately because as per Postgres docs, + // not using CONCURRENTLY takes less resources and completes faster + // at the expense of locking the mat view. Since viewing admin console + // is not a very frequent activity, we accept the tradeoff to let the + // refresh happen as fast as possible. + if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW posts_by_team_day"); err != nil { + return errors.Wrap(err, "error refreshing materialized view posts_by_team_day") + } + + if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW bot_posts_by_team_day"); err != nil { + return errors.Wrap(err, "error refreshing materialized view bot_posts_by_team_day") + } + } + + return nil +} diff --git a/server/channels/store/sqlstore/user_store.go b/server/channels/store/sqlstore/user_store.go index edd809d223..4bc5388d83 100644 --- a/server/channels/store/sqlstore/user_store.go +++ b/server/channels/store/sqlstore/user_store.go @@ -1696,7 +1696,6 @@ func (us SqlUserStore) performSearch(query sq.SelectBuilder, term string, option } func (us SqlUserStore) AnalyticsGetInactiveUsersCount() (int64, error) { - var count int64 query := us.getQueryBuilder(). Select("COUNT(Id)"). From("Users") @@ -1712,11 +1711,9 @@ func (us SqlUserStore) AnalyticsGetInactiveUsersCount() (int64, error) { sq.Gt{"Users.DeleteAt": 0}, }) } - queryStr, args, err := query.ToSql() - if err != nil { - return int64(0), errors.Wrap(err, "failed to create a SQL query to count inactive users") - } - err = us.GetReplica().Get(&count, queryStr, args...) + + var count int64 + err := us.GetReplica().GetBuilder(&count, query) if err != nil { return int64(0), errors.Wrap(err, "failed to count inactive Users") } diff --git a/server/channels/store/store.go b/server/channels/store/store.go index 59f790ff1f..eec2b9c4f1 100644 --- a/server/channels/store/store.go +++ b/server/channels/store/store.go @@ -260,6 +260,8 @@ type ChannelStore interface { CountUrgentPostsAfter(channelID string, timestamp int64, excludedUserID string) (int, error) IncrementMentionCount(channelID string, userIDs []string, isRoot, isUrgent bool) error AnalyticsTypeCount(teamID string, channelType model.ChannelType) (int64, error) + AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) + AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) GetMembersForUser(teamID string, userID string) (model.ChannelMembers, error) GetTeamMembersForChannel(channelID string) ([]string, error) GetMembersForUserWithPagination(userID string, page, perPage int) (model.ChannelMembersWithTeamData, error) @@ -275,7 +277,6 @@ type ChannelStore interface { GetMembersByIds(channelID string, userIds []string) (model.ChannelMembers, error) GetMembersByChannelIds(channelIds []string, userID string) (model.ChannelMembers, error) GetMembersInfoByChannelIds(channelIDs []string) (map[string][]*model.User, error) - AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) GetChannelUnread(channelID, userID string) (*model.ChannelUnread, error) GetChannelsWithUnreadsAndWithMentions(ctx context.Context, channelIDs []string, userID string, userNotifyProps model.StringMap) ([]string, []string, map[string]int64, error) ClearCaches() @@ -387,6 +388,7 @@ type PostStore interface { AnalyticsUserCountsWithPostsByDay(teamID string) (model.AnalyticsRows, error) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) + AnalyticsPostCountByTeam(teamID string) (int64, error) ClearCaches() InvalidateLastPostTimeCache(channelID string) GetPostsCreatedAt(channelID string, timestamp int64) ([]*model.Post, error) @@ -411,6 +413,8 @@ type PostStore interface { GetPostReminderMetadata(postID string) (*PostReminderMetadata, error) // GetNthRecentPostTime returns the CreateAt time of the nth most recent post. GetNthRecentPostTime(n int64) (int64, error) + // RefreshPostStats refreshes the various materialized views for admin console post stats. + RefreshPostStats() error } type UserStore interface { @@ -743,6 +747,8 @@ type FileInfoStore interface { GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error) // GetUptoNSizeFileTime returns the CreateAt time of the last accessible file with a running-total size upto n bytes. GetUptoNSizeFileTime(n int64) (int64, error) + // RefreshFileStats recomputes the fileinfo materialized views. + RefreshFileStats() error } type UploadSessionStore interface { diff --git a/server/channels/store/storetest/channel_store.go b/server/channels/store/storetest/channel_store.go index 781f5e7717..abd9651c3d 100644 --- a/server/channels/store/storetest/channel_store.go +++ b/server/channels/store/storetest/channel_store.go @@ -4148,18 +4148,34 @@ func testChannelStoreGetMoreChannels(t *testing.T, rctx request.CTX, ss store.St count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen) require.NoError(t, err) require.EqualValues(t, 4, count) + + counts, err := ss.Channel().AnalyticsCountAll(teamID) + require.NoError(t, err) + require.EqualValues(t, 4, counts[model.ChannelTypeOpen]) }) t.Run("verify analytics for private channels", func(t *testing.T) { count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate) require.NoError(t, err) require.EqualValues(t, 2, count) + + counts, err := ss.Channel().AnalyticsCountAll(teamID) + require.NoError(t, err) + require.EqualValues(t, 2, counts[model.ChannelTypePrivate]) }) t.Run("verify analytics for all channels", func(t *testing.T) { count, err := ss.Channel().AnalyticsTypeCount(teamID, "") require.NoError(t, err) require.EqualValues(t, 6, count) + + counts, err := ss.Channel().AnalyticsCountAll(teamID) + require.NoError(t, err) + total := int64(0) + for _, count := range counts { + total += count + } + require.EqualValues(t, 6, total) }) } @@ -4338,12 +4354,20 @@ func testChannelStoreGetPublicChannelsForTeam(t *testing.T, rctx request.CTX, ss count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen) require.NoError(t, err) require.EqualValues(t, 3, count) + + counts, err := ss.Channel().AnalyticsCountAll(teamID) + require.NoError(t, err) + require.EqualValues(t, 3, counts[model.ChannelTypeOpen]) }) t.Run("verify analytics for private channels", func(t *testing.T) { count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate) require.NoError(t, err) require.EqualValues(t, 1, count) + + counts, err := ss.Channel().AnalyticsCountAll(teamID) + require.NoError(t, err) + require.EqualValues(t, 1, counts[model.ChannelTypePrivate]) }) } diff --git a/server/channels/store/storetest/file_info_store.go b/server/channels/store/storetest/file_info_store.go index 8335045df4..4055e8657f 100644 --- a/server/channels/store/storetest/file_info_store.go +++ b/server/channels/store/storetest/file_info_store.go @@ -815,12 +815,15 @@ func testFileInfoStoreCountAll(t *testing.T, rctx request.CTX, ss store.Store) { }) require.NoError(t, err) + require.NoError(t, ss.FileInfo().RefreshFileStats()) count, err := ss.FileInfo().CountAll() require.NoError(t, err) require.Equal(t, int64(3), count) _, err = ss.FileInfo().DeleteForPost(rctx, f1.PostId) require.NoError(t, err) + + require.NoError(t, ss.FileInfo().RefreshFileStats()) count, err = ss.FileInfo().CountAll() require.NoError(t, err) require.Equal(t, int64(2), count) @@ -830,6 +833,7 @@ func testFileInfoGetStorageUsage(t *testing.T, rctx request.CTX, ss store.Store) _, err := ss.FileInfo().PermanentDeleteBatch(rctx, model.GetMillis(), 100000) require.NoError(t, err) + require.NoError(t, ss.FileInfo().RefreshFileStats()) usage, err := ss.FileInfo().GetStorageUsage(false, false) require.NoError(t, err) require.Equal(t, int64(0), usage) @@ -857,12 +861,14 @@ func testFileInfoGetStorageUsage(t *testing.T, rctx request.CTX, ss store.Store) }) require.NoError(t, err) + require.NoError(t, ss.FileInfo().RefreshFileStats()) usage, err = ss.FileInfo().GetStorageUsage(false, false) require.NoError(t, err) require.Equal(t, int64(30), usage) _, err = ss.FileInfo().DeleteForPost(rctx, f1.PostId) require.NoError(t, err) + require.NoError(t, ss.FileInfo().RefreshFileStats()) usage, err = ss.FileInfo().GetStorageUsage(false, false) require.NoError(t, err) require.Equal(t, int64(20), usage) diff --git a/server/channels/store/storetest/mocks/ChannelStore.go b/server/channels/store/storetest/mocks/ChannelStore.go index f8eee02b89..caca201fa7 100644 --- a/server/channels/store/storetest/mocks/ChannelStore.go +++ b/server/channels/store/storetest/mocks/ChannelStore.go @@ -20,6 +20,36 @@ type ChannelStore struct { mock.Mock } +// AnalyticsCountAll provides a mock function with given fields: teamID +func (_m *ChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) { + ret := _m.Called(teamID) + + if len(ret) == 0 { + panic("no return value specified for AnalyticsCountAll") + } + + var r0 map[model.ChannelType]int64 + var r1 error + if rf, ok := ret.Get(0).(func(string) (map[model.ChannelType]int64, error)); ok { + return rf(teamID) + } + if rf, ok := ret.Get(0).(func(string) map[model.ChannelType]int64); ok { + r0 = rf(teamID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[model.ChannelType]int64) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(teamID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // AnalyticsDeletedTypeCount provides a mock function with given fields: teamID, channelType func (_m *ChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) { ret := _m.Called(teamID, channelType) diff --git a/server/channels/store/storetest/mocks/FileInfoStore.go b/server/channels/store/storetest/mocks/FileInfoStore.go index a86eb04b31..a360c5e8d3 100644 --- a/server/channels/store/storetest/mocks/FileInfoStore.go +++ b/server/channels/store/storetest/mocks/FileInfoStore.go @@ -505,6 +505,24 @@ func (_m *FileInfoStore) PermanentDeleteForPost(rctx request.CTX, postID string) return r0 } +// RefreshFileStats provides a mock function with given fields: +func (_m *FileInfoStore) RefreshFileStats() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for RefreshFileStats") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + // RestoreForPostByIds provides a mock function with given fields: rctx, postId, fileIDs func (_m *FileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error { ret := _m.Called(rctx, postId, fileIDs) diff --git a/server/channels/store/storetest/mocks/PostStore.go b/server/channels/store/storetest/mocks/PostStore.go index 4d74b5e376..bc2d016885 100644 --- a/server/channels/store/storetest/mocks/PostStore.go +++ b/server/channels/store/storetest/mocks/PostStore.go @@ -48,6 +48,34 @@ func (_m *PostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, return r0, r1 } +// AnalyticsPostCountByTeam provides a mock function with given fields: teamID +func (_m *PostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) { + ret := _m.Called(teamID) + + if len(ret) == 0 { + panic("no return value specified for AnalyticsPostCountByTeam") + } + + var r0 int64 + var r1 error + if rf, ok := ret.Get(0).(func(string) (int64, error)); ok { + return rf(teamID) + } + if rf, ok := ret.Get(0).(func(string) int64); ok { + r0 = rf(teamID) + } else { + r0 = ret.Get(0).(int64) + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(teamID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // AnalyticsPostCountsByDay provides a mock function with given fields: options func (_m *PostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) { ret := _m.Called(options) @@ -1163,6 +1191,24 @@ func (_m *PostStore) PermanentDeleteByUser(rctx request.CTX, userID string) erro return r0 } +// RefreshPostStats provides a mock function with given fields: +func (_m *PostStore) RefreshPostStats() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for RefreshPostStats") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + // Save provides a mock function with given fields: rctx, post func (_m *PostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) { ret := _m.Called(rctx, post) diff --git a/server/channels/store/storetest/post_store.go b/server/channels/store/storetest/post_store.go index df6a55fc83..b69ea2969b 100644 --- a/server/channels/store/storetest/post_store.go +++ b/server/channels/store/storetest/post_store.go @@ -2895,6 +2895,8 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) { _, nErr = ss.Post().Save(rctx, b1a) require.NoError(t, nErr) + require.NoError(t, ss.Post().RefreshPostStats()) + time.Sleep(1 * time.Second) // summary of posts @@ -2907,6 +2909,8 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) { require.NoError(t, err) assert.Equal(t, float64(3), r1[0].Value) assert.Equal(t, float64(3), r1[1].Value) + assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name) + assert.Equal(t, utils.Yesterday().Add(-48*time.Hour).Format("2006-01-02"), r1[1].Name) // last 31 days, bots only postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: true, YesterdayOnly: false} @@ -2914,18 +2918,22 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) { require.NoError(t, err) assert.Equal(t, float64(1), r1[0].Value) assert.Equal(t, float64(1), r1[1].Value) + assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name) + assert.Equal(t, utils.Yesterday().Add(-48*time.Hour).Format("2006-01-02"), r1[1].Name) // yesterday only, all users (including bots) postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: false, YesterdayOnly: true} r1, err = ss.Post().AnalyticsPostCountsByDay(postCountsOptions) require.NoError(t, err) assert.Equal(t, float64(3), r1[0].Value) + assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name) // yesterday only, bots only postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: true, YesterdayOnly: true} r1, err = ss.Post().AnalyticsPostCountsByDay(postCountsOptions) require.NoError(t, err) assert.Equal(t, float64(1), r1[0].Value) + assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name) } func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) { @@ -3029,6 +3037,8 @@ func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) { _, nErr = ss.Post().Save(rctx, p7) require.NoError(t, nErr) + require.NoError(t, ss.Post().RefreshPostStats()) + // total across all teams c, err := ss.Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, err) @@ -3039,6 +3049,10 @@ func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) { require.NoError(t, err) assert.Equal(t, int64(7), c) + c, err = ss.Post().AnalyticsPostCountByTeam(t1.Id) + require.NoError(t, err) + assert.Equal(t, int64(7), c) + // with files c, err = ss.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: t1.Id, MustHaveFile: true}) require.NoError(t, err) diff --git a/server/channels/store/timerlayer/timerlayer.go b/server/channels/store/timerlayer/timerlayer.go index 27e5f48745..4c6b2c262f 100644 --- a/server/channels/store/timerlayer/timerlayer.go +++ b/server/channels/store/timerlayer/timerlayer.go @@ -671,6 +671,22 @@ func (s *TimerLayerBotStore) Update(bot *model.Bot) (*model.Bot, error) { return result, err } +func (s *TimerLayerChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) { + start := time.Now() + + result, err := s.ChannelStore.AnalyticsCountAll(teamID) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("ChannelStore.AnalyticsCountAll", success, elapsed) + } + return result, err +} + func (s *TimerLayerChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) { start := time.Now() @@ -3859,6 +3875,22 @@ func (s *TimerLayerFileInfoStore) PermanentDeleteForPost(rctx request.CTX, postI return err } +func (s *TimerLayerFileInfoStore) RefreshFileStats() error { + start := time.Now() + + err := s.FileInfoStore.RefreshFileStats() + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.RefreshFileStats", success, elapsed) + } + return err +} + func (s *TimerLayerFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error { start := time.Now() @@ -5811,6 +5843,22 @@ func (s *TimerLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions return result, err } +func (s *TimerLayerPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) { + start := time.Now() + + result, err := s.PostStore.AnalyticsPostCountByTeam(teamID) + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("PostStore.AnalyticsPostCountByTeam", success, elapsed) + } + return result, err +} + func (s *TimerLayerPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) { start := time.Now() @@ -6465,6 +6513,22 @@ func (s *TimerLayerPostStore) PermanentDeleteByUser(rctx request.CTX, userID str return err } +func (s *TimerLayerPostStore) RefreshPostStats() error { + start := time.Now() + + err := s.PostStore.RefreshPostStats() + + elapsed := float64(time.Since(start)) / float64(time.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("PostStore.RefreshPostStats", success, elapsed) + } + return err +} + func (s *TimerLayerPostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) { start := time.Now() diff --git a/server/i18n/en.json b/server/i18n/en.json index f2d191d9c8..43cf0ce975 100644 --- a/server/i18n/en.json +++ b/server/i18n/en.json @@ -5218,10 +5218,18 @@ "id": "app.file_info.get_by_post_id.app_error", "translation": "Failed to find files for post." }, + { + "id": "app.file_info.get_count.app_error", + "translation": "Failed to get count of all files." + }, { "id": "app.file_info.get_for_post.app_error", "translation": "Unable to get the file info for the post." }, + { + "id": "app.file_info.get_storage_usage.app_error", + "translation": "Failed to get storage usage of all files." + }, { "id": "app.file_info.get_with_options.app_error", "translation": "Unable to get the file info with options" diff --git a/server/public/model/job.go b/server/public/model/job.go index 815fe7d96d..a9ca699bf7 100644 --- a/server/public/model/job.go +++ b/server/public/model/job.go @@ -39,7 +39,7 @@ const ( JobTypeS3PathMigration = "s3_path_migration" JobTypeCleanupDesktopTokens = "cleanup_desktop_tokens" JobTypeDeleteEmptyDraftsMigration = "delete_empty_drafts_migration" - JobTypeRefreshPostStats = "refresh_post_stats" + JobTypeRefreshMaterializedViews = "refresh_materialized_views" JobTypeDeleteOrphanDraftsMigration = "delete_orphan_drafts_migration" JobTypeExportUsersToCSV = "export_users_to_csv" JobTypeDeleteDmsPreferencesMigration = "delete_dms_preferences_migration" @@ -75,7 +75,7 @@ var AllJobTypes = [...]string{ JobTypeLastAccessiblePost, JobTypeLastAccessibleFile, JobTypeCleanupDesktopTokens, - JobTypeRefreshPostStats, + JobTypeRefreshMaterializedViews, JobTypeMobileSessionMetadata, } diff --git a/webapp/channels/src/components/analytics/format.tsx b/webapp/channels/src/components/analytics/format.tsx index 7d8167f5bf..76036be888 100644 --- a/webapp/channels/src/components/analytics/format.tsx +++ b/webapp/channels/src/components/analytics/format.tsx @@ -19,23 +19,6 @@ export function formatChannelDoughtnutData(totalPublic: any, totalPrivate: any) return channelTypeData; } -export function formatPostDoughtnutData(filePosts: any, hashtagPosts: any, totalPosts: any) { - const postTypeData = { - labels: [ - Utils.localizeMessage({id: 'analytics.system.totalFilePosts', defaultMessage: 'Posts with Files'}), - Utils.localizeMessage({id: 'analytics.system.totalHashtagPosts', defaultMessage: 'Posts with Hashtags'}), - Utils.localizeMessage({id: 'analytics.system.textPosts', defaultMessage: 'Posts with Text-only'}), - ], - datasets: [{ - data: [filePosts, hashtagPosts, (totalPosts - filePosts - hashtagPosts)], - backgroundColor: ['#46BFBD', '#F7464A', '#FDB45C'], - hoverBackgroundColor: ['#5AD3D1', '#FF5A5E', '#FFC870'], - }], - }; - - return postTypeData; -} - export function formatPostsPerDayData(labels: string[], data: any) { const chartData = { labels: [] as string[], diff --git a/webapp/channels/src/components/analytics/statistic_count.test.tsx b/webapp/channels/src/components/analytics/statistic_count.test.tsx index eb023736ba..a4c13c5752 100644 --- a/webapp/channels/src/components/analytics/statistic_count.test.tsx +++ b/webapp/channels/src/components/analytics/statistic_count.test.tsx @@ -41,4 +41,19 @@ describe('components/analytics/statistic_count.tsx', () => { expect(wrapper).toMatchSnapshot(); }); + + test('should apply formatter function when provided', () => { + const mockFormatter = (value: number) => `${value}%`; + const wrapper = shallow( + , + ); + + expect(wrapper.find('[data-testid="test-stat"]').text()).toBe('42%'); + }); }); diff --git a/webapp/channels/src/components/analytics/statistic_count.tsx b/webapp/channels/src/components/analytics/statistic_count.tsx index e6c8ce9391..41c6c3e970 100644 --- a/webapp/channels/src/components/analytics/statistic_count.tsx +++ b/webapp/channels/src/components/analytics/statistic_count.tsx @@ -13,6 +13,7 @@ type Props = { id?: string; children?: React.ReactNode; status?: 'warning' | 'error'; + formatter?: (value: number) => string; } const StatisticCount = ({ @@ -22,6 +23,7 @@ const StatisticCount = ({ id, children, status, + formatter, }: Props) => { const loading = ( ); + const result = formatter ? formatter(count ?? 0) : count; + const displayValue = typeof count === 'undefined' || isNaN(count) ? loading : result; + return (
- {typeof count === 'undefined' || isNaN(count) ? loading : count} + {displayValue}
{children} diff --git a/webapp/channels/src/components/analytics/system_analytics/analytics.scss b/webapp/channels/src/components/analytics/system_analytics/analytics.scss new file mode 100644 index 0000000000..9ac9ec4f5d --- /dev/null +++ b/webapp/channels/src/components/analytics/system_analytics/analytics.scss @@ -0,0 +1,49 @@ +details { + padding: 12px; + border: 1px solid rgba(var(--center-channel-color-rgb), 0.16); + border-radius: 4px; + background-color: var(--center-channel-bg); +} + +summary { + position: relative; + padding: 12px 12px 12px 28px; + margin: -12px; + cursor: pointer; + font-weight: bold; + transition: background 0.15s ease-in-out; + + &:hover { + background: rgba(var(--center-channel-color-rgb), 0.04); + } + + &::marker { + display: none; + } + + &::before { + position: absolute; + top: 16px; + left: 12px; + content: '►'; + font-size: 10px; + transform: rotate(0deg); + transition: transform 0.15s ease-in-out; + } +} + +details[open] { + + summary { + border-bottom: 1px solid rgba(var(--center-channel-color-rgb), 0.16); + margin-bottom: 0; + + &::before { + transform: rotate(90deg); + } + } + + .row:last-child .total-count { + margin-bottom: 0; + } +} diff --git a/webapp/channels/src/components/analytics/system_analytics/system_analytics.test.tsx b/webapp/channels/src/components/analytics/system_analytics/system_analytics.test.tsx index 0f6f15a303..f526e9682c 100644 --- a/webapp/channels/src/components/analytics/system_analytics/system_analytics.test.tsx +++ b/webapp/channels/src/components/analytics/system_analytics/system_analytics.test.tsx @@ -1,6 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. +import {fireEvent} from '@testing-library/react'; import React from 'react'; import {FormattedMessage} from 'react-intl'; @@ -59,7 +60,7 @@ describe('components/analytics/system_analytics/system_analytics.tsx', () => { expect(screen.queryByTestId('totalPostsLineChart')).not.toBeInTheDocument(); }); - test('system data', () => { + test('system data', async () => { const state = { ...initialState, entities: { @@ -90,6 +91,11 @@ describe('components/analytics/system_analytics/system_analytics.tsx', () => { renderWithContext(, state, {useMockedStore: true}); + const detailsElement = screen.getByText('Load Advanced Statistics'); + fireEvent.click(detailsElement); + + await screen.findByTestId('totalPostsLineChart'); + expect(screen.getByTestId('totalPosts')).toHaveTextContent('45'); expect(screen.getByTestId('totalPostsLineChart')).toBeInTheDocument(); }); @@ -233,6 +239,11 @@ describe('components/analytics/system_analytics/system_analytics.tsx', () => { await new Promise(process.nextTick); + const detailsElement = screen.getByText('Load Advanced Statistics'); + fireEvent.click(detailsElement); + + await screen.findByTestId('totalPostsLineChart'); + expect(screen.getByTestId('totalPosts')).toHaveTextContent('45'); expect(screen.getByTestId('totalPostsLineChart')).toBeInTheDocument(); expect(screen.getByTestId('com.mattermost.playbooks.playbook_count')).toHaveTextContent('45'); diff --git a/webapp/channels/src/components/analytics/system_analytics/system_analytics.tsx b/webapp/channels/src/components/analytics/system_analytics/system_analytics.tsx index 7ab69f141f..34a8da0f4a 100644 --- a/webapp/channels/src/components/analytics/system_analytics/system_analytics.tsx +++ b/webapp/channels/src/components/analytics/system_analytics/system_analytics.tsx @@ -8,6 +8,8 @@ import type {AnalyticsRow, PluginAnalyticsRow, IndexedPluginAnalyticsRow, Analyt import {AnalyticsVisualizationType} from '@mattermost/types/admin'; import type {ClientLicense} from '@mattermost/types/config'; +import {getFormattedFileSize} from 'mattermost-redux/utils/file_utils'; + import * as AdminActions from 'actions/admin_actions.jsx'; import ActivatedUserCard from 'components/analytics/activated_users_card'; @@ -16,6 +18,8 @@ import AdminHeader from 'components/widgets/admin_console/admin_header'; import Constants from 'utils/constants'; +import './analytics.scss'; + import type {GlobalState} from 'types/store'; import DoughnutChart from '../doughnut_chart'; @@ -23,7 +27,6 @@ import { formatPostsPerDayData, formatUsersWithPostsPerDayData, formatChannelDoughtnutData, - formatPostDoughtnutData, synchronizeChartLabels, } from '../format'; import LineChart from '../line_chart'; @@ -40,6 +43,7 @@ type Props = { type State = { pluginSiteStats: Record; + lineChartsDataLoaded: boolean; } const messages = defineMessages({ @@ -59,6 +63,8 @@ const messages = defineMessages({ totalChannels: {id: 'analytics.system.totalChannels', defaultMessage: 'Total Channels'}, dailyActiveUsers: {id: 'analytics.system.dailyActiveUsers', defaultMessage: 'Daily Active Users'}, monthlyActiveUsers: {id: 'analytics.system.monthlyActiveUsers', defaultMessage: 'Monthly Active Users'}, + totalFiles: {id: 'analytics.system.totalFiles', defaultMessage: 'Total Files'}, + totalFilesSize: {id: 'analytics.system.totalFilesSize', defaultMessage: 'Total Files Size'}, }); export const searchableStrings = [ @@ -78,18 +84,18 @@ export const searchableStrings = [ messages.totalChannels, messages.dailyActiveUsers, messages.monthlyActiveUsers, + messages.totalFiles, + messages.totalFilesSize, ]; export default class SystemAnalytics extends React.PureComponent { state = { pluginSiteStats: {} as Record, + lineChartsDataLoaded: false, }; public async componentDidMount() { AdminActions.getStandardAnalytics(); - AdminActions.getPostsPerDayAnalytics(); - AdminActions.getBotPostsPerDayAnalytics(); - AdminActions.getUsersPerDayAnalytics(); if (this.props.isLicensed) { AdminActions.getAdvancedAnalytics(); @@ -97,6 +103,24 @@ export default class SystemAnalytics extends React.PureComponent { this.fetchPluginStats(); } + private loadLineChartData = async () => { + await Promise.allSettled([ + AdminActions.getPostsPerDayAnalytics(), + AdminActions.getBotPostsPerDayAnalytics(), + AdminActions.getUsersPerDayAnalytics(), + ]); + this.setState({lineChartsDataLoaded: true}); + }; + + private handleLineChartsToggle = (e: React.MouseEvent) => { + const details = e.currentTarget; + const isExpanding = details.open; + + if (isExpanding && !this.state.lineChartsDataLoaded) { + this.loadLineChartData(); + } + }; + // fetchPluginStats does a call for each one of the registered handlers, // wait and set the data in the state private async fetchPluginStats() { @@ -223,6 +247,8 @@ export default class SystemAnalytics extends React.PureComponent { let commandCount; let incomingCount; let outgoingCount; + let totalFiles; + let totalFilesSize; if (this.props.isLicensed) { sessionCount = ( { /> ); + totalFiles = ( + } + icon='fa-files-o' + count={this.getStatValue(stats[StatTypes.TOTAL_FILE_COUNT])} + /> + ); + + totalFilesSize = ( + } + icon='fa-files-o' + count={this.getStatValue(stats[StatTypes.TOTAL_FILE_SIZE])} + formatter={getFormattedFileSize} + /> + ); + advancedStats = ( <> { ); const channelTypeData = formatChannelDoughtnutData(stats[StatTypes.TOTAL_PUBLIC_CHANNELS], stats[StatTypes.TOTAL_PRIVATE_GROUPS]); - const postTypeData = formatPostDoughtnutData(stats[StatTypes.TOTAL_FILE_POSTS], stats[StatTypes.TOTAL_HASHTAG_POSTS], stats[StatTypes.TOTAL_POSTS]); - - let postTypeGraph; - if (stats[StatTypes.TOTAL_POSTS] !== -1) { - postTypeGraph = ( - - } - data={postTypeData} - width={300} - height={225} - /> - ); - } advancedGraphs = (
@@ -313,7 +344,6 @@ export default class SystemAnalytics extends React.PureComponent { width={300} height={225} /> - {postTypeGraph}
); } @@ -401,25 +431,33 @@ export default class SystemAnalytics extends React.PureComponent { switch (stat.visualizationType) { case AnalyticsVisualizationType.LineChart: pluginLineCharts.push(( - + > + + )); break; case AnalyticsVisualizationType.DoughnutChart: pluginDoughnutCharts.push(( - + > + + )); break; case AnalyticsVisualizationType.Count: @@ -449,6 +487,8 @@ export default class SystemAnalytics extends React.PureComponent { {commandCount} {incomingCount} {outgoingCount} + {totalFiles} + {totalFilesSize} ); } else if (!isLicensed) { @@ -480,10 +520,23 @@ export default class SystemAnalytics extends React.PureComponent { {advancedGraphs} {pluginDoughnutCharts} - {postTotalGraph} - {botPostTotalGraph} - {activeUserGraph} {pluginLineCharts} +
+ + + + <> + {postTotalGraph} + {botPostTotalGraph} + {activeUserGraph} + +
diff --git a/webapp/channels/src/i18n/en.json b/webapp/channels/src/i18n/en.json index 9f9c45f1fe..b1eb312626 100644 --- a/webapp/channels/src/i18n/en.json +++ b/webapp/channels/src/i18n/en.json @@ -2895,18 +2895,18 @@ "analytics.system.infoAndSkippedIntensiveQueries1": "Use data for only the chosen team. Exclude posts in direct message channels that are not tied to a team.", "analytics.system.infoAndSkippedIntensiveQueries2": "To maximize performance, some statistics are disabled. You can re-enable them in config.json.", "analytics.system.monthlyActiveUsers": "Monthly Active Users", + "analytics.system.perDayStatistics": "Load Advanced Statistics", "analytics.system.postTypes": "Posts, Files and Hashtags", "analytics.system.privateGroups": "Private Channels", "analytics.system.publicChannels": "Public Channels", "analytics.system.seatsPurchased": "Licensed Seats", "analytics.system.skippedIntensiveQueries": "To maximize performance, some statistics are disabled. You can re-enable them in config.json.", - "analytics.system.textPosts": "Posts with Text-only", "analytics.system.title": "System Statistics", "analytics.system.totalBotPosts": "Total Posts from Bots", "analytics.system.totalChannels": "Total Channels", "analytics.system.totalCommands": "Total Commands", - "analytics.system.totalFilePosts": "Posts with Files", - "analytics.system.totalHashtagPosts": "Posts with Hashtags", + "analytics.system.totalFiles": "Total Files", + "analytics.system.totalFilesSize": "Total Files Size", "analytics.system.totalIncomingWebhooks": "Incoming Webhooks", "analytics.system.totalMasterDbConnections": "Master DB Conns", "analytics.system.totalOutgoingWebhooks": "Outgoing Webhooks", diff --git a/webapp/channels/src/packages/mattermost-redux/src/constants/stats.ts b/webapp/channels/src/packages/mattermost-redux/src/constants/stats.ts index 82794e92b9..1314e8b510 100644 --- a/webapp/channels/src/packages/mattermost-redux/src/constants/stats.ts +++ b/webapp/channels/src/packages/mattermost-redux/src/constants/stats.ts @@ -10,8 +10,6 @@ export default keyMirror({ TOTAL_PRIVATE_GROUPS: null, TOTAL_POSTS: null, TOTAL_TEAMS: null, - TOTAL_FILE_POSTS: null, - TOTAL_HASHTAG_POSTS: null, TOTAL_IHOOKS: null, TOTAL_OHOOKS: null, TOTAL_COMMANDS: null, @@ -27,5 +25,7 @@ export default keyMirror({ DAILY_ACTIVE_USERS: null, MONTHLY_ACTIVE_USERS: null, REGISTERED_USERS: null, + TOTAL_FILE_COUNT: null, + TOTAL_FILE_SIZE: null, }); diff --git a/webapp/channels/src/packages/mattermost-redux/src/reducers/entities/admin.ts b/webapp/channels/src/packages/mattermost-redux/src/reducers/entities/admin.ts index a22f467235..e43ccd666a 100644 --- a/webapp/channels/src/packages/mattermost-redux/src/reducers/entities/admin.ts +++ b/webapp/channels/src/packages/mattermost-redux/src/reducers/entities/admin.ts @@ -218,12 +218,6 @@ export function convertAnalyticsRowsToStats(data: AnalyticsRow[], name: string): case 'monthly_active_users': key = Stats.MONTHLY_ACTIVE_USERS; break; - case 'file_post_count': - key = Stats.TOTAL_FILE_POSTS; - break; - case 'hashtag_post_count': - key = Stats.TOTAL_HASHTAG_POSTS; - break; case 'incoming_webhook_count': key = Stats.TOTAL_IHOOKS; break; @@ -239,6 +233,12 @@ export function convertAnalyticsRowsToStats(data: AnalyticsRow[], name: string): case 'registered_users': key = Stats.REGISTERED_USERS; break; + case 'total_file_count': + key = Stats.TOTAL_FILE_COUNT; + break; + case 'total_file_size': + key = Stats.TOTAL_FILE_SIZE; + break; } if (key) { diff --git a/webapp/channels/src/packages/mattermost-redux/src/utils/file_utils.ts b/webapp/channels/src/packages/mattermost-redux/src/utils/file_utils.ts index 080d0e3ecb..06aaa95699 100644 --- a/webapp/channels/src/packages/mattermost-redux/src/utils/file_utils.ts +++ b/webapp/channels/src/packages/mattermost-redux/src/utils/file_utils.ts @@ -7,8 +7,7 @@ import {Client4} from 'mattermost-redux/client'; import {Files, General} from '../constants'; -export function getFormattedFileSize(file: FileInfo): string { - const bytes = file.size; +export function getFormattedFileSize(bytes: number): string { const fileSizes = [ ['TB', 1024 * 1024 * 1024 * 1024], ['GB', 1024 * 1024 * 1024], diff --git a/webapp/channels/src/utils/constants.tsx b/webapp/channels/src/utils/constants.tsx index 11dd7b135f..c10b006a19 100644 --- a/webapp/channels/src/utils/constants.tsx +++ b/webapp/channels/src/utils/constants.tsx @@ -824,8 +824,6 @@ export const StatTypes = keyMirror({ TOTAL_PRIVATE_GROUPS: null, TOTAL_POSTS: null, TOTAL_TEAMS: null, - TOTAL_FILE_POSTS: null, - TOTAL_HASHTAG_POSTS: null, TOTAL_IHOOKS: null, TOTAL_OHOOKS: null, TOTAL_COMMANDS: null, @@ -840,6 +838,8 @@ export const StatTypes = keyMirror({ TOTAL_READ_DB_CONNECTIONS: null, DAILY_ACTIVE_USERS: null, MONTHLY_ACTIVE_USERS: null, + TOTAL_FILE_COUNT: null, + TOTAL_FILE_SIZE: null, }); export const SearchTypes = keyMirror({ diff --git a/webapp/platform/types/src/admin.ts b/webapp/platform/types/src/admin.ts index 3ec0753592..9fdacbac9e 100644 --- a/webapp/platform/types/src/admin.ts +++ b/webapp/platform/types/src/admin.ts @@ -87,13 +87,13 @@ export type AnalyticsState = { TOTAL_READ_DB_CONNECTIONS?: number; DAILY_ACTIVE_USERS?: number; MONTHLY_ACTIVE_USERS?: number; - TOTAL_FILE_POSTS?: number; - TOTAL_HASHTAG_POSTS?: number; TOTAL_IHOOKS?: number; TOTAL_OHOOKS?: number; TOTAL_COMMANDS?: number; TOTAL_SESSIONS?: number; REGISTERED_USERS?: number; + TOTAL_FILE_COUNT?: number; + TOTAL_FILE_SIZE?: number; } export type ClusterInfo = {