[AI assisted]: Improve system console statistics performance (#29899)

```release-note
NONE
```

Co-authored-by: Matthew Birtch <mattbirtch@gmail.com>
Этот коммит содержится в:
Agniva De Sarker
2025-02-04 21:54:01 +05:30
коммит произвёл GitHub
родитель 6046a304b2
Коммит ae9e6174e5
45 изменённых файлов: 1052 добавлений и 407 удалений

Просмотреть файл

@@ -128,25 +128,30 @@ describe('System Console > Site Statistics', () => {
cy.visit('/admin_console');
cy.wait('@resources');
// * Find site statistics and click it
cy.findByTestId('reporting.system_analytics', {timeout: TIMEOUTS.ONE_MIN}).click();
cy.dbRefreshPostStats().then(() => {
// * Find site statistics and click it
cy.findByTestId('reporting.system_analytics', {timeout: TIMEOUTS.ONE_MIN}).click();
let totalPostsDataSet;
let totalPostsFromBots;
let activeUsersWithPosts;
// * Expand the details
cy.findByTestId('details-expander', {timeout: TIMEOUTS.ONE_MIN}).click();
// # Grab all data from the 3 charts from there data labels
cy.findByTestId('totalPostsLineChart').then((el) => {
totalPostsDataSet = el[0].dataset.labels;
cy.findByTestId('totalPostsFromBotsLineChart').then((el2) => {
totalPostsFromBots = el2[0].dataset.labels;
cy.findByTestId('activeUsersWithPostsLineChart').then((el3) => {
activeUsersWithPosts = el3[0].dataset.labels;
let totalPostsDataSet;
let totalPostsFromBots;
let activeUsersWithPosts;
// * Assert that all the dates are the same
expect(totalPostsDataSet).equal(totalPostsFromBots);
expect(totalPostsDataSet).equal(activeUsersWithPosts);
expect(totalPostsFromBots).equal(activeUsersWithPosts);
// # Grab all data from the 3 charts from there data labels
cy.findByTestId('totalPostsLineChart').then((el) => {
totalPostsDataSet = el[0].dataset.labels;
cy.findByTestId('totalPostsFromBotsLineChart').then((el2) => {
totalPostsFromBots = el2[0].dataset.labels;
cy.findByTestId('activeUsersWithPostsLineChart').then((el3) => {
activeUsersWithPosts = el3[0].dataset.labels;
// * Assert that all the dates are the same
expect(totalPostsDataSet).equal(totalPostsFromBots);
expect(totalPostsDataSet).equal(activeUsersWithPosts);
expect(totalPostsFromBots).equal(activeUsersWithPosts);
});
});
});
});

Просмотреть файл

@@ -19,14 +19,16 @@ describe('System Console > Team Statistics', () => {
// # Create private channel.
cy.apiCreateChannel(team.id, 'mmt906-ch', 'mmt906-ch', 'P');
// # Visit team statistics page.
cy.visit('/admin_console/reporting/team_statistics');
cy.dbRefreshPostStats().then(() => {
// # Visit team statistics page.
cy.visit('/admin_console/reporting/team_statistics');
// # Select created team.
cy.get('select.team-statistics__team-filter__dropdown').select(team.id);
// # Select created team.
cy.get('select.team-statistics__team-filter__dropdown').select(team.id);
// # Explicit wait to allow stats to get loaded
cy.wait(TIMEOUTS.TWO_SEC);
// # Explicit wait to allow stats to get loaded
cy.wait(TIMEOUTS.TWO_SEC);
});
});
});

Просмотреть файл

@@ -127,9 +127,36 @@ function toLowerCase(config, name) {
return name.toLowerCase();
}
const dbRefreshPostStats = async ({dbConfig}) => {
if (!knexClient) {
knexClient = getKnexClient(dbConfig);
}
// Only run for PostgreSQL
if (dbConfig.client !== 'postgres') {
return {
skipped: true,
message: 'Refresh post stats is only supported for PostgreSQL',
};
}
try {
await knexClient.raw('REFRESH MATERIALIZED VIEW posts_by_team_day;');
await knexClient.raw('REFRESH MATERIALIZED VIEW bot_posts_by_team_day;');
return {
success: true,
};
} catch (error) {
const errorMessage = 'Failed to refresh post statistics materialized views.';
return {error, errorMessage};
}
};
module.exports = {
dbGetActiveUserSessions,
dbGetUser,
dbGetUserSession,
dbUpdateUserSession,
dbRefreshPostStats,
};

Просмотреть файл

@@ -11,6 +11,7 @@ const {
dbGetUser,
dbGetUserSession,
dbUpdateUserSession,
dbRefreshPostStats,
} = require('./db_request');
const externalRequest = require('./external_request').default;
const {fileExist, writeToFile} = require('./file_util');
@@ -42,6 +43,7 @@ module.exports = (on, config) => {
dbGetUser,
dbGetUserSession,
dbUpdateUserSession,
dbRefreshPostStats,
externalRequest,
fileExist,
writeToFile,

Просмотреть файл

@@ -96,6 +96,14 @@ function dbUpdateUserSession(params: UpdateUserSessionParam): ChainableT<UpdateU
}
Cypress.Commands.add('dbUpdateUserSession', dbUpdateUserSession);
function dbRefreshPostStats(): ChainableT<{success?: boolean; skipped?: boolean; message?: string}> {
return cy.task('dbRefreshPostStats', {dbConfig}).then(({success, skipped, message, errorMessage, error}) => {
verifyError(error, errorMessage);
return cy.wrap({success, skipped, message});
});
}
Cypress.Commands.add('dbRefreshPostStats', dbRefreshPostStats);
function verifyError(error, errorMessage) {
if (errorMessage) {
expect(errorMessage, `${errorMessage}\n\n${message}\n\n${JSON.stringify(error)}`).to.be.undefined;
@@ -150,6 +158,15 @@ declare global {
* @returns {Session} session
*/
dbUpdateUserSession: typeof dbUpdateUserSession;
/**
* Refreshes PostgreSQL materialized views for post statistics
* @returns {Object} result
* @returns {boolean} result.success - true if refresh was successful
* @returns {boolean} result.skipped - true if operation was skipped (non-PostgreSQL)
* @returns {string} result.message - message when operation is skipped
*/
dbRefreshPostStats: typeof dbRefreshPostStats;
}
}
}

Просмотреть файл

@@ -2649,6 +2649,7 @@ func TestPermanentDeleteAllUsers(t *testing.T) {
require.NoError(t, err)
require.Greater(t, len(users), 0)
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
postCount, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, err)
require.Greater(t, postCount, int64(0))
@@ -2662,6 +2663,7 @@ func TestPermanentDeleteAllUsers(t *testing.T) {
require.NoError(t, err)
require.Len(t, users, 0)
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
postCount, err = th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, err)
require.Equal(t, postCount, int64(0))

Просмотреть файл

@@ -34,287 +34,273 @@ func (a *App) getAnalytics(rctx request.CTX, name string, teamID string, forSupp
skipIntensiveQueries := false
// When generating a Support Packet, always run intensive queries.
if !forSupportPacket {
if systemUserCount > int64(*a.Config().AnalyticsSettings.MaxUsersForStatistics) {
rctx.Logger().Debug("More than limit users are on the system, intensive queries skipped", mlog.Int("limit", *a.Config().AnalyticsSettings.MaxUsersForStatistics))
skipIntensiveQueries = true
}
if !forSupportPacket && systemUserCount > int64(*a.Config().AnalyticsSettings.MaxUsersForStatistics) {
rctx.Logger().Warn("Number of users in the system is higher than the configured limit. Skipping intensive SQL queries.", mlog.Int("limit", *a.Config().AnalyticsSettings.MaxUsersForStatistics))
skipIntensiveQueries = true
}
if name == "standard" {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 11)
rows[0] = &model.AnalyticsRow{Name: "channel_open_count", Value: 0}
rows[1] = &model.AnalyticsRow{Name: "channel_private_count", Value: 0}
rows[2] = &model.AnalyticsRow{Name: "post_count", Value: 0}
rows[3] = &model.AnalyticsRow{Name: "unique_user_count", Value: 0}
rows[4] = &model.AnalyticsRow{Name: "team_count", Value: 0}
rows[5] = &model.AnalyticsRow{Name: "total_websocket_connections", Value: 0}
rows[6] = &model.AnalyticsRow{Name: "total_master_db_connections", Value: 0}
rows[7] = &model.AnalyticsRow{Name: "total_read_db_connections", Value: 0}
rows[8] = &model.AnalyticsRow{Name: "daily_active_users", Value: 0}
rows[9] = &model.AnalyticsRow{Name: "monthly_active_users", Value: 0}
rows[10] = &model.AnalyticsRow{Name: "inactive_user_count", Value: 0}
switch name {
case "standard":
return a.getStandardAnalytics(rctx, teamID, systemUserCount)
case "bot_post_counts_day":
return a.getBotPostCountsAnalytics(rctx, teamID)
case "post_counts_day":
return a.getPostCountsAnalytics(rctx, teamID)
case "user_counts_with_posts_day":
return a.getUserCountsWithPostsAnalytics(rctx, teamID, skipIntensiveQueries)
case "extra_counts":
return a.getExtraCountsAnalytics(rctx, teamID)
default:
return nil, nil
}
}
var g errgroup.Group
var openChannelsCount int64
func (a *App) getStandardAnalytics(rctx request.CTX, teamID string, systemUserCount int64) (model.AnalyticsRows, *model.AppError) {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 11)
rows[0] = &model.AnalyticsRow{Name: "channel_open_count", Value: 0}
rows[1] = &model.AnalyticsRow{Name: "channel_private_count", Value: 0}
rows[2] = &model.AnalyticsRow{Name: "post_count", Value: 0}
rows[3] = &model.AnalyticsRow{Name: "unique_user_count", Value: 0}
rows[4] = &model.AnalyticsRow{Name: "team_count", Value: 0}
rows[5] = &model.AnalyticsRow{Name: "total_websocket_connections", Value: 0}
rows[6] = &model.AnalyticsRow{Name: "total_master_db_connections", Value: 0}
rows[7] = &model.AnalyticsRow{Name: "total_read_db_connections", Value: 0}
rows[8] = &model.AnalyticsRow{Name: "daily_active_users", Value: 0}
rows[9] = &model.AnalyticsRow{Name: "monthly_active_users", Value: 0}
rows[10] = &model.AnalyticsRow{Name: "inactive_user_count", Value: 0}
var g errgroup.Group
g.SetLimit(2)
var channelCounts map[model.ChannelType]int64
g.Go(func() error {
var err error
if channelCounts, err = a.Srv().Store().Channel().AnalyticsCountAll(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var usersCount int64
var inactiveUsersCount int64
if teamID == "" {
g.Go(func() error {
var err error
if openChannelsCount, err = a.Srv().Store().Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen); err != nil {
return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
if inactiveUsersCount, err = a.Srv().Store().User().AnalyticsGetInactiveUsersCount(); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_get_inactive_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var privateChannelsCount int64
} else {
g.Go(func() error {
var err error
if privateChannelsCount, err = a.Srv().Store().Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate); err != nil {
return model.NewAppError("GetAnalytics", "app.channel.analytics_type_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
if usersCount, err = a.Srv().Store().User().Count(model.UserCountOptions{TeamId: teamID}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.get_total_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
}
var usersCount int64
var inactiveUsersCount int64
if teamID == "" {
g.Go(func() error {
var err error
if inactiveUsersCount, err = a.Srv().Store().User().AnalyticsGetInactiveUsersCount(); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_get_inactive_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
} else {
g.Go(func() error {
var err error
if usersCount, err = a.Srv().Store().User().Count(model.UserCountOptions{TeamId: teamID}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.get_total_users_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var postsCount int64
g.Go(func() error {
var err error
if postsCount, err = a.Srv().Store().Post().AnalyticsPostCountByTeam(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var teamsCount int64
g.Go(func() error {
var err error
if teamsCount, err = a.Srv().Store().Team().AnalyticsTeamCount(nil); err != nil {
return model.NewAppError("GetAnalytics", "app.team.analytics_team_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var dailyActiveUsersCount int64
g.Go(func() error {
var err error
if dailyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(DayMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var monthlyActiveUsersCount int64
g.Go(func() error {
var err error
if monthlyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(MonthMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
if err := g.Wait(); err != nil {
return nil, err.(*model.AppError)
}
rows[0].Value = float64(channelCounts[model.ChannelTypeOpen])
rows[1].Value = float64(channelCounts[model.ChannelTypePrivate])
rows[2].Value = float64(postsCount)
if teamID == "" {
rows[3].Value = float64(systemUserCount)
rows[10].Value = float64(inactiveUsersCount)
} else {
rows[3].Value = float64(usersCount)
rows[10].Value = -1
}
rows[4].Value = float64(teamsCount)
// If in HA mode then aggregate all the stats
if a.Cluster() != nil && *a.Config().ClusterSettings.Enable {
stats, err2 := a.Cluster().GetClusterStats(rctx)
if err2 != nil {
return nil, err2
}
var postsCount int64
if !skipIntensiveQueries {
g.Go(func() error {
var err error
if postsCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID}); err != nil {
return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
totalSockets := a.TotalWebsocketConnections()
totalMasterDb := a.Srv().Store().TotalMasterDbConnections()
totalReadDb := a.Srv().Store().TotalReadDbConnections()
for _, stat := range stats {
totalSockets = totalSockets + stat.TotalWebsocketConnections
totalMasterDb = totalMasterDb + stat.TotalMasterDbConnections
totalReadDb = totalReadDb + stat.TotalReadDbConnections
}
var teamsCount int64
g.Go(func() error {
var err error
if teamsCount, err = a.Srv().Store().Team().AnalyticsTeamCount(nil); err != nil {
return model.NewAppError("GetAnalytics", "app.team.analytics_team_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
rows[5].Value = float64(totalSockets)
rows[6].Value = float64(totalMasterDb)
rows[7].Value = float64(totalReadDb)
} else {
rows[5].Value = float64(a.TotalWebsocketConnections())
rows[6].Value = float64(a.Srv().Store().TotalMasterDbConnections())
rows[7].Value = float64(a.Srv().Store().TotalReadDbConnections())
}
var dailyActiveUsersCount int64
g.Go(func() error {
var err error
if dailyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(DayMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
rows[8].Value = float64(dailyActiveUsersCount)
rows[9].Value = float64(monthlyActiveUsersCount)
var monthlyActiveUsersCount int64
g.Go(func() error {
var err error
if monthlyActiveUsersCount, err = a.Srv().Store().User().AnalyticsActiveCount(MonthMilliseconds, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}); err != nil {
return model.NewAppError("GetAnalytics", "app.user.analytics_daily_active_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
return rows, nil
}
if err := g.Wait(); err != nil {
return nil, err.(*model.AppError)
}
func (a *App) getBotPostCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) {
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
TeamId: teamID,
BotsOnly: true,
})
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
rows[0].Value = float64(openChannelsCount)
rows[1].Value = float64(privateChannelsCount)
return analyticsRows, nil
}
if skipIntensiveQueries {
rows[2].Value = -1
} else {
rows[2].Value = float64(postsCount)
}
func (a *App) getPostCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) {
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
TeamId: teamID,
BotsOnly: false,
})
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
if teamID == "" {
rows[3].Value = float64(systemUserCount)
rows[10].Value = float64(inactiveUsersCount)
} else {
rows[10].Value = -1
rows[3].Value = float64(usersCount)
}
rows[4].Value = float64(teamsCount)
// If in HA mode then aggregate all the stats
if a.Cluster() != nil && *a.Config().ClusterSettings.Enable {
stats, err2 := a.Cluster().GetClusterStats(rctx)
if err2 != nil {
return nil, err2
}
totalSockets := a.TotalWebsocketConnections()
totalMasterDb := a.Srv().Store().TotalMasterDbConnections()
totalReadDb := a.Srv().Store().TotalReadDbConnections()
for _, stat := range stats {
totalSockets = totalSockets + stat.TotalWebsocketConnections
totalMasterDb = totalMasterDb + stat.TotalMasterDbConnections
totalReadDb = totalReadDb + stat.TotalReadDbConnections
}
rows[5].Value = float64(totalSockets)
rows[6].Value = float64(totalMasterDb)
rows[7].Value = float64(totalReadDb)
} else {
rows[5].Value = float64(a.TotalWebsocketConnections())
rows[6].Value = float64(a.Srv().Store().TotalMasterDbConnections())
rows[7].Value = float64(a.Srv().Store().TotalReadDbConnections())
}
rows[8].Value = float64(dailyActiveUsersCount)
rows[9].Value = float64(monthlyActiveUsersCount)
return rows, nil
} else if name == "bot_post_counts_day" {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
return rows, nil
}
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
TeamId: teamID,
BotsOnly: true,
YesterdayOnly: false,
})
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return analyticsRows, nil
} else if name == "post_counts_day" {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
return rows, nil
}
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
TeamId: teamID,
BotsOnly: false,
YesterdayOnly: false,
})
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_posts_count_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return analyticsRows, nil
} else if name == "user_counts_with_posts_day" {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
return rows, nil
}
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsUserCountsWithPostsByDay(teamID)
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_user_counts_posts_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return analyticsRows, nil
} else if name == "extra_counts" {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 6)
rows[0] = &model.AnalyticsRow{Name: "file_post_count", Value: 0}
rows[1] = &model.AnalyticsRow{Name: "hashtag_post_count", Value: 0}
rows[2] = &model.AnalyticsRow{Name: "incoming_webhook_count", Value: 0}
rows[3] = &model.AnalyticsRow{Name: "outgoing_webhook_count", Value: 0}
rows[4] = &model.AnalyticsRow{Name: "command_count", Value: 0}
rows[5] = &model.AnalyticsRow{Name: "session_count", Value: 0}
var g2 errgroup.Group
var incomingWebhookCount int64
g2.Go(func() error {
var err error
if incomingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsIncomingCount(teamID, ""); err != nil {
return model.NewAppError("GetAnalytics", "app.webhooks.analytics_incoming_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var outgoingWebhookCount int64
g2.Go(func() error {
var err error
if outgoingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsOutgoingCount(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.webhooks.analytics_outgoing_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var commandsCount int64
g2.Go(func() error {
var err error
if commandsCount, err = a.Srv().Store().Command().AnalyticsCommandCount(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.analytics.getanalytics.internal_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var sessionsCount int64
g2.Go(func() error {
var err error
if sessionsCount, err = a.Srv().Store().Session().AnalyticsSessionCount(); err != nil {
return model.NewAppError("GetAnalytics", "app.session.analytics_session_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var filesCount int64
var hashtagsCount int64
if !skipIntensiveQueries {
g2.Go(func() error {
var err error
if filesCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID, MustHaveFile: true}); err != nil {
return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
g2.Go(func() error {
var err error
if hashtagsCount, err = a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID, MustHaveHashtag: true}); err != nil {
return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
}
if err := g2.Wait(); err != nil {
return nil, err.(*model.AppError)
}
if skipIntensiveQueries {
rows[0].Value = -1
rows[1].Value = -1
} else {
rows[0].Value = float64(filesCount)
rows[1].Value = float64(hashtagsCount)
}
rows[2].Value = float64(incomingWebhookCount)
rows[3].Value = float64(outgoingWebhookCount)
rows[4].Value = float64(commandsCount)
rows[5].Value = float64(sessionsCount)
return analyticsRows, nil
}
func (a *App) getUserCountsWithPostsAnalytics(rctx request.CTX, teamID string, skipIntensiveQueries bool) (model.AnalyticsRows, *model.AppError) {
if skipIntensiveQueries {
rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
return rows, nil
}
return nil, nil
analyticsRows, nErr := a.Srv().Store().Post().AnalyticsUserCountsWithPostsByDay(teamID)
if nErr != nil {
return nil, model.NewAppError("GetAnalytics", "app.post.analytics_user_counts_posts_by_day.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return analyticsRows, nil
}
func (a *App) getExtraCountsAnalytics(rctx request.CTX, teamID string) (model.AnalyticsRows, *model.AppError) {
var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 6)
rows[0] = &model.AnalyticsRow{Name: "incoming_webhook_count", Value: 0}
rows[1] = &model.AnalyticsRow{Name: "outgoing_webhook_count", Value: 0}
rows[2] = &model.AnalyticsRow{Name: "command_count", Value: 0}
rows[3] = &model.AnalyticsRow{Name: "session_count", Value: 0}
rows[4] = &model.AnalyticsRow{Name: "total_file_count", Value: 0}
rows[5] = &model.AnalyticsRow{Name: "total_file_size", Value: 0}
var incomingWebhookCount int64
var g errgroup.Group
g.SetLimit(2)
g.Go(func() error {
var err error
if incomingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsIncomingCount(teamID, ""); err != nil {
return model.NewAppError("GetAnalytics", "app.webhooks.analytics_incoming_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var outgoingWebhookCount int64
g.Go(func() error {
var err error
if outgoingWebhookCount, err = a.Srv().Store().Webhook().AnalyticsOutgoingCount(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.webhooks.analytics_outgoing_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var commandsCount int64
g.Go(func() error {
var err error
if commandsCount, err = a.Srv().Store().Command().AnalyticsCommandCount(teamID); err != nil {
return model.NewAppError("GetAnalytics", "app.analytics.getanalytics.internal_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var sessionsCount int64
g.Go(func() error {
var err error
if sessionsCount, err = a.Srv().Store().Session().AnalyticsSessionCount(); err != nil {
return model.NewAppError("GetAnalytics", "app.session.analytics_session_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var fileCount int64
g.Go(func() error {
var err error
if fileCount, err = a.Srv().Store().FileInfo().CountAll(); err != nil {
return model.NewAppError("GetAnalytics", "app.file_info.get_count.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
var fileSize int64
g.Go(func() error {
var err error
if fileSize, err = a.Srv().Store().FileInfo().GetStorageUsage(false, false); err != nil {
return model.NewAppError("GetAnalytics", "app.file_info.get_storage_usage.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
return nil
})
if err := g.Wait(); err != nil {
return nil, err.(*model.AppError)
}
rows[0].Value = float64(incomingWebhookCount)
rows[1].Value = float64(outgoingWebhookCount)
rows[2].Value = float64(commandsCount)
rows[3].Value = float64(sessionsCount)
rows[4].Value = float64(fileCount)
rows[5].Value = float64(fileSize)
return rows, nil
}
func (a *App) GetRecentlyActiveUsersForTeam(rctx request.CTX, teamID string) (map[string]*model.User, *model.AppError) {

Просмотреть файл

@@ -2182,6 +2182,7 @@ func TestImportimportMultiplePostLines(t *testing.T) {
require.Nil(t, err, "Failed to get user from database.")
// Count the number of posts in the testing team.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
initialPostCount, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team.Id})
require.NoError(t, nErr)
@@ -2859,6 +2860,7 @@ func TestImportimportMultiplePostLines(t *testing.T) {
require.Nil(t, err, "Failed to get channel from database.")
// Count the number of posts in the team2.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
initialPostCountForTeam2, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team2.Id})
require.NoError(t, nErr)
@@ -3239,6 +3241,7 @@ func TestImportImportPost(t *testing.T) {
require.Nil(t, appErr, "Failed to get user from database.")
// Count the number of posts in the testing team.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
initialPostCount, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team.Id})
require.NoError(t, nErr)
@@ -4194,6 +4197,7 @@ func TestImportImportDirectPost(t *testing.T) {
directChannel = channel
// Get the number of posts in the system.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
result, err := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, err)
initialPostCount := result
@@ -4664,6 +4668,7 @@ func TestImportImportDirectPost(t *testing.T) {
groupChannel = channel
// Get the number of posts in the system.
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
result, nErr := th.App.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, nErr)
initialPostCount = result

Просмотреть файл

@@ -53,6 +53,8 @@ func checkNoError(t *testing.T, err *model.AppError) {
}
func AssertAllPostsCount(t *testing.T, a *App, initialCount int64, change int64, teamName string) {
t.Helper()
require.NoError(t, a.Srv().Store().Post().RefreshPostStats())
result, err := a.Srv().Store().Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamName})
require.NoError(t, err)
require.Equal(t, initialCount+change, result, "Did not find the expected number of posts.")

Просмотреть файл

@@ -60,7 +60,7 @@ import (
"github.com/mattermost/mattermost/server/v8/channels/jobs/plugins"
"github.com/mattermost/mattermost/server/v8/channels/jobs/post_persistent_notifications"
"github.com/mattermost/mattermost/server/v8/channels/jobs/product_notices"
"github.com/mattermost/mattermost/server/v8/channels/jobs/refresh_post_stats"
"github.com/mattermost/mattermost/server/v8/channels/jobs/refresh_materialized_views"
"github.com/mattermost/mattermost/server/v8/channels/jobs/resend_invitation_email"
"github.com/mattermost/mattermost/server/v8/channels/jobs/s3_path_migration"
"github.com/mattermost/mattermost/server/v8/channels/store"
@@ -1643,9 +1643,9 @@ func (s *Server) initJobs() {
)
s.Jobs.RegisterJobType(
model.JobTypeRefreshPostStats,
refresh_post_stats.MakeWorker(s.Jobs, *s.platform.Config().SqlSettings.DriverName),
refresh_post_stats.MakeScheduler(s.Jobs, *s.platform.Config().SqlSettings.DriverName),
model.JobTypeRefreshMaterializedViews,
refresh_materialized_views.MakeWorker(s.Jobs, *s.platform.Config().SqlSettings.DriverName),
refresh_materialized_views.MakeScheduler(s.Jobs, *s.platform.Config().SqlSettings.DriverName),
)
s.Jobs.RegisterJobType(

Просмотреть файл

@@ -337,6 +337,8 @@ func TestGetSupportPacketStats(t *testing.T) {
generateStats := func(t *testing.T) *model.SupportPacketStats {
t.Helper()
require.NoError(t, th.App.Srv().Store().Post().RefreshPostStats())
fileData, err := th.App.getSupportPacketStats(th.Context)
require.NotNil(t, fileData)
assert.Equal(t, "stats.yaml", fileData.Filename)

Просмотреть файл

@@ -255,6 +255,8 @@ channels/db/migrations/mysql/000128_create_scheduled_posts.down.sql
channels/db/migrations/mysql/000128_create_scheduled_posts.up.sql
channels/db/migrations/mysql/000129_add_property_system_architecture.down.sql
channels/db/migrations/mysql/000129_add_property_system_architecture.up.sql
channels/db/migrations/mysql/000130_system_console_stats.down.sql
channels/db/migrations/mysql/000130_system_console_stats.up.sql
channels/db/migrations/postgres/000001_create_teams.down.sql
channels/db/migrations/postgres/000001_create_teams.up.sql
channels/db/migrations/postgres/000002_create_team_members.down.sql
@@ -511,3 +513,5 @@ channels/db/migrations/postgres/000128_create_scheduled_posts.down.sql
channels/db/migrations/postgres/000128_create_scheduled_posts.up.sql
channels/db/migrations/postgres/000129_add_property_system_architecture.down.sql
channels/db/migrations/postgres/000129_add_property_system_architecture.up.sql
channels/db/migrations/postgres/000130_system_console_stats.down.sql
channels/db/migrations/postgres/000130_system_console_stats.up.sql

Просмотреть файл

@@ -0,0 +1 @@
-- Nothing to do for MySQL

Просмотреть файл

@@ -0,0 +1 @@
-- Nothing to do for MySQL

Просмотреть файл

@@ -0,0 +1,5 @@
DROP MATERIALIZED VIEW IF EXISTS posts_by_team_day;
DROP MATERIALIZED VIEW IF EXISTS bot_posts_by_team_day;
DROP MATERIALIZED VIEW IF EXISTS file_stats;

Просмотреть файл

@@ -0,0 +1,16 @@
CREATE MATERIALIZED VIEW IF NOT EXISTS posts_by_team_day as
SELECT to_timestamp(p.createat/1000)::date as day, COUNT(*) as num, teamid
FROM posts p JOIN channels c on p.channelid=c.id
GROUP BY day, c.teamid;
CREATE MATERIALIZED VIEW IF NOT EXISTS bot_posts_by_team_day as
SELECT to_timestamp(p.createat/1000)::date as day, COUNT(*) as num, teamid
FROM posts p
JOIN Bots b ON p.UserId = b.Userid
JOIN channels c on p.channelid=c.id
GROUP BY day, c.teamid;
CREATE MATERIALIZED VIEW IF NOT EXISTS file_stats as
SELECT COUNT(*) as num, COALESCE(SUM(Size), 0) as usage
FROM fileinfo
WHERE DeleteAt = 0;

Просмотреть файл

@@ -1,7 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package refresh_post_stats
package refresh_materialized_views
import (
"time"
@@ -21,5 +21,5 @@ func MakeScheduler(jobServer *jobs.JobServer, sqlDriverName string) *jobs.DailyS
isEnabled := func(cfg *model.Config) bool {
return sqlDriverName == model.DatabaseDriverPostgres
}
return jobs.NewDailyScheduler(jobServer, model.JobTypeRefreshPostStats, startTime, isEnabled)
return jobs.NewDailyScheduler(jobServer, model.JobTypeRefreshMaterializedViews, startTime, isEnabled)
}

Просмотреть файл

@@ -1,7 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package refresh_post_stats
package refresh_materialized_views
import (
"github.com/mattermost/mattermost/server/public/model"
@@ -9,7 +9,7 @@ import (
"github.com/mattermost/mattermost/server/v8/channels/jobs"
)
const jobName = "RefreshPostStats"
const jobName = "RefreshMaterializedViews"
func MakeWorker(jobServer *jobs.JobServer, sqlDriverName string) *jobs.SimpleWorker {
isEnabled := func(cfg *model.Config) bool {
@@ -18,8 +18,17 @@ func MakeWorker(jobServer *jobs.JobServer, sqlDriverName string) *jobs.SimpleWor
execute := func(logger mlog.LoggerIFace, job *model.Job) error {
defer jobServer.HandleJobPanic(logger, job)
if err := jobServer.Store.Post().RefreshPostStats(); err != nil {
return err
}
if err := jobServer.Store.FileInfo().RefreshFileStats(); err != nil {
return err
}
return jobServer.Store.User().RefreshPostStatsForUsers()
}
worker := jobs.NewSimpleWorker(jobName, jobServer, execute, isEnabled)
return worker
}

Просмотреть файл

@@ -741,6 +741,27 @@ func (s *RetryLayerBotStore) Update(bot *model.Bot) (*model.Bot, error) {
}
func (s *RetryLayerChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) {
tries := 0
for {
result, err := s.ChannelStore.AnalyticsCountAll(teamID)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) {
tries := 0
@@ -4746,6 +4767,27 @@ func (s *RetryLayerFileInfoStore) PermanentDeleteForPost(rctx request.CTX, postI
}
func (s *RetryLayerFileInfoStore) RefreshFileStats() error {
tries := 0
for {
err := s.FileInfoStore.RefreshFileStats()
if err == nil {
return nil
}
if !isRepeatableError(err) {
return err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error {
tries := 0
@@ -7308,6 +7350,27 @@ func (s *RetryLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions
}
func (s *RetryLayerPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
tries := 0
for {
result, err := s.PostStore.AnalyticsPostCountByTeam(teamID)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
tries := 0
@@ -8109,6 +8172,27 @@ func (s *RetryLayerPostStore) PermanentDeleteByUser(rctx request.CTX, userID str
}
func (s *RetryLayerPostStore) RefreshPostStats() error {
tries := 0
for {
err := s.PostStore.RefreshPostStats()
if err == nil {
return nil
}
if !isRepeatableError(err) {
return err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerPostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) {
tries := 0

Просмотреть файл

@@ -2961,6 +2961,40 @@ func (s SqlChannelStore) AnalyticsDeletedTypeCount(teamId string, channelType mo
return v, nil
}
func (s SqlChannelStore) AnalyticsCountAll(teamId string) (map[model.ChannelType]int64, error) {
query := s.getQueryBuilder().
Select("Type, COUNT(*) AS Count").
From("Channels").
GroupBy("Type")
if teamId != "" {
query = query.Where(sq.Eq{"TeamId": teamId})
}
sql, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "AnalyticsCountAll_ToSql")
}
rows, err := s.GetReplica().Query(sql, args...)
if err != nil {
return nil, errors.Wrap(err, "failed to count Channels by type")
}
defer rows.Close()
counts := make(map[model.ChannelType]int64)
for rows.Next() {
var channelType model.ChannelType
var count int64
if err := rows.Scan(&channelType, &count); err != nil {
return nil, errors.Wrap(err, "unable to scan row")
}
counts[channelType] = count
}
return counts, nil
}
func (s SqlChannelStore) GetMembersForUser(teamID string, userID string) (model.ChannelMembers, error) {
sql, args, err := s.channelMembersForTeamWithSchemeSelectQuery.
Where(sq.And{

Просмотреть файл

@@ -712,18 +712,20 @@ func (fs SqlFileInfoStore) Search(rctx request.CTX, paramsList []*model.SearchPa
}
func (fs SqlFileInfoStore) CountAll() (int64, error) {
query := fs.getQueryBuilder().
Select("COUNT(*)").
From("FileInfo").
Where("DeleteAt = 0")
queryString, args, err := query.ToSql()
if err != nil {
return int64(0), errors.Wrap(err, "count_tosql")
var query sq.SelectBuilder
if fs.DriverName() == model.DatabaseDriverPostgres {
query = fs.getQueryBuilder().
Select("num").
From("file_stats")
} else {
query = fs.getQueryBuilder().
Select("COUNT(*)").
From("FileInfo").
Where("DeleteAt = 0")
}
var count int64
err = fs.GetReplica().Get(&count, queryString, args...)
err := fs.GetReplica().GetBuilder(&count, query)
if err != nil {
return int64(0), errors.Wrap(err, "failed to count Files")
}
@@ -758,13 +760,20 @@ func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID
return files, nil
}
func (fs SqlFileInfoStore) GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error) {
query := fs.getQueryBuilder().
Select("COALESCE(SUM(Size), 0)").
From("FileInfo")
func (fs SqlFileInfoStore) GetStorageUsage(_, includeDeleted bool) (int64, error) {
var query sq.SelectBuilder
if fs.DriverName() == model.DatabaseDriverPostgres && !includeDeleted {
query = fs.getQueryBuilder().
Select("usage").
From("file_stats")
} else {
query = fs.getQueryBuilder().
Select("COALESCE(SUM(Size), 0)").
From("FileInfo")
if !includeDeleted {
query = query.Where("DeleteAt = 0")
if !includeDeleted {
query = query.Where("DeleteAt = 0")
}
}
var size int64
@@ -841,3 +850,18 @@ func (fs SqlFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string,
return nil
}
func (fs SqlFileInfoStore) RefreshFileStats() error {
if fs.DriverName() == model.DatabaseDriverPostgres {
// CONCURRENTLY is not used deliberately because as per Postgres docs,
// not using CONCURRENTLY takes less resources and completes faster
// at the expense of locking the mat view. Since viewing admin console
// is not a very frequent activity, we accept the tradeoff to let the
// refresh happen as fast as possible.
if _, err := fs.GetMaster().Exec("REFRESH MATERIALIZED VIEW file_stats"); err != nil {
return errors.Wrap(err, "error refreshing materialized view file_stats")
}
}
return nil
}

Просмотреть файл

@@ -2292,8 +2292,79 @@ func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.A
return rows, nil
}
func (s *SqlPostStore) countBotPostsByDay(teamID, startDay, endDay string) (model.AnalyticsRows, error) {
var query sq.SelectBuilder
if teamID != "" {
query = s.getQueryBuilder().
Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, num as Value").
From("bot_posts_by_team_day").
Where(sq.Eq{"teamid": teamID})
} else {
query = s.getQueryBuilder().
Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, COALESCE(SUM(num), 0) as Value").
From("bot_posts_by_team_day").
GroupBy("Name")
}
query = query.
Where(sq.GtOrEq{"day": startDay}).
Where(sq.LtOrEq{"day": endDay}).
OrderBy("Name DESC").
Limit(30)
rows := model.AnalyticsRows{}
err := s.GetReplica().SelectBuilder(&rows, query)
if err != nil {
return nil, errors.Wrapf(err, "failed to find bot posts with teamId=%s", teamID)
}
return rows, nil
}
func (s *SqlPostStore) countPostsByDay(teamID, startDay, endDay string) (model.AnalyticsRows, error) {
var query sq.SelectBuilder
if teamID != "" {
query = s.getQueryBuilder().
Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, num as Value").
From("posts_by_team_day").
Where(sq.Eq{"teamid": teamID})
} else {
query = s.getQueryBuilder().
Select("TO_CHAR(day, 'YYYY-MM-DD') as Name, COALESCE(SUM(num), 0) as Value").
From("posts_by_team_day").
GroupBy("Name")
}
query = query.
Where(sq.GtOrEq{"day": startDay}).
Where(sq.LtOrEq{"day": endDay}).
OrderBy("Name DESC").
Limit(30)
rows := model.AnalyticsRows{}
err := s.GetReplica().SelectBuilder(&rows, query)
if err != nil {
return nil, errors.Wrapf(err, "failed to find posts with teamId=%s", teamID)
}
return rows, nil
}
// TODO: convert to squirrel HW
func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
if s.DriverName() == model.DatabaseDriverPostgres {
endDay := utils.Yesterday().Format("2006-01-02")
startDay := utils.Yesterday().AddDate(0, 0, -31).Format("2006-01-02")
if options.YesterdayOnly {
startDay = utils.Yesterday().AddDate(0, 0, -1).Format("2006-01-02")
}
// Use materialized views
if options.BotsOnly {
return s.countBotPostsByDay(options.TeamId, startDay, endDay)
}
return s.countPostsByDay(options.TeamId, startDay, endDay)
}
var args []any
query :=
`SELECT
@@ -2318,30 +2389,6 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun
ORDER BY Name DESC
LIMIT 30`
if s.DriverName() == model.DatabaseDriverPostgres {
query =
`SELECT
TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, Count(Posts.Id) AS Value
FROM Posts`
if options.BotsOnly {
query += " INNER JOIN Bots ON Posts.UserId = Bots.Userid"
}
if options.TeamId != "" {
query += " INNER JOIN Channels ON Posts.ChannelId = Channels.Id AND Channels.TeamId = ? AND"
args = []any{options.TeamId}
} else {
query += " WHERE"
}
query += ` Posts.CreateAt <= ?
AND Posts.CreateAt >= ?
GROUP BY DATE(TO_TIMESTAMP(Posts.CreateAt / 1000))
ORDER BY Name DESC
LIMIT 30`
}
end := utils.MillisFromTime(utils.EndOfDay(utils.Yesterday()))
start := utils.MillisFromTime(utils.StartOfDay(utils.Yesterday().AddDate(0, 0, -31)))
if options.YesterdayOnly {
@@ -2350,16 +2397,39 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun
args = append(args, end, start)
rows := model.AnalyticsRows{}
err := s.GetReplica().Select(
&rows,
query,
args...)
err := s.GetReplica().Select(&rows, query, args...)
if err != nil {
return nil, errors.Wrapf(err, "failed to find Posts with teamId=%s", options.TeamId)
}
return rows, nil
}
func (s *SqlPostStore) countByTeam(teamID string) (int64, error) {
query := s.getQueryBuilder().
Select("COALESCE(SUM(num), 0) AS total").
From("posts_by_team_day")
if teamID != "" {
query = query.Where(sq.Eq{"teamid": teamID})
}
var v int64
err := s.GetReplica().GetBuilder(&v, query)
if err != nil {
return 0, fmt.Errorf("failed to count Posts by team: %w, teamID: %s", err, teamID)
}
return v, nil
}
func (s *SqlPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
if s.DriverName() == model.DatabaseDriverPostgres {
return s.countByTeam(teamID)
}
return s.AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID})
}
func (s *SqlPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) {
query := s.getQueryBuilder().
Select("COUNT(*) AS Value").
@@ -2553,7 +2623,7 @@ func (s *SqlPostStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolic
func (s *SqlPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
var query string
if s.DriverName() == "postgres" {
if s.DriverName() == model.DatabaseDriverPostgres {
query = "DELETE from Posts WHERE Id = any (array (SELECT Id FROM Posts WHERE CreateAt < ? LIMIT ?))"
} else {
query = "DELETE from Posts WHERE CreateAt < ? LIMIT ?"
@@ -3295,3 +3365,22 @@ func (s *SqlPostStore) GetPostReminderMetadata(postID string) (*store.PostRemind
return meta, nil
}
func (s *SqlPostStore) RefreshPostStats() error {
if s.DriverName() == model.DatabaseDriverPostgres {
// CONCURRENTLY is not used deliberately because as per Postgres docs,
// not using CONCURRENTLY takes less resources and completes faster
// at the expense of locking the mat view. Since viewing admin console
// is not a very frequent activity, we accept the tradeoff to let the
// refresh happen as fast as possible.
if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW posts_by_team_day"); err != nil {
return errors.Wrap(err, "error refreshing materialized view posts_by_team_day")
}
if _, err := s.GetMaster().Exec("REFRESH MATERIALIZED VIEW bot_posts_by_team_day"); err != nil {
return errors.Wrap(err, "error refreshing materialized view bot_posts_by_team_day")
}
}
return nil
}

Просмотреть файл

@@ -1696,7 +1696,6 @@ func (us SqlUserStore) performSearch(query sq.SelectBuilder, term string, option
}
func (us SqlUserStore) AnalyticsGetInactiveUsersCount() (int64, error) {
var count int64
query := us.getQueryBuilder().
Select("COUNT(Id)").
From("Users")
@@ -1712,11 +1711,9 @@ func (us SqlUserStore) AnalyticsGetInactiveUsersCount() (int64, error) {
sq.Gt{"Users.DeleteAt": 0},
})
}
queryStr, args, err := query.ToSql()
if err != nil {
return int64(0), errors.Wrap(err, "failed to create a SQL query to count inactive users")
}
err = us.GetReplica().Get(&count, queryStr, args...)
var count int64
err := us.GetReplica().GetBuilder(&count, query)
if err != nil {
return int64(0), errors.Wrap(err, "failed to count inactive Users")
}

Просмотреть файл

@@ -260,6 +260,8 @@ type ChannelStore interface {
CountUrgentPostsAfter(channelID string, timestamp int64, excludedUserID string) (int, error)
IncrementMentionCount(channelID string, userIDs []string, isRoot, isUrgent bool) error
AnalyticsTypeCount(teamID string, channelType model.ChannelType) (int64, error)
AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error)
AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error)
GetMembersForUser(teamID string, userID string) (model.ChannelMembers, error)
GetTeamMembersForChannel(channelID string) ([]string, error)
GetMembersForUserWithPagination(userID string, page, perPage int) (model.ChannelMembersWithTeamData, error)
@@ -275,7 +277,6 @@ type ChannelStore interface {
GetMembersByIds(channelID string, userIds []string) (model.ChannelMembers, error)
GetMembersByChannelIds(channelIds []string, userID string) (model.ChannelMembers, error)
GetMembersInfoByChannelIds(channelIDs []string) (map[string][]*model.User, error)
AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error)
GetChannelUnread(channelID, userID string) (*model.ChannelUnread, error)
GetChannelsWithUnreadsAndWithMentions(ctx context.Context, channelIDs []string, userID string, userNotifyProps model.StringMap) ([]string, []string, map[string]int64, error)
ClearCaches()
@@ -387,6 +388,7 @@ type PostStore interface {
AnalyticsUserCountsWithPostsByDay(teamID string) (model.AnalyticsRows, error)
AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error)
AnalyticsPostCount(options *model.PostCountOptions) (int64, error)
AnalyticsPostCountByTeam(teamID string) (int64, error)
ClearCaches()
InvalidateLastPostTimeCache(channelID string)
GetPostsCreatedAt(channelID string, timestamp int64) ([]*model.Post, error)
@@ -411,6 +413,8 @@ type PostStore interface {
GetPostReminderMetadata(postID string) (*PostReminderMetadata, error)
// GetNthRecentPostTime returns the CreateAt time of the nth most recent post.
GetNthRecentPostTime(n int64) (int64, error)
// RefreshPostStats refreshes the various materialized views for admin console post stats.
RefreshPostStats() error
}
type UserStore interface {
@@ -743,6 +747,8 @@ type FileInfoStore interface {
GetStorageUsage(allowFromCache, includeDeleted bool) (int64, error)
// GetUptoNSizeFileTime returns the CreateAt time of the last accessible file with a running-total size upto n bytes.
GetUptoNSizeFileTime(n int64) (int64, error)
// RefreshFileStats recomputes the fileinfo materialized views.
RefreshFileStats() error
}
type UploadSessionStore interface {

Просмотреть файл

@@ -4148,18 +4148,34 @@ func testChannelStoreGetMoreChannels(t *testing.T, rctx request.CTX, ss store.St
count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen)
require.NoError(t, err)
require.EqualValues(t, 4, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
require.EqualValues(t, 4, counts[model.ChannelTypeOpen])
})
t.Run("verify analytics for private channels", func(t *testing.T) {
count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate)
require.NoError(t, err)
require.EqualValues(t, 2, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
require.EqualValues(t, 2, counts[model.ChannelTypePrivate])
})
t.Run("verify analytics for all channels", func(t *testing.T) {
count, err := ss.Channel().AnalyticsTypeCount(teamID, "")
require.NoError(t, err)
require.EqualValues(t, 6, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
total := int64(0)
for _, count := range counts {
total += count
}
require.EqualValues(t, 6, total)
})
}
@@ -4338,12 +4354,20 @@ func testChannelStoreGetPublicChannelsForTeam(t *testing.T, rctx request.CTX, ss
count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypeOpen)
require.NoError(t, err)
require.EqualValues(t, 3, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
require.EqualValues(t, 3, counts[model.ChannelTypeOpen])
})
t.Run("verify analytics for private channels", func(t *testing.T) {
count, err := ss.Channel().AnalyticsTypeCount(teamID, model.ChannelTypePrivate)
require.NoError(t, err)
require.EqualValues(t, 1, count)
counts, err := ss.Channel().AnalyticsCountAll(teamID)
require.NoError(t, err)
require.EqualValues(t, 1, counts[model.ChannelTypePrivate])
})
}

Просмотреть файл

@@ -815,12 +815,15 @@ func testFileInfoStoreCountAll(t *testing.T, rctx request.CTX, ss store.Store) {
})
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
count, err := ss.FileInfo().CountAll()
require.NoError(t, err)
require.Equal(t, int64(3), count)
_, err = ss.FileInfo().DeleteForPost(rctx, f1.PostId)
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
count, err = ss.FileInfo().CountAll()
require.NoError(t, err)
require.Equal(t, int64(2), count)
@@ -830,6 +833,7 @@ func testFileInfoGetStorageUsage(t *testing.T, rctx request.CTX, ss store.Store)
_, err := ss.FileInfo().PermanentDeleteBatch(rctx, model.GetMillis(), 100000)
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
usage, err := ss.FileInfo().GetStorageUsage(false, false)
require.NoError(t, err)
require.Equal(t, int64(0), usage)
@@ -857,12 +861,14 @@ func testFileInfoGetStorageUsage(t *testing.T, rctx request.CTX, ss store.Store)
})
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
usage, err = ss.FileInfo().GetStorageUsage(false, false)
require.NoError(t, err)
require.Equal(t, int64(30), usage)
_, err = ss.FileInfo().DeleteForPost(rctx, f1.PostId)
require.NoError(t, err)
require.NoError(t, ss.FileInfo().RefreshFileStats())
usage, err = ss.FileInfo().GetStorageUsage(false, false)
require.NoError(t, err)
require.Equal(t, int64(20), usage)

Просмотреть файл

@@ -20,6 +20,36 @@ type ChannelStore struct {
mock.Mock
}
// AnalyticsCountAll provides a mock function with given fields: teamID
func (_m *ChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) {
ret := _m.Called(teamID)
if len(ret) == 0 {
panic("no return value specified for AnalyticsCountAll")
}
var r0 map[model.ChannelType]int64
var r1 error
if rf, ok := ret.Get(0).(func(string) (map[model.ChannelType]int64, error)); ok {
return rf(teamID)
}
if rf, ok := ret.Get(0).(func(string) map[model.ChannelType]int64); ok {
r0 = rf(teamID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(map[model.ChannelType]int64)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(teamID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// AnalyticsDeletedTypeCount provides a mock function with given fields: teamID, channelType
func (_m *ChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) {
ret := _m.Called(teamID, channelType)

Просмотреть файл

@@ -505,6 +505,24 @@ func (_m *FileInfoStore) PermanentDeleteForPost(rctx request.CTX, postID string)
return r0
}
// RefreshFileStats provides a mock function with given fields:
func (_m *FileInfoStore) RefreshFileStats() error {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for RefreshFileStats")
}
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// RestoreForPostByIds provides a mock function with given fields: rctx, postId, fileIDs
func (_m *FileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error {
ret := _m.Called(rctx, postId, fileIDs)

Просмотреть файл

@@ -48,6 +48,34 @@ func (_m *PostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64,
return r0, r1
}
// AnalyticsPostCountByTeam provides a mock function with given fields: teamID
func (_m *PostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
ret := _m.Called(teamID)
if len(ret) == 0 {
panic("no return value specified for AnalyticsPostCountByTeam")
}
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(string) (int64, error)); ok {
return rf(teamID)
}
if rf, ok := ret.Get(0).(func(string) int64); ok {
r0 = rf(teamID)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(teamID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// AnalyticsPostCountsByDay provides a mock function with given fields: options
func (_m *PostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
ret := _m.Called(options)
@@ -1163,6 +1191,24 @@ func (_m *PostStore) PermanentDeleteByUser(rctx request.CTX, userID string) erro
return r0
}
// RefreshPostStats provides a mock function with given fields:
func (_m *PostStore) RefreshPostStats() error {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for RefreshPostStats")
}
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// Save provides a mock function with given fields: rctx, post
func (_m *PostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) {
ret := _m.Called(rctx, post)

Просмотреть файл

@@ -2895,6 +2895,8 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) {
_, nErr = ss.Post().Save(rctx, b1a)
require.NoError(t, nErr)
require.NoError(t, ss.Post().RefreshPostStats())
time.Sleep(1 * time.Second)
// summary of posts
@@ -2907,6 +2909,8 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) {
require.NoError(t, err)
assert.Equal(t, float64(3), r1[0].Value)
assert.Equal(t, float64(3), r1[1].Value)
assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name)
assert.Equal(t, utils.Yesterday().Add(-48*time.Hour).Format("2006-01-02"), r1[1].Name)
// last 31 days, bots only
postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: true, YesterdayOnly: false}
@@ -2914,18 +2918,22 @@ func testPostCountsByDay(t *testing.T, rctx request.CTX, ss store.Store) {
require.NoError(t, err)
assert.Equal(t, float64(1), r1[0].Value)
assert.Equal(t, float64(1), r1[1].Value)
assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name)
assert.Equal(t, utils.Yesterday().Add(-48*time.Hour).Format("2006-01-02"), r1[1].Name)
// yesterday only, all users (including bots)
postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: false, YesterdayOnly: true}
r1, err = ss.Post().AnalyticsPostCountsByDay(postCountsOptions)
require.NoError(t, err)
assert.Equal(t, float64(3), r1[0].Value)
assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name)
// yesterday only, bots only
postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: true, YesterdayOnly: true}
r1, err = ss.Post().AnalyticsPostCountsByDay(postCountsOptions)
require.NoError(t, err)
assert.Equal(t, float64(1), r1[0].Value)
assert.Equal(t, utils.Yesterday().Format("2006-01-02"), r1[0].Name)
}
func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) {
@@ -3029,6 +3037,8 @@ func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) {
_, nErr = ss.Post().Save(rctx, p7)
require.NoError(t, nErr)
require.NoError(t, ss.Post().RefreshPostStats())
// total across all teams
c, err := ss.Post().AnalyticsPostCount(&model.PostCountOptions{})
require.NoError(t, err)
@@ -3039,6 +3049,10 @@ func testPostCounts(t *testing.T, rctx request.CTX, ss store.Store) {
require.NoError(t, err)
assert.Equal(t, int64(7), c)
c, err = ss.Post().AnalyticsPostCountByTeam(t1.Id)
require.NoError(t, err)
assert.Equal(t, int64(7), c)
// with files
c, err = ss.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: t1.Id, MustHaveFile: true})
require.NoError(t, err)

Просмотреть файл

@@ -671,6 +671,22 @@ func (s *TimerLayerBotStore) Update(bot *model.Bot) (*model.Bot, error) {
return result, err
}
func (s *TimerLayerChannelStore) AnalyticsCountAll(teamID string) (map[model.ChannelType]int64, error) {
start := time.Now()
result, err := s.ChannelStore.AnalyticsCountAll(teamID)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("ChannelStore.AnalyticsCountAll", success, elapsed)
}
return result, err
}
func (s *TimerLayerChannelStore) AnalyticsDeletedTypeCount(teamID string, channelType model.ChannelType) (int64, error) {
start := time.Now()
@@ -3859,6 +3875,22 @@ func (s *TimerLayerFileInfoStore) PermanentDeleteForPost(rctx request.CTX, postI
return err
}
func (s *TimerLayerFileInfoStore) RefreshFileStats() error {
start := time.Now()
err := s.FileInfoStore.RefreshFileStats()
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.RefreshFileStats", success, elapsed)
}
return err
}
func (s *TimerLayerFileInfoStore) RestoreForPostByIds(rctx request.CTX, postId string, fileIDs []string) error {
start := time.Now()
@@ -5811,6 +5843,22 @@ func (s *TimerLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions
return result, err
}
func (s *TimerLayerPostStore) AnalyticsPostCountByTeam(teamID string) (int64, error) {
start := time.Now()
result, err := s.PostStore.AnalyticsPostCountByTeam(teamID)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("PostStore.AnalyticsPostCountByTeam", success, elapsed)
}
return result, err
}
func (s *TimerLayerPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) {
start := time.Now()
@@ -6465,6 +6513,22 @@ func (s *TimerLayerPostStore) PermanentDeleteByUser(rctx request.CTX, userID str
return err
}
func (s *TimerLayerPostStore) RefreshPostStats() error {
start := time.Now()
err := s.PostStore.RefreshPostStats()
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("PostStore.RefreshPostStats", success, elapsed)
}
return err
}
func (s *TimerLayerPostStore) Save(rctx request.CTX, post *model.Post) (*model.Post, error) {
start := time.Now()

Просмотреть файл

@@ -5218,10 +5218,18 @@
"id": "app.file_info.get_by_post_id.app_error",
"translation": "Failed to find files for post."
},
{
"id": "app.file_info.get_count.app_error",
"translation": "Failed to get count of all files."
},
{
"id": "app.file_info.get_for_post.app_error",
"translation": "Unable to get the file info for the post."
},
{
"id": "app.file_info.get_storage_usage.app_error",
"translation": "Failed to get storage usage of all files."
},
{
"id": "app.file_info.get_with_options.app_error",
"translation": "Unable to get the file info with options"

Просмотреть файл

@@ -39,7 +39,7 @@ const (
JobTypeS3PathMigration = "s3_path_migration"
JobTypeCleanupDesktopTokens = "cleanup_desktop_tokens"
JobTypeDeleteEmptyDraftsMigration = "delete_empty_drafts_migration"
JobTypeRefreshPostStats = "refresh_post_stats"
JobTypeRefreshMaterializedViews = "refresh_materialized_views"
JobTypeDeleteOrphanDraftsMigration = "delete_orphan_drafts_migration"
JobTypeExportUsersToCSV = "export_users_to_csv"
JobTypeDeleteDmsPreferencesMigration = "delete_dms_preferences_migration"
@@ -75,7 +75,7 @@ var AllJobTypes = [...]string{
JobTypeLastAccessiblePost,
JobTypeLastAccessibleFile,
JobTypeCleanupDesktopTokens,
JobTypeRefreshPostStats,
JobTypeRefreshMaterializedViews,
JobTypeMobileSessionMetadata,
}

Просмотреть файл

@@ -19,23 +19,6 @@ export function formatChannelDoughtnutData(totalPublic: any, totalPrivate: any)
return channelTypeData;
}
export function formatPostDoughtnutData(filePosts: any, hashtagPosts: any, totalPosts: any) {
const postTypeData = {
labels: [
Utils.localizeMessage({id: 'analytics.system.totalFilePosts', defaultMessage: 'Posts with Files'}),
Utils.localizeMessage({id: 'analytics.system.totalHashtagPosts', defaultMessage: 'Posts with Hashtags'}),
Utils.localizeMessage({id: 'analytics.system.textPosts', defaultMessage: 'Posts with Text-only'}),
],
datasets: [{
data: [filePosts, hashtagPosts, (totalPosts - filePosts - hashtagPosts)],
backgroundColor: ['#46BFBD', '#F7464A', '#FDB45C'],
hoverBackgroundColor: ['#5AD3D1', '#FF5A5E', '#FFC870'],
}],
};
return postTypeData;
}
export function formatPostsPerDayData(labels: string[], data: any) {
const chartData = {
labels: [] as string[],

Просмотреть файл

@@ -41,4 +41,19 @@ describe('components/analytics/statistic_count.tsx', () => {
expect(wrapper).toMatchSnapshot();
});
test('should apply formatter function when provided', () => {
const mockFormatter = (value: number) => `${value}%`;
const wrapper = shallow(
<StatisticCount
title='Test'
icon='test-icon'
count={42}
id='test-stat'
formatter={mockFormatter}
/>,
);
expect(wrapper.find('[data-testid="test-stat"]').text()).toBe('42%');
});
});

Просмотреть файл

@@ -13,6 +13,7 @@ type Props = {
id?: string;
children?: React.ReactNode;
status?: 'warning' | 'error';
formatter?: (value: number) => string;
}
const StatisticCount = ({
@@ -22,6 +23,7 @@ const StatisticCount = ({
id,
children,
status,
formatter,
}: Props) => {
const loading = (
<FormattedMessage
@@ -30,6 +32,9 @@ const StatisticCount = ({
/>
);
const result = formatter ? formatter(count ?? 0) : count;
const displayValue = typeof count === 'undefined' || isNaN(count) ? loading : result;
return (
<div className='grid-statistics__card'>
<div
@@ -57,7 +62,7 @@ const StatisticCount = ({
'team_statistics--error': status === 'error',
})}
>
{typeof count === 'undefined' || isNaN(count) ? loading : count}
{displayValue}
</div>
</div>
{children}

Просмотреть файл

@@ -0,0 +1,49 @@
details {
padding: 12px;
border: 1px solid rgba(var(--center-channel-color-rgb), 0.16);
border-radius: 4px;
background-color: var(--center-channel-bg);
}
summary {
position: relative;
padding: 12px 12px 12px 28px;
margin: -12px;
cursor: pointer;
font-weight: bold;
transition: background 0.15s ease-in-out;
&:hover {
background: rgba(var(--center-channel-color-rgb), 0.04);
}
&::marker {
display: none;
}
&::before {
position: absolute;
top: 16px;
left: 12px;
content: '';
font-size: 10px;
transform: rotate(0deg);
transition: transform 0.15s ease-in-out;
}
}
details[open] {
summary {
border-bottom: 1px solid rgba(var(--center-channel-color-rgb), 0.16);
margin-bottom: 0;
&::before {
transform: rotate(90deg);
}
}
.row:last-child .total-count {
margin-bottom: 0;
}
}

Просмотреть файл

@@ -1,6 +1,7 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
import {fireEvent} from '@testing-library/react';
import React from 'react';
import {FormattedMessage} from 'react-intl';
@@ -59,7 +60,7 @@ describe('components/analytics/system_analytics/system_analytics.tsx', () => {
expect(screen.queryByTestId('totalPostsLineChart')).not.toBeInTheDocument();
});
test('system data', () => {
test('system data', async () => {
const state = {
...initialState,
entities: {
@@ -90,6 +91,11 @@ describe('components/analytics/system_analytics/system_analytics.tsx', () => {
renderWithContext(<SystemAnalytics {...baseProps}/>, state, {useMockedStore: true});
const detailsElement = screen.getByText('Load Advanced Statistics');
fireEvent.click(detailsElement);
await screen.findByTestId('totalPostsLineChart');
expect(screen.getByTestId('totalPosts')).toHaveTextContent('45');
expect(screen.getByTestId('totalPostsLineChart')).toBeInTheDocument();
});
@@ -233,6 +239,11 @@ describe('components/analytics/system_analytics/system_analytics.tsx', () => {
await new Promise(process.nextTick);
const detailsElement = screen.getByText('Load Advanced Statistics');
fireEvent.click(detailsElement);
await screen.findByTestId('totalPostsLineChart');
expect(screen.getByTestId('totalPosts')).toHaveTextContent('45');
expect(screen.getByTestId('totalPostsLineChart')).toBeInTheDocument();
expect(screen.getByTestId('com.mattermost.playbooks.playbook_count')).toHaveTextContent('45');

Просмотреть файл

@@ -8,6 +8,8 @@ import type {AnalyticsRow, PluginAnalyticsRow, IndexedPluginAnalyticsRow, Analyt
import {AnalyticsVisualizationType} from '@mattermost/types/admin';
import type {ClientLicense} from '@mattermost/types/config';
import {getFormattedFileSize} from 'mattermost-redux/utils/file_utils';
import * as AdminActions from 'actions/admin_actions.jsx';
import ActivatedUserCard from 'components/analytics/activated_users_card';
@@ -16,6 +18,8 @@ import AdminHeader from 'components/widgets/admin_console/admin_header';
import Constants from 'utils/constants';
import './analytics.scss';
import type {GlobalState} from 'types/store';
import DoughnutChart from '../doughnut_chart';
@@ -23,7 +27,6 @@ import {
formatPostsPerDayData,
formatUsersWithPostsPerDayData,
formatChannelDoughtnutData,
formatPostDoughtnutData,
synchronizeChartLabels,
} from '../format';
import LineChart from '../line_chart';
@@ -40,6 +43,7 @@ type Props = {
type State = {
pluginSiteStats: Record<string, PluginAnalyticsRow>;
lineChartsDataLoaded: boolean;
}
const messages = defineMessages({
@@ -59,6 +63,8 @@ const messages = defineMessages({
totalChannels: {id: 'analytics.system.totalChannels', defaultMessage: 'Total Channels'},
dailyActiveUsers: {id: 'analytics.system.dailyActiveUsers', defaultMessage: 'Daily Active Users'},
monthlyActiveUsers: {id: 'analytics.system.monthlyActiveUsers', defaultMessage: 'Monthly Active Users'},
totalFiles: {id: 'analytics.system.totalFiles', defaultMessage: 'Total Files'},
totalFilesSize: {id: 'analytics.system.totalFilesSize', defaultMessage: 'Total Files Size'},
});
export const searchableStrings = [
@@ -78,18 +84,18 @@ export const searchableStrings = [
messages.totalChannels,
messages.dailyActiveUsers,
messages.monthlyActiveUsers,
messages.totalFiles,
messages.totalFilesSize,
];
export default class SystemAnalytics extends React.PureComponent<Props, State> {
state = {
pluginSiteStats: {} as Record<string, PluginAnalyticsRow>,
lineChartsDataLoaded: false,
};
public async componentDidMount() {
AdminActions.getStandardAnalytics();
AdminActions.getPostsPerDayAnalytics();
AdminActions.getBotPostsPerDayAnalytics();
AdminActions.getUsersPerDayAnalytics();
if (this.props.isLicensed) {
AdminActions.getAdvancedAnalytics();
@@ -97,6 +103,24 @@ export default class SystemAnalytics extends React.PureComponent<Props, State> {
this.fetchPluginStats();
}
private loadLineChartData = async () => {
await Promise.allSettled([
AdminActions.getPostsPerDayAnalytics(),
AdminActions.getBotPostsPerDayAnalytics(),
AdminActions.getUsersPerDayAnalytics(),
]);
this.setState({lineChartsDataLoaded: true});
};
private handleLineChartsToggle = (e: React.MouseEvent<HTMLDetailsElement>) => {
const details = e.currentTarget;
const isExpanding = details.open;
if (isExpanding && !this.state.lineChartsDataLoaded) {
this.loadLineChartData();
}
};
// fetchPluginStats does a call for each one of the registered handlers,
// wait and set the data in the state
private async fetchPluginStats() {
@@ -223,6 +247,8 @@ export default class SystemAnalytics extends React.PureComponent<Props, State> {
let commandCount;
let incomingCount;
let outgoingCount;
let totalFiles;
let totalFilesSize;
if (this.props.isLicensed) {
sessionCount = (
<StatisticCount
@@ -262,6 +288,25 @@ export default class SystemAnalytics extends React.PureComponent<Props, State> {
/>
);
totalFiles = (
<StatisticCount
id='totalFiles'
title={<FormattedMessage {...messages.totalFiles}/>}
icon='fa-files-o'
count={this.getStatValue(stats[StatTypes.TOTAL_FILE_COUNT])}
/>
);
totalFilesSize = (
<StatisticCount
id='totalFilesSize'
title={<FormattedMessage {...messages.totalFilesSize}/>}
icon='fa-files-o'
count={this.getStatValue(stats[StatTypes.TOTAL_FILE_SIZE])}
formatter={getFormattedFileSize}
/>
);
advancedStats = (
<>
<StatisticCount
@@ -289,20 +334,6 @@ export default class SystemAnalytics extends React.PureComponent<Props, State> {
);
const channelTypeData = formatChannelDoughtnutData(stats[StatTypes.TOTAL_PUBLIC_CHANNELS], stats[StatTypes.TOTAL_PRIVATE_GROUPS]);
const postTypeData = formatPostDoughtnutData(stats[StatTypes.TOTAL_FILE_POSTS], stats[StatTypes.TOTAL_HASHTAG_POSTS], stats[StatTypes.TOTAL_POSTS]);
let postTypeGraph;
if (stats[StatTypes.TOTAL_POSTS] !== -1) {
postTypeGraph = (
<DoughnutChart
title={<FormattedMessage {...messages.postTypes}/>
}
data={postTypeData}
width={300}
height={225}
/>
);
}
advancedGraphs = (
<div className='row'>
@@ -313,7 +344,6 @@ export default class SystemAnalytics extends React.PureComponent<Props, State> {
width={300}
height={225}
/>
{postTypeGraph}
</div>
);
}
@@ -401,25 +431,33 @@ export default class SystemAnalytics extends React.PureComponent<Props, State> {
switch (stat.visualizationType) {
case AnalyticsVisualizationType.LineChart:
pluginLineCharts.push((
<LineChart
id={key}
<div
className='row'
key={'pluginstat.' + key}
title={stat.name}
data={stat.value}
width={740}
height={225}
/>
>
<LineChart
id={key}
title={stat.name}
data={stat.value}
width={740}
height={225}
/>
</div>
));
break;
case AnalyticsVisualizationType.DoughnutChart:
pluginDoughnutCharts.push((
<DoughnutChart
<div
className='row'
key={'pluginstat.' + key}
title={stat.name}
data={stat.value}
width={300}
height={225}
/>
>
<DoughnutChart
title={stat.name}
data={stat.value}
width={300}
height={225}
/>
</div>
));
break;
case AnalyticsVisualizationType.Count:
@@ -449,6 +487,8 @@ export default class SystemAnalytics extends React.PureComponent<Props, State> {
{commandCount}
{incomingCount}
{outgoingCount}
{totalFiles}
{totalFilesSize}
</>
);
} else if (!isLicensed) {
@@ -480,10 +520,23 @@ export default class SystemAnalytics extends React.PureComponent<Props, State> {
</div>
{advancedGraphs}
{pluginDoughnutCharts}
{postTotalGraph}
{botPostTotalGraph}
{activeUserGraph}
{pluginLineCharts}
<details
onToggle={this.handleLineChartsToggle}
data-testid='details-expander'
>
<summary>
<FormattedMessage
id='analytics.system.perDayStatistics'
defaultMessage='Load Advanced Statistics'
/>
</summary>
<>
{postTotalGraph}
{botPostTotalGraph}
{activeUserGraph}
</>
</details>
</div>
</div>
</div>

Просмотреть файл

@@ -2895,18 +2895,18 @@
"analytics.system.infoAndSkippedIntensiveQueries1": "Use data for only the chosen team. Exclude posts in direct message channels that are not tied to a team.",
"analytics.system.infoAndSkippedIntensiveQueries2": "To maximize performance, some statistics are disabled. You can <link>re-enable them in config.json</link>.",
"analytics.system.monthlyActiveUsers": "Monthly Active Users",
"analytics.system.perDayStatistics": "Load Advanced Statistics",
"analytics.system.postTypes": "Posts, Files and Hashtags",
"analytics.system.privateGroups": "Private Channels",
"analytics.system.publicChannels": "Public Channels",
"analytics.system.seatsPurchased": "Licensed Seats",
"analytics.system.skippedIntensiveQueries": "To maximize performance, some statistics are disabled. You can <link>re-enable them in config.json</link>.",
"analytics.system.textPosts": "Posts with Text-only",
"analytics.system.title": "System Statistics",
"analytics.system.totalBotPosts": "Total Posts from Bots",
"analytics.system.totalChannels": "Total Channels",
"analytics.system.totalCommands": "Total Commands",
"analytics.system.totalFilePosts": "Posts with Files",
"analytics.system.totalHashtagPosts": "Posts with Hashtags",
"analytics.system.totalFiles": "Total Files",
"analytics.system.totalFilesSize": "Total Files Size",
"analytics.system.totalIncomingWebhooks": "Incoming Webhooks",
"analytics.system.totalMasterDbConnections": "Master DB Conns",
"analytics.system.totalOutgoingWebhooks": "Outgoing Webhooks",

Просмотреть файл

@@ -10,8 +10,6 @@ export default keyMirror({
TOTAL_PRIVATE_GROUPS: null,
TOTAL_POSTS: null,
TOTAL_TEAMS: null,
TOTAL_FILE_POSTS: null,
TOTAL_HASHTAG_POSTS: null,
TOTAL_IHOOKS: null,
TOTAL_OHOOKS: null,
TOTAL_COMMANDS: null,
@@ -27,5 +25,7 @@ export default keyMirror({
DAILY_ACTIVE_USERS: null,
MONTHLY_ACTIVE_USERS: null,
REGISTERED_USERS: null,
TOTAL_FILE_COUNT: null,
TOTAL_FILE_SIZE: null,
});

Просмотреть файл

@@ -218,12 +218,6 @@ export function convertAnalyticsRowsToStats(data: AnalyticsRow[], name: string):
case 'monthly_active_users':
key = Stats.MONTHLY_ACTIVE_USERS;
break;
case 'file_post_count':
key = Stats.TOTAL_FILE_POSTS;
break;
case 'hashtag_post_count':
key = Stats.TOTAL_HASHTAG_POSTS;
break;
case 'incoming_webhook_count':
key = Stats.TOTAL_IHOOKS;
break;
@@ -239,6 +233,12 @@ export function convertAnalyticsRowsToStats(data: AnalyticsRow[], name: string):
case 'registered_users':
key = Stats.REGISTERED_USERS;
break;
case 'total_file_count':
key = Stats.TOTAL_FILE_COUNT;
break;
case 'total_file_size':
key = Stats.TOTAL_FILE_SIZE;
break;
}
if (key) {

Просмотреть файл

@@ -7,8 +7,7 @@ import {Client4} from 'mattermost-redux/client';
import {Files, General} from '../constants';
export function getFormattedFileSize(file: FileInfo): string {
const bytes = file.size;
export function getFormattedFileSize(bytes: number): string {
const fileSizes = [
['TB', 1024 * 1024 * 1024 * 1024],
['GB', 1024 * 1024 * 1024],

Просмотреть файл

@@ -824,8 +824,6 @@ export const StatTypes = keyMirror({
TOTAL_PRIVATE_GROUPS: null,
TOTAL_POSTS: null,
TOTAL_TEAMS: null,
TOTAL_FILE_POSTS: null,
TOTAL_HASHTAG_POSTS: null,
TOTAL_IHOOKS: null,
TOTAL_OHOOKS: null,
TOTAL_COMMANDS: null,
@@ -840,6 +838,8 @@ export const StatTypes = keyMirror({
TOTAL_READ_DB_CONNECTIONS: null,
DAILY_ACTIVE_USERS: null,
MONTHLY_ACTIVE_USERS: null,
TOTAL_FILE_COUNT: null,
TOTAL_FILE_SIZE: null,
});
export const SearchTypes = keyMirror({

Просмотреть файл

@@ -87,13 +87,13 @@ export type AnalyticsState = {
TOTAL_READ_DB_CONNECTIONS?: number;
DAILY_ACTIVE_USERS?: number;
MONTHLY_ACTIVE_USERS?: number;
TOTAL_FILE_POSTS?: number;
TOTAL_HASHTAG_POSTS?: number;
TOTAL_IHOOKS?: number;
TOTAL_OHOOKS?: number;
TOTAL_COMMANDS?: number;
TOTAL_SESSIONS?: number;
REGISTERED_USERS?: number;
TOTAL_FILE_COUNT?: number;
TOTAL_FILE_SIZE?: number;
}
export type ClusterInfo = {