From fd703a365bac5634fc85119ccd90ac9096728973 Mon Sep 17 00:00:00 2001 From: Vishal Date: Tue, 17 May 2022 17:00:40 +0530 Subject: [PATCH] [MM-43917] Cloud Freemium limits API: messages/posts (#20152) * WIP - Add api and app funcs * Add test cases * Add utils testcases * Exclude deleted posts * Add doc for func * Move api from cloud to usage * Allow api access to authenticated users * Change int to int64 * Fix lint issue * Simplify err check Co-authored-by: Ashish Bhate Co-authored-by: Mattermod Co-authored-by: Ashish Bhate --- api4/api.go | 5 ++ api4/cloud_test.go | 2 +- api4/usage.go | 32 +++++++++ api4/usage_test.go | 39 +++++++++++ api4/user_test.go | 4 +- app/analytics.go | 6 +- app/app_iface.go | 2 + app/import_functions_test.go | 10 +-- app/import_test.go | 2 +- app/migrations.go | 2 +- app/opentracing/opentracing_layer.go | 22 +++++++ app/product_notices.go | 2 +- app/usage.go | 21 ++++++ app/usage_test.go | 49 ++++++++++++++ model/client4.go | 19 ++++++ model/post.go | 8 +++ model/usage.go | 8 +++ .../bleveengine/indexer/indexing_job.go | 2 +- services/telemetry/telemetry.go | 2 +- services/telemetry/telemetry_test.go | 2 +- store/opentracinglayer/opentracinglayer.go | 4 +- store/retrylayer/retrylayer.go | 4 +- store/sqlstore/post_store.go | 14 ++-- store/store.go | 2 +- store/storetest/mocks/PostStore.go | 14 ++-- store/storetest/post_store.go | 23 +++++-- store/timerlayer/timerlayer.go | 4 +- utils/utils.go | 14 ++++ utils/utils_test.go | 66 +++++++++++++++++++ 29 files changed, 341 insertions(+), 43 deletions(-) create mode 100644 api4/usage.go create mode 100644 api4/usage_test.go create mode 100644 app/usage.go create mode 100644 app/usage_test.go create mode 100644 model/usage.go diff --git a/api4/api.go b/api4/api.go index 9f38eac19b..28114af4fe 100644 --- a/api4/api.go +++ b/api4/api.go @@ -137,6 +137,8 @@ type Routes struct { InsightsForTeam *mux.Router // 'api/v4/teams/{team_id:[A-Za-z0-9]+}/top' InsightsForUser *mux.Router // 'api/v4/users/me/top' + + Usage *mux.Router // 'api/v4/usage' } type API struct { @@ -261,6 +263,8 @@ func Init(srv *app.Server) (*API, error) { api.BaseRoutes.InsightsForTeam = api.BaseRoutes.Team.PathPrefix("/top").Subrouter() api.BaseRoutes.InsightsForUser = api.BaseRoutes.Users.PathPrefix("/me/top").Subrouter() + api.BaseRoutes.Usage = api.BaseRoutes.APIRoot.PathPrefix("/usage").Subrouter() + api.InitUser() api.InitBot() api.InitTeam() @@ -303,6 +307,7 @@ func Init(srv *app.Server) (*API, error) { api.InitPermissions() api.InitExport() api.InitInsights() + api.InitUsage() if err := api.InitGraphQL(); err != nil { return nil, err } diff --git a/api4/cloud_test.go b/api4/cloud_test.go index 628230bbc8..753a0e08ec 100644 --- a/api4/cloud_test.go +++ b/api4/cloud_test.go @@ -9,11 +9,11 @@ import ( "os" "testing" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/mattermost/mattermost-server/v6/einterfaces/mocks" "github.com/mattermost/mattermost-server/v6/model" - "github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock" ) func Test_getCloudLimits(t *testing.T) { diff --git a/api4/usage.go b/api4/usage.go new file mode 100644 index 0000000000..475812538e --- /dev/null +++ b/api4/usage.go @@ -0,0 +1,32 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package api4 + +import ( + "encoding/json" + "net/http" + + "github.com/mattermost/mattermost-server/v6/model" +) + +func (api *API) InitUsage() { + // GET /api/v4/usage/posts + api.BaseRoutes.Usage.Handle("/posts", api.APISessionRequired(getPostsUsage)).Methods("GET") +} + +func getPostsUsage(c *Context, w http.ResponseWriter, r *http.Request) { + count, appErr := c.App.GetPostsUsage() + if appErr != nil { + c.Err = model.NewAppError("Api4.getPostsUsage", "app.post.analytics_posts_count.app_error", nil, appErr.Error(), http.StatusInternalServerError) + return + } + + json, err := json.Marshal(&model.PostsUsage{Count: count}) + if err != nil { + c.Err = model.NewAppError("Api4.getPostsUsage", "api.marshal_error", nil, err.Error(), http.StatusInternalServerError) + return + } + + w.Write(json) +} diff --git a/api4/usage_test.go b/api4/usage_test.go new file mode 100644 index 0000000000..c1f28f4fb0 --- /dev/null +++ b/api4/usage_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package api4 + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetPostsUsage(t *testing.T) { + t.Run("unauthenticated users can not access", func(t *testing.T) { + th := Setup(t) + defer th.TearDown() + + th.Client.Logout() + + usage, r, err := th.Client.GetPostsUsage() + assert.Error(t, err) + assert.Nil(t, usage) + assert.Equal(t, http.StatusUnauthorized, r.StatusCode) + }) + + t.Run("good request returns response", func(t *testing.T) { + // Following calls create a total of 15 posts + th := Setup(t).InitBasic() + defer th.TearDown() + th.CreatePost() + th.CreatePost() + + usage, r, err := th.Client.GetPostsUsage() + assert.NoError(t, err) + assert.Equal(t, http.StatusOK, r.StatusCode) + assert.NotNil(t, usage) + assert.Equal(t, int64(10), usage.Count) + }) +} diff --git a/api4/user_test.go b/api4/user_test.go index 75aec717e1..ef04227abe 100644 --- a/api4/user_test.go +++ b/api4/user_test.go @@ -2102,7 +2102,7 @@ func TestPermanentDeleteAllUsers(t *testing.T) { require.NoError(t, err) require.Greater(t, len(users), 0) - postCount, err := th.App.Srv().Store.Post().AnalyticsPostCount("", false, false) + postCount, err := th.App.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, err) require.Greater(t, postCount, int64(0)) @@ -2115,7 +2115,7 @@ func TestPermanentDeleteAllUsers(t *testing.T) { require.NoError(t, err) require.Len(t, users, 0) - postCount, err = th.App.Srv().Store.Post().AnalyticsPostCount("", false, false) + postCount, err = th.App.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, err) require.Equal(t, postCount, int64(0)) diff --git a/app/analytics.go b/app/analytics.go index 747a7b80c8..b7394db4a3 100644 --- a/app/analytics.go +++ b/app/analytics.go @@ -87,7 +87,7 @@ func (a *App) GetAnalytics(name string, teamID string) (model.AnalyticsRows, *mo if !skipIntensiveQueries { g.Go(func() error { var err error - if postsCount, err = a.Srv().Store.Post().AnalyticsPostCount(teamID, false, false); err != nil { + if postsCount, err = a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID}); err != nil { return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, err.Error(), http.StatusInternalServerError) } return nil @@ -269,7 +269,7 @@ func (a *App) GetAnalytics(name string, teamID string) (model.AnalyticsRows, *mo if !skipIntensiveQueries { g2.Go(func() error { var err error - if filesCount, err = a.Srv().Store.Post().AnalyticsPostCount(teamID, true, false); err != nil { + if filesCount, err = a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID, MustHaveFile: true}); err != nil { return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, err.Error(), http.StatusInternalServerError) } return nil @@ -277,7 +277,7 @@ func (a *App) GetAnalytics(name string, teamID string) (model.AnalyticsRows, *mo g2.Go(func() error { var err error - if hashtagsCount, err = a.Srv().Store.Post().AnalyticsPostCount(teamID, false, true); err != nil { + if hashtagsCount, err = a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamID, MustHaveHashtag: true}); err != nil { return model.NewAppError("GetAnalytics", "app.post.analytics_posts_count.app_error", nil, err.Error(), http.StatusInternalServerError) } return nil diff --git a/app/app_iface.go b/app/app_iface.go index 058cd5b640..bfcd595fb8 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -187,6 +187,8 @@ type AppIface interface { // To get the plugins environment when the plugins are disabled, manually acquire the plugins // lock instead. GetPluginsEnvironment() *plugin.Environment + // GetPostsUsage returns "rounded off" total posts count like returns 900 instead of 987 + GetPostsUsage() (int64, *model.AppError) // GetProductNotices is called from the frontend to fetch the product notices that are relevant to the caller GetProductNotices(c *request.Context, userID, teamID string, client model.NoticeClientType, clientVersion string, locale string) (model.NoticeMessages, *model.AppError) // GetPublicKey will return the actual public key saved in the `name` file. diff --git a/app/import_functions_test.go b/app/import_functions_test.go index 9c4c59bc48..0b029e4245 100644 --- a/app/import_functions_test.go +++ b/app/import_functions_test.go @@ -1964,7 +1964,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { require.Nil(t, err, "Failed to get user from database.") // Count the number of posts in the testing team. - initialPostCount, nErr := th.App.Srv().Store.Post().AnalyticsPostCount(team.Id, false, false) + initialPostCount, nErr := th.App.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team.Id}) require.NoError(t, nErr) // Try adding an invalid post in dry run mode. @@ -2470,7 +2470,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { require.Nil(t, err, "Failed to get channel from database.") // Count the number of posts in the team2. - initialPostCountForTeam2, nErr := th.App.Srv().Store.Post().AnalyticsPostCount(team2.Id, false, false) + initialPostCountForTeam2, nErr := th.App.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team2.Id}) require.NoError(t, nErr) // Try adding two valid posts in apply mode. @@ -2576,7 +2576,7 @@ func TestImportImportPost(t *testing.T) { require.Nil(t, appErr, "Failed to get user from database.") // Count the number of posts in the testing team. - initialPostCount, nErr := th.App.Srv().Store.Post().AnalyticsPostCount(team.Id, false, false) + initialPostCount, nErr := th.App.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: team.Id}) require.NoError(t, nErr) time := model.GetMillis() @@ -3283,7 +3283,7 @@ func TestImportImportDirectPost(t *testing.T) { directChannel = channel // Get the number of posts in the system. - result, err := th.App.Srv().Store.Post().AnalyticsPostCount("", false, false) + result, err := th.App.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, err) initialPostCount := result initialDate := model.GetMillis() @@ -3644,7 +3644,7 @@ func TestImportImportDirectPost(t *testing.T) { groupChannel = channel // Get the number of posts in the system. - result, nErr := th.App.Srv().Store.Post().AnalyticsPostCount("", false, false) + result, nErr := th.App.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, nErr) initialPostCount = result diff --git a/app/import_test.go b/app/import_test.go index 356d69aa5d..3286f3f03a 100644 --- a/app/import_test.go +++ b/app/import_test.go @@ -68,7 +68,7 @@ func checkNoError(t *testing.T, err *model.AppError) { } func AssertAllPostsCount(t *testing.T, a *App, initialCount int64, change int64, teamName string) { - result, err := a.Srv().Store.Post().AnalyticsPostCount(teamName, false, false) + result, err := a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: teamName}) require.NoError(t, err) require.Equal(t, initialCount+change, result, "Did not find the expected number of posts.") } diff --git a/app/migrations.go b/app/migrations.go index 680687eb13..ac188958b3 100644 --- a/app/migrations.go +++ b/app/migrations.go @@ -464,7 +464,7 @@ func (s *Server) doFirstAdminSetupCompleteMigration() { } // if there are teams, then if this isn't a new installation, there should be posts - postCount, err := s.Store.Post().AnalyticsPostCount("", false, false) + postCount, err := s.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}) if err != nil || postCount < existingInstallationPostsThreshold { return } diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index dd5f8d033c..d87942d047 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -7832,6 +7832,28 @@ func (a *OpenTracingAppLayer) GetPostsSince(options model.GetPostsSinceOptions) return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) GetPostsUsage() (int64, *model.AppError) { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetPostsUsage") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0, resultVar1 := a.app.GetPostsUsage() + + if resultVar1 != nil { + span.LogFields(spanlog.Error(resultVar1)) + ext.Error.Set(span, true) + } + + return resultVar0, resultVar1 +} + func (a *OpenTracingAppLayer) GetPreferenceByCategoryAndNameForUser(userID string, category string, preferenceName string) (*model.Preference, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetPreferenceByCategoryAndNameForUser") diff --git a/app/product_notices.go b/app/product_notices.go index 515e1f6b79..a3692db342 100644 --- a/app/product_notices.go +++ b/app/product_notices.go @@ -343,7 +343,7 @@ func (a *App) UpdateProductNotices() *model.AppError { skip := *a.Config().AnnouncementSettings.NoticesSkipCache mlog.Debug("Will fetch notices from", mlog.String("url", url), mlog.Bool("skip_cache", skip)) var err error - a.ch.cachedPostCount, err = a.Srv().Store.Post().AnalyticsPostCount("", false, false) + a.ch.cachedPostCount, err = a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{}) if err != nil { mlog.Warn("Failed to fetch post count", mlog.String("error", err.Error())) } diff --git a/app/usage.go b/app/usage.go new file mode 100644 index 0000000000..ee1985a589 --- /dev/null +++ b/app/usage.go @@ -0,0 +1,21 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package app + +import ( + "net/http" + + "github.com/mattermost/mattermost-server/v6/model" + "github.com/mattermost/mattermost-server/v6/utils" +) + +// GetPostsUsage returns "rounded off" total posts count like returns 900 instead of 987 +func (a *App) GetPostsUsage() (int64, *model.AppError) { + count, err := a.Srv().Store.Post().AnalyticsPostCount(&model.PostCountOptions{ExcludeDeleted: true}) + if err != nil { + return 0, model.NewAppError("GetPostsUsage", "app.post.analytics_posts_count.app_error", nil, err.Error(), http.StatusInternalServerError) + } + + return utils.RoundOffToZeroes(float64(count)), nil +} diff --git a/app/usage_test.go b/app/usage_test.go new file mode 100644 index 0000000000..54f8bd59b9 --- /dev/null +++ b/app/usage_test.go @@ -0,0 +1,49 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package app + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "github.com/mattermost/mattermost-server/v6/store/storetest/mocks" +) + +func TestGetPostsUsage(t *testing.T) { + t.Run("returns error when AnalyticsPostCount fails", func(t *testing.T) { + th := SetupWithStoreMock(t) + defer th.TearDown() + + errMsg := "Test posts count error" + + mockStore := th.App.Srv().Store.(*mocks.Store) + mockPostStore := mocks.PostStore{} + mockPostStore.On("AnalyticsPostCount", mock.Anything).Return(int64(0), errors.New(errMsg)) + mockStore.On("Post").Return(&mockPostStore) + + usage, appErr := th.App.GetPostsUsage() + assert.Zero(t, usage) + assert.ErrorContains(t, appErr, errMsg) + }) + + t.Run("returns rounded off count when AnalyticsPostCount returns valid count", func(t *testing.T) { + th := SetupWithStoreMock(t) + defer th.TearDown() + + var mockCount int64 = 4321 + var expected int64 = 4000 + + mockStore := th.App.Srv().Store.(*mocks.Store) + mockPostStore := mocks.PostStore{} + mockPostStore.On("AnalyticsPostCount", mock.Anything).Return(mockCount, nil) + mockStore.On("Post").Return(&mockPostStore) + + count, appErr := th.App.GetPostsUsage() + assert.Nil(t, appErr) + assert.Equal(t, expected, count) + }) +} diff --git a/model/client4.go b/model/client4.go index fec114e173..04f499172b 100644 --- a/model/client4.go +++ b/model/client4.go @@ -324,6 +324,10 @@ func (c *Client4) cloudRoute() string { return "/cloud" } +func (c *Client4) usageRoute() string { + return "/usage" +} + func (c *Client4) testEmailRoute() string { return "/email/test" } @@ -8078,3 +8082,18 @@ func (c *Client4) GetAppliedSchemaMigrations() ([]AppliedMigration, *Response, e } return list, BuildResponse(r), nil } + +// Usage Section + +// GetPostsUsage returns rounded off total usage of posts for the instance +func (c *Client4) GetPostsUsage() (*PostsUsage, *Response, error) { + r, err := c.DoAPIGet(c.usageRoute()+"/posts", "") + if err != nil { + return nil, BuildResponse(r), err + } + defer closeBody(r) + + var usage *PostsUsage + err = json.NewDecoder(r.Body).Decode(&usage) + return usage, BuildResponse(r), err +} diff --git a/model/post.go b/model/post.go index f7ffcb12c4..4858953055 100644 --- a/model/post.go +++ b/model/post.go @@ -274,6 +274,14 @@ type GetPostsOptions struct { Direction string // Only accepts up|down. Indicates the order in which to send the items. } +type PostCountOptions struct { + // Only include posts on a specific team. "" for any team. + TeamId string + MustHaveFile bool + MustHaveHashtag bool + ExcludeDeleted bool +} + func (o *Post) Etag() string { return Etag(o.Id, o.UpdateAt) } diff --git a/model/usage.go b/model/usage.go new file mode 100644 index 0000000000..09fd566cbf --- /dev/null +++ b/model/usage.go @@ -0,0 +1,8 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package model + +type PostsUsage struct { + Count int64 `json:"count"` +} diff --git a/services/searchengine/bleveengine/indexer/indexing_job.go b/services/searchengine/bleveengine/indexer/indexing_job.go index 0a4851577f..155167abb4 100644 --- a/services/searchengine/bleveengine/indexer/indexing_job.go +++ b/services/searchengine/bleveengine/indexer/indexing_job.go @@ -212,7 +212,7 @@ func (worker *BleveIndexerWorker) DoJob(job *model.Job) { // Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry // on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate. - if count, err := worker.jobServer.Store.Post().AnalyticsPostCount("", false, false); err != nil { + if count, err := worker.jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}); err != nil { mlog.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err)) progress.TotalPostsCount = estimatedPostCount } else { diff --git a/services/telemetry/telemetry.go b/services/telemetry/telemetry.go index 7974d77ee0..1b05c2d827 100644 --- a/services/telemetry/telemetry.go +++ b/services/telemetry/telemetry.go @@ -297,7 +297,7 @@ func (ts *TelemetryService) trackActivity() { deletedPrivateChannelCount = dpccr } - postsCount, _ = ts.dbStore.Post().AnalyticsPostCount("", false, false) + postsCount, _ = ts.dbStore.Post().AnalyticsPostCount(&model.PostCountOptions{}) postCountsOptions := &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: false, YesterdayOnly: true} postCountsYesterday, _ := ts.dbStore.Post().AnalyticsPostCountsByDay(postCountsOptions) diff --git a/services/telemetry/telemetry_test.go b/services/telemetry/telemetry_test.go index 60985f2577..97947e14f1 100644 --- a/services/telemetry/telemetry_test.go +++ b/services/telemetry/telemetry_test.go @@ -112,7 +112,7 @@ func initializeMocks(cfg *model.Config) (*mocks.ServerIface, *storeMocks.Store, channelStore.On("GroupSyncedChannelCount").Return(int64(17), nil) postStore := storeMocks.PostStore{} - postStore.On("AnalyticsPostCount", "", false, false).Return(int64(1000), nil) + postStore.On("AnalyticsPostCount", &model.PostCountOptions{}).Return(int64(1000), nil) postStore.On("AnalyticsPostCountsByDay", &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: false, YesterdayOnly: true}).Return(model.AnalyticsRows{}, nil) postStore.On("AnalyticsPostCountsByDay", &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: true, YesterdayOnly: true}).Return(model.AnalyticsRows{}, nil) diff --git a/store/opentracinglayer/opentracinglayer.go b/store/opentracinglayer/opentracinglayer.go index 4b7a0e6eb2..fec9dd46ff 100644 --- a/store/opentracinglayer/opentracinglayer.go +++ b/store/opentracinglayer/opentracinglayer.go @@ -5365,7 +5365,7 @@ func (s *OpenTracingLayerPluginStore) SetWithOptions(pluginID string, key string return result, err } -func (s *OpenTracingLayerPostStore) AnalyticsPostCount(teamID string, mustHaveFile bool, mustHaveHashtag bool) (int64, error) { +func (s *OpenTracingLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.AnalyticsPostCount") s.Root.Store.SetContext(newCtx) @@ -5374,7 +5374,7 @@ func (s *OpenTracingLayerPostStore) AnalyticsPostCount(teamID string, mustHaveFi }() defer span.Finish() - result, err := s.PostStore.AnalyticsPostCount(teamID, mustHaveFile, mustHaveHashtag) + result, err := s.PostStore.AnalyticsPostCount(options) if err != nil { span.LogFields(spanlog.Error(err)) ext.Error.Set(span, true) diff --git a/store/retrylayer/retrylayer.go b/store/retrylayer/retrylayer.go index 09917f12ba..32ce4f6037 100644 --- a/store/retrylayer/retrylayer.go +++ b/store/retrylayer/retrylayer.go @@ -6094,11 +6094,11 @@ func (s *RetryLayerPluginStore) SetWithOptions(pluginID string, key string, valu } -func (s *RetryLayerPostStore) AnalyticsPostCount(teamID string, mustHaveFile bool, mustHaveHashtag bool) (int64, error) { +func (s *RetryLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) { tries := 0 for { - result, err := s.PostStore.AnalyticsPostCount(teamID, mustHaveFile, mustHaveHashtag) + result, err := s.PostStore.AnalyticsPostCount(options) if err == nil { return result, nil } diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 9ae4b158a1..2368dded54 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -2128,25 +2128,29 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCoun return rows, nil } -func (s *SqlPostStore) AnalyticsPostCount(teamId string, mustHaveFile bool, mustHaveHashtag bool) (int64, error) { +func (s *SqlPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) { query := s.getQueryBuilder(). Select("COUNT(p.Id) AS Value"). From("Posts p") - if teamId != "" { + if options.TeamId != "" { query = query. Join("Channels c ON (c.Id = p.ChannelId)"). - Where(sq.Eq{"c.TeamId": teamId}) + Where(sq.Eq{"c.TeamId": options.TeamId}) } - if mustHaveFile { + if options.MustHaveFile { query = query.Where(sq.Or{sq.NotEq{"p.FileIds": "[]"}, sq.NotEq{"p.Filenames": "[]"}}) } - if mustHaveHashtag { + if options.MustHaveHashtag { query = query.Where(sq.NotEq{"p.Hashtags": ""}) } + if options.ExcludeDeleted { + query = query.Where(sq.Eq{"p.DeleteAt": 0}) + } + queryString, args, err := query.ToSql() if err != nil { return 0, errors.Wrap(err, "post_tosql") diff --git a/store/store.go b/store/store.go index 7330f5fa20..ff4c6fee55 100644 --- a/store/store.go +++ b/store/store.go @@ -348,7 +348,7 @@ type PostStore interface { Search(teamID string, userID string, params *model.SearchParams) (*model.PostList, error) AnalyticsUserCountsWithPostsByDay(teamID string) (model.AnalyticsRows, error) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, error) - AnalyticsPostCount(teamID string, mustHaveFile bool, mustHaveHashtag bool) (int64, error) + AnalyticsPostCount(options *model.PostCountOptions) (int64, error) ClearCaches() InvalidateLastPostTimeCache(channelID string) GetLastPostRowCreateAt() (int64, error) diff --git a/store/storetest/mocks/PostStore.go b/store/storetest/mocks/PostStore.go index 21ca52de92..0f67d1a338 100644 --- a/store/storetest/mocks/PostStore.go +++ b/store/storetest/mocks/PostStore.go @@ -16,20 +16,20 @@ type PostStore struct { mock.Mock } -// AnalyticsPostCount provides a mock function with given fields: teamID, mustHaveFile, mustHaveHashtag -func (_m *PostStore) AnalyticsPostCount(teamID string, mustHaveFile bool, mustHaveHashtag bool) (int64, error) { - ret := _m.Called(teamID, mustHaveFile, mustHaveHashtag) +// AnalyticsPostCount provides a mock function with given fields: options +func (_m *PostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) { + ret := _m.Called(options) var r0 int64 - if rf, ok := ret.Get(0).(func(string, bool, bool) int64); ok { - r0 = rf(teamID, mustHaveFile, mustHaveHashtag) + if rf, ok := ret.Get(0).(func(*model.PostCountOptions) int64); ok { + r0 = rf(options) } else { r0 = ret.Get(0).(int64) } var r1 error - if rf, ok := ret.Get(1).(func(string, bool, bool) error); ok { - r1 = rf(teamID, mustHaveFile, mustHaveHashtag) + if rf, ok := ret.Get(1).(func(*model.PostCountOptions) error); ok { + r1 = rf(options) } else { r1 = ret.Error(1) } diff --git a/store/storetest/post_store.go b/store/storetest/post_store.go index eb10ac9b91..1d98dc9a31 100644 --- a/store/storetest/post_store.go +++ b/store/storetest/post_store.go @@ -2115,30 +2115,39 @@ func testPostCountsByDay(t *testing.T, ss store.Store) { require.NoError(t, err) assert.Equal(t, float64(1), r1[0].Value) - // total - r2, err := ss.Post().AnalyticsPostCount(t1.Id, false, false) + // total for single team + r2, err := ss.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: t1.Id}) require.NoError(t, err) assert.Equal(t, int64(6), r2) // total across teams - r2, err = ss.Post().AnalyticsPostCount("", false, false) + r2, err = ss.Post().AnalyticsPostCount(&model.PostCountOptions{}) require.NoError(t, err) assert.GreaterOrEqual(t, r2, int64(6)) // total across teams with files - r2, err = ss.Post().AnalyticsPostCount("", true, false) + r2, err = ss.Post().AnalyticsPostCount(&model.PostCountOptions{MustHaveFile: true}) require.NoError(t, err) assert.GreaterOrEqual(t, r2, int64(3)) - // total across teams with hastags - r2, err = ss.Post().AnalyticsPostCount("", false, true) + // total across teams with hashtags + r2, err = ss.Post().AnalyticsPostCount(&model.PostCountOptions{MustHaveHashtag: true}) require.NoError(t, err) assert.GreaterOrEqual(t, r2, int64(2)) // total across teams with hastags and files - r2, err = ss.Post().AnalyticsPostCount("", true, true) + r2, err = ss.Post().AnalyticsPostCount(&model.PostCountOptions{MustHaveFile: true, MustHaveHashtag: true}) require.NoError(t, err) assert.GreaterOrEqual(t, r2, int64(1)) + + // delete 1 post + err = ss.Post().Delete(o1.Id, 1, o1.UserId) + require.NoError(t, err) + + // total for single team with the deleted post excluded + r2, err = ss.Post().AnalyticsPostCount(&model.PostCountOptions{TeamId: t1.Id, ExcludeDeleted: true}) + require.NoError(t, err) + assert.Equal(t, int64(5), r2) } func testPostStoreGetFlaggedPostsForTeam(t *testing.T, ss store.Store, s SqlStore) { diff --git a/store/timerlayer/timerlayer.go b/store/timerlayer/timerlayer.go index 22caf13918..8eeaa492ef 100644 --- a/store/timerlayer/timerlayer.go +++ b/store/timerlayer/timerlayer.go @@ -4859,10 +4859,10 @@ func (s *TimerLayerPluginStore) SetWithOptions(pluginID string, key string, valu return result, err } -func (s *TimerLayerPostStore) AnalyticsPostCount(teamID string, mustHaveFile bool, mustHaveHashtag bool) (int64, error) { +func (s *TimerLayerPostStore) AnalyticsPostCount(options *model.PostCountOptions) (int64, error) { start := timemodule.Now() - result, err := s.PostStore.AnalyticsPostCount(teamID, mustHaveFile, mustHaveHashtag) + result, err := s.PostStore.AnalyticsPostCount(options) elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) if s.Root.Metrics != nil { diff --git a/utils/utils.go b/utils/utils.go index 52cd3d2da0..0fa3157957 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -5,6 +5,7 @@ package utils import ( "io/ioutil" + "math" "net" "net/http" "net/url" @@ -215,3 +216,16 @@ func IsValidMobileAuthRedirectURL(config *model.Config, redirectURL string) bool } return false } + +// RoundOffToZeroes converts all digits to 0 except the 1st one. +// Special case: If there is only 1 digit, then returns 0. +func RoundOffToZeroes(n float64) int64 { + if n >= -9 && n <= 9 { + return 0 + } + + zeroes := int(math.Log10(math.Abs(n))) + tens := int64(math.Pow10(zeroes)) + firstDigit := int64(n) / tens + return firstDigit * tens +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 39bf4c545e..13f56ea097 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -170,3 +170,69 @@ func TestAppendQueryParamsToURL(t *testing.T) { expected := url + "?key1=value1&key2=value2" assert.Equal(t, redirectURL, expected) } + +func TestRoundOffToZeroes(t *testing.T) { + testCases := []struct { + desc string + n float64 + expected int64 + }{ + { + desc: "returns 0 when n is 0", + n: 0, + expected: 0, + }, + { + desc: "returns 0 when n is 9", + n: 9, + expected: 0, + }, + { + desc: "returns 10 when n is 10", + n: 10, + expected: 10, + }, + { + desc: "returns 90 when n is 99", + n: 99, + expected: 90, + }, + { + desc: "returns 100 when n is 100", + n: 100, + expected: 100, + }, + { + desc: "returns 100 when n is 101", + n: 101, + expected: 100, + }, + { + desc: "returns 4000 when n is 4321", + n: 4321, + expected: 4000, + }, + { + desc: "returns 0 when n is -9", + n: -9, + expected: 0, + }, + { + desc: "returns -4000 when n is -4321", + n: -4321, + expected: -4000, + }, + { + desc: "returns 4000 when n is 4321.235", + n: 4321.235, + expected: 4000, + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + res := RoundOffToZeroes(tc.n) + assert.Equal(t, tc.expected, res) + }) + } +}