[MM-14720] Add Posts Per Day Analytics and Number Posts Previous Day Functionality (#11402)

* Add ability to get bot counts per day

Modify tests using AnalyticsPostCountsByDay to include extra input
(botsOnly)

Correctly generate the mock file with 'make store-mocks' target.  Didn't
see these comments earlier

* Initial commit for calculating total posts previous day and total posts
from bots previous day

* - Refactor to use asserts instead of if statements in tests
- Capture inputs to AnalyticsPostCountsByDay() function into an
  options struct
- Remove bot creation from diagnostics_test.go.

* Remove utils library

* create AnalyticsPostCountsOptions struct which is accepted as an input
to AnalyticsPostCountsByDay method

* Go vet fixes

app/analytics.go

@@ -181,13 +181,26 @@ func (a *App) GetAnalytics(name string, teamId string) (model.AnalyticsRows, *mo
 		rows[9].Value = float64(r.Data.(int64))
 
 		return rows, nil
+	} else if name == "bot_post_counts_day" {
+		if skipIntensiveQueries {
+			rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
+			return rows, nil
+		}
+		return a.Srv.Store.Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
+			TeamId:        teamId,
+			BotsOnly:      true,
+			YesterdayOnly: false,
+		})
 	} else if name == "post_counts_day" {
 		if skipIntensiveQueries {
 			rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}
 			return rows, nil
 		}
-
+		return a.Srv.Store.Post().AnalyticsPostCountsByDay(&model.AnalyticsPostCountsOptions{
-		return a.Srv.Store.Post().AnalyticsPostCountsByDay(teamId)
+			TeamId:        teamId,
+			BotsOnly:      false,
+			YesterdayOnly: false,
+		})
 	} else if name == "user_counts_with_posts_day" {
 		if skipIntensiveQueries {
 			rows := model.AnalyticsRows{&model.AnalyticsRow{Name: "", Value: -1}}

app/diagnostics.go

@@ -134,6 +134,8 @@ func (a *App) trackActivity() {
 	var deletedPublicChannelCount int64
 	var deletedPrivateChannelCount int64
 	var postsCount int64
+	var postsCountPreviousDay int64
+	var botPostsCountPreviousDay int64
 	var slashCommandsCount int64
 	var incomingWebhooksCount int64
 	var outgoingWebhooksCount int64
@@ -188,6 +190,20 @@ func (a *App) trackActivity() {
 
 	postsCount, _ = a.Srv.Store.Post().AnalyticsPostCount("", false, false)
 
+	postCountsOptions := &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: false, YesterdayOnly: true}
+	postCountsYesterday, _ := a.Srv.Store.Post().AnalyticsPostCountsByDay(postCountsOptions)
+	postsCountPreviousDay = 0
+	if len(postCountsYesterday) > 0 {
+		postsCountPreviousDay = int64(postCountsYesterday[0].Value)
+	}
+
+	postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: true, YesterdayOnly: true}
+	botPostCountsYesterday, _ := a.Srv.Store.Post().AnalyticsPostCountsByDay(postCountsOptions)
+	botPostsCountPreviousDay = 0
+	if len(botPostCountsYesterday) > 0 {
+		botPostsCountPreviousDay = int64(botPostCountsYesterday[0].Value)
+	}
+
 	slashCommandsCount, _ = a.Srv.Store.Command().AnalyticsCommandCount("")
 
 	if c, err := a.Srv.Store.Webhook().AnalyticsIncomingCount(""); err == nil {
@@ -208,6 +224,8 @@ func (a *App) trackActivity() {
 		"direct_message_channels":      directChannelCount,
 		"public_channels_deleted":      deletedPublicChannelCount,
 		"private_channels_deleted":     deletedPrivateChannelCount,
+		"posts_previous_day":           postsCountPreviousDay,
+		"bot_posts_previous_day":       botPostsCountPreviousDay,
 		"posts":                        postsCount,
 		"slash_commands":               slashCommandsCount,
 		"incoming_webhooks":            incomingWebhooksCount,

model/post.go

@@ -108,6 +108,12 @@ type SearchParameter struct {
 	IncludeDeletedChannels *bool   `json:"include_deleted_channels"`
 }
 
+type AnalyticsPostCountsOptions struct {
+	TeamId        string
+	BotsOnly      bool
+	YesterdayOnly bool
+}
+
 func (o *PostPatch) WithRewrittenImageURLs(f func(string) string) *PostPatch {
 	copy := *o
 	if copy.Message != nil {

store/sqlstore/post_store.go

@@ -991,14 +991,19 @@ func (s *SqlPostStore) AnalyticsUserCountsWithPostsByDay(teamId string) (model.A
 	return rows, nil
 }
 
-func (s *SqlPostStore) AnalyticsPostCountsByDay(teamId string) (model.AnalyticsRows, *model.AppError) {
+func (s *SqlPostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, *model.AppError) {
+
 	query :=
 		`SELECT
 		        DATE(FROM_UNIXTIME(Posts.CreateAt / 1000)) AS Name,
 		        COUNT(Posts.Id) AS Value
 		    FROM Posts`
 
-	if len(teamId) > 0 {
+	if options.BotsOnly {
+		query += " INNER JOIN Bots ON Posts.UserId = Bots.Userid"
+	}
+
+	if len(options.TeamId) > 0 {
 		query += " INNER JOIN Channels ON Posts.ChannelId = Channels.Id AND Channels.TeamId = :TeamId AND"
 	} else {
 		query += " WHERE"
@@ -1016,7 +1021,11 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(teamId string) (model.AnalyticsR
 				TO_CHAR(DATE(TO_TIMESTAMP(Posts.CreateAt / 1000)), 'YYYY-MM-DD') AS Name, Count(Posts.Id) AS Value
 			FROM Posts`
 
-		if len(teamId) > 0 {
+		if options.BotsOnly {
+			query += " INNER JOIN Bots ON Posts.UserId = Bots.Userid"
+		}
+
+		if len(options.TeamId) > 0 {
 			query += " INNER JOIN Channels ON Posts.ChannelId = Channels.Id  AND Channels.TeamId = :TeamId AND"
 		} else {
 			query += " WHERE"
@@ -1031,12 +1040,15 @@ func (s *SqlPostStore) AnalyticsPostCountsByDay(teamId string) (model.AnalyticsR
 
 	end := utils.MillisFromTime(utils.EndOfDay(utils.Yesterday()))
 	start := utils.MillisFromTime(utils.StartOfDay(utils.Yesterday().AddDate(0, 0, -31)))
+	if options.YesterdayOnly {
+		start = utils.MillisFromTime(utils.StartOfDay(utils.Yesterday().AddDate(0, 0, -1)))
+	}
 
 	var rows model.AnalyticsRows
 	_, err := s.GetReplica().Select(
 		&rows,
 		query,
-		map[string]interface{}{"TeamId": teamId, "StartTime": start, "EndTime": end})
+		map[string]interface{}{"TeamId": options.TeamId, "StartTime": start, "EndTime": end})
 	if err != nil {
 		return nil, model.NewAppError("SqlPostStore.AnalyticsPostCountsByDay", "store.sql_post.analytics_posts_count_by_day.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}

store/store.go

@@ -229,7 +229,7 @@ type PostStore interface {
 	GetEtag(channelId string, allowFromCache bool) string
 	Search(teamId string, userId string, params *model.SearchParams) StoreChannel
 	AnalyticsUserCountsWithPostsByDay(teamId string) (model.AnalyticsRows, *model.AppError)
-	AnalyticsPostCountsByDay(teamId string) (model.AnalyticsRows, *model.AppError)
+	AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, *model.AppError)
 	AnalyticsPostCount(teamId string, mustHaveFile bool, mustHaveHashtag bool) (int64, *model.AppError)
 	ClearCaches()
 	InvalidateLastPostTimeCache(channelId string)

store/storetest/mocks/PostStore.go

@@ -36,13 +36,13 @@ func (_m *PostStore) AnalyticsPostCount(teamId string, mustHaveFile bool, mustHa
 	return r0, r1
 }
 
-// AnalyticsPostCountsByDay provides a mock function with given fields: teamId
+// AnalyticsPostCountsByDay provides a mock function with given fields: options
-func (_m *PostStore) AnalyticsPostCountsByDay(teamId string) (model.AnalyticsRows, *model.AppError) {
+func (_m *PostStore) AnalyticsPostCountsByDay(options *model.AnalyticsPostCountsOptions) (model.AnalyticsRows, *model.AppError) {
-	ret := _m.Called(teamId)
+	ret := _m.Called(options)
 
 	var r0 model.AnalyticsRows
-	if rf, ok := ret.Get(0).(func(string) model.AnalyticsRows); ok {
+	if rf, ok := ret.Get(0).(func(*model.AnalyticsPostCountsOptions) model.AnalyticsRows); ok {
-		r0 = rf(teamId)
+		r0 = rf(options)
 	} else {
 		if ret.Get(0) != nil {
 			r0 = ret.Get(0).(model.AnalyticsRows)
@@ -50,8 +50,8 @@ func (_m *PostStore) AnalyticsPostCountsByDay(teamId string) (model.AnalyticsRow
 	}
 
 	var r1 *model.AppError
-	if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
+	if rf, ok := ret.Get(1).(func(*model.AnalyticsPostCountsOptions) *model.AppError); ok {
-		r1 = rf(teamId)
+		r1 = rf(options)
 	} else {
 		if ret.Get(1) != nil {
 			r1 = ret.Get(1).(*model.AppError)

store/storetest/post_store.go

@@ -1352,28 +1352,75 @@ func testPostCountsByDay(t *testing.T, ss store.Store) {
 	_, err = ss.Post().Save(o2a)
 	require.Nil(t, err)
 
+	bot1 := &model.Bot{
+		Username:    "username",
+		Description: "a bot",
+		OwnerId:     model.NewId(),
+		UserId:      model.NewId(),
+	}
+	_, err = ss.Bot().Save(bot1)
+
+	b1 := &model.Post{}
+	b1.Message = "bot message one"
+	b1.ChannelId = c1.Id
+	b1.UserId = bot1.UserId
+	b1.CreateAt = utils.MillisFromTime(utils.Yesterday())
+	_, err = ss.Post().Save(b1)
+	require.Nil(t, err)
+
+	b1a := &model.Post{}
+	b1a.Message = "bot message two"
+	b1a.ChannelId = c1.Id
+	b1a.UserId = bot1.UserId
+	b1a.CreateAt = utils.MillisFromTime(utils.Yesterday()) - (1000 * 60 * 60 * 24 * 2)
+	_, err = ss.Post().Save(b1a)
+	require.Nil(t, err)
+
 	time.Sleep(1 * time.Second)
 
-	if r1, err := ss.Post().AnalyticsPostCountsByDay(t1.Id); err != nil {
+	// summary of posts
+	// yesterday - 2 non-bot user posts, 1 bot user post
+	// 3 days ago - 2 non-bot user posts, 1 bot user post
+
+	// last 31 days, all users (including bots)
+	postCountsOptions := &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: false, YesterdayOnly: false}
+	if r1, err := ss.Post().AnalyticsPostCountsByDay(postCountsOptions); err != nil {
 		t.Fatal(err)
 	} else {
-		row1 := r1[0]
+		assert.Equal(t, float64(3), r1[0].Value)
-		if row1.Value != 2 {
+		assert.Equal(t, float64(3), r1[1].Value)
-			t.Fatal(row1)
-		}
-
-		row2 := r1[1]
-		if row2.Value != 2 {
-			t.Fatal("wrong value")
-		}
 	}
 
+	// last 31 days, bots only
+	postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: true, YesterdayOnly: false}
+	if r1, err := ss.Post().AnalyticsPostCountsByDay(postCountsOptions); err != nil {
+		t.Fatal(err)
+	} else {
+		assert.Equal(t, float64(1), r1[0].Value)
+		assert.Equal(t, float64(1), r1[1].Value)
+	}
+
+	// yesterday only, all users (including bots)
+	postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: false, YesterdayOnly: true}
+	if r1, err := ss.Post().AnalyticsPostCountsByDay(postCountsOptions); err != nil {
+		t.Fatal(err)
+	} else {
+		assert.Equal(t, float64(3), r1[0].Value)
+	}
+
+	// yesterday only, bots only
+	postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: t1.Id, BotsOnly: true, YesterdayOnly: true}
+	if r1, err := ss.Post().AnalyticsPostCountsByDay(postCountsOptions); err != nil {
+		t.Fatal(err)
+	} else {
+		assert.Equal(t, float64(1), r1[0].Value)
+	}
+
+	// total posts
 	if r1, err := ss.Post().AnalyticsPostCount(t1.Id, false, false); err != nil {
 		t.Fatal(err)
 	} else {
-		if r1 != 4 {
+		assert.Equal(t, int64(6), r1)
-			t.Fatal("wrong value")
-		}
 	}
 }