Skip intensive stat DB queries when more than a set number of users on the system (#4876)

api/admin.go

@@ -367,6 +367,19 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
 	teamId := params["id"]
 	name := params["name"]
 
+	skipIntensiveQueries := false
+	var systemUserCount int64
+	if r := <-Srv.Store.User().AnalyticsUniqueUserCount(""); r.Err != nil {
+		c.Err = r.Err
+		return
+	} else {
+		systemUserCount = r.Data.(int64)
+		if systemUserCount > int64(*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics) {
+			l4g.Debug("More than %v users on the system, intensive queries skipped", *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics)
+			skipIntensiveQueries = true
+		}
+	}
+
 	if name == "standard" {
 		var rows model.AnalyticsRows = make([]*model.AnalyticsRow, 8)
 		rows[0] = &model.AnalyticsRow{"channel_open_count", 0}
@@ -380,10 +393,18 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
 
 		openChan := Srv.Store.Channel().AnalyticsTypeCount(teamId, model.CHANNEL_OPEN)
 		privateChan := Srv.Store.Channel().AnalyticsTypeCount(teamId, model.CHANNEL_PRIVATE)
-		postChan := Srv.Store.Post().AnalyticsPostCount(teamId, false, false)
-		userChan := Srv.Store.User().AnalyticsUniqueUserCount(teamId)
 		teamChan := Srv.Store.Team().AnalyticsTeamCount()
 
+		var userChan store.StoreChannel
+		if teamId != "" {
+			userChan = Srv.Store.User().AnalyticsUniqueUserCount(teamId)
+		}
+
+		var postChan store.StoreChannel
+		if !skipIntensiveQueries {
+			postChan = Srv.Store.Post().AnalyticsPostCount(teamId, false, false)
+		}
+
 		if r := <-openChan; r.Err != nil {
 			c.Err = r.Err
 			return
@@ -398,19 +419,27 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
 			rows[1].Value = float64(r.Data.(int64))
 		}
 
+		if postChan == nil {
+			rows[2].Value = -1
+		} else {
 			if r := <-postChan; r.Err != nil {
 				c.Err = r.Err
 				return
 			} else {
 				rows[2].Value = float64(r.Data.(int64))
 			}
+		}
 
+		if userChan == nil {
+			rows[3].Value = float64(systemUserCount)
+		} else {
 			if r := <-userChan; r.Err != nil {
 				c.Err = r.Err
 				return
 			} else {
 				rows[3].Value = float64(r.Data.(int64))
 			}
+		}
 
 		if r := <-teamChan; r.Err != nil {
 			c.Err = r.Err
@@ -449,6 +478,12 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
 
 		w.Write([]byte(rows.ToJson()))
 	} else if name == "post_counts_day" {
+		if skipIntensiveQueries {
+			rows := model.AnalyticsRows{&model.AnalyticsRow{"", -1}}
+			w.Write([]byte(rows.ToJson()))
+			return
+		}
+
 		if r := <-Srv.Store.Post().AnalyticsPostCountsByDay(teamId); r.Err != nil {
 			c.Err = r.Err
 			return
@@ -456,6 +491,12 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
 			w.Write([]byte(r.Data.(model.AnalyticsRows).ToJson()))
 		}
 	} else if name == "user_counts_with_posts_day" {
+		if skipIntensiveQueries {
+			rows := model.AnalyticsRows{&model.AnalyticsRow{"", -1}}
+			w.Write([]byte(rows.ToJson()))
+			return
+		}
+
 		if r := <-Srv.Store.Post().AnalyticsUserCountsWithPostsByDay(teamId); r.Err != nil {
 			c.Err = r.Err
 			return
@@ -471,26 +512,39 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) {
 		rows[4] = &model.AnalyticsRow{"command_count", 0}
 		rows[5] = &model.AnalyticsRow{"session_count", 0}
 
-		fileChan := Srv.Store.Post().AnalyticsPostCount(teamId, true, false)
-		hashtagChan := Srv.Store.Post().AnalyticsPostCount(teamId, false, true)
 		iHookChan := Srv.Store.Webhook().AnalyticsIncomingCount(teamId)
 		oHookChan := Srv.Store.Webhook().AnalyticsOutgoingCount(teamId)
 		commandChan := Srv.Store.Command().AnalyticsCommandCount(teamId)
 		sessionChan := Srv.Store.Session().AnalyticsSessionCount()
 
+		var fileChan store.StoreChannel
+		var hashtagChan store.StoreChannel
+		if !skipIntensiveQueries {
+			fileChan = Srv.Store.Post().AnalyticsPostCount(teamId, true, false)
+			hashtagChan = Srv.Store.Post().AnalyticsPostCount(teamId, false, true)
+		}
+
+		if fileChan == nil {
+			rows[0].Value = -1
+		} else {
 			if r := <-fileChan; r.Err != nil {
 				c.Err = r.Err
 				return
 			} else {
 				rows[0].Value = float64(r.Data.(int64))
 			}
+		}
 
+		if hashtagChan == nil {
+			rows[1].Value = -1
+		} else {
 			if r := <-hashtagChan; r.Err != nil {
 				c.Err = r.Err
 				return
 			} else {
 				rows[1].Value = float64(r.Data.(int64))
 			}
+		}
 
 		if r := <-iHookChan; r.Err != nil {
 			c.Err = r.Err

api/admin_test.go

@@ -192,6 +192,12 @@ func TestGetTeamAnalyticsStandard(t *testing.T) {
 		t.Fatal("Shouldn't have permissions")
 	}
 
+	maxUsersForStats := *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics
+	defer func() {
+		*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = maxUsersForStats
+	}()
+	*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1000000
+
 	if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "standard"); err != nil {
 		t.Fatal(err)
 	} else {
@@ -303,6 +309,24 @@ func TestGetTeamAnalyticsStandard(t *testing.T) {
 			t.Fatal()
 		}
 	}
+
+	*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1
+
+	if result, err := th.SystemAdminClient.GetSystemAnalytics("standard"); err != nil {
+		t.Fatal(err)
+	} else {
+		rows := result.Data.(model.AnalyticsRows)
+
+		if rows[2].Name != "post_count" {
+			t.Log(rows.ToJson())
+			t.Fatal()
+		}
+
+		if rows[2].Value != -1 {
+			t.Log(rows.ToJson())
+			t.Fatal()
+		}
+	}
 }
 
 func TestGetPostCount(t *testing.T) {
@@ -316,6 +340,12 @@ func TestGetPostCount(t *testing.T) {
 		t.Fatal("Shouldn't have permissions")
 	}
 
+	maxUsersForStats := *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics
+	defer func() {
+		*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = maxUsersForStats
+	}()
+	*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1000000
+
 	if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "post_counts_day"); err != nil {
 		t.Fatal(err)
 	} else {
@@ -326,6 +356,19 @@ func TestGetPostCount(t *testing.T) {
 			t.Fatal()
 		}
 	}
+
+	*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1
+
+	if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "post_counts_day"); err != nil {
+		t.Fatal(err)
+	} else {
+		rows := result.Data.(model.AnalyticsRows)
+
+		if rows[0].Value != -1 {
+			t.Log(rows.ToJson())
+			t.Fatal()
+		}
+	}
 }
 
 func TestUserCountsWithPostsByDay(t *testing.T) {
@@ -339,6 +382,12 @@ func TestUserCountsWithPostsByDay(t *testing.T) {
 		t.Fatal("Shouldn't have permissions")
 	}
 
+	maxUsersForStats := *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics
+	defer func() {
+		*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = maxUsersForStats
+	}()
+	*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1000000
+
 	if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "user_counts_with_posts_day"); err != nil {
 		t.Fatal(err)
 	} else {
@@ -349,6 +398,19 @@ func TestUserCountsWithPostsByDay(t *testing.T) {
 			t.Fatal()
 		}
 	}
+
+	*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1
+
+	if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "user_counts_with_posts_day"); err != nil {
+		t.Fatal(err)
+	} else {
+		rows := result.Data.(model.AnalyticsRows)
+
+		if rows[0].Value != -1 {
+			t.Log(rows.ToJson())
+			t.Fatal()
+		}
+	}
 }
 
 func TestGetTeamAnalyticsExtra(t *testing.T) {
@@ -360,6 +422,12 @@ func TestGetTeamAnalyticsExtra(t *testing.T) {
 		t.Fatal("Shouldn't have permissions")
 	}
 
+	maxUsersForStats := *utils.Cfg.AnalyticsSettings.MaxUsersForStatistics
+	defer func() {
+		*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = maxUsersForStats
+	}()
+	*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1000000
+
 	if result, err := th.SystemAdminClient.GetTeamAnalytics(th.BasicTeam.Id, "extra_counts"); err != nil {
 		t.Fatal(err)
 	} else {
@@ -461,6 +529,24 @@ func TestGetTeamAnalyticsExtra(t *testing.T) {
 			t.Fatal()
 		}
 	}
+
+	*utils.Cfg.AnalyticsSettings.MaxUsersForStatistics = 1
+
+	if result, err := th.SystemAdminClient.GetSystemAnalytics("extra_counts"); err != nil {
+		t.Fatal(err)
+	} else {
+		rows := result.Data.(model.AnalyticsRows)
+
+		if rows[0].Value != -1 {
+			t.Log(rows.ToJson())
+			t.Fatal()
+		}
+
+		if rows[1].Value != -1 {
+			t.Log(rows.ToJson())
+			t.Fatal()
+		}
+	}
 }
 
 func TestAdminResetMfa(t *testing.T) {

config/config.json

@@ -241,6 +241,9 @@
         "BlockProfileRate": 0,
         "ListenAddress": ":8067"
     },
+    "AnalyticsSettings": {
+        "MaxUsersForStatistics": 2500
+    },
     "WebrtcSettings": {
         "Enable": false,
         "GatewayWebsocketUrl": "",

model/config.go

@@ -105,6 +105,10 @@ type MetricsSettings struct {
 	ListenAddress    *string
 }
 
+type AnalyticsSettings struct {
+	MaxUsersForStatistics *int
+}
+
 type SSOSettings struct {
 	Enable          bool
 	Secret          string
@@ -345,6 +349,7 @@ type Config struct {
 	NativeAppSettings    NativeAppSettings
 	ClusterSettings      ClusterSettings
 	MetricsSettings      MetricsSettings
+	AnalyticsSettings    AnalyticsSettings
 	WebrtcSettings       WebrtcSettings
 }
 
@@ -845,6 +850,11 @@ func (o *Config) SetDefaults() {
 		*o.MetricsSettings.Enable = false
 	}
 
+	if o.AnalyticsSettings.MaxUsersForStatistics == nil {
+		o.AnalyticsSettings.MaxUsersForStatistics = new(int)
+		*o.AnalyticsSettings.MaxUsersForStatistics = 2500
+	}
+
 	if o.ComplianceSettings.Enable == nil {
 		o.ComplianceSettings.Enable = new(bool)
 		*o.ComplianceSettings.Enable = false