Moving diagnostics into a service (#14832)

* Moving diagnostics into a service

* Fixing golint checks

* Fixing tests

* Renaming from diagnostics to telemetry

* Adding missing files

* Initializing telemetry earlier in the server startup

* Fixing tests

* Adding a log for the telemetryID initialization error

* Addressing PR review comments

* Fixing merge problem

* Removing some extra Diagnostics mentions

* Making tests pass

app/app.go

@@ -127,12 +127,8 @@ func (a *App) initJobs() {
 	a.srv.Jobs.Schedulers = a.srv.Jobs.InitSchedulers()
 }
 
-func (a *App) DiagnosticId() string {
-	return a.Srv().diagnosticId
-}
-
-func (a *App) SetDiagnosticId(id string) {
-	a.Srv().diagnosticId = id
+func (a *App) TelemetryId() string {
+	return a.Srv().TelemetryId()
 }
 
 func (s *Server) HTMLTemplates() *template.Template {
@@ -374,8 +370,8 @@ func (a *App) NotifyAndSetWarnMetricAck(warnMetricId string, sender *model.User,
 			}
 			bodyPage.Props["SiteURLHeader"] = T("api.templates.warn_metric_ack.body.site_url_header")
 			bodyPage.Props["SiteURL"] = a.GetSiteURL()
-			bodyPage.Props["DiagnosticIdHeader"] = T("api.templates.warn_metric_ack.body.diagnostic_id_header")
-			bodyPage.Props["DiagnosticIdValue"] = a.DiagnosticId()
+			bodyPage.Props["TelemetryIdHeader"] = T("api.templates.warn_metric_ack.body.diagnostic_id_header")
+			bodyPage.Props["TelemetryIdValue"] = a.TelemetryId()
 			bodyPage.Props["Footer"] = T("api.templates.warn_metric_ack.footer")
 
 			warnMetricStatus, warnMetricDisplayTexts := a.getWarnMetricStatusAndDisplayTextsForId(warnMetricId, T)

app/app_iface.go

@@ -457,7 +457,6 @@ type AppIface interface {
 	DeleteScheme(schemeId string) (*model.Scheme, *model.AppError)
 	DeleteSidebarCategory(userId, teamId, categoryId string) *model.AppError
 	DeleteToken(token *model.Token) *model.AppError
-	DiagnosticId() string
 	DisableAutoResponder(userId string, asAdmin bool) *model.AppError
 	DisableUserAccessToken(token *model.UserAccessToken) *model.AppError
 	DoAppMigrations()
@@ -885,7 +884,6 @@ type AppIface interface {
 	SetAutoResponderStatus(user *model.User, oldNotifyProps model.StringMap)
 	SetContext(c context.Context)
 	SetDefaultProfileImage(user *model.User) *model.AppError
-	SetDiagnosticId(id string)
 	SetIpAddress(s string)
 	SetLog(l *mlog.Logger)
 	SetPath(s string)
@@ -925,6 +923,7 @@ type AppIface interface {
 	T(translationID string, args ...interface{}) string
 	TeamMembersToAdd(since int64, teamID *string) ([]*model.UserTeamIDPair, *model.AppError)
 	TeamMembersToRemove(teamID *string) ([]*model.TeamMember, *model.AppError)
+	TelemetryId() string
 	TestElasticsearch(cfg *model.Config) *model.AppError
 	TestEmail(userId string, cfg *model.Config) *model.AppError
 	TestLdap() *model.AppError

app/config.go

@@ -297,8 +297,8 @@ func (a *App) PostActionCookieSecret() []byte {
 }
 
 func (s *Server) regenerateClientConfig() {
-	clientConfig := config.GenerateClientConfig(s.Config(), s.diagnosticId, s.License())
-	limitedClientConfig := config.GenerateLimitedClientConfig(s.Config(), s.diagnosticId, s.License())
+	clientConfig := config.GenerateClientConfig(s.Config(), s.TelemetryId(), s.License())
+	limitedClientConfig := config.GenerateLimitedClientConfig(s.Config(), s.TelemetryId(), s.License())
 
 	if clientConfig["EnableCustomTermsOfService"] == "true" {
 		termsOfService, err := s.Store.TermsOfService().GetLatest(true)

app/diagnostics_test.go

@@ -1,350 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.txt for license information.
-
-package app
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/mattermost/mattermost-server/v5/model"
-)
-
-func TestPluginSetting(t *testing.T) {
-	settings := &model.PluginSettings{
-		Plugins: map[string]map[string]interface{}{
-			"test": {
-				"foo": "bar",
-			},
-		},
-	}
-	assert.Equal(t, "bar", pluginSetting(settings, "test", "foo", "asd"))
-	assert.Equal(t, "asd", pluginSetting(settings, "test", "qwe", "asd"))
-}
-
-func TestPluginActivated(t *testing.T) {
-	states := map[string]*model.PluginState{
-		"foo": {
-			Enable: true,
-		},
-		"bar": {
-			Enable: false,
-		},
-	}
-	assert.True(t, pluginActivated(states, "foo"))
-	assert.False(t, pluginActivated(states, "bar"))
-	assert.False(t, pluginActivated(states, "none"))
-}
-
-func TestPluginVersion(t *testing.T) {
-	plugins := []*model.BundleInfo{
-		{
-			Manifest: &model.Manifest{
-				Id:      "test.plugin",
-				Version: "1.2.3",
-			},
-		},
-		{
-			Manifest: &model.Manifest{
-				Id:      "test.plugin2",
-				Version: "4.5.6",
-			},
-		},
-	}
-	assert.Equal(t, "1.2.3", pluginVersion(plugins, "test.plugin"))
-	assert.Equal(t, "4.5.6", pluginVersion(plugins, "test.plugin2"))
-	assert.Empty(t, pluginVersion(plugins, "unknown.plugin"))
-}
-
-func TestRudderDiagnostics(t *testing.T) {
-	if testing.Short() {
-		t.SkipNow()
-	}
-
-	th := SetupWithCustomConfig(t, func(config *model.Config) {
-		*config.PluginSettings.Enable = false
-	})
-	defer th.TearDown()
-
-	type batch struct {
-		MessageId  string
-		UserId     string
-		Event      string
-		Timestamp  time.Time
-		Properties map[string]interface{}
-	}
-
-	type payload struct {
-		MessageId string
-		SentAt    time.Time
-		Batch     []batch
-		Context   struct {
-			Library struct {
-				Name    string
-				Version string
-			}
-		}
-	}
-
-	data := make(chan payload, 100)
-	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		body, err := ioutil.ReadAll(r.Body)
-		require.NoError(t, err)
-
-		var p payload
-		err = json.Unmarshal(body, &p)
-		require.NoError(t, err)
-
-		data <- p
-	}))
-	defer server.Close()
-
-	marketplaceServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
-		res.WriteHeader(http.StatusOK)
-		json, err := json.Marshal([]*model.MarketplacePlugin{{
-			BaseMarketplacePlugin: &model.BaseMarketplacePlugin{
-				Manifest: &model.Manifest{
-					Id: "testplugin",
-				},
-			},
-		}})
-		require.NoError(t, err)
-		res.Write(json)
-	}))
-
-	defer func() { marketplaceServer.Close() }()
-
-	diagnosticID := "test-diagnostic-id-12345"
-	th.App.SetDiagnosticId(diagnosticID)
-	th.Server.initDiagnostics(server.URL, RUDDER_KEY)
-
-	assertPayload := func(t *testing.T, actual payload, event string, properties map[string]interface{}) {
-		t.Helper()
-		assert.NotEmpty(t, actual.MessageId)
-		assert.False(t, actual.SentAt.IsZero())
-		if assert.Len(t, actual.Batch, 1) {
-			assert.NotEmpty(t, actual.Batch[0].MessageId, "message id should not be empty")
-			assert.Equal(t, diagnosticID, actual.Batch[0].UserId)
-			if event != "" {
-				assert.Equal(t, event, actual.Batch[0].Event)
-			}
-			assert.False(t, actual.Batch[0].Timestamp.IsZero(), "batch timestamp should not be the zero value")
-			if properties != nil {
-				assert.Equal(t, properties, actual.Batch[0].Properties)
-			}
-		}
-		assert.Equal(t, "analytics-go", actual.Context.Library.Name)
-		assert.Equal(t, "3.0.0", actual.Context.Library.Version)
-	}
-
-	collectInfo := func(info *[]string) {
-		t.Helper()
-		for {
-			select {
-			case result := <-data:
-				assertPayload(t, result, "", nil)
-				*info = append(*info, result.Batch[0].Event)
-			case <-time.After(time.Second * 1):
-				return
-			}
-		}
-	}
-
-	collectBatches := func(info *[]batch) {
-		t.Helper()
-		for {
-			select {
-			case result := <-data:
-				assertPayload(t, result, "", nil)
-				*info = append(*info, result.Batch[0])
-			case <-time.After(time.Second * 1):
-				return
-			}
-		}
-	}
-
-	// Should send a client identify message
-	select {
-	case identifyMessage := <-data:
-		assertPayload(t, identifyMessage, "", nil)
-	case <-time.After(time.Second * 1):
-		require.Fail(t, "Did not receive ID message")
-	}
-
-	t.Run("Send", func(t *testing.T) {
-		testValue := "test-send-value-6789"
-		th.App.Srv().SendDiagnostic("Testing Diagnostic", map[string]interface{}{
-			"hey": testValue,
-		})
-		select {
-		case result := <-data:
-			assertPayload(t, result, "Testing Diagnostic", map[string]interface{}{
-				"hey": testValue,
-			})
-		case <-time.After(time.Second * 1):
-			require.Fail(t, "Did not receive diagnostic")
-		}
-	})
-
-	// Plugins remain disabled at this point
-	t.Run("SendDailyDiagnosticsPluginsDisabled", func(t *testing.T) {
-		th.App.Srv().sendDailyDiagnostics(true)
-
-		var info []string
-		// Collect the info sent.
-		collectInfo(&info)
-
-		for _, item := range []string{
-			TRACK_CONFIG_SERVICE,
-			TRACK_CONFIG_TEAM,
-			TRACK_CONFIG_SQL,
-			TRACK_CONFIG_LOG,
-			TRACK_CONFIG_NOTIFICATION_LOG,
-			TRACK_CONFIG_FILE,
-			TRACK_CONFIG_RATE,
-			TRACK_CONFIG_EMAIL,
-			TRACK_CONFIG_PRIVACY,
-			TRACK_CONFIG_OAUTH,
-			TRACK_CONFIG_LDAP,
-			TRACK_CONFIG_COMPLIANCE,
-			TRACK_CONFIG_LOCALIZATION,
-			TRACK_CONFIG_SAML,
-			TRACK_CONFIG_PASSWORD,
-			TRACK_CONFIG_CLUSTER,
-			TRACK_CONFIG_METRICS,
-			TRACK_CONFIG_SUPPORT,
-			TRACK_CONFIG_NATIVEAPP,
-			TRACK_CONFIG_EXPERIMENTAL,
-			TRACK_CONFIG_ANALYTICS,
-			TRACK_CONFIG_PLUGIN,
-			TRACK_ACTIVITY,
-			TRACK_SERVER,
-			TRACK_CONFIG_MESSAGE_EXPORT,
-			// TRACK_PLUGINS,
-		} {
-			require.Contains(t, info, item)
-		}
-	})
-
-	// Enable plugins for the remainder of the tests.
-	th.App.UpdateConfig(func(cfg *model.Config) { *cfg.PluginSettings.Enable = true })
-
-	t.Run("SendDailyDiagnostics", func(t *testing.T) {
-		th.App.Srv().sendDailyDiagnostics(true)
-
-		var info []string
-		// Collect the info sent.
-		collectInfo(&info)
-
-		for _, item := range []string{
-			TRACK_CONFIG_SERVICE,
-			TRACK_CONFIG_TEAM,
-			TRACK_CONFIG_SQL,
-			TRACK_CONFIG_LOG,
-			TRACK_CONFIG_NOTIFICATION_LOG,
-			TRACK_CONFIG_FILE,
-			TRACK_CONFIG_RATE,
-			TRACK_CONFIG_EMAIL,
-			TRACK_CONFIG_PRIVACY,
-			TRACK_CONFIG_OAUTH,
-			TRACK_CONFIG_LDAP,
-			TRACK_CONFIG_COMPLIANCE,
-			TRACK_CONFIG_LOCALIZATION,
-			TRACK_CONFIG_SAML,
-			TRACK_CONFIG_PASSWORD,
-			TRACK_CONFIG_CLUSTER,
-			TRACK_CONFIG_METRICS,
-			TRACK_CONFIG_SUPPORT,
-			TRACK_CONFIG_NATIVEAPP,
-			TRACK_CONFIG_EXPERIMENTAL,
-			TRACK_CONFIG_ANALYTICS,
-			TRACK_CONFIG_PLUGIN,
-			TRACK_ACTIVITY,
-			TRACK_SERVER,
-			TRACK_CONFIG_MESSAGE_EXPORT,
-			TRACK_PLUGINS,
-		} {
-			require.Contains(t, info, item)
-		}
-	})
-
-	t.Run("Diagnostics for Marketplace plugins is returned", func(t *testing.T) {
-		th.App.Srv().trackPluginConfig(th.App.Srv().Config(), marketplaceServer.URL)
-
-		var batches []batch
-		collectBatches(&batches)
-
-		for _, b := range batches {
-			if b.Event == TRACK_CONFIG_PLUGIN {
-				assert.Contains(t, b.Properties, "enable_testplugin")
-				assert.Contains(t, b.Properties, "version_testplugin")
-
-				// Confirm known plugins are not present
-				assert.NotContains(t, b.Properties, "enable_jira")
-				assert.NotContains(t, b.Properties, "version_jira")
-			}
-		}
-	})
-
-	t.Run("Diagnostics for known plugins is returned, if request to Marketplace fails", func(t *testing.T) {
-		th.App.Srv().trackPluginConfig(th.App.Srv().Config(), "http://some.random.invalid.url")
-
-		var batches []batch
-		collectBatches(&batches)
-
-		for _, b := range batches {
-			if b.Event == TRACK_CONFIG_PLUGIN {
-				assert.NotContains(t, b.Properties, "enable_testplugin")
-				assert.NotContains(t, b.Properties, "version_testplugin")
-
-				// Confirm known plugins are present
-				assert.Contains(t, b.Properties, "enable_jira")
-				assert.Contains(t, b.Properties, "version_jira")
-			}
-		}
-	})
-
-	t.Run("SendDailyDiagnosticsNoRudderKey", func(t *testing.T) {
-		th.App.Srv().SendDailyDiagnostics()
-
-		select {
-		case <-data:
-			require.Fail(t, "Should not send diagnostics when the rudder key is not set")
-		case <-time.After(time.Second * 1):
-			// Did not receive diagnostics
-		}
-	})
-
-	t.Run("SendDailyDiagnosticsDisabled", func(t *testing.T) {
-		th.App.UpdateConfig(func(cfg *model.Config) { *cfg.LogSettings.EnableDiagnostics = false })
-
-		th.App.Srv().sendDailyDiagnostics(true)
-
-		select {
-		case <-data:
-			require.Fail(t, "Should not send diagnostics when they are disabled")
-		case <-time.After(time.Second * 1):
-			// Did not receive diagnostics
-		}
-	})
-
-	t.Run("RudderConfigUsesConfigForValues", func(t *testing.T) {
-		os.Setenv("RUDDER_KEY", "abc123")
-		os.Setenv("RUDDER_DATAPLANE_URL", "arudderstackplace")
-		defer os.Unsetenv("RUDDER_KEY")
-		defer os.Unsetenv("RUDDER_DATAPLANE_URL")
-
-		config := th.App.Srv().getRudderConfig()
-
-		assert.Equal(t, "arudderstackplace", config.DataplaneUrl)
-		assert.Equal(t, "abc123", config.RudderKey)
-	})
-}

app/helper_test.go

@@ -167,17 +167,6 @@ func SetupEnterpriseWithStoreMock(tb testing.TB) *TestHelper {
 	return th
 }
 
-func SetupWithCustomConfig(tb testing.TB, configSet func(*model.Config)) *TestHelper {
-	if testing.Short() {
-		tb.SkipNow()
-	}
-	dbStore := mainHelper.GetStore()
-	dbStore.DropAllTables()
-	dbStore.MarkSystemRanUnitTests()
-
-	return setupTestHelper(dbStore, false, true, tb, configSet)
-}
-
 var initBasicOnce sync.Once
 var userCache struct {
 	SystemAdminUser *model.User
@@ -583,7 +572,7 @@ func (me *TestHelper) ResetRoleMigration() {
 
 	mainHelper.GetClusterInterface().SendClearRoleCacheMessage()
 
-	if _, err := sqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil {
+	if _, err := sqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": model.ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil {
 		panic(err)
 	}
 }

app/integration_action.go

@@ -528,7 +528,7 @@ func (a *App) buildWarnMetricMailtoLink(warnMetricId string, user *model.User) s
 	mailBody += T("api.server.warn_metric.bot_response.mailto_site_url_header", map[string]interface{}{"SiteUrl": a.GetSiteURL()})
 	mailBody += "\r\n"
 
-	mailBody += T("api.server.warn_metric.bot_response.mailto_diagnostic_id_header", map[string]interface{}{"DiagnosticId": a.DiagnosticId()})
+	mailBody += T("api.server.warn_metric.bot_response.mailto_diagnostic_id_header", map[string]interface{}{"DiagnosticId": a.TelemetryId()})
 	mailBody += "\r\n"
 
 	mailBody += T("api.server.warn_metric.bot_response.mailto_footer")

app/migrations.go

@@ -12,7 +12,6 @@ import (
 	"github.com/mattermost/mattermost-server/v5/utils"
 )
 
-const ADVANCED_PERMISSIONS_MIGRATION_KEY = "AdvancedPermissionsMigrationComplete"
 const EMOJIS_PERMISSIONS_MIGRATION_KEY = "EmojisPermissionsMigrationComplete"
 const GUEST_ROLES_CREATION_MIGRATION_KEY = "GuestRolesCreationMigrationComplete"
 const SYSTEM_CONSOLE_ROLES_CREATION_MIGRATION_KEY = "SystemConsoleRolesCreationMigrationComplete"
@@ -20,7 +19,7 @@ const SYSTEM_CONSOLE_ROLES_CREATION_MIGRATION_KEY = "SystemConsoleRolesCreationM
 // This function migrates the default built in roles from code/config to the database.
 func (a *App) DoAdvancedPermissionsMigration() {
 	// If the migration is already marked as completed, don't do it again.
-	if _, err := a.Srv().Store.System().GetByName(ADVANCED_PERMISSIONS_MIGRATION_KEY); err == nil {
+	if _, err := a.Srv().Store.System().GetByName(model.ADVANCED_PERMISSIONS_MIGRATION_KEY); err == nil {
 		return
 	}
 
@@ -71,7 +70,7 @@ func (a *App) DoAdvancedPermissionsMigration() {
 	}
 
 	system := model.System{
-		Name:  ADVANCED_PERMISSIONS_MIGRATION_KEY,
+		Name:  model.ADVANCED_PERMISSIONS_MIGRATION_KEY,
 		Value: "true",
 	}
 

app/notification_push.go

@@ -332,7 +332,7 @@ func (s *Server) StopPushNotificationsHubWorkers() {
 }
 
 func (a *App) sendToPushProxy(msg *model.PushNotification, session *model.Session) error {
-	msg.ServerId = a.DiagnosticId()
+	msg.ServerId = a.TelemetryId()
 
 	a.NotificationsLog().Info("Notification will be sent",
 		mlog.String("ackId", msg.AckId),

app/opentracing/opentracing_layer.go

@@ -2901,23 +2901,6 @@ func (a *OpenTracingAppLayer) DemoteUserToGuest(user *model.User) *model.AppErro
 	return resultVar0
 }
 
-func (a *OpenTracingAppLayer) DiagnosticId() string {
-	origCtx := a.ctx
-	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DiagnosticId")
-
-	a.ctx = newCtx
-	a.app.Srv().Store.SetContext(newCtx)
-	defer func() {
-		a.app.Srv().Store.SetContext(origCtx)
-		a.ctx = origCtx
-	}()
-
-	defer span.Finish()
-	resultVar0 := a.app.DiagnosticId()
-
-	return resultVar0
-}
-
 func (a *OpenTracingAppLayer) DisableAutoResponder(userId string, asAdmin bool) *model.AppError {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DisableAutoResponder")
@@ -13069,21 +13052,6 @@ func (a *OpenTracingAppLayer) SetDefaultProfileImage(user *model.User) *model.Ap
 	return resultVar0
 }
 
-func (a *OpenTracingAppLayer) SetDiagnosticId(id string) {
-	origCtx := a.ctx
-	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SetDiagnosticId")
-
-	a.ctx = newCtx
-	a.app.Srv().Store.SetContext(newCtx)
-	defer func() {
-		a.app.Srv().Store.SetContext(origCtx)
-		a.ctx = origCtx
-	}()
-
-	defer span.Finish()
-	a.app.SetDiagnosticId(id)
-}
-
 func (a *OpenTracingAppLayer) SetLog(l *mlog.Logger) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SetLog")
@@ -13785,6 +13753,23 @@ func (a *OpenTracingAppLayer) TeamMembersToRemove(teamID *string) ([]*model.Team
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) TelemetryId() string {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.TelemetryId")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0 := a.app.TelemetryId()
+
+	return resultVar0
+}
+
 func (a *OpenTracingAppLayer) TestElasticsearch(cfg *model.Config) *model.AppError {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.TestElasticsearch")

app/permissions.go

@@ -54,7 +54,7 @@ func (a *App) ResetPermissionsSystem() *model.AppError {
 	}
 
 	// Remove the "System" table entry that marks the advanced permissions migration as done.
-	if _, err := a.Srv().Store.System().PermanentDeleteByName(ADVANCED_PERMISSIONS_MIGRATION_KEY); err != nil {
+	if _, err := a.Srv().Store.System().PermanentDeleteByName(model.ADVANCED_PERMISSIONS_MIGRATION_KEY); err != nil {
 		return model.NewAppError("ResetPermissionSystem", "app.system.permanent_delete_by_name.app_error", nil, err.Error(), http.StatusInternalServerError)
 	}
 

app/plugin_api.go

@@ -144,7 +144,11 @@ func (api *PluginAPI) GetSystemInstallDate() (int64, *model.AppError) {
 }
 
 func (api *PluginAPI) GetDiagnosticId() string {
-	return api.app.DiagnosticId()
+	return api.app.TelemetryId()
+}
+
+func (api *PluginAPI) GetTelemetryId() string {
+	return api.app.TelemetryId()
 }
 
 func (api *PluginAPI) CreateTeam(team *model.Team) (*model.Team, *model.AppError) {

app/security_update_check.go

@@ -49,7 +49,7 @@ func (s *Server) DoSecurityUpdateCheck() {
 
 		v := url.Values{}
 
-		v.Set(PROP_SECURITY_ID, s.diagnosticId)
+		v.Set(PROP_SECURITY_ID, s.TelemetryId())
 		v.Set(PROP_SECURITY_BUILD, model.CurrentVersion+"."+model.BuildNumber)
 		v.Set(PROP_SECURITY_ENTERPRISE_READY, model.BuildEnterpriseReady)
 		v.Set(PROP_SECURITY_DATABASE, *s.Config().SqlSettings.DriverName)

app/server.go

@@ -26,7 +26,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 	"github.com/rs/cors"
-	rudder "github.com/rudderlabs/analytics-go"
 
 	"golang.org/x/crypto/acme/autocert"
 
@@ -44,6 +43,7 @@ import (
 	"github.com/mattermost/mattermost-server/v5/services/mailservice"
 	"github.com/mattermost/mattermost-server/v5/services/searchengine"
 	"github.com/mattermost/mattermost-server/v5/services/searchengine/bleveengine"
+	"github.com/mattermost/mattermost-server/v5/services/telemetry"
 	"github.com/mattermost/mattermost-server/v5/services/timezones"
 	"github.com/mattermost/mattermost-server/v5/services/tracing"
 	"github.com/mattermost/mattermost-server/v5/services/upgrader"
@@ -58,6 +58,9 @@ import (
 
 var MaxNotificationsPerChannelDefault int64 = 1000000
 
+// declaring this as var to allow overriding in tests
+var SENTRY_DSN = "placeholder_sentry_dsn"
+
 type Server struct {
 	sqlStore           *sqlstore.SqlSupplier
 	Store              store.Store
@@ -136,8 +139,7 @@ type Server struct {
 	clientConfigHash    atomic.Value
 	limitedClientConfig atomic.Value
 
-	diagnosticId string
-	rudderClient rudder.Client
+	telemetryService *telemetry.TelemetryService
 
 	phase2PermissionsMigrationComplete bool
 
@@ -167,8 +169,7 @@ type Server struct {
 
 	CacheProvider cache.Provider
 
-	tracer                      *tracing.Tracer
-	timestampLastDiagnosticSent time.Time
+	tracer *tracing.Tracer
 }
 
 func NewServer(options ...Option) (*Server, error) {
@@ -331,6 +332,8 @@ func NewServer(options ...Option) (*Server, error) {
 
 	s.Store = s.newStore()
 
+	s.telemetryService = telemetry.New(s, s.Store, s.SearchEngine, s.Log)
+
 	emailService, err := NewEmailService(s)
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to initialize email service")
@@ -363,7 +366,6 @@ func NewServer(options ...Option) (*Server, error) {
 		return nil, errors.Wrapf(err, "unable to ensure first run timestamp")
 	}
 
-	s.ensureDiagnosticId()
 	s.regenerateClientConfig()
 
 	s.clusterLeaderListenerId = s.AddClusterLeaderChangedListener(func() {
@@ -519,7 +521,13 @@ func (s *Server) RunJobs() {
 			runSecurityJob(s)
 		})
 		s.Go(func() {
-			runDiagnosticsJob(s)
+			firstRun, err := s.getFirstServerRunTimestamp()
+			if err != nil {
+				mlog.Warn("Fetching time of first server run failed. Setting to 'now'.")
+				s.ensureFirstServerRunTimestamp()
+				firstRun = utils.MillisFromTime(time.Now())
+			}
+			s.telemetryService.RunTelemetryJob(firstRun)
 		})
 		s.Go(func() {
 			runSessionCleanupJob(s)
@@ -676,9 +684,9 @@ func (s *Server) Shutdown() error {
 		}
 	}
 
-	err := s.shutdownDiagnostics()
+	err := s.telemetryService.Shutdown()
 	if err != nil {
-		mlog.Error("Unable to cleanly shutdown diagnostic client", mlog.Err(err))
+		mlog.Error("Unable to cleanly shutdown telemetry client", mlog.Err(err))
 	}
 
 	s.StopHTTPServer()
@@ -1105,34 +1113,6 @@ func runSecurityJob(s *Server) {
 	}, time.Hour*4)
 }
 
-func doDiagnosticsIfNeeded(s *Server, firstRun time.Time) {
-	hoursSinceFirstServerRun := time.Since(firstRun).Hours()
-	// Send once every 10 minutes for the first hour
-	// Send once every hour thereafter for the first 12 hours
-	// Send at the 24 hour mark and every 24 hours after
-	if hoursSinceFirstServerRun < 1 {
-		doDiagnostics(s)
-	} else if hoursSinceFirstServerRun <= 12 && time.Since(s.timestampLastDiagnosticSent) >= time.Hour {
-		doDiagnostics(s)
-	} else if hoursSinceFirstServerRun > 12 && time.Since(s.timestampLastDiagnosticSent) >= 24*time.Hour {
-		doDiagnostics(s)
-	}
-}
-
-func runDiagnosticsJob(s *Server) {
-	// Send on boot
-	doDiagnostics(s)
-	firstRun, err := s.getFirstServerRunTimestamp()
-	if err != nil {
-		mlog.Warn("Fetching time of first server run failed. Setting to 'now'.")
-		s.ensureFirstServerRunTimestamp()
-		firstRun = utils.MillisFromTime(time.Now())
-	}
-	model.CreateRecurringTask("Diagnostics", func() {
-		doDiagnosticsIfNeeded(s, utils.TimeFromMillis(firstRun))
-	}, time.Minute*10)
-}
-
 func runTokenCleanupJob(s *Server) {
 	doTokenCleanup(s)
 	model.CreateRecurringTask("Token Cleanup", func() {
@@ -1172,13 +1152,6 @@ func doSecurity(s *Server) {
 	s.DoSecurityUpdateCheck()
 }
 
-func doDiagnostics(s *Server) {
-	if *s.Config().LogSettings.EnableDiagnostics {
-		s.timestampLastDiagnosticSent = time.Now()
-		s.SendDailyDiagnostics()
-	}
-}
-
 func doTokenCleanup(s *Server) {
 	s.Store.Token().Cleanup()
 }
@@ -1368,39 +1341,6 @@ func (s *Server) stopSearchEngine() {
 	}
 }
 
-// initDiagnostics initialises the Rudder client for the diagnostics system.
-func (s *Server) initDiagnostics(endpoint string, rudderKey string) {
-	if s.rudderClient == nil {
-		config := rudder.Config{}
-		config.Logger = rudder.StdLogger(s.Log.StdLog(mlog.String("source", "rudder")))
-		config.Endpoint = endpoint
-		// For testing
-		if endpoint != RUDDER_DATAPLANE_URL {
-			config.Verbose = true
-			config.BatchSize = 1
-		}
-		client, err := rudder.NewWithConfig(rudderKey, endpoint, config)
-		if err != nil {
-			mlog.Error("Failed to create Rudder instance", mlog.Err(err))
-			return
-		}
-		client.Enqueue(rudder.Identify{
-			UserId: s.diagnosticId,
-		})
-
-		s.rudderClient = client
-	}
-}
-
-// shutdownDiagnostics closes the diagnostics system Rudder client.
-func (s *Server) shutdownDiagnostics() error {
-	if s.rudderClient != nil {
-		return s.rudderClient.Close()
-	}
-
-	return nil
-}
-
 func (s *Server) FileBackend() (filesstore.FileBackend, *model.AppError) {
 	license := s.License()
 	return filesstore.NewFileBackend(&s.Config().FileSettings, license != nil && *license.Features.Compliance)
@@ -1421,25 +1361,6 @@ func (s *Server) ClusterHealthScore() int {
 	return s.Cluster.HealthScore()
 }
 
-func (s *Server) ensureDiagnosticId() {
-	if s.diagnosticId != "" {
-		return
-	}
-	props, err := s.Store.System().Get()
-	if err != nil {
-		return
-	}
-
-	id := props[model.SYSTEM_DIAGNOSTIC_ID]
-	if len(id) == 0 {
-		id = model.NewId()
-		systemID := &model.System{Name: model.SYSTEM_DIAGNOSTIC_ID, Value: id}
-		s.Store.System().Save(systemID)
-	}
-
-	s.diagnosticId = id
-}
-
 func (s *Server) configOrLicenseListener() {
 	s.regenerateClientConfig()
 }
@@ -1469,3 +1390,14 @@ func (s *Server) initJobs() {
 		s.Jobs.Migrations = jobsMigrationsInterface(s)
 	}
 }
+
+func (s *Server) TelemetryId() string {
+	if s.telemetryService == nil {
+		return ""
+	}
+	return s.telemetryService.TelemetryID
+}
+
+func (s *Server) HttpService() httpservice.HTTPService {
+	return s.HTTPService
+}