From 44079785eb126b13f593562f7450ad49abf1bd8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Tue, 8 Sep 2020 20:30:54 +0200 Subject: [PATCH] Moving diagnostics into a service (#14832) * Moving diagnostics into a service * Fixing golint checks * Fixing tests * Renaming from diagnostics to telemetry * Adding missing files * Initializing telemetry earlier in the server startup * Fixing tests * Adding a log for the telemetryID initialization error * Addressing PR review comments * Fixing merge problem * Removing some extra Diagnostics mentions * Making tests pass --- Makefile | 4 + api4/license.go | 2 +- app/app.go | 12 +- app/app_iface.go | 3 +- app/config.go | 4 +- app/diagnostics_test.go | 350 ------------- app/helper_test.go | 13 +- app/integration_action.go | 2 +- app/migrations.go | 5 +- app/notification_push.go | 2 +- app/opentracing/opentracing_layer.go | 49 +- app/permissions.go | 2 +- app/plugin_api.go | 6 +- app/security_update_check.go | 2 +- app/server.go | 124 ++--- config/client.go | 9 +- config/client_test.go | 8 +- migrations/helper_test.go | 2 +- model/migration.go | 1 + model/system.go | 2 +- plugin/api.go | 6 + plugin/api_timer_layer_generated.go | 7 + plugin/client_rpc_generated.go | 27 + plugin/plugintest/api.go | 14 + services/telemetry/mocks/ServerIface.go | 147 ++++++ .../telemetry/telemetry.go | 422 ++++++++++------ services/telemetry/telemetry_test.go | 476 ++++++++++++++++++ templates/warn_metric_ack.html | 6 +- 28 files changed, 1019 insertions(+), 688 deletions(-) delete mode 100644 app/diagnostics_test.go create mode 100644 services/telemetry/mocks/ServerIface.go rename app/diagnostics.go => services/telemetry/telemetry.go (79%) create mode 100644 services/telemetry/telemetry_test.go diff --git a/Makefile b/Makefile index c200a5a623..5001f7d9ea 100644 --- a/Makefile +++ b/Makefile @@ -240,6 +240,10 @@ store-mocks: ## Creates mock files. $(GO) get -modfile=go.tools.mod github.com/vektra/mockery/... $(GOBIN)/mockery -dir store -all -output store/storetest/mocks -note 'Regenerate this file using `make store-mocks`.' +telemetry-mocks: ## Creates mock files. + $(GO) get -modfile=go.tools.mod github.com/vektra/mockery/... + $(GOBIN)/mockery -dir services/telemetry -all -output services/telemetry/mocks -note 'Regenerate this file using `make telemetry-mocks`.' + store-layers: ## Generate layers for the store $(GO) generate $(GOFLAGS) ./store diff --git a/api4/license.go b/api4/license.go index 4b1202819c..b8ee89153a 100644 --- a/api4/license.go +++ b/api4/license.go @@ -185,7 +185,7 @@ func requestTrialLicense(c *Context, w http.ResponseWriter, r *http.Request) { } trialLicenseRequest := &model.TrialLicenseRequest{ - ServerID: c.App.DiagnosticId(), + ServerID: c.App.TelemetryId(), Name: currentUser.GetDisplayName(model.SHOW_FULLNAME), Email: currentUser.Email, SiteName: *c.App.Config().TeamSettings.SiteName, diff --git a/app/app.go b/app/app.go index e25a010d58..0276629624 100644 --- a/app/app.go +++ b/app/app.go @@ -127,12 +127,8 @@ func (a *App) initJobs() { a.srv.Jobs.Schedulers = a.srv.Jobs.InitSchedulers() } -func (a *App) DiagnosticId() string { - return a.Srv().diagnosticId -} - -func (a *App) SetDiagnosticId(id string) { - a.Srv().diagnosticId = id +func (a *App) TelemetryId() string { + return a.Srv().TelemetryId() } func (s *Server) HTMLTemplates() *template.Template { @@ -374,8 +370,8 @@ func (a *App) NotifyAndSetWarnMetricAck(warnMetricId string, sender *model.User, } bodyPage.Props["SiteURLHeader"] = T("api.templates.warn_metric_ack.body.site_url_header") bodyPage.Props["SiteURL"] = a.GetSiteURL() - bodyPage.Props["DiagnosticIdHeader"] = T("api.templates.warn_metric_ack.body.diagnostic_id_header") - bodyPage.Props["DiagnosticIdValue"] = a.DiagnosticId() + bodyPage.Props["TelemetryIdHeader"] = T("api.templates.warn_metric_ack.body.diagnostic_id_header") + bodyPage.Props["TelemetryIdValue"] = a.TelemetryId() bodyPage.Props["Footer"] = T("api.templates.warn_metric_ack.footer") warnMetricStatus, warnMetricDisplayTexts := a.getWarnMetricStatusAndDisplayTextsForId(warnMetricId, T) diff --git a/app/app_iface.go b/app/app_iface.go index d66feaf7fe..deafbc50e4 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -457,7 +457,6 @@ type AppIface interface { DeleteScheme(schemeId string) (*model.Scheme, *model.AppError) DeleteSidebarCategory(userId, teamId, categoryId string) *model.AppError DeleteToken(token *model.Token) *model.AppError - DiagnosticId() string DisableAutoResponder(userId string, asAdmin bool) *model.AppError DisableUserAccessToken(token *model.UserAccessToken) *model.AppError DoAppMigrations() @@ -885,7 +884,6 @@ type AppIface interface { SetAutoResponderStatus(user *model.User, oldNotifyProps model.StringMap) SetContext(c context.Context) SetDefaultProfileImage(user *model.User) *model.AppError - SetDiagnosticId(id string) SetIpAddress(s string) SetLog(l *mlog.Logger) SetPath(s string) @@ -925,6 +923,7 @@ type AppIface interface { T(translationID string, args ...interface{}) string TeamMembersToAdd(since int64, teamID *string) ([]*model.UserTeamIDPair, *model.AppError) TeamMembersToRemove(teamID *string) ([]*model.TeamMember, *model.AppError) + TelemetryId() string TestElasticsearch(cfg *model.Config) *model.AppError TestEmail(userId string, cfg *model.Config) *model.AppError TestLdap() *model.AppError diff --git a/app/config.go b/app/config.go index b4a0c5a78e..29a84206c1 100644 --- a/app/config.go +++ b/app/config.go @@ -297,8 +297,8 @@ func (a *App) PostActionCookieSecret() []byte { } func (s *Server) regenerateClientConfig() { - clientConfig := config.GenerateClientConfig(s.Config(), s.diagnosticId, s.License()) - limitedClientConfig := config.GenerateLimitedClientConfig(s.Config(), s.diagnosticId, s.License()) + clientConfig := config.GenerateClientConfig(s.Config(), s.TelemetryId(), s.License()) + limitedClientConfig := config.GenerateLimitedClientConfig(s.Config(), s.TelemetryId(), s.License()) if clientConfig["EnableCustomTermsOfService"] == "true" { termsOfService, err := s.Store.TermsOfService().GetLatest(true) diff --git a/app/diagnostics_test.go b/app/diagnostics_test.go deleted file mode 100644 index 054bb265bc..0000000000 --- a/app/diagnostics_test.go +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See LICENSE.txt for license information. - -package app - -import ( - "encoding/json" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/mattermost/mattermost-server/v5/model" -) - -func TestPluginSetting(t *testing.T) { - settings := &model.PluginSettings{ - Plugins: map[string]map[string]interface{}{ - "test": { - "foo": "bar", - }, - }, - } - assert.Equal(t, "bar", pluginSetting(settings, "test", "foo", "asd")) - assert.Equal(t, "asd", pluginSetting(settings, "test", "qwe", "asd")) -} - -func TestPluginActivated(t *testing.T) { - states := map[string]*model.PluginState{ - "foo": { - Enable: true, - }, - "bar": { - Enable: false, - }, - } - assert.True(t, pluginActivated(states, "foo")) - assert.False(t, pluginActivated(states, "bar")) - assert.False(t, pluginActivated(states, "none")) -} - -func TestPluginVersion(t *testing.T) { - plugins := []*model.BundleInfo{ - { - Manifest: &model.Manifest{ - Id: "test.plugin", - Version: "1.2.3", - }, - }, - { - Manifest: &model.Manifest{ - Id: "test.plugin2", - Version: "4.5.6", - }, - }, - } - assert.Equal(t, "1.2.3", pluginVersion(plugins, "test.plugin")) - assert.Equal(t, "4.5.6", pluginVersion(plugins, "test.plugin2")) - assert.Empty(t, pluginVersion(plugins, "unknown.plugin")) -} - -func TestRudderDiagnostics(t *testing.T) { - if testing.Short() { - t.SkipNow() - } - - th := SetupWithCustomConfig(t, func(config *model.Config) { - *config.PluginSettings.Enable = false - }) - defer th.TearDown() - - type batch struct { - MessageId string - UserId string - Event string - Timestamp time.Time - Properties map[string]interface{} - } - - type payload struct { - MessageId string - SentAt time.Time - Batch []batch - Context struct { - Library struct { - Name string - Version string - } - } - } - - data := make(chan payload, 100) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - body, err := ioutil.ReadAll(r.Body) - require.NoError(t, err) - - var p payload - err = json.Unmarshal(body, &p) - require.NoError(t, err) - - data <- p - })) - defer server.Close() - - marketplaceServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - res.WriteHeader(http.StatusOK) - json, err := json.Marshal([]*model.MarketplacePlugin{{ - BaseMarketplacePlugin: &model.BaseMarketplacePlugin{ - Manifest: &model.Manifest{ - Id: "testplugin", - }, - }, - }}) - require.NoError(t, err) - res.Write(json) - })) - - defer func() { marketplaceServer.Close() }() - - diagnosticID := "test-diagnostic-id-12345" - th.App.SetDiagnosticId(diagnosticID) - th.Server.initDiagnostics(server.URL, RUDDER_KEY) - - assertPayload := func(t *testing.T, actual payload, event string, properties map[string]interface{}) { - t.Helper() - assert.NotEmpty(t, actual.MessageId) - assert.False(t, actual.SentAt.IsZero()) - if assert.Len(t, actual.Batch, 1) { - assert.NotEmpty(t, actual.Batch[0].MessageId, "message id should not be empty") - assert.Equal(t, diagnosticID, actual.Batch[0].UserId) - if event != "" { - assert.Equal(t, event, actual.Batch[0].Event) - } - assert.False(t, actual.Batch[0].Timestamp.IsZero(), "batch timestamp should not be the zero value") - if properties != nil { - assert.Equal(t, properties, actual.Batch[0].Properties) - } - } - assert.Equal(t, "analytics-go", actual.Context.Library.Name) - assert.Equal(t, "3.0.0", actual.Context.Library.Version) - } - - collectInfo := func(info *[]string) { - t.Helper() - for { - select { - case result := <-data: - assertPayload(t, result, "", nil) - *info = append(*info, result.Batch[0].Event) - case <-time.After(time.Second * 1): - return - } - } - } - - collectBatches := func(info *[]batch) { - t.Helper() - for { - select { - case result := <-data: - assertPayload(t, result, "", nil) - *info = append(*info, result.Batch[0]) - case <-time.After(time.Second * 1): - return - } - } - } - - // Should send a client identify message - select { - case identifyMessage := <-data: - assertPayload(t, identifyMessage, "", nil) - case <-time.After(time.Second * 1): - require.Fail(t, "Did not receive ID message") - } - - t.Run("Send", func(t *testing.T) { - testValue := "test-send-value-6789" - th.App.Srv().SendDiagnostic("Testing Diagnostic", map[string]interface{}{ - "hey": testValue, - }) - select { - case result := <-data: - assertPayload(t, result, "Testing Diagnostic", map[string]interface{}{ - "hey": testValue, - }) - case <-time.After(time.Second * 1): - require.Fail(t, "Did not receive diagnostic") - } - }) - - // Plugins remain disabled at this point - t.Run("SendDailyDiagnosticsPluginsDisabled", func(t *testing.T) { - th.App.Srv().sendDailyDiagnostics(true) - - var info []string - // Collect the info sent. - collectInfo(&info) - - for _, item := range []string{ - TRACK_CONFIG_SERVICE, - TRACK_CONFIG_TEAM, - TRACK_CONFIG_SQL, - TRACK_CONFIG_LOG, - TRACK_CONFIG_NOTIFICATION_LOG, - TRACK_CONFIG_FILE, - TRACK_CONFIG_RATE, - TRACK_CONFIG_EMAIL, - TRACK_CONFIG_PRIVACY, - TRACK_CONFIG_OAUTH, - TRACK_CONFIG_LDAP, - TRACK_CONFIG_COMPLIANCE, - TRACK_CONFIG_LOCALIZATION, - TRACK_CONFIG_SAML, - TRACK_CONFIG_PASSWORD, - TRACK_CONFIG_CLUSTER, - TRACK_CONFIG_METRICS, - TRACK_CONFIG_SUPPORT, - TRACK_CONFIG_NATIVEAPP, - TRACK_CONFIG_EXPERIMENTAL, - TRACK_CONFIG_ANALYTICS, - TRACK_CONFIG_PLUGIN, - TRACK_ACTIVITY, - TRACK_SERVER, - TRACK_CONFIG_MESSAGE_EXPORT, - // TRACK_PLUGINS, - } { - require.Contains(t, info, item) - } - }) - - // Enable plugins for the remainder of the tests. - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.PluginSettings.Enable = true }) - - t.Run("SendDailyDiagnostics", func(t *testing.T) { - th.App.Srv().sendDailyDiagnostics(true) - - var info []string - // Collect the info sent. - collectInfo(&info) - - for _, item := range []string{ - TRACK_CONFIG_SERVICE, - TRACK_CONFIG_TEAM, - TRACK_CONFIG_SQL, - TRACK_CONFIG_LOG, - TRACK_CONFIG_NOTIFICATION_LOG, - TRACK_CONFIG_FILE, - TRACK_CONFIG_RATE, - TRACK_CONFIG_EMAIL, - TRACK_CONFIG_PRIVACY, - TRACK_CONFIG_OAUTH, - TRACK_CONFIG_LDAP, - TRACK_CONFIG_COMPLIANCE, - TRACK_CONFIG_LOCALIZATION, - TRACK_CONFIG_SAML, - TRACK_CONFIG_PASSWORD, - TRACK_CONFIG_CLUSTER, - TRACK_CONFIG_METRICS, - TRACK_CONFIG_SUPPORT, - TRACK_CONFIG_NATIVEAPP, - TRACK_CONFIG_EXPERIMENTAL, - TRACK_CONFIG_ANALYTICS, - TRACK_CONFIG_PLUGIN, - TRACK_ACTIVITY, - TRACK_SERVER, - TRACK_CONFIG_MESSAGE_EXPORT, - TRACK_PLUGINS, - } { - require.Contains(t, info, item) - } - }) - - t.Run("Diagnostics for Marketplace plugins is returned", func(t *testing.T) { - th.App.Srv().trackPluginConfig(th.App.Srv().Config(), marketplaceServer.URL) - - var batches []batch - collectBatches(&batches) - - for _, b := range batches { - if b.Event == TRACK_CONFIG_PLUGIN { - assert.Contains(t, b.Properties, "enable_testplugin") - assert.Contains(t, b.Properties, "version_testplugin") - - // Confirm known plugins are not present - assert.NotContains(t, b.Properties, "enable_jira") - assert.NotContains(t, b.Properties, "version_jira") - } - } - }) - - t.Run("Diagnostics for known plugins is returned, if request to Marketplace fails", func(t *testing.T) { - th.App.Srv().trackPluginConfig(th.App.Srv().Config(), "http://some.random.invalid.url") - - var batches []batch - collectBatches(&batches) - - for _, b := range batches { - if b.Event == TRACK_CONFIG_PLUGIN { - assert.NotContains(t, b.Properties, "enable_testplugin") - assert.NotContains(t, b.Properties, "version_testplugin") - - // Confirm known plugins are present - assert.Contains(t, b.Properties, "enable_jira") - assert.Contains(t, b.Properties, "version_jira") - } - } - }) - - t.Run("SendDailyDiagnosticsNoRudderKey", func(t *testing.T) { - th.App.Srv().SendDailyDiagnostics() - - select { - case <-data: - require.Fail(t, "Should not send diagnostics when the rudder key is not set") - case <-time.After(time.Second * 1): - // Did not receive diagnostics - } - }) - - t.Run("SendDailyDiagnosticsDisabled", func(t *testing.T) { - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.LogSettings.EnableDiagnostics = false }) - - th.App.Srv().sendDailyDiagnostics(true) - - select { - case <-data: - require.Fail(t, "Should not send diagnostics when they are disabled") - case <-time.After(time.Second * 1): - // Did not receive diagnostics - } - }) - - t.Run("RudderConfigUsesConfigForValues", func(t *testing.T) { - os.Setenv("RUDDER_KEY", "abc123") - os.Setenv("RUDDER_DATAPLANE_URL", "arudderstackplace") - defer os.Unsetenv("RUDDER_KEY") - defer os.Unsetenv("RUDDER_DATAPLANE_URL") - - config := th.App.Srv().getRudderConfig() - - assert.Equal(t, "arudderstackplace", config.DataplaneUrl) - assert.Equal(t, "abc123", config.RudderKey) - }) -} diff --git a/app/helper_test.go b/app/helper_test.go index 0ce9d97f65..ea4e8f1998 100644 --- a/app/helper_test.go +++ b/app/helper_test.go @@ -167,17 +167,6 @@ func SetupEnterpriseWithStoreMock(tb testing.TB) *TestHelper { return th } -func SetupWithCustomConfig(tb testing.TB, configSet func(*model.Config)) *TestHelper { - if testing.Short() { - tb.SkipNow() - } - dbStore := mainHelper.GetStore() - dbStore.DropAllTables() - dbStore.MarkSystemRanUnitTests() - - return setupTestHelper(dbStore, false, true, tb, configSet) -} - var initBasicOnce sync.Once var userCache struct { SystemAdminUser *model.User @@ -583,7 +572,7 @@ func (me *TestHelper) ResetRoleMigration() { mainHelper.GetClusterInterface().SendClearRoleCacheMessage() - if _, err := sqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil { + if _, err := sqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": model.ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil { panic(err) } } diff --git a/app/integration_action.go b/app/integration_action.go index ddc02a2430..ef27b66b2b 100644 --- a/app/integration_action.go +++ b/app/integration_action.go @@ -528,7 +528,7 @@ func (a *App) buildWarnMetricMailtoLink(warnMetricId string, user *model.User) s mailBody += T("api.server.warn_metric.bot_response.mailto_site_url_header", map[string]interface{}{"SiteUrl": a.GetSiteURL()}) mailBody += "\r\n" - mailBody += T("api.server.warn_metric.bot_response.mailto_diagnostic_id_header", map[string]interface{}{"DiagnosticId": a.DiagnosticId()}) + mailBody += T("api.server.warn_metric.bot_response.mailto_diagnostic_id_header", map[string]interface{}{"DiagnosticId": a.TelemetryId()}) mailBody += "\r\n" mailBody += T("api.server.warn_metric.bot_response.mailto_footer") diff --git a/app/migrations.go b/app/migrations.go index f8e7b12041..a0571e013b 100644 --- a/app/migrations.go +++ b/app/migrations.go @@ -12,7 +12,6 @@ import ( "github.com/mattermost/mattermost-server/v5/utils" ) -const ADVANCED_PERMISSIONS_MIGRATION_KEY = "AdvancedPermissionsMigrationComplete" const EMOJIS_PERMISSIONS_MIGRATION_KEY = "EmojisPermissionsMigrationComplete" const GUEST_ROLES_CREATION_MIGRATION_KEY = "GuestRolesCreationMigrationComplete" const SYSTEM_CONSOLE_ROLES_CREATION_MIGRATION_KEY = "SystemConsoleRolesCreationMigrationComplete" @@ -20,7 +19,7 @@ const SYSTEM_CONSOLE_ROLES_CREATION_MIGRATION_KEY = "SystemConsoleRolesCreationM // This function migrates the default built in roles from code/config to the database. func (a *App) DoAdvancedPermissionsMigration() { // If the migration is already marked as completed, don't do it again. - if _, err := a.Srv().Store.System().GetByName(ADVANCED_PERMISSIONS_MIGRATION_KEY); err == nil { + if _, err := a.Srv().Store.System().GetByName(model.ADVANCED_PERMISSIONS_MIGRATION_KEY); err == nil { return } @@ -71,7 +70,7 @@ func (a *App) DoAdvancedPermissionsMigration() { } system := model.System{ - Name: ADVANCED_PERMISSIONS_MIGRATION_KEY, + Name: model.ADVANCED_PERMISSIONS_MIGRATION_KEY, Value: "true", } diff --git a/app/notification_push.go b/app/notification_push.go index 21f2f5c757..4a4f3502f1 100644 --- a/app/notification_push.go +++ b/app/notification_push.go @@ -332,7 +332,7 @@ func (s *Server) StopPushNotificationsHubWorkers() { } func (a *App) sendToPushProxy(msg *model.PushNotification, session *model.Session) error { - msg.ServerId = a.DiagnosticId() + msg.ServerId = a.TelemetryId() a.NotificationsLog().Info("Notification will be sent", mlog.String("ackId", msg.AckId), diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index 62d04ab534..9f2336cba6 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -2901,23 +2901,6 @@ func (a *OpenTracingAppLayer) DemoteUserToGuest(user *model.User) *model.AppErro return resultVar0 } -func (a *OpenTracingAppLayer) DiagnosticId() string { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DiagnosticId") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0 := a.app.DiagnosticId() - - return resultVar0 -} - func (a *OpenTracingAppLayer) DisableAutoResponder(userId string, asAdmin bool) *model.AppError { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DisableAutoResponder") @@ -13069,21 +13052,6 @@ func (a *OpenTracingAppLayer) SetDefaultProfileImage(user *model.User) *model.Ap return resultVar0 } -func (a *OpenTracingAppLayer) SetDiagnosticId(id string) { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SetDiagnosticId") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - a.app.SetDiagnosticId(id) -} - func (a *OpenTracingAppLayer) SetLog(l *mlog.Logger) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SetLog") @@ -13785,6 +13753,23 @@ func (a *OpenTracingAppLayer) TeamMembersToRemove(teamID *string) ([]*model.Team return resultVar0, resultVar1 } +func (a *OpenTracingAppLayer) TelemetryId() string { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.TelemetryId") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0 := a.app.TelemetryId() + + return resultVar0 +} + func (a *OpenTracingAppLayer) TestElasticsearch(cfg *model.Config) *model.AppError { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.TestElasticsearch") diff --git a/app/permissions.go b/app/permissions.go index 9b719f3d8a..a8c514d6b7 100644 --- a/app/permissions.go +++ b/app/permissions.go @@ -54,7 +54,7 @@ func (a *App) ResetPermissionsSystem() *model.AppError { } // Remove the "System" table entry that marks the advanced permissions migration as done. - if _, err := a.Srv().Store.System().PermanentDeleteByName(ADVANCED_PERMISSIONS_MIGRATION_KEY); err != nil { + if _, err := a.Srv().Store.System().PermanentDeleteByName(model.ADVANCED_PERMISSIONS_MIGRATION_KEY); err != nil { return model.NewAppError("ResetPermissionSystem", "app.system.permanent_delete_by_name.app_error", nil, err.Error(), http.StatusInternalServerError) } diff --git a/app/plugin_api.go b/app/plugin_api.go index 02c54e9d17..7bdb8cc66c 100644 --- a/app/plugin_api.go +++ b/app/plugin_api.go @@ -144,7 +144,11 @@ func (api *PluginAPI) GetSystemInstallDate() (int64, *model.AppError) { } func (api *PluginAPI) GetDiagnosticId() string { - return api.app.DiagnosticId() + return api.app.TelemetryId() +} + +func (api *PluginAPI) GetTelemetryId() string { + return api.app.TelemetryId() } func (api *PluginAPI) CreateTeam(team *model.Team) (*model.Team, *model.AppError) { diff --git a/app/security_update_check.go b/app/security_update_check.go index d918623614..ac37175a71 100644 --- a/app/security_update_check.go +++ b/app/security_update_check.go @@ -49,7 +49,7 @@ func (s *Server) DoSecurityUpdateCheck() { v := url.Values{} - v.Set(PROP_SECURITY_ID, s.diagnosticId) + v.Set(PROP_SECURITY_ID, s.TelemetryId()) v.Set(PROP_SECURITY_BUILD, model.CurrentVersion+"."+model.BuildNumber) v.Set(PROP_SECURITY_ENTERPRISE_READY, model.BuildEnterpriseReady) v.Set(PROP_SECURITY_DATABASE, *s.Config().SqlSettings.DriverName) diff --git a/app/server.go b/app/server.go index eb3cea761e..cc6baba532 100644 --- a/app/server.go +++ b/app/server.go @@ -26,7 +26,6 @@ import ( "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/rs/cors" - rudder "github.com/rudderlabs/analytics-go" "golang.org/x/crypto/acme/autocert" @@ -44,6 +43,7 @@ import ( "github.com/mattermost/mattermost-server/v5/services/mailservice" "github.com/mattermost/mattermost-server/v5/services/searchengine" "github.com/mattermost/mattermost-server/v5/services/searchengine/bleveengine" + "github.com/mattermost/mattermost-server/v5/services/telemetry" "github.com/mattermost/mattermost-server/v5/services/timezones" "github.com/mattermost/mattermost-server/v5/services/tracing" "github.com/mattermost/mattermost-server/v5/services/upgrader" @@ -58,6 +58,9 @@ import ( var MaxNotificationsPerChannelDefault int64 = 1000000 +// declaring this as var to allow overriding in tests +var SENTRY_DSN = "placeholder_sentry_dsn" + type Server struct { sqlStore *sqlstore.SqlSupplier Store store.Store @@ -136,8 +139,7 @@ type Server struct { clientConfigHash atomic.Value limitedClientConfig atomic.Value - diagnosticId string - rudderClient rudder.Client + telemetryService *telemetry.TelemetryService phase2PermissionsMigrationComplete bool @@ -167,8 +169,7 @@ type Server struct { CacheProvider cache.Provider - tracer *tracing.Tracer - timestampLastDiagnosticSent time.Time + tracer *tracing.Tracer } func NewServer(options ...Option) (*Server, error) { @@ -331,6 +332,8 @@ func NewServer(options ...Option) (*Server, error) { s.Store = s.newStore() + s.telemetryService = telemetry.New(s, s.Store, s.SearchEngine, s.Log) + emailService, err := NewEmailService(s) if err != nil { return nil, errors.Wrapf(err, "unable to initialize email service") @@ -363,7 +366,6 @@ func NewServer(options ...Option) (*Server, error) { return nil, errors.Wrapf(err, "unable to ensure first run timestamp") } - s.ensureDiagnosticId() s.regenerateClientConfig() s.clusterLeaderListenerId = s.AddClusterLeaderChangedListener(func() { @@ -519,7 +521,13 @@ func (s *Server) RunJobs() { runSecurityJob(s) }) s.Go(func() { - runDiagnosticsJob(s) + firstRun, err := s.getFirstServerRunTimestamp() + if err != nil { + mlog.Warn("Fetching time of first server run failed. Setting to 'now'.") + s.ensureFirstServerRunTimestamp() + firstRun = utils.MillisFromTime(time.Now()) + } + s.telemetryService.RunTelemetryJob(firstRun) }) s.Go(func() { runSessionCleanupJob(s) @@ -676,9 +684,9 @@ func (s *Server) Shutdown() error { } } - err := s.shutdownDiagnostics() + err := s.telemetryService.Shutdown() if err != nil { - mlog.Error("Unable to cleanly shutdown diagnostic client", mlog.Err(err)) + mlog.Error("Unable to cleanly shutdown telemetry client", mlog.Err(err)) } s.StopHTTPServer() @@ -1105,34 +1113,6 @@ func runSecurityJob(s *Server) { }, time.Hour*4) } -func doDiagnosticsIfNeeded(s *Server, firstRun time.Time) { - hoursSinceFirstServerRun := time.Since(firstRun).Hours() - // Send once every 10 minutes for the first hour - // Send once every hour thereafter for the first 12 hours - // Send at the 24 hour mark and every 24 hours after - if hoursSinceFirstServerRun < 1 { - doDiagnostics(s) - } else if hoursSinceFirstServerRun <= 12 && time.Since(s.timestampLastDiagnosticSent) >= time.Hour { - doDiagnostics(s) - } else if hoursSinceFirstServerRun > 12 && time.Since(s.timestampLastDiagnosticSent) >= 24*time.Hour { - doDiagnostics(s) - } -} - -func runDiagnosticsJob(s *Server) { - // Send on boot - doDiagnostics(s) - firstRun, err := s.getFirstServerRunTimestamp() - if err != nil { - mlog.Warn("Fetching time of first server run failed. Setting to 'now'.") - s.ensureFirstServerRunTimestamp() - firstRun = utils.MillisFromTime(time.Now()) - } - model.CreateRecurringTask("Diagnostics", func() { - doDiagnosticsIfNeeded(s, utils.TimeFromMillis(firstRun)) - }, time.Minute*10) -} - func runTokenCleanupJob(s *Server) { doTokenCleanup(s) model.CreateRecurringTask("Token Cleanup", func() { @@ -1172,13 +1152,6 @@ func doSecurity(s *Server) { s.DoSecurityUpdateCheck() } -func doDiagnostics(s *Server) { - if *s.Config().LogSettings.EnableDiagnostics { - s.timestampLastDiagnosticSent = time.Now() - s.SendDailyDiagnostics() - } -} - func doTokenCleanup(s *Server) { s.Store.Token().Cleanup() } @@ -1368,39 +1341,6 @@ func (s *Server) stopSearchEngine() { } } -// initDiagnostics initialises the Rudder client for the diagnostics system. -func (s *Server) initDiagnostics(endpoint string, rudderKey string) { - if s.rudderClient == nil { - config := rudder.Config{} - config.Logger = rudder.StdLogger(s.Log.StdLog(mlog.String("source", "rudder"))) - config.Endpoint = endpoint - // For testing - if endpoint != RUDDER_DATAPLANE_URL { - config.Verbose = true - config.BatchSize = 1 - } - client, err := rudder.NewWithConfig(rudderKey, endpoint, config) - if err != nil { - mlog.Error("Failed to create Rudder instance", mlog.Err(err)) - return - } - client.Enqueue(rudder.Identify{ - UserId: s.diagnosticId, - }) - - s.rudderClient = client - } -} - -// shutdownDiagnostics closes the diagnostics system Rudder client. -func (s *Server) shutdownDiagnostics() error { - if s.rudderClient != nil { - return s.rudderClient.Close() - } - - return nil -} - func (s *Server) FileBackend() (filesstore.FileBackend, *model.AppError) { license := s.License() return filesstore.NewFileBackend(&s.Config().FileSettings, license != nil && *license.Features.Compliance) @@ -1421,25 +1361,6 @@ func (s *Server) ClusterHealthScore() int { return s.Cluster.HealthScore() } -func (s *Server) ensureDiagnosticId() { - if s.diagnosticId != "" { - return - } - props, err := s.Store.System().Get() - if err != nil { - return - } - - id := props[model.SYSTEM_DIAGNOSTIC_ID] - if len(id) == 0 { - id = model.NewId() - systemID := &model.System{Name: model.SYSTEM_DIAGNOSTIC_ID, Value: id} - s.Store.System().Save(systemID) - } - - s.diagnosticId = id -} - func (s *Server) configOrLicenseListener() { s.regenerateClientConfig() } @@ -1469,3 +1390,14 @@ func (s *Server) initJobs() { s.Jobs.Migrations = jobsMigrationsInterface(s) } } + +func (s *Server) TelemetryId() string { + if s.telemetryService == nil { + return "" + } + return s.telemetryService.TelemetryID +} + +func (s *Server) HttpService() httpservice.HTTPService { + return s.HTTPService +} diff --git a/config/client.go b/config/client.go index b060b3d609..9b5632b5e3 100644 --- a/config/client.go +++ b/config/client.go @@ -12,8 +12,8 @@ import ( ) // GenerateClientConfig renders the given configuration for a client. -func GenerateClientConfig(c *model.Config, diagnosticID string, license *model.License) map[string]string { - props := GenerateLimitedClientConfig(c, diagnosticID, license) +func GenerateClientConfig(c *model.Config, telemetryID string, license *model.License) map[string]string { + props := GenerateLimitedClientConfig(c, telemetryID, license) props["SiteURL"] = strings.TrimRight(*c.ServiceSettings.SiteURL, "/") props["EnableUserDeactivation"] = strconv.FormatBool(*c.TeamSettings.EnableUserDeactivation) @@ -199,7 +199,7 @@ func GenerateClientConfig(c *model.Config, diagnosticID string, license *model.L } // GenerateLimitedClientConfig renders the given configuration for an untrusted client. -func GenerateLimitedClientConfig(c *model.Config, diagnosticID string, license *model.License) map[string]string { +func GenerateLimitedClientConfig(c *model.Config, telemetryID string, license *model.License) map[string]string { props := make(map[string]string) props["Version"] = model.CurrentVersion @@ -252,7 +252,8 @@ func GenerateLimitedClientConfig(c *model.Config, diagnosticID string, license * props["AndroidAppDownloadLink"] = *c.NativeAppSettings.AndroidAppDownloadLink props["IosAppDownloadLink"] = *c.NativeAppSettings.IosAppDownloadLink - props["DiagnosticId"] = diagnosticID + props["DiagnosticId"] = telemetryID + props["TelemetryId"] = telemetryID props["DiagnosticsEnabled"] = strconv.FormatBool(*c.LogSettings.EnableDiagnostics) props["HasImageProxy"] = strconv.FormatBool(*c.ImageProxySettings.Enable) diff --git a/config/client_test.go b/config/client_test.go index b0ce3dfcfb..7c60a21013 100644 --- a/config/client_test.go +++ b/config/client_test.go @@ -18,7 +18,7 @@ func TestGetClientConfig(t *testing.T) { testCases := []struct { description string config *model.Config - diagnosticID string + telemetryID string license *model.License expectedFields map[string]string }{ @@ -190,7 +190,7 @@ func TestGetClientConfig(t *testing.T) { testCase.license.Features.SetDefaults() } - configMap := config.GenerateClientConfig(testCase.config, testCase.diagnosticID, testCase.license) + configMap := config.GenerateClientConfig(testCase.config, testCase.telemetryID, testCase.license) for expectedField, expectedValue := range testCase.expectedFields { actualValue, ok := configMap[expectedField] if assert.True(t, ok, fmt.Sprintf("config does not contain %v", expectedField)) { @@ -206,7 +206,7 @@ func TestGetLimitedClientConfig(t *testing.T) { testCases := []struct { description string config *model.Config - diagnosticID string + telemetryID string license *model.License expectedFields map[string]string }{ @@ -269,7 +269,7 @@ func TestGetLimitedClientConfig(t *testing.T) { testCase.license.Features.SetDefaults() } - configMap := config.GenerateLimitedClientConfig(testCase.config, testCase.diagnosticID, testCase.license) + configMap := config.GenerateLimitedClientConfig(testCase.config, testCase.telemetryID, testCase.license) for expectedField, expectedValue := range testCase.expectedFields { actualValue, ok := configMap[expectedField] if assert.True(t, ok, fmt.Sprintf("config does not contain %v", expectedField)) { diff --git a/migrations/helper_test.go b/migrations/helper_test.go index 89c6463206..fae9a3256b 100644 --- a/migrations/helper_test.go +++ b/migrations/helper_test.go @@ -263,7 +263,7 @@ func (me *TestHelper) ResetRoleMigration() { mainHelper.GetClusterInterface().SendClearRoleCacheMessage() - if _, err := sqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": app.ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil { + if _, err := sqlSupplier.GetMaster().Exec("DELETE from Systems where Name = :Name", map[string]interface{}{"Name": model.ADVANCED_PERMISSIONS_MIGRATION_KEY}); err != nil { panic(err) } } diff --git a/model/migration.go b/model/migration.go index 1bcfdb0e94..2e7a0f71e7 100644 --- a/model/migration.go +++ b/model/migration.go @@ -4,6 +4,7 @@ package model const ( + ADVANCED_PERMISSIONS_MIGRATION_KEY = "AdvancedPermissionsMigrationComplete" MIGRATION_KEY_ADVANCED_PERMISSIONS_PHASE_2 = "migration_advanced_permissions_phase_2" MIGRATION_KEY_EMOJI_PERMISSIONS_SPLIT = "emoji_permissions_split" diff --git a/model/system.go b/model/system.go index 9563d7b056..11730fb43f 100644 --- a/model/system.go +++ b/model/system.go @@ -10,7 +10,7 @@ import ( ) const ( - SYSTEM_DIAGNOSTIC_ID = "DiagnosticId" + SYSTEM_TELEMETRY_ID = "DiagnosticId" SYSTEM_RAN_UNIT_TESTS = "RanUnitTests" SYSTEM_LAST_SECURITY_TIME = "LastSecurityTime" SYSTEM_ACTIVE_LICENSE_ID = "ActiveLicenseId" diff --git a/plugin/api.go b/plugin/api.go index c5d8317747..d53a82567f 100644 --- a/plugin/api.go +++ b/plugin/api.go @@ -109,6 +109,12 @@ type API interface { // Minimum server version: 5.10 GetDiagnosticId() string + // GetTelemetryId returns a unique identifier used by the server for telemetry reports. + // + // @tag Server + // Minimum server version: 5.28 + GetTelemetryId() string + // CreateUser creates a user. // // @tag User diff --git a/plugin/api_timer_layer_generated.go b/plugin/api_timer_layer_generated.go index 5e668e0267..ddda506dc2 100644 --- a/plugin/api_timer_layer_generated.go +++ b/plugin/api_timer_layer_generated.go @@ -133,6 +133,13 @@ func (api *apiTimerLayer) GetDiagnosticId() string { return _returnsA } +func (api *apiTimerLayer) GetTelemetryId() string { + startTime := timePkg.Now() + _returnsA := api.apiImpl.GetTelemetryId() + api.recordTime(startTime, "GetTelemetryId", true) + return _returnsA +} + func (api *apiTimerLayer) CreateUser(user *model.User) (*model.User, *model.AppError) { startTime := timePkg.Now() _returnsA, _returnsB := api.apiImpl.CreateUser(user) diff --git a/plugin/client_rpc_generated.go b/plugin/client_rpc_generated.go index 7335e0fccd..b26b74d259 100644 --- a/plugin/client_rpc_generated.go +++ b/plugin/client_rpc_generated.go @@ -857,6 +857,33 @@ func (s *apiRPCServer) GetDiagnosticId(args *Z_GetDiagnosticIdArgs, returns *Z_G return nil } +type Z_GetTelemetryIdArgs struct { +} + +type Z_GetTelemetryIdReturns struct { + A string +} + +func (g *apiRPCClient) GetTelemetryId() string { + _args := &Z_GetTelemetryIdArgs{} + _returns := &Z_GetTelemetryIdReturns{} + if err := g.client.Call("Plugin.GetTelemetryId", _args, _returns); err != nil { + log.Printf("RPC call to GetTelemetryId API failed: %s", err.Error()) + } + return _returns.A +} + +func (s *apiRPCServer) GetTelemetryId(args *Z_GetTelemetryIdArgs, returns *Z_GetTelemetryIdReturns) error { + if hook, ok := s.impl.(interface { + GetTelemetryId() string + }); ok { + returns.A = hook.GetTelemetryId() + } else { + return encodableError(fmt.Errorf("API GetTelemetryId called but not implemented.")) + } + return nil +} + type Z_CreateUserArgs struct { A *model.User } diff --git a/plugin/plugintest/api.go b/plugin/plugintest/api.go index c97499ac09..f2aa7c3e0b 100644 --- a/plugin/plugintest/api.go +++ b/plugin/plugintest/api.go @@ -1916,6 +1916,20 @@ func (_m *API) GetTeamsUnreadForUser(userId string) ([]*model.TeamUnread, *model return r0, r1 } +// GetTelemetryId provides a mock function with given fields: +func (_m *API) GetTelemetryId() string { + ret := _m.Called() + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + // GetUnsanitizedConfig provides a mock function with given fields: func (_m *API) GetUnsanitizedConfig() *model.Config { ret := _m.Called() diff --git a/services/telemetry/mocks/ServerIface.go b/services/telemetry/mocks/ServerIface.go new file mode 100644 index 0000000000..91663cf369 --- /dev/null +++ b/services/telemetry/mocks/ServerIface.go @@ -0,0 +1,147 @@ +// Code generated by mockery v1.0.0. DO NOT EDIT. + +// Regenerate this file using `make telemetry-mocks`. + +package mocks + +import ( + httpservice "github.com/mattermost/mattermost-server/v5/services/httpservice" + mock "github.com/stretchr/testify/mock" + + model "github.com/mattermost/mattermost-server/v5/model" + + plugin "github.com/mattermost/mattermost-server/v5/plugin" +) + +// ServerIface is an autogenerated mock type for the ServerIface type +type ServerIface struct { + mock.Mock +} + +// Config provides a mock function with given fields: +func (_m *ServerIface) Config() *model.Config { + ret := _m.Called() + + var r0 *model.Config + if rf, ok := ret.Get(0).(func() *model.Config); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Config) + } + } + + return r0 +} + +// GetPluginsEnvironment provides a mock function with given fields: +func (_m *ServerIface) GetPluginsEnvironment() *plugin.Environment { + ret := _m.Called() + + var r0 *plugin.Environment + if rf, ok := ret.Get(0).(func() *plugin.Environment); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*plugin.Environment) + } + } + + return r0 +} + +// GetRoleByName provides a mock function with given fields: _a0 +func (_m *ServerIface) GetRoleByName(_a0 string) (*model.Role, *model.AppError) { + ret := _m.Called(_a0) + + var r0 *model.Role + if rf, ok := ret.Get(0).(func(string) *model.Role); ok { + r0 = rf(_a0) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.Role) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string) *model.AppError); ok { + r1 = rf(_a0) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// GetSchemes provides a mock function with given fields: _a0, _a1, _a2 +func (_m *ServerIface) GetSchemes(_a0 string, _a1 int, _a2 int) ([]*model.Scheme, *model.AppError) { + ret := _m.Called(_a0, _a1, _a2) + + var r0 []*model.Scheme + if rf, ok := ret.Get(0).(func(string, int, int) []*model.Scheme); ok { + r0 = rf(_a0, _a1, _a2) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.Scheme) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(string, int, int) *model.AppError); ok { + r1 = rf(_a0, _a1, _a2) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + +// HttpService provides a mock function with given fields: +func (_m *ServerIface) HttpService() httpservice.HTTPService { + ret := _m.Called() + + var r0 httpservice.HTTPService + if rf, ok := ret.Get(0).(func() httpservice.HTTPService); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(httpservice.HTTPService) + } + } + + return r0 +} + +// IsLeader provides a mock function with given fields: +func (_m *ServerIface) IsLeader() bool { + ret := _m.Called() + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// License provides a mock function with given fields: +func (_m *ServerIface) License() *model.License { + ret := _m.Called() + + var r0 *model.License + if rf, ok := ret.Get(0).(func() *model.License); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.License) + } + } + + return r0 +} diff --git a/app/diagnostics.go b/services/telemetry/telemetry.go similarity index 79% rename from app/diagnostics.go rename to services/telemetry/telemetry.go index d2cd31ced5..cda067a5dd 100644 --- a/app/diagnostics.go +++ b/services/telemetry/telemetry.go @@ -1,22 +1,31 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package app +package telemetry import ( "os" "path/filepath" "runtime" "strings" + "time" "github.com/mattermost/mattermost-server/v5/mlog" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/plugin" + "github.com/mattermost/mattermost-server/v5/services/httpservice" "github.com/mattermost/mattermost-server/v5/services/marketplace" + "github.com/mattermost/mattermost-server/v5/services/searchengine" "github.com/mattermost/mattermost-server/v5/store" + "github.com/mattermost/mattermost-server/v5/utils" + rudder "github.com/rudderlabs/analytics-go" ) const ( + DAY_MILLISECONDS = 24 * 60 * 60 * 1000 + MONTH_MILLISECONDS = 31 * DAY_MILLISECONDS + RUDDER_KEY = "placeholder_rudder_key" RUDDER_DATAPLANE_URL = "placeholder_rudder_dataplane_url" @@ -67,59 +76,84 @@ const ( TRACK_PLUGINS = "plugins" ) -// declaring this as var to allow overriding in tests -var SENTRY_DSN = "placeholder_sentry_dsn" - -type RudderConfig struct { - RudderKey string - DataplaneUrl string +type ServerIface interface { + Config() *model.Config + IsLeader() bool + HttpService() httpservice.HTTPService + GetPluginsEnvironment() *plugin.Environment + License() *model.License + GetRoleByName(string) (*model.Role, *model.AppError) + GetSchemes(string, int, int) ([]*model.Scheme, *model.AppError) } -func (s *Server) SendDailyDiagnostics() { - s.sendDailyDiagnostics(false) +type TelemetryService struct { + srv ServerIface + dbStore store.Store + searchEngine *searchengine.Broker + log *mlog.Logger + rudderClient rudder.Client + TelemetryID string + timestampLastTelemetrySent time.Time } -func (s *Server) getRudderConfig() RudderConfig { - if !strings.Contains(RUDDER_KEY, "placeholder") && !strings.Contains(RUDDER_DATAPLANE_URL, "placeholder") { - return RudderConfig{RUDDER_KEY, RUDDER_DATAPLANE_URL} - } else if os.Getenv("RUDDER_KEY") != "" && os.Getenv("RUDDER_DATAPLANE_URL") != "" { - return RudderConfig{os.Getenv("RUDDER_KEY"), os.Getenv("RUDDER_DATAPLANE_URL")} - } else { - return RudderConfig{} +func New(srv ServerIface, dbStore store.Store, searchEngine *searchengine.Broker, log *mlog.Logger) *TelemetryService { + service := &TelemetryService{ + srv: srv, + dbStore: dbStore, + searchEngine: searchEngine, + log: log, + } + service.ensureTelemetryID() + service.initRudder(RUDDER_DATAPLANE_URL, RUDDER_KEY) + return service +} + +func (ts *TelemetryService) ensureTelemetryID() { + if ts.TelemetryID != "" { + return + } + props, err := ts.dbStore.System().Get() + if err != nil { + mlog.Error("unable to get the telemetry ID", mlog.Err(err)) + return + } + + id := props[model.SYSTEM_TELEMETRY_ID] + if len(id) == 0 { + id = model.NewId() + systemID := &model.System{Name: model.SYSTEM_TELEMETRY_ID, Value: id} + ts.dbStore.System().Save(systemID) + } + + ts.TelemetryID = id +} + +func (ts *TelemetryService) sendDailyTelemetry(override bool) { + if *ts.srv.Config().LogSettings.EnableDiagnostics && ts.srv.IsLeader() && ((!strings.Contains(RUDDER_KEY, "placeholder") && !strings.Contains(RUDDER_DATAPLANE_URL, "placeholder")) || override) { + ts.initRudder(RUDDER_DATAPLANE_URL, RUDDER_KEY) + ts.trackActivity() + ts.trackConfig() + ts.trackLicense() + ts.trackPlugins() + ts.trackServer() + ts.trackPermissions() + ts.trackElasticsearch() + ts.trackGroups() + ts.trackChannelModeration() + ts.trackWarnMetrics() } } -func (s *Server) diagnosticsEnabled() bool { - return *s.Config().LogSettings.EnableDiagnostics && s.IsLeader() -} - -func (s *Server) sendDailyDiagnostics(override bool) { - config := s.getRudderConfig() - if s.diagnosticsEnabled() && ((config.DataplaneUrl != "" && config.RudderKey != "") || override) { - s.initDiagnostics(config.DataplaneUrl, config.RudderKey) - s.trackActivity() - s.trackConfig() - s.trackLicense() - s.trackPlugins() - s.trackServer() - s.trackPermissions() - s.trackElasticsearch() - s.trackGroups() - s.trackChannelModeration() - s.trackWarnMetrics() - } -} - -func (s *Server) SendDiagnostic(event string, properties map[string]interface{}) { - if s.rudderClient != nil { +func (ts *TelemetryService) sendTelemetry(event string, properties map[string]interface{}) { + if ts.rudderClient != nil { var context *rudder.Context // if we are part of a cloud installation, add it's ID to the tracked event's context if installationId := os.Getenv("MM_CLOUD_INSTALLATION_ID"); installationId != "" { context = &rudder.Context{Traits: map[string]interface{}{"installationId": installationId}} } - s.rudderClient.Enqueue(rudder.Track{ + ts.rudderClient.Enqueue(rudder.Track{ Event: event, - UserId: s.diagnosticId, + UserId: ts.TelemetryID, Properties: properties, Context: context, }) @@ -158,7 +192,7 @@ func pluginVersion(pluginsAvailable []*model.BundleInfo, pluginId string) string return "" } -func (s *Server) trackActivity() { +func (ts *TelemetryService) trackActivity() { var userCount int64 var guestAccountsCount int64 var botAccountsCount int64 @@ -177,82 +211,82 @@ func (s *Server) trackActivity() { activeUsersDailyCountChan := make(chan store.StoreResult, 1) go func() { - count, err := s.Store.User().AnalyticsActiveCount(DAY_MILLISECONDS, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}) + count, err := ts.dbStore.User().AnalyticsActiveCount(DAY_MILLISECONDS, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}) activeUsersDailyCountChan <- store.StoreResult{Data: count, Err: err} close(activeUsersDailyCountChan) }() activeUsersMonthlyCountChan := make(chan store.StoreResult, 1) go func() { - count, err := s.Store.User().AnalyticsActiveCount(MONTH_MILLISECONDS, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}) + count, err := ts.dbStore.User().AnalyticsActiveCount(MONTH_MILLISECONDS, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false}) activeUsersMonthlyCountChan <- store.StoreResult{Data: count, Err: err} close(activeUsersMonthlyCountChan) }() - if count, err := s.Store.User().Count(model.UserCountOptions{IncludeDeleted: true}); err == nil { + if count, err := ts.dbStore.User().Count(model.UserCountOptions{IncludeDeleted: true}); err == nil { userCount = count } - if count, err := s.Store.User().AnalyticsGetGuestCount(); err == nil { + if count, err := ts.dbStore.User().AnalyticsGetGuestCount(); err == nil { guestAccountsCount = count } - if count, err := s.Store.User().Count(model.UserCountOptions{IncludeBotAccounts: true, ExcludeRegularUsers: true}); err == nil { + if count, err := ts.dbStore.User().Count(model.UserCountOptions{IncludeBotAccounts: true, ExcludeRegularUsers: true}); err == nil { botAccountsCount = count } - if iucr, err := s.Store.User().AnalyticsGetInactiveUsersCount(); err == nil { + if iucr, err := ts.dbStore.User().AnalyticsGetInactiveUsersCount(); err == nil { inactiveUserCount = iucr } - teamCount, err := s.Store.Team().AnalyticsTeamCount(false) + teamCount, err := ts.dbStore.Team().AnalyticsTeamCount(false) if err != nil { mlog.Error(err.Error()) } - if ucc, err := s.Store.Channel().AnalyticsTypeCount("", "O"); err == nil { + if ucc, err := ts.dbStore.Channel().AnalyticsTypeCount("", "O"); err == nil { publicChannelCount = ucc } - if pcc, err := s.Store.Channel().AnalyticsTypeCount("", "P"); err == nil { + if pcc, err := ts.dbStore.Channel().AnalyticsTypeCount("", "P"); err == nil { privateChannelCount = pcc } - if dcc, err := s.Store.Channel().AnalyticsTypeCount("", "D"); err == nil { + if dcc, err := ts.dbStore.Channel().AnalyticsTypeCount("", "D"); err == nil { directChannelCount = dcc } - if duccr, err := s.Store.Channel().AnalyticsDeletedTypeCount("", "O"); err == nil { + if duccr, err := ts.dbStore.Channel().AnalyticsDeletedTypeCount("", "O"); err == nil { deletedPublicChannelCount = duccr } - if dpccr, err := s.Store.Channel().AnalyticsDeletedTypeCount("", "P"); err == nil { + if dpccr, err := ts.dbStore.Channel().AnalyticsDeletedTypeCount("", "P"); err == nil { deletedPrivateChannelCount = dpccr } - postsCount, _ = s.Store.Post().AnalyticsPostCount("", false, false) + postsCount, _ = ts.dbStore.Post().AnalyticsPostCount("", false, false) postCountsOptions := &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: false, YesterdayOnly: true} - postCountsYesterday, _ := s.Store.Post().AnalyticsPostCountsByDay(postCountsOptions) + postCountsYesterday, _ := ts.dbStore.Post().AnalyticsPostCountsByDay(postCountsOptions) postsCountPreviousDay = 0 if len(postCountsYesterday) > 0 { postsCountPreviousDay = int64(postCountsYesterday[0].Value) } postCountsOptions = &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: true, YesterdayOnly: true} - botPostCountsYesterday, _ := s.Store.Post().AnalyticsPostCountsByDay(postCountsOptions) + botPostCountsYesterday, _ := ts.dbStore.Post().AnalyticsPostCountsByDay(postCountsOptions) botPostsCountPreviousDay = 0 if len(botPostCountsYesterday) > 0 { botPostsCountPreviousDay = int64(botPostCountsYesterday[0].Value) } - slashCommandsCount, _ = s.Store.Command().AnalyticsCommandCount("") + slashCommandsCount, _ = ts.dbStore.Command().AnalyticsCommandCount("") - if c, err := s.Store.Webhook().AnalyticsIncomingCount(""); err == nil { + if c, err := ts.dbStore.Webhook().AnalyticsIncomingCount(""); err == nil { incomingWebhooksCount = c } - outgoingWebhooksCount, _ = s.Store.Webhook().AnalyticsOutgoingCount("") + outgoingWebhooksCount, _ = ts.dbStore.Webhook().AnalyticsOutgoingCount("") var activeUsersDailyCount int64 if r := <-activeUsersDailyCountChan; r.Err == nil { @@ -264,7 +298,7 @@ func (s *Server) trackActivity() { activeUsersMonthlyCount = r.Data.(int64) } - s.SendDiagnostic(TRACK_ACTIVITY, map[string]interface{}{ + ts.sendTelemetry(TRACK_ACTIVITY, map[string]interface{}{ "registered_users": userCount, "bot_accounts": botAccountsCount, "guest_accounts": guestAccountsCount, @@ -286,9 +320,9 @@ func (s *Server) trackActivity() { }) } -func (s *Server) trackConfig() { - cfg := s.Config() - s.SendDiagnostic(TRACK_CONFIG_SERVICE, map[string]interface{}{ +func (ts *TelemetryService) trackConfig() { + cfg := ts.srv.Config() + ts.sendTelemetry(TRACK_CONFIG_SERVICE, map[string]interface{}{ "web_server_mode": *cfg.ServiceSettings.WebserverMode, "enable_security_fix_alert": *cfg.ServiceSettings.EnableSecurityFixAlert, "enable_insecure_outgoing_connections": *cfg.ServiceSettings.EnableInsecureOutgoingConnections, @@ -367,7 +401,7 @@ func (s *Server) trackConfig() { "enable_local_mode": *cfg.ServiceSettings.EnableLocalMode, }) - s.SendDiagnostic(TRACK_CONFIG_TEAM, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_TEAM, map[string]interface{}{ "enable_user_creation": cfg.TeamSettings.EnableUserCreation, "enable_team_creation": *cfg.TeamSettings.DEPRECATED_DO_NOT_USE_EnableTeamCreation, "restrict_team_invite": *cfg.TeamSettings.DEPRECATED_DO_NOT_USE_RestrictTeamInvite, @@ -401,7 +435,7 @@ func (s *Server) trackConfig() { "experimental_default_channels": len(cfg.TeamSettings.ExperimentalDefaultChannels), }) - s.SendDiagnostic(TRACK_CONFIG_CLIENT_REQ, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_CLIENT_REQ, map[string]interface{}{ "android_latest_version": cfg.ClientRequirements.AndroidLatestVersion, "android_min_version": cfg.ClientRequirements.AndroidMinVersion, "desktop_latest_version": cfg.ClientRequirements.DesktopLatestVersion, @@ -410,7 +444,7 @@ func (s *Server) trackConfig() { "ios_min_version": cfg.ClientRequirements.IosMinVersion, }) - s.SendDiagnostic(TRACK_CONFIG_SQL, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_SQL, map[string]interface{}{ "driver_name": *cfg.SqlSettings.DriverName, "trace": cfg.SqlSettings.Trace, "max_idle_conns": *cfg.SqlSettings.MaxIdleConns, @@ -422,7 +456,7 @@ func (s *Server) trackConfig() { "disable_database_search": *cfg.SqlSettings.DisableDatabaseSearch, }) - s.SendDiagnostic(TRACK_CONFIG_LOG, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_LOG, map[string]interface{}{ "enable_console": cfg.LogSettings.EnableConsole, "console_level": cfg.LogSettings.ConsoleLevel, "console_json": *cfg.LogSettings.ConsoleJson, @@ -434,7 +468,7 @@ func (s *Server) trackConfig() { "advanced_logging_config": *cfg.LogSettings.AdvancedLoggingConfig != "", }) - s.SendDiagnostic(TRACK_CONFIG_AUDIT, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_AUDIT, map[string]interface{}{ "file_enabled": *cfg.ExperimentalAuditSettings.FileEnabled, "file_max_size_mb": *cfg.ExperimentalAuditSettings.FileMaxSizeMB, "file_max_age_days": *cfg.ExperimentalAuditSettings.FileMaxAgeDays, @@ -444,7 +478,7 @@ func (s *Server) trackConfig() { "advanced_logging_config": *cfg.ExperimentalAuditSettings.AdvancedLoggingConfig != "", }) - s.SendDiagnostic(TRACK_CONFIG_NOTIFICATION_LOG, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_NOTIFICATION_LOG, map[string]interface{}{ "enable_console": *cfg.NotificationLogSettings.EnableConsole, "console_level": *cfg.NotificationLogSettings.ConsoleLevel, "console_json": *cfg.NotificationLogSettings.ConsoleJson, @@ -455,7 +489,7 @@ func (s *Server) trackConfig() { "advanced_logging_config": *cfg.NotificationLogSettings.AdvancedLoggingConfig != "", }) - s.SendDiagnostic(TRACK_CONFIG_PASSWORD, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_PASSWORD, map[string]interface{}{ "minimum_length": *cfg.PasswordSettings.MinimumLength, "lowercase": *cfg.PasswordSettings.Lowercase, "number": *cfg.PasswordSettings.Number, @@ -463,7 +497,7 @@ func (s *Server) trackConfig() { "symbol": *cfg.PasswordSettings.Symbol, }) - s.SendDiagnostic(TRACK_CONFIG_FILE, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_FILE, map[string]interface{}{ "enable_public_links": cfg.FileSettings.EnablePublicLink, "driver_name": *cfg.FileSettings.DriverName, "isdefault_directory": isDefault(*cfg.FileSettings.Directory, model.FILE_SETTINGS_DEFAULT_DIRECTORY), @@ -478,7 +512,7 @@ func (s *Server) trackConfig() { "enable_mobile_download": *cfg.FileSettings.EnableMobileDownload, }) - s.SendDiagnostic(TRACK_CONFIG_EMAIL, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_EMAIL, map[string]interface{}{ "enable_sign_up_with_email": cfg.EmailSettings.EnableSignUpWithEmail, "enable_sign_in_with_email": *cfg.EmailSettings.EnableSignInWithEmail, "enable_sign_in_with_username": *cfg.EmailSettings.EnableSignInWithUsername, @@ -505,7 +539,7 @@ func (s *Server) trackConfig() { "smtp_server_timeout": *cfg.EmailSettings.SMTPServerTimeout, }) - s.SendDiagnostic(TRACK_CONFIG_RATE, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_RATE, map[string]interface{}{ "enable_rate_limiter": *cfg.RateLimitSettings.Enable, "vary_by_remote_address": *cfg.RateLimitSettings.VaryByRemoteAddr, "vary_by_user": *cfg.RateLimitSettings.VaryByUser, @@ -515,25 +549,25 @@ func (s *Server) trackConfig() { "isdefault_vary_by_header": isDefault(cfg.RateLimitSettings.VaryByHeader, ""), }) - s.SendDiagnostic(TRACK_CONFIG_PRIVACY, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_PRIVACY, map[string]interface{}{ "show_email_address": cfg.PrivacySettings.ShowEmailAddress, "show_full_name": cfg.PrivacySettings.ShowFullName, }) - s.SendDiagnostic(TRACK_CONFIG_THEME, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_THEME, map[string]interface{}{ "enable_theme_selection": *cfg.ThemeSettings.EnableThemeSelection, "isdefault_default_theme": isDefault(*cfg.ThemeSettings.DefaultTheme, model.TEAM_SETTINGS_DEFAULT_TEAM_TEXT), "allow_custom_themes": *cfg.ThemeSettings.AllowCustomThemes, "allowed_themes": len(cfg.ThemeSettings.AllowedThemes), }) - s.SendDiagnostic(TRACK_CONFIG_OAUTH, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_OAUTH, map[string]interface{}{ "enable_gitlab": cfg.GitLabSettings.Enable, "enable_google": cfg.GoogleSettings.Enable, "enable_office365": cfg.Office365Settings.Enable, }) - s.SendDiagnostic(TRACK_CONFIG_SUPPORT, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_SUPPORT, map[string]interface{}{ "isdefault_terms_of_service_link": isDefault(*cfg.SupportSettings.TermsOfServiceLink, model.SUPPORT_SETTINGS_DEFAULT_TERMS_OF_SERVICE_LINK), "isdefault_privacy_policy_link": isDefault(*cfg.SupportSettings.PrivacyPolicyLink, model.SUPPORT_SETTINGS_DEFAULT_PRIVACY_POLICY_LINK), "isdefault_about_link": isDefault(*cfg.SupportSettings.AboutLink, model.SUPPORT_SETTINGS_DEFAULT_ABOUT_LINK), @@ -545,7 +579,7 @@ func (s *Server) trackConfig() { "enable_ask_community_link": *cfg.SupportSettings.EnableAskCommunityLink, }) - s.SendDiagnostic(TRACK_CONFIG_LDAP, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_LDAP, map[string]interface{}{ "enable": *cfg.LdapSettings.Enable, "enable_sync": *cfg.LdapSettings.EnableSync, "enable_admin_filter": *cfg.LdapSettings.EnableAdminFilter, @@ -574,18 +608,18 @@ func (s *Server) trackConfig() { "isnotempty_picture_attribute": !isDefault(*cfg.LdapSettings.PictureAttribute, ""), }) - s.SendDiagnostic(TRACK_CONFIG_COMPLIANCE, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_COMPLIANCE, map[string]interface{}{ "enable": *cfg.ComplianceSettings.Enable, "enable_daily": *cfg.ComplianceSettings.EnableDaily, }) - s.SendDiagnostic(TRACK_CONFIG_LOCALIZATION, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_LOCALIZATION, map[string]interface{}{ "default_server_locale": *cfg.LocalizationSettings.DefaultServerLocale, "default_client_locale": *cfg.LocalizationSettings.DefaultClientLocale, "available_locales": *cfg.LocalizationSettings.AvailableLocales, }) - s.SendDiagnostic(TRACK_CONFIG_SAML, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_SAML, map[string]interface{}{ "enable": *cfg.SamlSettings.Enable, "enable_sync_with_ldap": *cfg.SamlSettings.EnableSyncWithLdap, "enable_sync_with_ldap_include_auth": *cfg.SamlSettings.EnableSyncWithLdapIncludeAuth, @@ -613,7 +647,7 @@ func (s *Server) trackConfig() { "isdefault_login_button_text_color": isDefault(*cfg.SamlSettings.LoginButtonTextColor, ""), }) - s.SendDiagnostic(TRACK_CONFIG_CLUSTER, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_CLUSTER, map[string]interface{}{ "enable": *cfg.ClusterSettings.Enable, "network_interface": isDefault(*cfg.ClusterSettings.NetworkInterface, ""), "bind_address": isDefault(*cfg.ClusterSettings.BindAddress, ""), @@ -624,18 +658,18 @@ func (s *Server) trackConfig() { "read_only_config": *cfg.ClusterSettings.ReadOnlyConfig, }) - s.SendDiagnostic(TRACK_CONFIG_METRICS, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_METRICS, map[string]interface{}{ "enable": *cfg.MetricsSettings.Enable, "block_profile_rate": *cfg.MetricsSettings.BlockProfileRate, }) - s.SendDiagnostic(TRACK_CONFIG_NATIVEAPP, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_NATIVEAPP, map[string]interface{}{ "isdefault_app_download_link": isDefault(*cfg.NativeAppSettings.AppDownloadLink, model.NATIVEAPP_SETTINGS_DEFAULT_APP_DOWNLOAD_LINK), "isdefault_android_app_download_link": isDefault(*cfg.NativeAppSettings.AndroidAppDownloadLink, model.NATIVEAPP_SETTINGS_DEFAULT_ANDROID_APP_DOWNLOAD_LINK), "isdefault_iosapp_download_link": isDefault(*cfg.NativeAppSettings.IosAppDownloadLink, model.NATIVEAPP_SETTINGS_DEFAULT_IOS_APP_DOWNLOAD_LINK), }) - s.SendDiagnostic(TRACK_CONFIG_EXPERIMENTAL, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_EXPERIMENTAL, map[string]interface{}{ "client_side_cert_enable": *cfg.ExperimentalSettings.ClientSideCertEnable, "isdefault_client_side_cert_check": isDefault(*cfg.ExperimentalSettings.ClientSideCertCheck, model.CLIENT_SIDE_CERT_CHECK_PRIMARY_AUTH), "link_metadata_timeout_milliseconds": *cfg.ExperimentalSettings.LinkMetadataTimeoutMilliseconds, @@ -645,18 +679,18 @@ func (s *Server) trackConfig() { "cloud_billing": *cfg.ExperimentalSettings.CloudBilling, }) - s.SendDiagnostic(TRACK_CONFIG_ANALYTICS, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_ANALYTICS, map[string]interface{}{ "isdefault_max_users_for_statistics": isDefault(*cfg.AnalyticsSettings.MaxUsersForStatistics, model.ANALYTICS_SETTINGS_DEFAULT_MAX_USERS_FOR_STATISTICS), }) - s.SendDiagnostic(TRACK_CONFIG_ANNOUNCEMENT, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_ANNOUNCEMENT, map[string]interface{}{ "enable_banner": *cfg.AnnouncementSettings.EnableBanner, "isdefault_banner_color": isDefault(*cfg.AnnouncementSettings.BannerColor, model.ANNOUNCEMENT_SETTINGS_DEFAULT_BANNER_COLOR), "isdefault_banner_text_color": isDefault(*cfg.AnnouncementSettings.BannerTextColor, model.ANNOUNCEMENT_SETTINGS_DEFAULT_BANNER_TEXT_COLOR), "allow_banner_dismissal": *cfg.AnnouncementSettings.AllowBannerDismissal, }) - s.SendDiagnostic(TRACK_CONFIG_ELASTICSEARCH, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_ELASTICSEARCH, map[string]interface{}{ "isdefault_connection_url": isDefault(*cfg.ElasticsearchSettings.ConnectionUrl, model.ELASTICSEARCH_SETTINGS_DEFAULT_CONNECTION_URL), "isdefault_username": isDefault(*cfg.ElasticsearchSettings.Username, model.ELASTICSEARCH_SETTINGS_DEFAULT_USERNAME), "isdefault_password": isDefault(*cfg.ElasticsearchSettings.Password, model.ELASTICSEARCH_SETTINGS_DEFAULT_PASSWORD), @@ -678,9 +712,9 @@ func (s *Server) trackConfig() { "trace": *cfg.ElasticsearchSettings.Trace, }) - s.trackPluginConfig(cfg, model.PLUGIN_SETTINGS_DEFAULT_MARKETPLACE_URL) + ts.trackPluginConfig(cfg, model.PLUGIN_SETTINGS_DEFAULT_MARKETPLACE_URL) - s.SendDiagnostic(TRACK_CONFIG_DATA_RETENTION, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_DATA_RETENTION, map[string]interface{}{ "enable_message_deletion": *cfg.DataRetentionSettings.EnableMessageDeletion, "enable_file_deletion": *cfg.DataRetentionSettings.EnableFileDeletion, "message_retention_days": *cfg.DataRetentionSettings.MessageRetentionDays, @@ -688,7 +722,7 @@ func (s *Server) trackConfig() { "deletion_job_start_time": *cfg.DataRetentionSettings.DeletionJobStartTime, }) - s.SendDiagnostic(TRACK_CONFIG_MESSAGE_EXPORT, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_MESSAGE_EXPORT, map[string]interface{}{ "enable_message_export": *cfg.MessageExportSettings.EnableExport, "export_format": *cfg.MessageExportSettings.ExportFormat, "daily_run_time": *cfg.MessageExportSettings.DailyRunTime, @@ -701,26 +735,26 @@ func (s *Server) trackConfig() { "global_relay_smtp_server_timeout": *cfg.EmailSettings.SMTPServerTimeout, }) - s.SendDiagnostic(TRACK_CONFIG_DISPLAY, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_DISPLAY, map[string]interface{}{ "experimental_timezone": *cfg.DisplaySettings.ExperimentalTimezone, "isdefault_custom_url_schemes": len(cfg.DisplaySettings.CustomUrlSchemes) != 0, }) - s.SendDiagnostic(TRACK_CONFIG_GUEST_ACCOUNTS, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_GUEST_ACCOUNTS, map[string]interface{}{ "enable": *cfg.GuestAccountsSettings.Enable, "allow_email_accounts": *cfg.GuestAccountsSettings.AllowEmailAccounts, "enforce_multifactor_authentication": *cfg.GuestAccountsSettings.EnforceMultifactorAuthentication, "isdefault_restrict_creation_to_domains": isDefault(*cfg.GuestAccountsSettings.RestrictCreationToDomains, ""), }) - s.SendDiagnostic(TRACK_CONFIG_IMAGE_PROXY, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_IMAGE_PROXY, map[string]interface{}{ "enable": *cfg.ImageProxySettings.Enable, "image_proxy_type": *cfg.ImageProxySettings.ImageProxyType, "isdefault_remote_image_proxy_url": isDefault(*cfg.ImageProxySettings.RemoteImageProxyURL, ""), "isdefault_remote_image_proxy_options": isDefault(*cfg.ImageProxySettings.RemoteImageProxyOptions, ""), }) - s.SendDiagnostic(TRACK_CONFIG_BLEVE, map[string]interface{}{ + ts.sendTelemetry(TRACK_CONFIG_BLEVE, map[string]interface{}{ "enable_indexing": *cfg.BleveSettings.EnableIndexing, "enable_searching": *cfg.BleveSettings.EnableSearching, "enable_autocomplete": *cfg.BleveSettings.EnableAutocomplete, @@ -728,8 +762,8 @@ func (s *Server) trackConfig() { }) } -func (s *Server) trackLicense() { - if license := s.License(); license != nil { +func (ts *TelemetryService) trackLicense() { + if license := ts.srv.License(); license != nil { data := map[string]interface{}{ "customer_id": license.Customer.Id, "license_id": license.Id, @@ -745,12 +779,12 @@ func (s *Server) trackLicense() { data["feature_"+featureName] = featureValue } - s.SendDiagnostic(TRACK_LICENSE, data) + ts.sendTelemetry(TRACK_LICENSE, data) } } -func (s *Server) trackPlugins() { - pluginsEnvironment := s.GetPluginsEnvironment() +func (ts *TelemetryService) trackPlugins() { + pluginsEnvironment := ts.srv.GetPluginsEnvironment() if pluginsEnvironment == nil { return } @@ -764,7 +798,7 @@ func (s *Server) trackPlugins() { brokenManifestCount := 0 settingsCount := 0 - pluginStates := s.Config().PluginSettings.PluginStates + pluginStates := ts.srv.Config().PluginSettings.PluginStates plugins, _ := pluginsEnvironment.Available() if pluginStates != nil && plugins != nil { @@ -800,7 +834,7 @@ func (s *Server) trackPlugins() { totalDisabledCount = -1 // -1 to indicate disabled or error } - s.SendDiagnostic(TRACK_PLUGINS, map[string]interface{}{ + ts.sendTelemetry(TRACK_PLUGINS, map[string]interface{}{ "enabled_plugins": totalEnabledCount, "enabled_webapp_plugins": webappEnabledCount, "enabled_backend_plugins": backendEnabledCount, @@ -812,82 +846,82 @@ func (s *Server) trackPlugins() { }) } -func (s *Server) trackServer() { +func (ts *TelemetryService) trackServer() { data := map[string]interface{}{ "edition": model.BuildEnterpriseReady, "version": model.CurrentVersion, - "database_type": *s.Config().SqlSettings.DriverName, + "database_type": *ts.srv.Config().SqlSettings.DriverName, "operating_system": runtime.GOOS, } - if scr, err := s.Store.User().AnalyticsGetSystemAdminCount(); err == nil { + if scr, err := ts.dbStore.User().AnalyticsGetSystemAdminCount(); err == nil { data["system_admins"] = scr } - if scr, err := s.Store.GetDbVersion(); err == nil { + if scr, err := ts.dbStore.GetDbVersion(); err == nil { data["database_version"] = scr } - s.SendDiagnostic(TRACK_SERVER, data) + ts.sendTelemetry(TRACK_SERVER, data) } -func (s *Server) trackPermissions() { +func (ts *TelemetryService) trackPermissions() { phase1Complete := false - if _, err := s.Store.System().GetByName(ADVANCED_PERMISSIONS_MIGRATION_KEY); err == nil { + if _, err := ts.dbStore.System().GetByName(model.ADVANCED_PERMISSIONS_MIGRATION_KEY); err == nil { phase1Complete = true } phase2Complete := false - if _, err := s.Store.System().GetByName(model.MIGRATION_KEY_ADVANCED_PERMISSIONS_PHASE_2); err == nil { + if _, err := ts.dbStore.System().GetByName(model.MIGRATION_KEY_ADVANCED_PERMISSIONS_PHASE_2); err == nil { phase2Complete = true } - s.SendDiagnostic(TRACK_PERMISSIONS_GENERAL, map[string]interface{}{ + ts.sendTelemetry(TRACK_PERMISSIONS_GENERAL, map[string]interface{}{ "phase_1_migration_complete": phase1Complete, "phase_2_migration_complete": phase2Complete, }) systemAdminPermissions := "" - if role, err := s.GetRoleByName(model.SYSTEM_ADMIN_ROLE_ID); err == nil { + if role, err := ts.srv.GetRoleByName(model.SYSTEM_ADMIN_ROLE_ID); err == nil { systemAdminPermissions = strings.Join(role.Permissions, " ") } systemUserPermissions := "" - if role, err := s.GetRoleByName(model.SYSTEM_USER_ROLE_ID); err == nil { + if role, err := ts.srv.GetRoleByName(model.SYSTEM_USER_ROLE_ID); err == nil { systemUserPermissions = strings.Join(role.Permissions, " ") } teamAdminPermissions := "" - if role, err := s.GetRoleByName(model.TEAM_ADMIN_ROLE_ID); err == nil { + if role, err := ts.srv.GetRoleByName(model.TEAM_ADMIN_ROLE_ID); err == nil { teamAdminPermissions = strings.Join(role.Permissions, " ") } teamUserPermissions := "" - if role, err := s.GetRoleByName(model.TEAM_USER_ROLE_ID); err == nil { + if role, err := ts.srv.GetRoleByName(model.TEAM_USER_ROLE_ID); err == nil { teamUserPermissions = strings.Join(role.Permissions, " ") } teamGuestPermissions := "" - if role, err := s.GetRoleByName(model.TEAM_GUEST_ROLE_ID); err == nil { + if role, err := ts.srv.GetRoleByName(model.TEAM_GUEST_ROLE_ID); err == nil { teamGuestPermissions = strings.Join(role.Permissions, " ") } channelAdminPermissions := "" - if role, err := s.GetRoleByName(model.CHANNEL_ADMIN_ROLE_ID); err == nil { + if role, err := ts.srv.GetRoleByName(model.CHANNEL_ADMIN_ROLE_ID); err == nil { channelAdminPermissions = strings.Join(role.Permissions, " ") } channelUserPermissions := "" - if role, err := s.GetRoleByName(model.CHANNEL_USER_ROLE_ID); err == nil { + if role, err := ts.srv.GetRoleByName(model.CHANNEL_USER_ROLE_ID); err == nil { channelUserPermissions = strings.Join(role.Permissions, " ") } channelGuestPermissions := "" - if role, err := s.GetRoleByName(model.CHANNEL_GUEST_ROLE_ID); err == nil { + if role, err := ts.srv.GetRoleByName(model.CHANNEL_GUEST_ROLE_ID); err == nil { channelGuestPermissions = strings.Join(role.Permissions, " ") } - s.SendDiagnostic(TRACK_PERMISSIONS_SYSTEM_SCHEME, map[string]interface{}{ + ts.sendTelemetry(TRACK_PERMISSIONS_SYSTEM_SCHEME, map[string]interface{}{ "system_admin_permissions": systemAdminPermissions, "system_user_permissions": systemUserPermissions, "team_admin_permissions": teamAdminPermissions, @@ -898,41 +932,41 @@ func (s *Server) trackPermissions() { "channel_guest_permissions": channelGuestPermissions, }) - if schemes, err := s.GetSchemes(model.SCHEME_SCOPE_TEAM, 0, 100); err == nil { + if schemes, err := ts.srv.GetSchemes(model.SCHEME_SCOPE_TEAM, 0, 100); err == nil { for _, scheme := range schemes { teamAdminPermissions := "" - if role, err := s.GetRoleByName(scheme.DefaultTeamAdminRole); err == nil { + if role, err := ts.srv.GetRoleByName(scheme.DefaultTeamAdminRole); err == nil { teamAdminPermissions = strings.Join(role.Permissions, " ") } teamUserPermissions := "" - if role, err := s.GetRoleByName(scheme.DefaultTeamUserRole); err == nil { + if role, err := ts.srv.GetRoleByName(scheme.DefaultTeamUserRole); err == nil { teamUserPermissions = strings.Join(role.Permissions, " ") } teamGuestPermissions := "" - if role, err := s.GetRoleByName(scheme.DefaultTeamGuestRole); err == nil { + if role, err := ts.srv.GetRoleByName(scheme.DefaultTeamGuestRole); err == nil { teamGuestPermissions = strings.Join(role.Permissions, " ") } channelAdminPermissions := "" - if role, err := s.GetRoleByName(scheme.DefaultChannelAdminRole); err == nil { + if role, err := ts.srv.GetRoleByName(scheme.DefaultChannelAdminRole); err == nil { channelAdminPermissions = strings.Join(role.Permissions, " ") } channelUserPermissions := "" - if role, err := s.GetRoleByName(scheme.DefaultChannelUserRole); err == nil { + if role, err := ts.srv.GetRoleByName(scheme.DefaultChannelUserRole); err == nil { channelUserPermissions = strings.Join(role.Permissions, " ") } channelGuestPermissions := "" - if role, err := s.GetRoleByName(scheme.DefaultChannelGuestRole); err == nil { + if role, err := ts.srv.GetRoleByName(scheme.DefaultChannelGuestRole); err == nil { channelGuestPermissions = strings.Join(role.Permissions, " ") } - count, _ := s.Store.Team().AnalyticsGetTeamCountForScheme(scheme.Id) + count, _ := ts.dbStore.Team().AnalyticsGetTeamCountForScheme(scheme.Id) - s.SendDiagnostic(TRACK_PERMISSIONS_TEAM_SCHEMES, map[string]interface{}{ + ts.sendTelemetry(TRACK_PERMISSIONS_TEAM_SCHEMES, map[string]interface{}{ "scheme_id": scheme.Id, "team_admin_permissions": teamAdminPermissions, "team_user_permissions": teamUserPermissions, @@ -946,60 +980,60 @@ func (s *Server) trackPermissions() { } } -func (s *Server) trackElasticsearch() { +func (ts *TelemetryService) trackElasticsearch() { data := map[string]interface{}{} - for _, engine := range s.SearchEngine.GetActiveEngines() { + for _, engine := range ts.searchEngine.GetActiveEngines() { if engine.GetVersion() != 0 && engine.GetName() == "elasticsearch" { data["elasticsearch_server_version"] = engine.GetVersion() } } - s.SendDiagnostic(TRACK_ELASTICSEARCH, data) + ts.sendTelemetry(TRACK_ELASTICSEARCH, data) } -func (s *Server) trackGroups() { - groupCount, err := s.Store.Group().GroupCount() +func (ts *TelemetryService) trackGroups() { + groupCount, err := ts.dbStore.Group().GroupCount() if err != nil { mlog.Error(err.Error()) } - groupTeamCount, err := s.Store.Group().GroupTeamCount() + groupTeamCount, err := ts.dbStore.Group().GroupTeamCount() if err != nil { mlog.Error(err.Error()) } - groupChannelCount, err := s.Store.Group().GroupChannelCount() + groupChannelCount, err := ts.dbStore.Group().GroupChannelCount() if err != nil { mlog.Error(err.Error()) } - groupSyncedTeamCount, err := s.Store.Team().GroupSyncedTeamCount() + groupSyncedTeamCount, err := ts.dbStore.Team().GroupSyncedTeamCount() if err != nil { mlog.Error(err.Error()) } - groupSyncedChannelCount, err := s.Store.Channel().GroupSyncedChannelCount() + groupSyncedChannelCount, err := ts.dbStore.Channel().GroupSyncedChannelCount() if err != nil { mlog.Error(err.Error()) } - groupMemberCount, err := s.Store.Group().GroupMemberCount() + groupMemberCount, err := ts.dbStore.Group().GroupMemberCount() if err != nil { mlog.Error(err.Error()) } - distinctGroupMemberCount, err := s.Store.Group().DistinctGroupMemberCount() + distinctGroupMemberCount, err := ts.dbStore.Group().DistinctGroupMemberCount() if err != nil { mlog.Error(err.Error()) } - groupCountWithAllowReference, err := s.Store.Group().GroupCountWithAllowReference() + groupCountWithAllowReference, err := ts.dbStore.Group().GroupCountWithAllowReference() if err != nil { mlog.Error(err.Error()) } - s.SendDiagnostic(TRACK_GROUPS, map[string]interface{}{ + ts.sendTelemetry(TRACK_GROUPS, map[string]interface{}{ "group_count": groupCount, "group_team_count": groupTeamCount, "group_channel_count": groupChannelCount, @@ -1011,50 +1045,50 @@ func (s *Server) trackGroups() { }) } -func (s *Server) trackChannelModeration() { - channelSchemeCount, err := s.Store.Scheme().CountByScope(model.SCHEME_SCOPE_CHANNEL) +func (ts *TelemetryService) trackChannelModeration() { + channelSchemeCount, err := ts.dbStore.Scheme().CountByScope(model.SCHEME_SCOPE_CHANNEL) if err != nil { mlog.Error(err.Error()) } - createPostUser, err := s.Store.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_CREATE_POST.Id, model.RoleScopeChannel, model.RoleTypeUser) + createPostUser, err := ts.dbStore.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_CREATE_POST.Id, model.RoleScopeChannel, model.RoleTypeUser) if err != nil { mlog.Error(err.Error()) } - createPostGuest, err := s.Store.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_CREATE_POST.Id, model.RoleScopeChannel, model.RoleTypeGuest) + createPostGuest, err := ts.dbStore.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_CREATE_POST.Id, model.RoleScopeChannel, model.RoleTypeGuest) if err != nil { mlog.Error(err.Error()) } // only need to track one of 'add_reaction' or 'remove_reaction` because they're both toggled together by the channel moderation feature - postReactionsUser, err := s.Store.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_ADD_REACTION.Id, model.RoleScopeChannel, model.RoleTypeUser) + postReactionsUser, err := ts.dbStore.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_ADD_REACTION.Id, model.RoleScopeChannel, model.RoleTypeUser) if err != nil { mlog.Error(err.Error()) } - postReactionsGuest, err := s.Store.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_ADD_REACTION.Id, model.RoleScopeChannel, model.RoleTypeGuest) + postReactionsGuest, err := ts.dbStore.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_ADD_REACTION.Id, model.RoleScopeChannel, model.RoleTypeGuest) if err != nil { mlog.Error(err.Error()) } // only need to track one of 'manage_public_channel_members' or 'manage_private_channel_members` because they're both toggled together by the channel moderation feature - manageMembersUser, err := s.Store.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_MANAGE_PUBLIC_CHANNEL_MEMBERS.Id, model.RoleScopeChannel, model.RoleTypeUser) + manageMembersUser, err := ts.dbStore.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_MANAGE_PUBLIC_CHANNEL_MEMBERS.Id, model.RoleScopeChannel, model.RoleTypeUser) if err != nil { mlog.Error(err.Error()) } - useChannelMentionsUser, err := s.Store.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_USE_CHANNEL_MENTIONS.Id, model.RoleScopeChannel, model.RoleTypeUser) + useChannelMentionsUser, err := ts.dbStore.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_USE_CHANNEL_MENTIONS.Id, model.RoleScopeChannel, model.RoleTypeUser) if err != nil { mlog.Error(err.Error()) } - useChannelMentionsGuest, err := s.Store.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_USE_CHANNEL_MENTIONS.Id, model.RoleScopeChannel, model.RoleTypeGuest) + useChannelMentionsGuest, err := ts.dbStore.Scheme().CountWithoutPermission(model.SCHEME_SCOPE_CHANNEL, model.PERMISSION_USE_CHANNEL_MENTIONS.Id, model.RoleScopeChannel, model.RoleTypeGuest) if err != nil { mlog.Error(err.Error()) } - s.SendDiagnostic(TRACK_CHANNEL_MODERATION, map[string]interface{}{ + ts.sendTelemetry(TRACK_CHANNEL_MODERATION, map[string]interface{}{ "channel_scheme_count": channelSchemeCount, "create_post_user_disabled_count": createPostUser, @@ -1070,15 +1104,75 @@ func (s *Server) trackChannelModeration() { }) } -func (s *Server) trackWarnMetrics() { - systemDataList, nErr := s.Store.System().Get() +func (ts *TelemetryService) initRudder(endpoint string, rudderKey string) { + if ts.rudderClient == nil { + config := rudder.Config{} + config.Logger = rudder.StdLogger(ts.log.StdLog(mlog.String("source", "rudder"))) + config.Endpoint = endpoint + // For testing + if endpoint != RUDDER_DATAPLANE_URL { + config.Verbose = true + config.BatchSize = 1 + } + client, err := rudder.NewWithConfig(rudderKey, endpoint, config) + if err != nil { + mlog.Error("Failed to create Rudder instance", mlog.Err(err)) + return + } + client.Enqueue(rudder.Identify{ + UserId: ts.TelemetryID, + }) + + ts.rudderClient = client + } +} + +func (ts *TelemetryService) doTelemetryIfNeeded(firstRun time.Time) { + hoursSinceFirstServerRun := time.Since(firstRun).Hours() + // Send once every 10 minutes for the first hour + // Send once every hour thereafter for the first 12 hours + // Send at the 24 hour mark and every 24 hours after + if hoursSinceFirstServerRun < 1 { + ts.doTelemetry() + } else if hoursSinceFirstServerRun <= 12 && time.Since(ts.timestampLastTelemetrySent) >= time.Hour { + ts.doTelemetry() + } else if hoursSinceFirstServerRun > 12 && time.Since(ts.timestampLastTelemetrySent) >= 24*time.Hour { + ts.doTelemetry() + } +} + +func (ts *TelemetryService) RunTelemetryJob(firstRun int64) { + // Send on boot + ts.doTelemetry() + model.CreateRecurringTask("Telemetry", func() { + ts.doTelemetryIfNeeded(utils.TimeFromMillis(firstRun)) + }, time.Minute*10) +} + +func (ts *TelemetryService) doTelemetry() { + if *ts.srv.Config().LogSettings.EnableDiagnostics { + ts.timestampLastTelemetrySent = time.Now() + ts.sendDailyTelemetry(false) + } +} + +// Shutdown closes the telemetry client. +func (ts *TelemetryService) Shutdown() error { + if ts.rudderClient != nil { + return ts.rudderClient.Close() + } + return nil +} + +func (ts *TelemetryService) trackWarnMetrics() { + systemDataList, nErr := ts.dbStore.System().Get() if nErr != nil { return } for key, value := range systemDataList { if strings.HasPrefix(key, model.WARN_METRIC_STATUS_STORE_PREFIX) { if _, ok := model.WarnMetricsTable[key]; ok { - s.SendDiagnostic(TRACK_WARN_METRICS, map[string]interface{}{ + ts.sendTelemetry(TRACK_WARN_METRICS, map[string]interface{}{ key: value != "false", }) } @@ -1086,7 +1180,7 @@ func (s *Server) trackWarnMetrics() { } } -func (s *Server) trackPluginConfig(cfg *model.Config, marketplaceURL string) { +func (ts *TelemetryService) trackPluginConfig(cfg *model.Config, marketplaceURL string) { pluginConfigData := map[string]interface{}{ "enable_nps_survey": pluginSetting(&cfg.PluginSettings, "com.mattermost.nps", "enablesurvey", true), "enable": *cfg.PluginSettings.Enable, @@ -1126,7 +1220,7 @@ func (s *Server) trackPluginConfig(cfg *model.Config, marketplaceURL string) { "zoom", } - marketplacePlugins, err := s.getAllMarketplaceplugins(marketplaceURL) + marketplacePlugins, err := ts.getAllMarketplaceplugins(marketplaceURL) if err != nil { mlog.Info("Failed to fetch marketplace plugins for telemetry. Using predefined list.", mlog.Err(err)) @@ -1141,10 +1235,10 @@ func (s *Server) trackPluginConfig(cfg *model.Config, marketplaceURL string) { } } - pluginsEnvironment := s.GetPluginsEnvironment() + pluginsEnvironment := ts.srv.GetPluginsEnvironment() if pluginsEnvironment != nil { if plugins, appErr := pluginsEnvironment.Available(); appErr != nil { - mlog.Error("Unable to add plugin versions to diagnostics", mlog.Err(appErr)) + mlog.Error("Unable to add plugin versions to telemetry", mlog.Err(appErr)) } else { // If marketplace request failed, use predefined list if marketplacePlugins == nil { @@ -1161,13 +1255,13 @@ func (s *Server) trackPluginConfig(cfg *model.Config, marketplaceURL string) { } } - s.SendDiagnostic(TRACK_CONFIG_PLUGIN, pluginConfigData) + ts.sendTelemetry(TRACK_CONFIG_PLUGIN, pluginConfigData) } -func (s *Server) getAllMarketplaceplugins(marketplaceURL string) ([]*model.BaseMarketplacePlugin, error) { +func (ts *TelemetryService) getAllMarketplaceplugins(marketplaceURL string) ([]*model.BaseMarketplacePlugin, error) { marketplaceClient, err := marketplace.NewClient( marketplaceURL, - s.HTTPService, + ts.srv.HttpService(), ) if err != nil { return nil, err @@ -1179,7 +1273,7 @@ func (s *Server) getAllMarketplaceplugins(marketplaceURL string) ([]*model.BaseM ServerVersion: model.CurrentVersion, } - license := s.License() + license := ts.srv.License() if license != nil && *license.Features.EnterprisePlugins { filter.EnterprisePlugins = true } diff --git a/services/telemetry/telemetry_test.go b/services/telemetry/telemetry_test.go new file mode 100644 index 0000000000..0b55700c6d --- /dev/null +++ b/services/telemetry/telemetry_test.go @@ -0,0 +1,476 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package telemetry + +import ( + "crypto/ecdsa" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/plugin" + "github.com/mattermost/mattermost-server/v5/plugin/plugintest" + "github.com/mattermost/mattermost-server/v5/services/httpservice" + "github.com/mattermost/mattermost-server/v5/services/searchengine" + "github.com/mattermost/mattermost-server/v5/services/telemetry/mocks" + storeMocks "github.com/mattermost/mattermost-server/v5/store/storetest/mocks" +) + +type FakeConfigService struct { + cfg *model.Config +} + +func (fcs *FakeConfigService) Config() *model.Config { return fcs.cfg } +func (fcs *FakeConfigService) AddConfigListener(f func(old, current *model.Config)) string { return "" } +func (fcs *FakeConfigService) RemoveConfigListener(key string) {} +func (fcs *FakeConfigService) AsymmetricSigningKey() *ecdsa.PrivateKey { return nil } + +func initializeMocks(cfg *model.Config) (*mocks.ServerIface, *storeMocks.Store, func(t *testing.T), func()) { + serverIfaceMock := &mocks.ServerIface{} + + configService := &FakeConfigService{cfg} + serverIfaceMock.On("Config").Return(cfg) + serverIfaceMock.On("IsLeader").Return(true) + + pluginDir, _ := ioutil.TempDir("", "") + webappPluginDir, _ := ioutil.TempDir("", "") + cleanUp := func() { + os.RemoveAll(pluginDir) + os.RemoveAll(webappPluginDir) + } + pluginsAPIMock := &plugintest.API{} + pluginEnv, _ := plugin.NewEnvironment(func(m *model.Manifest) plugin.API { return pluginsAPIMock }, pluginDir, webappPluginDir, mlog.NewLogger(&mlog.LoggerConfiguration{}), nil) + serverIfaceMock.On("GetPluginsEnvironment").Return(pluginEnv, nil) + + serverIfaceMock.On("License").Return(model.NewTestLicense(), nil) + serverIfaceMock.On("GetRoleByName", "system_admin").Return(&model.Role{Permissions: []string{"sa-test1", "sa-test2"}}, nil) + serverIfaceMock.On("GetRoleByName", "system_user").Return(&model.Role{Permissions: []string{"su-test1", "su-test2"}}, nil) + serverIfaceMock.On("GetRoleByName", "team_admin").Return(&model.Role{Permissions: []string{"ta-test1", "ta-test2"}}, nil) + serverIfaceMock.On("GetRoleByName", "team_user").Return(&model.Role{Permissions: []string{"tu-test1", "tu-test2"}}, nil) + serverIfaceMock.On("GetRoleByName", "team_guest").Return(&model.Role{Permissions: []string{"tg-test1", "tg-test2"}}, nil) + serverIfaceMock.On("GetRoleByName", "channel_admin").Return(&model.Role{Permissions: []string{"ca-test1", "ca-test2"}}, nil) + serverIfaceMock.On("GetRoleByName", "channel_user").Return(&model.Role{Permissions: []string{"cu-test1", "cu-test2"}}, nil) + serverIfaceMock.On("GetRoleByName", "channel_guest").Return(&model.Role{Permissions: []string{"cg-test1", "cg-test2"}}, nil) + serverIfaceMock.On("GetSchemes", "team", 0, 100).Return([]*model.Scheme{}, nil) + serverIfaceMock.On("HttpService").Return(httpservice.MakeHTTPService(configService)) + + storeMock := &storeMocks.Store{} + storeMock.On("GetDbVersion").Return("5.24.0", nil) + + systemStore := storeMocks.SystemStore{} + props := model.StringMap{} + props[model.SYSTEM_TELEMETRY_ID] = "test" + systemStore.On("Get").Return(props, nil) + systemStore.On("GetByName", model.ADVANCED_PERMISSIONS_MIGRATION_KEY).Return(nil, nil) + systemStore.On("GetByName", model.MIGRATION_KEY_ADVANCED_PERMISSIONS_PHASE_2).Return(nil, nil) + + userStore := storeMocks.UserStore{} + userStore.On("Count", model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: true, ExcludeRegularUsers: false, TeamId: "", ViewRestrictions: nil}).Return(int64(10), nil) + userStore.On("Count", model.UserCountOptions{IncludeBotAccounts: true, IncludeDeleted: false, ExcludeRegularUsers: true, TeamId: "", ViewRestrictions: nil}).Return(int64(100), nil) + userStore.On("AnalyticsGetGuestCount").Return(int64(11), nil) + userStore.On("AnalyticsActiveCount", mock.Anything, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false, ExcludeRegularUsers: false, TeamId: "", ViewRestrictions: nil}).Return(int64(5), nil) + userStore.On("AnalyticsGetInactiveUsersCount").Return(int64(8), nil) + userStore.On("AnalyticsGetSystemAdminCount").Return(int64(9), nil) + + teamStore := storeMocks.TeamStore{} + teamStore.On("AnalyticsTeamCount", false).Return(int64(3), nil) + teamStore.On("GroupSyncedTeamCount").Return(int64(16), nil) + + channelStore := storeMocks.ChannelStore{} + channelStore.On("AnalyticsTypeCount", "", "O").Return(int64(25), nil) + channelStore.On("AnalyticsTypeCount", "", "P").Return(int64(26), nil) + channelStore.On("AnalyticsTypeCount", "", "D").Return(int64(27), nil) + channelStore.On("AnalyticsDeletedTypeCount", "", "O").Return(int64(22), nil) + channelStore.On("AnalyticsDeletedTypeCount", "", "P").Return(int64(23), nil) + channelStore.On("GroupSyncedChannelCount").Return(int64(17), nil) + + postStore := storeMocks.PostStore{} + postStore.On("AnalyticsPostCount", "", false, false).Return(int64(1000), nil) + postStore.On("AnalyticsPostCountsByDay", &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: false, YesterdayOnly: true}).Return(model.AnalyticsRows{}, nil) + postStore.On("AnalyticsPostCountsByDay", &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: true, YesterdayOnly: true}).Return(model.AnalyticsRows{}, nil) + + commandStore := storeMocks.CommandStore{} + commandStore.On("AnalyticsCommandCount", "").Return(int64(15), nil) + + webhookStore := storeMocks.WebhookStore{} + webhookStore.On("AnalyticsIncomingCount", "").Return(int64(16), nil) + webhookStore.On("AnalyticsOutgoingCount", "").Return(int64(17), nil) + + groupStore := storeMocks.GroupStore{} + groupStore.On("GroupCount").Return(int64(25), nil) + groupStore.On("GroupTeamCount").Return(int64(26), nil) + groupStore.On("GroupChannelCount").Return(int64(27), nil) + groupStore.On("GroupMemberCount").Return(int64(32), nil) + groupStore.On("DistinctGroupMemberCount").Return(int64(22), nil) + groupStore.On("GroupCountWithAllowReference").Return(int64(13), nil) + + schemeStore := storeMocks.SchemeStore{} + schemeStore.On("CountByScope", "channel").Return(int64(8), nil) + schemeStore.On("CountByScope", "team").Return(int64(7), nil) + schemeStore.On("CountWithoutPermission", "channel", "create_post", model.RoleScopeChannel, model.RoleTypeUser).Return(int64(6), nil) + schemeStore.On("CountWithoutPermission", "channel", "create_post", model.RoleScopeChannel, model.RoleTypeGuest).Return(int64(7), nil) + schemeStore.On("CountWithoutPermission", "channel", "add_reaction", model.RoleScopeChannel, model.RoleTypeUser).Return(int64(8), nil) + schemeStore.On("CountWithoutPermission", "channel", "add_reaction", model.RoleScopeChannel, model.RoleTypeGuest).Return(int64(9), nil) + schemeStore.On("CountWithoutPermission", "channel", "manage_public_channel_members", model.RoleScopeChannel, model.RoleTypeUser).Return(int64(10), nil) + schemeStore.On("CountWithoutPermission", "channel", "use_channel_mentions", model.RoleScopeChannel, model.RoleTypeUser).Return(int64(11), nil) + schemeStore.On("CountWithoutPermission", "channel", "use_channel_mentions", model.RoleScopeChannel, model.RoleTypeGuest).Return(int64(12), nil) + + storeMock.On("System").Return(&systemStore) + storeMock.On("User").Return(&userStore) + storeMock.On("Team").Return(&teamStore) + storeMock.On("Channel").Return(&channelStore) + storeMock.On("Post").Return(&postStore) + storeMock.On("Command").Return(&commandStore) + storeMock.On("Webhook").Return(&webhookStore) + storeMock.On("Group").Return(&groupStore) + storeMock.On("Scheme").Return(&schemeStore) + + return serverIfaceMock, storeMock, func(t *testing.T) { + serverIfaceMock.AssertExpectations(t) + storeMock.AssertExpectations(t) + systemStore.AssertExpectations(t) + pluginsAPIMock.AssertExpectations(t) + }, cleanUp +} + +func TestPluginSetting(t *testing.T) { + settings := &model.PluginSettings{ + Plugins: map[string]map[string]interface{}{ + "test": { + "foo": "bar", + }, + }, + } + assert.Equal(t, "bar", pluginSetting(settings, "test", "foo", "asd")) + assert.Equal(t, "asd", pluginSetting(settings, "test", "qwe", "asd")) +} + +func TestPluginActivated(t *testing.T) { + states := map[string]*model.PluginState{ + "foo": { + Enable: true, + }, + "bar": { + Enable: false, + }, + } + assert.True(t, pluginActivated(states, "foo")) + assert.False(t, pluginActivated(states, "bar")) + assert.False(t, pluginActivated(states, "none")) +} + +func TestPluginVersion(t *testing.T) { + plugins := []*model.BundleInfo{ + { + Manifest: &model.Manifest{ + Id: "test.plugin", + Version: "1.2.3", + }, + }, + { + Manifest: &model.Manifest{ + Id: "test.plugin2", + Version: "4.5.6", + }, + }, + } + assert.Equal(t, "1.2.3", pluginVersion(plugins, "test.plugin")) + assert.Equal(t, "4.5.6", pluginVersion(plugins, "test.plugin2")) + assert.Empty(t, pluginVersion(plugins, "unknown.plugin")) +} + +func TestRudderTelemetry(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + type batch struct { + MessageId string + UserId string + Event string + Timestamp time.Time + Properties map[string]interface{} + } + + type payload struct { + MessageId string + SentAt time.Time + Batch []struct { + MessageId string + UserId string + Event string + Timestamp time.Time + Properties map[string]interface{} + } + Context struct { + Library struct { + Name string + Version string + } + } + } + + data := make(chan payload, 100) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + + var p payload + err = json.Unmarshal(body, &p) + require.NoError(t, err) + + data <- p + })) + defer server.Close() + + marketplaceServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + res.WriteHeader(http.StatusOK) + json, err := json.Marshal([]*model.MarketplacePlugin{{ + BaseMarketplacePlugin: &model.BaseMarketplacePlugin{ + Manifest: &model.Manifest{ + Id: "testplugin", + }, + }, + }}) + require.NoError(t, err) + res.Write(json) + })) + + defer func() { marketplaceServer.Close() }() + + telemetryID := "test-telemetry-id-12345" + + cfg := &model.Config{} + cfg.SetDefaults() + serverIfaceMock, storeMock, deferredAssertions, cleanUp := initializeMocks(cfg) + defer cleanUp() + defer deferredAssertions(t) + + telemetryService := New(serverIfaceMock, storeMock, searchengine.NewBroker(cfg, nil), mlog.NewLogger(&mlog.LoggerConfiguration{})) + telemetryService.TelemetryID = telemetryID + telemetryService.rudderClient = nil + telemetryService.initRudder(server.URL, "") + + assertPayload := func(t *testing.T, actual payload, event string, properties map[string]interface{}) { + t.Helper() + assert.NotEmpty(t, actual.MessageId) + assert.False(t, actual.SentAt.IsZero()) + if assert.Len(t, actual.Batch, 1) { + assert.NotEmpty(t, actual.Batch[0].MessageId, "message id should not be empty") + assert.Equal(t, telemetryID, actual.Batch[0].UserId) + if event != "" { + assert.Equal(t, event, actual.Batch[0].Event) + } + assert.False(t, actual.Batch[0].Timestamp.IsZero(), "batch timestamp should not be the zero value") + if properties != nil { + assert.Equal(t, properties, actual.Batch[0].Properties) + } + } + assert.Equal(t, "analytics-go", actual.Context.Library.Name) + assert.Equal(t, "3.0.0", actual.Context.Library.Version) + } + + collectInfo := func(info *[]string) { + t.Helper() + for { + select { + case result := <-data: + assertPayload(t, result, "", nil) + *info = append(*info, result.Batch[0].Event) + case <-time.After(time.Second * 1): + return + } + } + } + + collectBatches := func(info *[]batch) { + t.Helper() + for { + select { + case result := <-data: + assertPayload(t, result, "", nil) + *info = append(*info, result.Batch[0]) + case <-time.After(time.Second * 1): + return + } + } + } + + // Should send a client identify message + select { + case identifyMessage := <-data: + assertPayload(t, identifyMessage, "", nil) + case <-time.After(time.Second * 1): + require.Fail(t, "Did not receive ID message") + } + + t.Run("Send", func(t *testing.T) { + testValue := "test-send-value-6789" + telemetryService.sendTelemetry("Testing Telemetry", map[string]interface{}{ + "hey": testValue, + }) + select { + case result := <-data: + assertPayload(t, result, "Testing Telemetry", map[string]interface{}{ + "hey": testValue, + }) + case <-time.After(time.Second * 1): + require.Fail(t, "Did not receive telemetry") + } + }) + + // Plugins remain disabled at this point + t.Run("SendDailyTelemetryPluginsDisabled", func(t *testing.T) { + telemetryService.sendDailyTelemetry(true) + + var info []string + // Collect the info sent. + collectInfo(&info) + + for _, item := range []string{ + TRACK_CONFIG_SERVICE, + TRACK_CONFIG_TEAM, + TRACK_CONFIG_SQL, + TRACK_CONFIG_LOG, + TRACK_CONFIG_NOTIFICATION_LOG, + TRACK_CONFIG_FILE, + TRACK_CONFIG_RATE, + TRACK_CONFIG_EMAIL, + TRACK_CONFIG_PRIVACY, + TRACK_CONFIG_OAUTH, + TRACK_CONFIG_LDAP, + TRACK_CONFIG_COMPLIANCE, + TRACK_CONFIG_LOCALIZATION, + TRACK_CONFIG_SAML, + TRACK_CONFIG_PASSWORD, + TRACK_CONFIG_CLUSTER, + TRACK_CONFIG_METRICS, + TRACK_CONFIG_SUPPORT, + TRACK_CONFIG_NATIVEAPP, + TRACK_CONFIG_EXPERIMENTAL, + TRACK_CONFIG_ANALYTICS, + TRACK_CONFIG_PLUGIN, + TRACK_ACTIVITY, + TRACK_SERVER, + TRACK_CONFIG_MESSAGE_EXPORT, + TRACK_PLUGINS, + } { + require.Contains(t, info, item) + } + }) + + // Enable plugins for the remainder of the tests. + // th.Server.UpdateConfig(func(cfg *model.Config) { *cfg.PluginSettings.Enable = true }) + + t.Run("SendDailyTelemetry", func(t *testing.T) { + telemetryService.sendDailyTelemetry(true) + + var info []string + // Collect the info sent. + collectInfo(&info) + + for _, item := range []string{ + TRACK_CONFIG_SERVICE, + TRACK_CONFIG_TEAM, + TRACK_CONFIG_SQL, + TRACK_CONFIG_LOG, + TRACK_CONFIG_NOTIFICATION_LOG, + TRACK_CONFIG_FILE, + TRACK_CONFIG_RATE, + TRACK_CONFIG_EMAIL, + TRACK_CONFIG_PRIVACY, + TRACK_CONFIG_OAUTH, + TRACK_CONFIG_LDAP, + TRACK_CONFIG_COMPLIANCE, + TRACK_CONFIG_LOCALIZATION, + TRACK_CONFIG_SAML, + TRACK_CONFIG_PASSWORD, + TRACK_CONFIG_CLUSTER, + TRACK_CONFIG_METRICS, + TRACK_CONFIG_SUPPORT, + TRACK_CONFIG_NATIVEAPP, + TRACK_CONFIG_EXPERIMENTAL, + TRACK_CONFIG_ANALYTICS, + TRACK_CONFIG_PLUGIN, + TRACK_ACTIVITY, + TRACK_SERVER, + TRACK_CONFIG_MESSAGE_EXPORT, + TRACK_PLUGINS, + } { + require.Contains(t, info, item) + } + }) + t.Run("Telemetry for Marketplace plugins is returned", func(t *testing.T) { + telemetryService.trackPluginConfig(telemetryService.srv.Config(), marketplaceServer.URL) + + var batches []batch + collectBatches(&batches) + + for _, b := range batches { + if b.Event == TRACK_CONFIG_PLUGIN { + assert.Contains(t, b.Properties, "enable_testplugin") + assert.Contains(t, b.Properties, "version_testplugin") + + // Confirm known plugins are not present + assert.NotContains(t, b.Properties, "enable_jira") + assert.NotContains(t, b.Properties, "version_jira") + } + } + }) + + t.Run("Telemetry for known plugins is returned, if request to Marketplace fails", func(t *testing.T) { + telemetryService.trackPluginConfig(telemetryService.srv.Config(), "http://some.random.invalid.url") + + var batches []batch + collectBatches(&batches) + + for _, b := range batches { + if b.Event == TRACK_CONFIG_PLUGIN { + assert.NotContains(t, b.Properties, "enable_testplugin") + assert.NotContains(t, b.Properties, "version_testplugin") + + // Confirm known plugins are present + assert.Contains(t, b.Properties, "enable_jira") + assert.Contains(t, b.Properties, "version_jira") + } + } + }) + + t.Run("SendDailyTelemetryNoRudderKey", func(t *testing.T) { + telemetryService.sendDailyTelemetry(false) + + select { + case <-data: + require.Fail(t, "Should not send telemetry when the rudder key is not set") + case <-time.After(time.Second * 1): + // Did not receive telemetry + } + }) + + t.Run("SendDailyTelemetryDisabled", func(t *testing.T) { + *cfg.LogSettings.EnableDiagnostics = false + defer func() { + *cfg.LogSettings.EnableDiagnostics = true + }() + + telemetryService.sendDailyTelemetry(true) + + select { + case <-data: + require.Fail(t, "Should not send telemetry when they are disabled") + case <-time.After(time.Second * 1): + // Did not receive telemetry + } + }) +} diff --git a/templates/warn_metric_ack.html b/templates/warn_metric_ack.html index 0365b58612..0ed306b0ae 100644 --- a/templates/warn_metric_ack.html +++ b/templates/warn_metric_ack.html @@ -55,12 +55,12 @@ {{end}} - {{if .Props.DiagnosticIdValue}} + {{if .Props.TelemetryIdValue}}

- {{.Props.DiagnosticIdHeader}} - {{.Props.DiagnosticIdValue}}

+ {{.Props.TelemetryIdHeader}} + {{.Props.TelemetryIdValue}}

{{end}}