From f4ccc7061ca01ad41f3789d7d01443b2161f1c66 Mon Sep 17 00:00:00 2001 From: Eli Yukelzon Date: Mon, 7 Sep 2020 16:26:25 +0300 Subject: [PATCH] MM-27199 - Add metrics to MM server for total enabled user count (#15116) * Add metrics to MM server for total enabled user count and include installation ID from a new env var * diagnostics context Co-authored-by: Mattermod --- app/app.go | 4 +- app/diagnostics.go | 6 ++ app/enterprise.go | 6 ++ einterfaces/metrics.go | 1 + einterfaces/mocks/MetricsInterface.go | 5 + imports/placeholder.go | 3 + jobs/active_users/scheduler.go | 51 ++++++++++ jobs/active_users/worker.go | 117 ++++++++++++++++++++++ jobs/interfaces/active_users_interface.go | 11 ++ jobs/jobs_watcher.go | 7 ++ jobs/schedulers.go | 3 + jobs/server.go | 1 + jobs/workers.go | 12 +++ model/job.go | 2 + 14 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 jobs/active_users/scheduler.go create mode 100644 jobs/active_users/worker.go create mode 100644 jobs/interfaces/active_users_interface.go diff --git a/app/app.go b/app/app.go index bf167d35e3..e25a010d58 100644 --- a/app/app.go +++ b/app/app.go @@ -120,7 +120,9 @@ func (a *App) initJobs() { if jobsExpiryNotifyInterface != nil { a.srv.Jobs.ExpiryNotify = jobsExpiryNotifyInterface(a) } - + if jobsActiveUsersInterface != nil { + a.srv.Jobs.ActiveUsers = jobsActiveUsersInterface(a) + } a.srv.Jobs.Workers = a.srv.Jobs.InitWorkers() a.srv.Jobs.Schedulers = a.srv.Jobs.InitSchedulers() } diff --git a/app/diagnostics.go b/app/diagnostics.go index 1850d13cc1..d2cd31ced5 100644 --- a/app/diagnostics.go +++ b/app/diagnostics.go @@ -112,10 +112,16 @@ func (s *Server) sendDailyDiagnostics(override bool) { func (s *Server) SendDiagnostic(event string, properties map[string]interface{}) { if s.rudderClient != nil { + var context *rudder.Context + // if we are part of a cloud installation, add it's ID to the tracked event's context + if installationId := os.Getenv("MM_CLOUD_INSTALLATION_ID"); installationId != "" { + context = &rudder.Context{Traits: map[string]interface{}{"installationId": installationId}} + } s.rudderClient.Enqueue(rudder.Track{ Event: event, UserId: s.diagnosticId, Properties: properties, + Context: context, }) } } diff --git a/app/enterprise.go b/app/enterprise.go index 655a2ff641..3e4a92abe8 100644 --- a/app/enterprise.go +++ b/app/enterprise.go @@ -90,6 +90,12 @@ func RegisterJobsBleveIndexerInterface(f func(*Server) tjobs.IndexerJobInterface jobsBleveIndexerInterface = f } +var jobsActiveUsersInterface func(*App) tjobs.ActiveUsersJobInterface + +func RegisterJobsActiveUsersInterface(f func(*App) tjobs.ActiveUsersJobInterface) { + jobsActiveUsersInterface = f +} + var jobsExpiryNotifyInterface func(*App) tjobs.ExpiryNotifyJobInterface func RegisterJobsExpiryNotifyJobInterface(f func(*App) tjobs.ExpiryNotifyJobInterface) { diff --git a/einterfaces/metrics.go b/einterfaces/metrics.go index 921fadca73..63f6521f94 100644 --- a/einterfaces/metrics.go +++ b/einterfaces/metrics.go @@ -59,5 +59,6 @@ type MetricsInterface interface { ObservePluginMultiHookDuration(elapsed float64) ObservePluginApiDuration(pluginID, apiName string, success bool, elapsed float64) + ObserveEnabledUsers(users int64) GetLoggerMetricsCollector() logr.MetricsCollector } diff --git a/einterfaces/mocks/MetricsInterface.go b/einterfaces/mocks/MetricsInterface.go index b7b9564b06..69560fcebb 100644 --- a/einterfaces/mocks/MetricsInterface.go +++ b/einterfaces/mocks/MetricsInterface.go @@ -200,6 +200,11 @@ func (_m *MetricsInterface) ObserveClusterRequestDuration(elapsed float64) { _m.Called(elapsed) } +// ObserveEnabledUsers provides a mock function with given fields: users +func (_m *MetricsInterface) ObserveEnabledUsers(users int64) { + _m.Called(users) +} + // ObservePluginApiDuration provides a mock function with given fields: pluginID, apiName, success, elapsed func (_m *MetricsInterface) ObservePluginApiDuration(pluginID string, apiName string, success bool, elapsed float64) { _m.Called(pluginID, apiName, success, elapsed) diff --git a/imports/placeholder.go b/imports/placeholder.go index 8d86c74b89..656f3176d2 100644 --- a/imports/placeholder.go +++ b/imports/placeholder.go @@ -15,4 +15,7 @@ import ( // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. _ "github.com/mattermost/mattermost-server/v5/jobs/expirynotify" + + // This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty. + _ "github.com/mattermost/mattermost-server/v5/jobs/active_users" ) diff --git a/jobs/active_users/scheduler.go b/jobs/active_users/scheduler.go new file mode 100644 index 0000000000..af15861879 --- /dev/null +++ b/jobs/active_users/scheduler.go @@ -0,0 +1,51 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package active_users + +import ( + "time" + + "github.com/mattermost/mattermost-server/v5/app" + "github.com/mattermost/mattermost-server/v5/model" +) + +const ( + SchedFreqMinutes = 10 +) + +type Scheduler struct { + App *app.App +} + +func (m *ActiveUsersJobInterfaceImpl) MakeScheduler() model.Scheduler { + return &Scheduler{m.App} +} + +func (scheduler *Scheduler) Name() string { + return JobName + "Scheduler" +} + +func (scheduler *Scheduler) JobType() string { + return model.JOB_TYPE_ACTIVE_USERS +} + +func (scheduler *Scheduler) Enabled(cfg *model.Config) bool { + // Only enabled when Metrics are enabled. + return *cfg.MetricsSettings.Enable +} + +func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { + nextTime := time.Now().Add(SchedFreqMinutes * time.Minute) + return &nextTime +} + +func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) { + data := map[string]string{} + + if job, err := scheduler.App.Srv().Jobs.CreateJob(model.JOB_TYPE_ACTIVE_USERS, data); err != nil { + return nil, err + } else { + return job, nil + } +} diff --git a/jobs/active_users/worker.go b/jobs/active_users/worker.go new file mode 100644 index 0000000000..6d2feca919 --- /dev/null +++ b/jobs/active_users/worker.go @@ -0,0 +1,117 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package active_users + +import ( + "github.com/mattermost/mattermost-server/v5/app" + "github.com/mattermost/mattermost-server/v5/jobs" + tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces" + "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/model" +) + +const ( + JobName = "ActiveUsers" +) + +type Worker struct { + name string + stop chan bool + stopped chan bool + jobs chan model.Job + jobServer *jobs.JobServer + app *app.App +} + +func init() { + app.RegisterJobsActiveUsersInterface(func(a *app.App) tjobs.ActiveUsersJobInterface { + return &ActiveUsersJobInterfaceImpl{a} + }) +} + +type ActiveUsersJobInterfaceImpl struct { + App *app.App +} + +func (m *ActiveUsersJobInterfaceImpl) MakeWorker() model.Worker { + worker := Worker{ + name: JobName, + stop: make(chan bool, 1), + stopped: make(chan bool, 1), + jobs: make(chan model.Job), + jobServer: m.App.Srv().Jobs, + app: m.App, + } + return &worker +} + +func (worker *Worker) Run() { + mlog.Debug("Worker started", mlog.String("worker", worker.name)) + + defer func() { + mlog.Debug("Worker finished", mlog.String("worker", worker.name)) + worker.stopped <- true + }() + + for { + select { + case <-worker.stop: + mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name)) + return + case job := <-worker.jobs: + mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name)) + worker.DoJob(&job) + } + } +} + +func (worker *Worker) Stop() { + mlog.Debug("Worker stopping", mlog.String("worker", worker.name)) + worker.stop <- true + <-worker.stopped +} + +func (worker *Worker) JobChannel() chan<- model.Job { + return worker.jobs +} + +func (worker *Worker) DoJob(job *model.Job) { + if claimed, err := worker.jobServer.ClaimJob(job); err != nil { + mlog.Warn("Worker experienced an error while trying to claim job", + mlog.String("worker", worker.name), + mlog.String("job_id", job.Id), + mlog.String("error", err.Error())) + return + } else if !claimed { + return + } + + count, err := worker.app.Srv().Store.User().Count(model.UserCountOptions{IncludeDeleted: false}) + + if err != nil { + mlog.Error("Worker: Failed to get active user count", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + worker.setJobError(job, err) + return + } + + if worker.app.Metrics() != nil { + worker.app.Metrics().ObserveEnabledUsers(count) + } + + mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id)) + worker.setJobSuccess(job) +} + +func (worker *Worker) setJobSuccess(job *model.Job) { + if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil { + mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + worker.setJobError(job, err) + } +} + +func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) { + if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil { + mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error())) + } +} diff --git a/jobs/interfaces/active_users_interface.go b/jobs/interfaces/active_users_interface.go new file mode 100644 index 0000000000..dad1450b32 --- /dev/null +++ b/jobs/interfaces/active_users_interface.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package interfaces + +import "github.com/mattermost/mattermost-server/v5/model" + +type ActiveUsersJobInterface interface { + MakeWorker() model.Worker + MakeScheduler() model.Scheduler +} diff --git a/jobs/jobs_watcher.go b/jobs/jobs_watcher.go index f047d2a922..21a64abb08 100644 --- a/jobs/jobs_watcher.go +++ b/jobs/jobs_watcher.go @@ -135,6 +135,13 @@ func (watcher *Watcher) PollAndNotify() { default: } } + } else if job.Type == model.JOB_TYPE_ACTIVE_USERS { + if watcher.workers.ActiveUsers != nil { + select { + case watcher.workers.ActiveUsers.JobChannel() <- *job: + default: + } + } } } } diff --git a/jobs/schedulers.go b/jobs/schedulers.go index fbc9c0c020..fb9fa1fa6d 100644 --- a/jobs/schedulers.go +++ b/jobs/schedulers.go @@ -66,6 +66,9 @@ func (srv *JobServer) InitSchedulers() *Schedulers { schedulers.schedulers = append(schedulers.schedulers, expiryNotifyInterface.MakeScheduler()) } + if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil { + schedulers.schedulers = append(schedulers.schedulers, activeUsersInterface.MakeScheduler()) + } schedulers.nextRunTimes = make([]*time.Time, len(schedulers.schedulers)) return schedulers } diff --git a/jobs/server.go b/jobs/server.go index 359c2cdc0c..9498746175 100644 --- a/jobs/server.go +++ b/jobs/server.go @@ -26,6 +26,7 @@ type JobServer struct { Plugins tjobs.PluginsJobInterface BleveIndexer tjobs.IndexerJobInterface ExpiryNotify tjobs.ExpiryNotifyJobInterface + ActiveUsers tjobs.ActiveUsersJobInterface } func NewJobServer(configService configservice.ConfigService, store store.Store) *JobServer { diff --git a/jobs/workers.go b/jobs/workers.go index 2cd751a7fc..378b006ec5 100644 --- a/jobs/workers.go +++ b/jobs/workers.go @@ -25,6 +25,7 @@ type Workers struct { Plugins model.Worker BleveIndexing model.Worker ExpiryNotify model.Worker + ActiveUsers model.Worker listenerId string } @@ -70,6 +71,10 @@ func (srv *JobServer) InitWorkers() *Workers { if expiryNotifyInterface := srv.ExpiryNotify; expiryNotifyInterface != nil { workers.ExpiryNotify = expiryNotifyInterface.MakeWorker() } + + if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil { + workers.ActiveUsers = activeUsersInterface.MakeWorker() + } return workers } @@ -113,6 +118,10 @@ func (workers *Workers) Start() *Workers { go workers.ExpiryNotify.Run() } + if workers.ActiveUsers != nil { + go workers.ActiveUsers.Run() + } + go workers.Watcher.Start() }) @@ -214,6 +223,9 @@ func (workers *Workers) Stop() *Workers { workers.ExpiryNotify.Stop() } + if workers.ActiveUsers != nil { + workers.ActiveUsers.Stop() + } mlog.Info("Stopped workers") return workers diff --git a/model/job.go b/model/job.go index 85c1e9f835..90be02c824 100644 --- a/model/job.go +++ b/model/job.go @@ -20,6 +20,7 @@ const ( JOB_TYPE_MIGRATIONS = "migrations" JOB_TYPE_PLUGINS = "plugins" JOB_TYPE_EXPIRY_NOTIFY = "expiry_notify" + JOB_TYPE_ACTIVE_USERS = "active_users" JOB_STATUS_PENDING = "pending" JOB_STATUS_IN_PROGRESS = "in_progress" @@ -61,6 +62,7 @@ func (j *Job) IsValid() *AppError { case JOB_TYPE_MIGRATIONS: case JOB_TYPE_PLUGINS: case JOB_TYPE_EXPIRY_NOTIFY: + case JOB_TYPE_ACTIVE_USERS: default: return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest) }