MM-27199 - Add metrics to MM server for total enabled user count (#15116)

* Add metrics to MM server for total enabled user count and include installation ID from a new env var

* diagnostics context

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

app/app.go

@@ -120,7 +120,9 @@ func (a *App) initJobs() {
 	if jobsExpiryNotifyInterface != nil {
 		a.srv.Jobs.ExpiryNotify = jobsExpiryNotifyInterface(a)
 	}
-
+	if jobsActiveUsersInterface != nil {
+		a.srv.Jobs.ActiveUsers = jobsActiveUsersInterface(a)
+	}
 	a.srv.Jobs.Workers = a.srv.Jobs.InitWorkers()
 	a.srv.Jobs.Schedulers = a.srv.Jobs.InitSchedulers()
 }

app/diagnostics.go

@@ -112,10 +112,16 @@ func (s *Server) sendDailyDiagnostics(override bool) {
 
 func (s *Server) SendDiagnostic(event string, properties map[string]interface{}) {
 	if s.rudderClient != nil {
+		var context *rudder.Context
+		// if we are part of a cloud installation, add it's ID to the tracked event's context
+		if installationId := os.Getenv("MM_CLOUD_INSTALLATION_ID"); installationId != "" {
+			context = &rudder.Context{Traits: map[string]interface{}{"installationId": installationId}}
+		}
 		s.rudderClient.Enqueue(rudder.Track{
 			Event:      event,
 			UserId:     s.diagnosticId,
 			Properties: properties,
+			Context:    context,
 		})
 	}
 }

app/enterprise.go

@@ -90,6 +90,12 @@ func RegisterJobsBleveIndexerInterface(f func(*Server) tjobs.IndexerJobInterface
 	jobsBleveIndexerInterface = f
 }
 
+var jobsActiveUsersInterface func(*App) tjobs.ActiveUsersJobInterface
+
+func RegisterJobsActiveUsersInterface(f func(*App) tjobs.ActiveUsersJobInterface) {
+	jobsActiveUsersInterface = f
+}
+
 var jobsExpiryNotifyInterface func(*App) tjobs.ExpiryNotifyJobInterface
 
 func RegisterJobsExpiryNotifyJobInterface(f func(*App) tjobs.ExpiryNotifyJobInterface) {

einterfaces/metrics.go

@@ -59,5 +59,6 @@ type MetricsInterface interface {
 	ObservePluginMultiHookDuration(elapsed float64)
 	ObservePluginApiDuration(pluginID, apiName string, success bool, elapsed float64)
 
+	ObserveEnabledUsers(users int64)
 	GetLoggerMetricsCollector() logr.MetricsCollector
 }

einterfaces/mocks/MetricsInterface.go

@@ -200,6 +200,11 @@ func (_m *MetricsInterface) ObserveClusterRequestDuration(elapsed float64) {
 	_m.Called(elapsed)
 }
 
+// ObserveEnabledUsers provides a mock function with given fields: users
+func (_m *MetricsInterface) ObserveEnabledUsers(users int64) {
+	_m.Called(users)
+}
+
 // ObservePluginApiDuration provides a mock function with given fields: pluginID, apiName, success, elapsed
 func (_m *MetricsInterface) ObservePluginApiDuration(pluginID string, apiName string, success bool, elapsed float64) {
 	_m.Called(pluginID, apiName, success, elapsed)

imports/placeholder.go

@@ -15,4 +15,7 @@ import (
 
 	// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
 	_ "github.com/mattermost/mattermost-server/v5/jobs/expirynotify"
+
+	// This is a placeholder so this package can be imported in Team Edition when it will be otherwise empty.
+	_ "github.com/mattermost/mattermost-server/v5/jobs/active_users"
 )

jobs/active_users/scheduler.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package active_users
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/app"
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+const (
+	SchedFreqMinutes = 10
+)
+
+type Scheduler struct {
+	App *app.App
+}
+
+func (m *ActiveUsersJobInterfaceImpl) MakeScheduler() model.Scheduler {
+	return &Scheduler{m.App}
+}
+
+func (scheduler *Scheduler) Name() string {
+	return JobName + "Scheduler"
+}
+
+func (scheduler *Scheduler) JobType() string {
+	return model.JOB_TYPE_ACTIVE_USERS
+}
+
+func (scheduler *Scheduler) Enabled(cfg *model.Config) bool {
+	// Only enabled when Metrics are enabled.
+	return *cfg.MetricsSettings.Enable
+}
+
+func (scheduler *Scheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	nextTime := time.Now().Add(SchedFreqMinutes * time.Minute)
+	return &nextTime
+}
+
+func (scheduler *Scheduler) ScheduleJob(cfg *model.Config, pendingJobs bool, lastSuccessfulJob *model.Job) (*model.Job, *model.AppError) {
+	data := map[string]string{}
+
+	if job, err := scheduler.App.Srv().Jobs.CreateJob(model.JOB_TYPE_ACTIVE_USERS, data); err != nil {
+		return nil, err
+	} else {
+		return job, nil
+	}
+}

jobs/active_users/worker.go

@@ -0,0 +1,117 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package active_users
+
+import (
+	"github.com/mattermost/mattermost-server/v5/app"
+	"github.com/mattermost/mattermost-server/v5/jobs"
+	tjobs "github.com/mattermost/mattermost-server/v5/jobs/interfaces"
+	"github.com/mattermost/mattermost-server/v5/mlog"
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+const (
+	JobName = "ActiveUsers"
+)
+
+type Worker struct {
+	name      string
+	stop      chan bool
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	app       *app.App
+}
+
+func init() {
+	app.RegisterJobsActiveUsersInterface(func(a *app.App) tjobs.ActiveUsersJobInterface {
+		return &ActiveUsersJobInterfaceImpl{a}
+	})
+}
+
+type ActiveUsersJobInterfaceImpl struct {
+	App *app.App
+}
+
+func (m *ActiveUsersJobInterfaceImpl) MakeWorker() model.Worker {
+	worker := Worker{
+		name:      JobName,
+		stop:      make(chan bool, 1),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: m.App.Srv().Jobs,
+		app:       m.App,
+	}
+	return &worker
+}
+
+func (worker *Worker) Run() {
+	mlog.Debug("Worker started", mlog.String("worker", worker.name))
+
+	defer func() {
+		mlog.Debug("Worker finished", mlog.String("worker", worker.name))
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
+			return
+		case job := <-worker.jobs:
+			mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *Worker) Stop() {
+	mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
+	worker.stop <- true
+	<-worker.stopped
+}
+
+func (worker *Worker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *Worker) DoJob(job *model.Job) {
+	if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
+		mlog.Warn("Worker experienced an error while trying to claim job",
+			mlog.String("worker", worker.name),
+			mlog.String("job_id", job.Id),
+			mlog.String("error", err.Error()))
+		return
+	} else if !claimed {
+		return
+	}
+
+	count, err := worker.app.Srv().Store.User().Count(model.UserCountOptions{IncludeDeleted: false})
+
+	if err != nil {
+		mlog.Error("Worker: Failed to get active user count", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+		worker.setJobError(job, err)
+		return
+	}
+
+	if worker.app.Metrics() != nil {
+		worker.app.Metrics().ObserveEnabledUsers(count)
+	}
+
+	mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+	worker.setJobSuccess(job)
+}
+
+func (worker *Worker) setJobSuccess(job *model.Job) {
+	if err := worker.app.Srv().Jobs.SetJobSuccess(job); err != nil {
+		mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+		worker.setJobError(job, err)
+	}
+}
+
+func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
+	if err := worker.app.Srv().Jobs.SetJobError(job, appError); err != nil {
+		mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+	}
+}

jobs/interfaces/active_users_interface.go

@@ -0,0 +1,11 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package interfaces
+
+import "github.com/mattermost/mattermost-server/v5/model"
+
+type ActiveUsersJobInterface interface {
+	MakeWorker() model.Worker
+	MakeScheduler() model.Scheduler
+}

jobs/jobs_watcher.go

@@ -135,6 +135,13 @@ func (watcher *Watcher) PollAndNotify() {
 				default:
 				}
 			}
+		} else if job.Type == model.JOB_TYPE_ACTIVE_USERS {
+			if watcher.workers.ActiveUsers != nil {
+				select {
+				case watcher.workers.ActiveUsers.JobChannel() <- *job:
+				default:
+				}
+			}
 		}
 	}
 }

jobs/schedulers.go

@@ -66,6 +66,9 @@ func (srv *JobServer) InitSchedulers() *Schedulers {
 		schedulers.schedulers = append(schedulers.schedulers, expiryNotifyInterface.MakeScheduler())
 	}
 
+	if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil {
+		schedulers.schedulers = append(schedulers.schedulers, activeUsersInterface.MakeScheduler())
+	}
 	schedulers.nextRunTimes = make([]*time.Time, len(schedulers.schedulers))
 	return schedulers
 }

jobs/server.go

@@ -26,6 +26,7 @@ type JobServer struct {
 	Plugins                 tjobs.PluginsJobInterface
 	BleveIndexer            tjobs.IndexerJobInterface
 	ExpiryNotify            tjobs.ExpiryNotifyJobInterface
+	ActiveUsers             tjobs.ActiveUsersJobInterface
 }
 
 func NewJobServer(configService configservice.ConfigService, store store.Store) *JobServer {

jobs/workers.go

@@ -25,6 +25,7 @@ type Workers struct {
 	Plugins                  model.Worker
 	BleveIndexing            model.Worker
 	ExpiryNotify             model.Worker
+	ActiveUsers              model.Worker
 
 	listenerId string
 }
@@ -70,6 +71,10 @@ func (srv *JobServer) InitWorkers() *Workers {
 	if expiryNotifyInterface := srv.ExpiryNotify; expiryNotifyInterface != nil {
 		workers.ExpiryNotify = expiryNotifyInterface.MakeWorker()
 	}
+
+	if activeUsersInterface := srv.ActiveUsers; activeUsersInterface != nil {
+		workers.ActiveUsers = activeUsersInterface.MakeWorker()
+	}
 	return workers
 }
 
@@ -113,6 +118,10 @@ func (workers *Workers) Start() *Workers {
 			go workers.ExpiryNotify.Run()
 		}
 
+		if workers.ActiveUsers != nil {
+			go workers.ActiveUsers.Run()
+		}
+
 		go workers.Watcher.Start()
 	})
 
@@ -214,6 +223,9 @@ func (workers *Workers) Stop() *Workers {
 		workers.ExpiryNotify.Stop()
 	}
 
+	if workers.ActiveUsers != nil {
+		workers.ActiveUsers.Stop()
+	}
 	mlog.Info("Stopped workers")
 
 	return workers

model/job.go

@@ -20,6 +20,7 @@ const (
 	JOB_TYPE_MIGRATIONS                     = "migrations"
 	JOB_TYPE_PLUGINS                        = "plugins"
 	JOB_TYPE_EXPIRY_NOTIFY                  = "expiry_notify"
+	JOB_TYPE_ACTIVE_USERS                   = "active_users"
 
 	JOB_STATUS_PENDING          = "pending"
 	JOB_STATUS_IN_PROGRESS      = "in_progress"
@@ -61,6 +62,7 @@ func (j *Job) IsValid() *AppError {
 	case JOB_TYPE_MIGRATIONS:
 	case JOB_TYPE_PLUGINS:
 	case JOB_TYPE_EXPIRY_NOTIFY:
+	case JOB_TYPE_ACTIVE_USERS:
 	default:
 		return NewAppError("Job.IsValid", "model.job.is_valid.type.app_error", nil, "id="+j.Id, http.StatusBadRequest)
 	}