From 2a9b3e1a86298e9fe17185844d8d607d66467469 Mon Sep 17 00:00:00 2001
From: Agniva De Sarker <agnivade@yahoo.co.in>
Date: Thu, 27 Oct 2022 10:30:56 +0530
Subject: [PATCH] MM-47736: Add jitter to job scheduler run time (#21516)

There can be a case where if a config change happened
at the same time for a large number of installations,
it can trigger a job to re-run all at the same time,
therefore causing a thundering herd issue.

To prevent this, we add a jitter.

https://mattermost.atlassian.net/browse/MM-47736

```release-note
NONE
```
---
 jobs/base_schedulers.go | 14 +++++++++++++-
 jobs/schedulers_test.go |  9 +++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/jobs/base_schedulers.go b/jobs/base_schedulers.go
index b6416b3bbd..0de90b2d2f 100644
--- a/jobs/base_schedulers.go
+++ b/jobs/base_schedulers.go
@@ -4,6 +4,8 @@
 package jobs
 
 import (
+	"crypto/rand"
+	"math/big"
 	"time"
 
 	"github.com/mattermost/mattermost-server/v6/model"
@@ -30,7 +32,7 @@ func (scheduler *PeriodicScheduler) Enabled(cfg *model.Config) bool {
 }
 
 func (scheduler *PeriodicScheduler) NextScheduleTime(_ *model.Config, _ time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time {
-	nextTime := time.Now().Add(scheduler.period)
+	nextTime := time.Now().Add(getRandomDelay(jitterRange)).Add(scheduler.period)
 	return &nextTime
 }
 
@@ -70,3 +72,13 @@ func (scheduler *DailyScheduler) NextScheduleTime(cfg *model.Config, now time.Ti
 func (scheduler *DailyScheduler) ScheduleJob(_ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) {
 	return scheduler.jobs.CreateJob(scheduler.jobType, nil)
 }
+
+const jitterRange = 2000 // milliseconds
+
+func getRandomDelay(limit int64) time.Duration {
+	num, err := rand.Int(rand.Reader, big.NewInt(limit))
+	if err != nil {
+		return time.Millisecond
+	}
+	return time.Millisecond * time.Duration(num.Int64())
+}
diff --git a/jobs/schedulers_test.go b/jobs/schedulers_test.go
index 8421d4f0f4..d5facefa2d 100644
--- a/jobs/schedulers_test.go
+++ b/jobs/schedulers_test.go
@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v6/model"
 	"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
@@ -143,3 +144,11 @@ func TestScheduler(t *testing.T) {
 		wg.Wait()
 	})
 }
+
+func TestRandomDelay(t *testing.T) {
+	cases := []int64{5, 10, 100}
+	for _, c := range cases {
+		out := getRandomDelay(c)
+		require.Less(t, out.Milliseconds(), c)
+	}
+}