MM-47736: Add jitter to job scheduler run time (#21516)

There can be a case where if a config change happened
at the same time for a large number of installations,
it can trigger a job to re-run all at the same time,
therefore causing a thundering herd issue.

To prevent this, we add a jitter.

https://mattermost.atlassian.net/browse/MM-47736

```release-note
NONE
```

jobs/base_schedulers.go

@@ -4,6 +4,8 @@
 package jobs
 
 import (
+	"crypto/rand"
+	"math/big"
 	"time"
 
 	"github.com/mattermost/mattermost-server/v6/model"
@@ -30,7 +32,7 @@ func (scheduler *PeriodicScheduler) Enabled(cfg *model.Config) bool {
 }
 
 func (scheduler *PeriodicScheduler) NextScheduleTime(_ *model.Config, _ time.Time /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) *time.Time {
-	nextTime := time.Now().Add(scheduler.period)
+	nextTime := time.Now().Add(getRandomDelay(jitterRange)).Add(scheduler.period)
 	return &nextTime
 }
 
@@ -70,3 +72,13 @@ func (scheduler *DailyScheduler) NextScheduleTime(cfg *model.Config, now time.Ti
 func (scheduler *DailyScheduler) ScheduleJob(_ *model.Config /* pendingJobs */, _ bool /* lastSuccessfulJob */, _ *model.Job) (*model.Job, *model.AppError) {
 	return scheduler.jobs.CreateJob(scheduler.jobType, nil)
 }
+
+const jitterRange = 2000 // milliseconds
+
+func getRandomDelay(limit int64) time.Duration {
+	num, err := rand.Int(rand.Reader, big.NewInt(limit))
+	if err != nil {
+		return time.Millisecond
+	}
+	return time.Millisecond * time.Duration(num.Int64())
+}

jobs/schedulers_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v6/model"
 	"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
@@ -143,3 +144,11 @@ func TestScheduler(t *testing.T) {
 		wg.Wait()
 	})
 }
+
+func TestRandomDelay(t *testing.T) {
+	cases := []int64{5, 10, 100}
+	for _, c := range cases {
+		out := getRandomDelay(c)
+		require.Less(t, out.Milliseconds(), c)
+	}
+}