[MM-55726] Create batch report worker, add batch report job for exporting users to CSV (#25832)

* Split out migration logic and create generic BatchWorker

* WIP

* WIP

* POC batch reporting

* Oops

* Job hookup

* Working export to file

* PR feedback

* Merge'd

* Fix error handling

* Add API to start report, translations, couple fixes

* Add DMs to send reports to users

* Merge'd

* Update types

* A bit of cleanup

* Some fixes

* Add missing API doc

* PR feedback

* Fix generated

* Fix bug with post creation

* PR feedback

* Add some tests

* PR feedback

* Fix lint

* Some test changes

* Fix tests

* Add comment to explain why we forcibly stop

* Rework of some tests

* Batch report test

* Restrict batch exports to Pro and Enterprise licenses

* Fix erroneous comment

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

server/channels/jobs/batch_worker_test.go

@@ -0,0 +1,147 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package jobs_test
+
+import (
+	"strconv"
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBatchWorker(t *testing.T) {
+	createBatchWorker := func(t *testing.T, th *TestHelper, doBatch func(rctx *request.Context, job *model.Job) bool) (*jobs.BatchWorker, *model.Job) {
+		t.Helper()
+
+		worker := jobs.MakeBatchWorker(th.Server.Jobs, th.Server.Store(), 1*time.Second, doBatch)
+		job := th.SetupBatchWorker(t, worker)
+		return worker, job
+	}
+
+	getBatchNumberFromData := func(t *testing.T, data model.StringMap) int {
+		t.Helper()
+
+		batchNumber, err := strconv.Atoi(data["batch_number"])
+		require.NoError(t, err)
+
+		return batchNumber
+	}
+
+	incrementBatchNumber := func(t *testing.T, th *TestHelper, job *model.Job) {
+		t.Helper()
+
+		batchNumber, err := strconv.Atoi(job.Data["batch_number"])
+		require.NoError(t, err)
+
+		batchNumber++
+		job.Data["batch_number"] = strconv.Itoa(batchNumber)
+		th.Server.Jobs.SetJobProgress(job, 0)
+	}
+
+	t.Run("stop after first batch", func(t *testing.T) {
+		th := Setup(t).InitBasic()
+		defer th.TearDown()
+
+		var worker *jobs.BatchWorker
+		worker, job := createBatchWorker(t, th, func(rctx *request.Context, job *model.Job) bool {
+			batchNumber := getBatchNumberFromData(t, job.Data)
+
+			require.Equal(t, 1, batchNumber, "only batch 1 should have run")
+
+			// Shut down the worker after the first batch to prevent subsequent ones.
+			if batchNumber >= 1 {
+				go worker.Stop()
+			} else {
+				incrementBatchNumber(t, th, job)
+			}
+
+			return false
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusPending)
+		th.WaitForBatchNumber(t, job, 1)
+	})
+
+	t.Run("stop after second batch", func(t *testing.T) {
+		th := Setup(t).InitBasic()
+		defer th.TearDown()
+
+		var worker *jobs.BatchWorker
+		worker, job := createBatchWorker(t, th, func(rctx *request.Context, job *model.Job) bool {
+			batchNumber := getBatchNumberFromData(t, job.Data)
+
+			require.LessOrEqual(t, batchNumber, 2, "only batches 1 and 2 should have run")
+
+			// Shut down the worker after the second batch to prevent subsequent ones.
+			if batchNumber >= 2 {
+				go worker.Stop()
+			} else {
+				incrementBatchNumber(t, th, job)
+			}
+
+			return false
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForJobStatus(t, job, model.JobStatusPending)
+		th.WaitForBatchNumber(t, job, 2)
+	})
+
+	t.Run("done after first batch", func(t *testing.T) {
+		th := Setup(t).InitBasic()
+		defer th.TearDown()
+
+		var worker *jobs.BatchWorker
+		worker, job := createBatchWorker(t, th, func(rctx *request.Context, job *model.Job) bool {
+			batchNumber := getBatchNumberFromData(t, job.Data)
+			require.Equal(t, 1, batchNumber, "only batch 1 should have run")
+
+			if batchNumber >= 1 {
+				go worker.Stop() // Shut down the worker when the job is done
+				return true
+			}
+
+			incrementBatchNumber(t, th, job)
+			return false
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForBatchNumber(t, job, 1)
+	})
+
+	t.Run("done after three batches", func(t *testing.T) {
+		th := Setup(t).InitBasic()
+		defer th.TearDown()
+
+		var worker *jobs.BatchWorker
+		worker, job := createBatchWorker(t, th, func(rctx *request.Context, job *model.Job) bool {
+			batchNumber := getBatchNumberFromData(t, job.Data)
+			require.LessOrEqual(t, batchNumber, 3, "only 3 batches should have run")
+
+			if batchNumber >= 3 {
+				go worker.Stop() // Shut down the worker when the job is done
+				return true
+			}
+
+			incrementBatchNumber(t, th, job)
+			return false
+		})
+
+		// Queue the work to be done
+		worker.JobChannel() <- *job
+
+		th.WaitForBatchNumber(t, job, 3)
+	})
+}