[MM-63557] mmctl: Add compliance export create cmd (#30594)

* Refactor job retrieval to support multiple statuses & multiple types - Updated job retrieval functions to handle multiple job statuses. - Renamed `GetJobsByTypeAndStatus` to `GetJobsByTypesAndStatuses` for consistency across the codebase. - Adjusted related function signatures and implementations in the job store and retry layer to accommodate the new method. - Updated tests to reflect changes in job retrieval logic and ensure proper functionality. * Add compliance export create command and tests - Introduced `ComplianceExportCreateCmd` to facilitate the creation of compliance export jobs with options for date, start, and end timestamps. - Added unit tests for the new command, covering various scenarios including valid and invalid inputs. - Updated documentation to include usage examples and options for the new command. - Enhanced existing tests to ensure proper functionality of compliance export job handling. * update docs * update tests for new logic * Refactor message export job tests to use DefaultPreviousJobPageSize - Updated all test cases in worker_test.go to replace hardcoded page size of 100 with DefaultPreviousJobPageSize for consistency. - Adjusted the worker.go file to define DefaultPreviousJobPageSize and use it in job retrieval logic. - Ensured that the changes maintain the functionality of job data initialization and retrieval tests. * PR comments * PR comments, simplifications, clarifications, formatting * prefer hypen over underscore in command names * merge conflict * update mmctl docs
2025-06-24 17:38:30 -04:00
--- a/server/channels/api4/job.go
+++ b/server/channels/api4/job.go
@@ -234,7 +234,7 @@ func getJobs(c *Context, w http.ResponseWriter, r *http.Request) {
 	if status == "" {
 		jobs, appErr = c.App.GetJobsByTypesPage(c.AppContext, validJobTypes, c.Params.Page, c.Params.PerPage)
 	} else {
-		jobs, appErr = c.App.GetJobsByTypeAndStatus(c.AppContext, validJobTypes, status, c.Params.Page, c.Params.PerPage)
+		jobs, appErr = c.App.GetJobsByTypesAndStatuses(c.AppContext, validJobTypes, []string{status}, c.Params.Page, c.Params.PerPage)
 	}

 	if appErr != nil {
--- a/server/channels/app/job.go
+++ b/server/channels/app/job.go
@@ -43,10 +43,10 @@ func (a *App) GetJobsByTypesPage(c request.CTX, jobType []string, page int, perP
 	return jobs, nil
 }

-func (a *App) GetJobsByTypeAndStatus(c request.CTX, jobTypes []string, status string, page int, perPage int) ([]*model.Job, *model.AppError) {
-	jobs, err := a.Srv().Store().Job().GetAllByTypeAndStatusPage(c, jobTypes, status, page, perPage)
+func (a *App) GetJobsByTypesAndStatuses(c request.CTX, jobTypes []string, status []string, page int, perPage int) ([]*model.Job, *model.AppError) {
+	jobs, err := a.Srv().Store().Job().GetAllByTypesAndStatusesPage(c, jobTypes, status, page*perPage, perPage)
 	if err != nil {
-		return nil, model.NewAppError("GetAllByTypeAndStatusPage", "app.job.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		return nil, model.NewAppError("GetAllByTypesAndStatusesPage", "app.job.get_all.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
 	}
 	return jobs, nil
 }
--- a/server/channels/store/retrylayer/retrylayer.go
+++ b/server/channels/store/retrylayer/retrylayer.go
@@ -6404,11 +6404,11 @@ func (s *RetryLayerJobStore) GetAllByTypeAndStatus(c request.CTX, jobType string

 }

-func (s *RetryLayerJobStore) GetAllByTypeAndStatusPage(c request.CTX, jobType []string, status string, offset int, limit int) ([]*model.Job, error) {
+func (s *RetryLayerJobStore) GetAllByTypePage(c request.CTX, jobType string, offset int, limit int) ([]*model.Job, error) {

 	tries := 0
 	for {
-		result, err := s.JobStore.GetAllByTypeAndStatusPage(c, jobType, status, offset, limit)
+		result, err := s.JobStore.GetAllByTypePage(c, jobType, offset, limit)
 		if err == nil {
 			return result, nil
 		}
@@ -6425,11 +6425,11 @@ func (s *RetryLayerJobStore) GetAllByTypeAndStatusPage(c request.CTX, jobType []

 }

-func (s *RetryLayerJobStore) GetAllByTypePage(c request.CTX, jobType string, offset int, limit int) ([]*model.Job, error) {
+func (s *RetryLayerJobStore) GetAllByTypesAndStatusesPage(c request.CTX, jobType []string, status []string, offset int, limit int) ([]*model.Job, error) {

 	tries := 0
 	for {
-		result, err := s.JobStore.GetAllByTypePage(c, jobType, offset, limit)
+		result, err := s.JobStore.GetAllByTypesAndStatusesPage(c, jobType, status, offset, limit)
 		if err == nil {
 			return result, nil
 		}
--- a/server/channels/store/sqlstore/job_store.go
+++ b/server/channels/store/sqlstore/job_store.go
@@ -378,12 +378,11 @@ func (jss SqlJobStore) GetAllByStatus(c request.CTX, status string) ([]*model.Jo
 	return statuses, nil
 }

-func (jss SqlJobStore) GetAllByTypeAndStatusPage(c request.CTX, jobType []string, status string, page int, perPage int) ([]*model.Job, error) {
-	offset := page * perPage
+func (jss SqlJobStore) GetAllByTypesAndStatusesPage(c request.CTX, jobType []string, status []string, offset int, limit int) ([]*model.Job, error) {
 	query, args, err := jss.jobQuery.
 		Where(sq.Eq{"Type": jobType, "Status": status}).
 		OrderBy("CreateAt DESC").
-		Limit(uint64(perPage)).
+		Limit(uint64(limit)).
 		Offset(uint64(offset)).ToSql()
 	if err != nil {
 		return nil, errors.Wrap(err, "job_tosql")
@@ -391,7 +390,7 @@ func (jss SqlJobStore) GetAllByTypeAndStatusPage(c request.CTX, jobType []string

 	jobs := []*model.Job{}
 	if err = jss.GetReplica().Select(&jobs, query, args...); err != nil {
-		return nil, errors.Wrapf(err, "failed to find Jobs with type=%s and status=%s", strings.Join(jobType, ","), status)
+		return nil, errors.Wrapf(err, "failed to find Jobs with types=%s and statuses=%s", strings.Join(jobType, ","), strings.Join(status, ","))
 	}

 	return jobs, nil
--- a/server/channels/store/store.go
+++ b/server/channels/store/store.go
@@ -796,7 +796,7 @@ type JobStore interface {
 	GetAllByTypePage(c request.CTX, jobType string, offset int, limit int) ([]*model.Job, error)
 	GetAllByTypesPage(c request.CTX, jobTypes []string, offset int, limit int) ([]*model.Job, error)
 	GetAllByStatus(c request.CTX, status string) ([]*model.Job, error)
-	GetAllByTypeAndStatusPage(c request.CTX, jobType []string, status string, offset int, limit int) ([]*model.Job, error)
+	GetAllByTypesAndStatusesPage(c request.CTX, jobType []string, status []string, offset int, limit int) ([]*model.Job, error)
 	GetNewestJobByStatusAndType(status string, jobType string) (*model.Job, error)
 	GetNewestJobByStatusesAndType(statuses []string, jobType string) (*model.Job, error)
 	GetCountByStatusAndType(status string, jobType string) (int64, error)
--- a/server/channels/store/storetest/job_store.go
+++ b/server/channels/store/storetest/job_store.go
@@ -25,6 +25,7 @@ func TestJobStore(t *testing.T, rctx request.CTX, ss store.Store) {
 	t.Run("JobGetAllByTypePage", func(t *testing.T) { testJobGetAllByTypePage(t, rctx, ss) })
 	t.Run("JobGetAllByTypesPage", func(t *testing.T) { testJobGetAllByTypesPage(t, rctx, ss) })
 	t.Run("JobGetAllByTypeAndStatusPage", func(t *testing.T) { testJobGetAllByTypeAndStatusPage(t, rctx, ss) })
+	t.Run("JobGetAllByTypesAndStatusesPage", func(t *testing.T) { testJobGetAllByTypesAndStatusesPage(t, rctx, ss) })
 	t.Run("JobGetAllByStatus", func(t *testing.T) { testJobGetAllByStatus(t, rctx, ss) })
 	t.Run("GetNewestJobByStatusAndType", func(t *testing.T) { testJobStoreGetNewestJobByStatusAndType(t, rctx, ss) })
 	t.Run("GetNewestJobByStatusesAndType", func(t *testing.T) { testJobStoreGetNewestJobByStatusesAndType(t, rctx, ss) })
@@ -299,23 +300,133 @@ func testJobGetAllByTypeAndStatusPage(t *testing.T, rctx request.CTX, ss store.S
 	}

 	jobTypes := []string{jobType, jobType2}
-	received, err := ss.Job().GetAllByTypeAndStatusPage(rctx, jobTypes, model.JobStatusPending, 0, 4)
+	received, err := ss.Job().GetAllByTypesAndStatusesPage(rctx, jobTypes, []string{model.JobStatusPending}, 0, 4)
 	require.NoError(t, err)
 	require.Len(t, received, 2)
 	require.Equal(t, received[0].Id, jobs[1].Id, "should've received newest job first")
 	require.Equal(t, received[1].Id, jobs[0].Id, "should've received oldest job last")

-	received, err = ss.Job().GetAllByTypeAndStatusPage(rctx, jobTypes, model.JobStatusPending, 1, 1)
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, jobTypes, []string{model.JobStatusPending}, 1, 1)
 	require.NoError(t, err)
 	require.Len(t, received, 1)
 	require.Equal(t, received[0].Id, jobs[0].Id, "should've received the oldest pending job")

-	received, err = ss.Job().GetAllByTypeAndStatusPage(rctx, []string{jobType2}, model.JobStatusCanceled, 1, 1)
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, []string{jobType2}, []string{model.JobStatusCanceled}, 1, 1)
 	require.NoError(t, err)
 	require.Len(t, received, 1)
 	require.Equal(t, received[0].Id, jobs[2].Id, "should've received the oldest canceled job")
 }

+func testJobGetAllByTypesAndStatusesPage(t *testing.T, rctx request.CTX, ss store.Store) {
+	jobType1 := model.NewId()
+	jobType2 := model.NewId()
+	jobType3 := model.NewId()
+	status1 := model.JobStatusPending
+	status2 := model.JobStatusInProgress
+	status3 := model.JobStatusSuccess
+	t0 := model.GetMillis()
+
+	jobs := []*model.Job{
+		{
+			Id:       model.NewId(), // 0: type1, status1, t0
+			Type:     jobType1,
+			Status:   status1,
+			CreateAt: t0,
+		},
+		{
+			Id:       model.NewId(), // 1: type1, status2, t0+1
+			Type:     jobType1,
+			Status:   status2,
+			CreateAt: t0 + 1,
+		},
+		{
+			Id:       model.NewId(), // 2: type2, status1, t0+2
+			Type:     jobType2,
+			Status:   status1,
+			CreateAt: t0 + 2,
+		},
+		{
+			Id:       model.NewId(), // 3: type2, status2, t0+3
+			Type:     jobType2,
+			Status:   status2,
+			CreateAt: t0 + 3,
+		},
+		{
+			Id:       model.NewId(), // 4: type1, status3, t0+4
+			Type:     jobType1,
+			Status:   status3,
+			CreateAt: t0 + 4,
+		},
+		{
+			Id:       model.NewId(), // 5: type3, status1, t0+5
+			Type:     jobType3,
+			Status:   status1,
+			CreateAt: t0 + 5,
+		},
+	}
+
+	for _, job := range jobs {
+		_, err := ss.Job().Save(job)
+		require.NoError(t, err)
+		defer ss.Job().Delete(job.Id)
+	}
+
+	// Test case 1: Get jobs of type1 or type2 with status1 or status2, limit 4, offset 0
+	types1 := []string{jobType1, jobType2}
+	statuses1 := []string{status1, status2}
+	received, err := ss.Job().GetAllByTypesAndStatusesPage(rctx, types1, statuses1, 0, 4)
+	require.NoError(t, err)
+	require.Len(t, received, 4)
+	require.Equal(t, jobs[3].Id, received[0].Id, "case 1: newest job type2/status2")
+	require.Equal(t, jobs[2].Id, received[1].Id, "case 1: second newest job type2/status1")
+	require.Equal(t, jobs[1].Id, received[2].Id, "case 1: third newest job type1/status2")
+	require.Equal(t, jobs[0].Id, received[3].Id, "case 1: oldest job type1/status1")
+
+	// Test case 2: Get jobs of type1 or type2 with status1 or status2, limit 2, offset 2
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, types1, statuses1, 2, 2)
+	require.NoError(t, err)
+	require.Len(t, received, 2)
+	require.Equal(t, jobs[1].Id, received[0].Id, "case 2: third newest job type1/status2")
+	require.Equal(t, jobs[0].Id, received[1].Id, "case 2: oldest job type1/status1")
+
+	// Test case 3: Get jobs of type1 with status1 or status3, limit 5, offset 0
+	types2 := []string{jobType1}
+	statuses2 := []string{status1, status3}
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, types2, statuses2, 0, 5)
+	require.NoError(t, err)
+	require.Len(t, received, 2)
+	require.Equal(t, jobs[4].Id, received[0].Id, "case 3: newest job type1/status3")
+	require.Equal(t, jobs[0].Id, received[1].Id, "case 3: oldest job type1/status1")
+
+	// Test case 4: Get jobs of type3 with status1, limit 1, offset 0
+	types3 := []string{jobType3}
+	statuses3 := []string{status1}
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, types3, statuses3, 0, 1)
+	require.NoError(t, err)
+	require.Len(t, received, 1)
+	require.Equal(t, jobs[5].Id, received[0].Id, "case 4: only job type3/status1")
+
+	// Test case 5: Get jobs with non-existent type
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, []string{model.NewId()}, statuses1, 0, 5)
+	require.NoError(t, err)
+	require.Len(t, received, 0, "case 5: no jobs with non-existent type")
+
+	// Test case 6: Get jobs with non-existent status
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, types1, []string{model.NewId()}, 0, 5)
+	require.NoError(t, err)
+	require.Len(t, received, 0, "case 6: no jobs with non-existent status")
+
+	// Test case 7: Empty types slice
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, []string{}, statuses1, 0, 5)
+	require.NoError(t, err)
+	require.Len(t, received, 0, "case 7: empty types slice should return no jobs")
+
+	// Test case 8: Empty statuses slice
+	received, err = ss.Job().GetAllByTypesAndStatusesPage(rctx, types1, []string{}, 0, 5)
+	require.NoError(t, err)
+	require.Len(t, received, 0, "case 8: empty statuses slice should return no jobs")
+}
+
 func testJobGetAllByStatus(t *testing.T, rctx request.CTX, ss store.Store) {
 	jobType := model.NewId()
 	status := model.NewId()
--- a/server/channels/store/storetest/mocks/JobStore.go
+++ b/server/channels/store/storetest/mocks/JobStore.go
@@ -181,36 +181,6 @@ func (_m *JobStore) GetAllByTypeAndStatus(c request.CTX, jobType string, status
 	return r0, r1
 }

-// GetAllByTypeAndStatusPage provides a mock function with given fields: c, jobType, status, offset, limit
-func (_m *JobStore) GetAllByTypeAndStatusPage(c request.CTX, jobType []string, status string, offset int, limit int) ([]*model.Job, error) {
-	ret := _m.Called(c, jobType, status, offset, limit)
-
-	if len(ret) == 0 {
-		panic("no return value specified for GetAllByTypeAndStatusPage")
-	}
-
-	var r0 []*model.Job
-	var r1 error
-	if rf, ok := ret.Get(0).(func(request.CTX, []string, string, int, int) ([]*model.Job, error)); ok {
-		return rf(c, jobType, status, offset, limit)
-	}
-	if rf, ok := ret.Get(0).(func(request.CTX, []string, string, int, int) []*model.Job); ok {
-		r0 = rf(c, jobType, status, offset, limit)
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]*model.Job)
-		}
-	}
-
-	if rf, ok := ret.Get(1).(func(request.CTX, []string, string, int, int) error); ok {
-		r1 = rf(c, jobType, status, offset, limit)
-	} else {
-		r1 = ret.Error(1)
-	}
-
-	return r0, r1
-}
-
 // GetAllByTypePage provides a mock function with given fields: c, jobType, offset, limit
 func (_m *JobStore) GetAllByTypePage(c request.CTX, jobType string, offset int, limit int) ([]*model.Job, error) {
 	ret := _m.Called(c, jobType, offset, limit)
@@ -241,6 +211,36 @@ func (_m *JobStore) GetAllByTypePage(c request.CTX, jobType string, offset int,
 	return r0, r1
 }

+// GetAllByTypesAndStatusesPage provides a mock function with given fields: c, jobType, status, offset, limit
+func (_m *JobStore) GetAllByTypesAndStatusesPage(c request.CTX, jobType []string, status []string, offset int, limit int) ([]*model.Job, error) {
+	ret := _m.Called(c, jobType, status, offset, limit)
+
+	if len(ret) == 0 {
+		panic("no return value specified for GetAllByTypesAndStatusesPage")
+	}
+
+	var r0 []*model.Job
+	var r1 error
+	if rf, ok := ret.Get(0).(func(request.CTX, []string, []string, int, int) ([]*model.Job, error)); ok {
+		return rf(c, jobType, status, offset, limit)
+	}
+	if rf, ok := ret.Get(0).(func(request.CTX, []string, []string, int, int) []*model.Job); ok {
+		r0 = rf(c, jobType, status, offset, limit)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.Job)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(request.CTX, []string, []string, int, int) error); ok {
+		r1 = rf(c, jobType, status, offset, limit)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // GetAllByTypesPage provides a mock function with given fields: c, jobTypes, offset, limit
 func (_m *JobStore) GetAllByTypesPage(c request.CTX, jobTypes []string, offset int, limit int) ([]*model.Job, error) {
 	ret := _m.Called(c, jobTypes, offset, limit)
--- a/server/channels/store/timerlayer/timerlayer.go
+++ b/server/channels/store/timerlayer/timerlayer.go
@@ -5127,22 +5127,6 @@ func (s *TimerLayerJobStore) GetAllByTypeAndStatus(c request.CTX, jobType string
 	return result, err
 }

-func (s *TimerLayerJobStore) GetAllByTypeAndStatusPage(c request.CTX, jobType []string, status string, offset int, limit int) ([]*model.Job, error) {
-	start := time.Now()
-
-	result, err := s.JobStore.GetAllByTypeAndStatusPage(c, jobType, status, offset, limit)
-
-	elapsed := float64(time.Since(start)) / float64(time.Second)
-	if s.Root.Metrics != nil {
-		success := "false"
-		if err == nil {
-			success = "true"
-		}
-		s.Root.Metrics.ObserveStoreMethodDuration("JobStore.GetAllByTypeAndStatusPage", success, elapsed)
-	}
-	return result, err
-}
-
 func (s *TimerLayerJobStore) GetAllByTypePage(c request.CTX, jobType string, offset int, limit int) ([]*model.Job, error) {
 	start := time.Now()

@@ -5159,6 +5143,22 @@ func (s *TimerLayerJobStore) GetAllByTypePage(c request.CTX, jobType string, off
 	return result, err
 }

+func (s *TimerLayerJobStore) GetAllByTypesAndStatusesPage(c request.CTX, jobType []string, status []string, offset int, limit int) ([]*model.Job, error) {
+	start := time.Now()
+
+	result, err := s.JobStore.GetAllByTypesAndStatusesPage(c, jobType, status, offset, limit)
+
+	elapsed := float64(time.Since(start)) / float64(time.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("JobStore.GetAllByTypesAndStatusesPage", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerJobStore) GetAllByTypesPage(c request.CTX, jobTypes []string, offset int, limit int) ([]*model.Job, error) {
 	start := time.Now()