[MM-63557] mmctl: Add compliance export create cmd (#30594)
* Refactor job retrieval to support multiple statuses & multiple types - Updated job retrieval functions to handle multiple job statuses. - Renamed `GetJobsByTypeAndStatus` to `GetJobsByTypesAndStatuses` for consistency across the codebase. - Adjusted related function signatures and implementations in the job store and retry layer to accommodate the new method. - Updated tests to reflect changes in job retrieval logic and ensure proper functionality. * Add compliance export create command and tests - Introduced `ComplianceExportCreateCmd` to facilitate the creation of compliance export jobs with options for date, start, and end timestamps. - Added unit tests for the new command, covering various scenarios including valid and invalid inputs. - Updated documentation to include usage examples and options for the new command. - Enhanced existing tests to ensure proper functionality of compliance export job handling. * update docs * update tests for new logic * Refactor message export job tests to use DefaultPreviousJobPageSize - Updated all test cases in worker_test.go to replace hardcoded page size of 100 with DefaultPreviousJobPageSize for consistency. - Adjusted the worker.go file to define DefaultPreviousJobPageSize and use it in job retrieval logic. - Ensured that the changes maintain the functionality of job data initialization and retrieval tests. * PR comments * PR comments, simplifications, clarifications, formatting * prefer hypen over underscore in command names * merge conflict * update mmctl docs
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
b33a7e362f
Коммит
9b1e03a6b8
@@ -22,7 +22,10 @@ import (
|
||||
"github.com/mattermost/mattermost/server/v8/platform/shared/templates"
|
||||
)
|
||||
|
||||
const TimeBetweenBatchesMs = 100
|
||||
const (
|
||||
TimeBetweenBatchesMs = 100
|
||||
DefaultPreviousJobPageSize = 5
|
||||
)
|
||||
|
||||
// testEndOfBatchCb is only used for testing
|
||||
var testEndOfBatchCb func(worker *MessageExportWorker)
|
||||
@@ -158,8 +161,9 @@ func (w *MessageExportWorker) DoJob(job *model.Job) {
|
||||
go w.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
|
||||
defer cancelCancelWatcher()
|
||||
|
||||
rctx := request.EmptyContext(logger).WithContext(w.context)
|
||||
// if job data is missing, we'll do our best to recover
|
||||
w.initJobData(logger, job, time.Now())
|
||||
w.initJobData(rctx, logger, job, time.Now())
|
||||
data, err := extractJobData(logger, job.Data)
|
||||
if err != nil {
|
||||
// Error in conversion. Not much we can do about that. But it shouldn't happen, unless someone edited the db.
|
||||
@@ -167,7 +171,6 @@ func (w *MessageExportWorker) DoJob(job *model.Job) {
|
||||
return
|
||||
}
|
||||
|
||||
rctx := request.EmptyContext(logger).WithContext(w.context)
|
||||
reportProgress := func(message string) {
|
||||
logger.Debug(message)
|
||||
// Don't fail because we couldn't update progress.
|
||||
@@ -260,7 +263,7 @@ func (w *MessageExportWorker) finishExport(rctx request.CTX, logger *mlog.Logger
|
||||
}
|
||||
|
||||
// initializes job data if it's missing, allows us to recover from failed or improperly configured jobs
|
||||
func (w *MessageExportWorker) initJobData(logger mlog.LoggerIFace, job *model.Job, now time.Time) {
|
||||
func (w *MessageExportWorker) initJobData(rctx request.CTX, logger mlog.LoggerIFace, job *model.Job, now time.Time) {
|
||||
if job.Data == nil {
|
||||
job.Data = make(map[string]string)
|
||||
}
|
||||
@@ -307,7 +310,8 @@ func (w *MessageExportWorker) initJobData(logger mlog.LoggerIFace, job *model.Jo
|
||||
}
|
||||
|
||||
if _, exists := job.Data[shared.JobDataBatchStartTime]; !exists {
|
||||
previousJob, err := w.jobServer.Store.Job().GetNewestJobByStatusesAndType([]string{model.JobStatusWarning, model.JobStatusSuccess}, model.JobTypeMessageExport)
|
||||
previousJob, err := w.getPreviousNonCliJob(rctx)
|
||||
|
||||
if err != nil {
|
||||
exportFromTimestamp := strconv.FormatInt(*w.jobServer.Config().MessageExportSettings.ExportFromTimestamp, 10)
|
||||
logger.Info("Worker: No previously successful job found, falling back to configured MessageExportSettings.ExportFromTimestamp", mlog.String("export_from_timestamp", exportFromTimestamp))
|
||||
@@ -362,6 +366,36 @@ func (w *MessageExportWorker) initJobData(logger mlog.LoggerIFace, job *model.Jo
|
||||
job.Data[shared.JobDataExportDir] = getJobExportDir(logger, job.Data, job.Data[shared.JobDataJobStartTime], job.Data[shared.JobDataJobEndTime])
|
||||
}
|
||||
|
||||
// getPreviousNonCliJob returns the most recent job that was not initiated by mmctl
|
||||
func (w *MessageExportWorker) getPreviousNonCliJob(rctx request.CTX) (*model.Job, error) {
|
||||
offset := 0
|
||||
|
||||
for {
|
||||
jobs, err := w.jobServer.Store.Job().GetAllByTypesAndStatusesPage(rctx,
|
||||
[]string{model.JobTypeMessageExport},
|
||||
[]string{model.JobStatusWarning, model.JobStatusSuccess},
|
||||
offset, DefaultPreviousJobPageSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Find the first job not initiated by mmctl
|
||||
for _, job := range jobs {
|
||||
if job.Data == nil || job.Data[shared.JobDataInitiatedBy] != "mmctl" {
|
||||
return job, nil
|
||||
}
|
||||
}
|
||||
|
||||
// If we didn't get a full page of jobs, we've reached the end
|
||||
if len(jobs) < DefaultPreviousJobPageSize {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// If we didn't find a non-mmctl job in this page, continue to the next page
|
||||
offset += DefaultPreviousJobPageSize
|
||||
}
|
||||
}
|
||||
|
||||
func extractJobData(logger *mlog.Logger, strmap map[string]string) (shared.JobData, error) {
|
||||
data, err := shared.StringMapToJobDataWithZeroValues(strmap)
|
||||
if err != nil {
|
||||
|
||||
Ссылка в новой задаче
Block a user