[MM-63557] mmctl: Add compliance export create cmd (#30594)
* Refactor job retrieval to support multiple statuses & multiple types - Updated job retrieval functions to handle multiple job statuses. - Renamed `GetJobsByTypeAndStatus` to `GetJobsByTypesAndStatuses` for consistency across the codebase. - Adjusted related function signatures and implementations in the job store and retry layer to accommodate the new method. - Updated tests to reflect changes in job retrieval logic and ensure proper functionality. * Add compliance export create command and tests - Introduced `ComplianceExportCreateCmd` to facilitate the creation of compliance export jobs with options for date, start, and end timestamps. - Added unit tests for the new command, covering various scenarios including valid and invalid inputs. - Updated documentation to include usage examples and options for the new command. - Enhanced existing tests to ensure proper functionality of compliance export job handling. * update docs * update tests for new logic * Refactor message export job tests to use DefaultPreviousJobPageSize - Updated all test cases in worker_test.go to replace hardcoded page size of 100 with DefaultPreviousJobPageSize for consistency. - Adjusted the worker.go file to define DefaultPreviousJobPageSize and use it in job retrieval logic. - Ensured that the changes maintain the functionality of job data initialization and retrieval tests. * PR comments * PR comments, simplifications, clarifications, formatting * prefer hypen over underscore in command names * merge conflict * update mmctl docs
Этот коммит содержится в:
коммит произвёл
GitHub
родитель
b33a7e362f
Коммит
9b1e03a6b8
@@ -5,11 +5,17 @@ package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/v8/cmd/mmctl/client"
|
||||
"github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer"
|
||||
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -44,12 +50,22 @@ var ComplianceExportCancelCmd = &cobra.Command{
|
||||
|
||||
var ComplianceExportDownloadCmd = &cobra.Command{
|
||||
Use: "download [complianceExportJobID] [output filepath (optional)]",
|
||||
Example: " compliance_export download o98rj3ur83dp5dppfyk5yk6osy",
|
||||
Example: "compliance-export download o98rj3ur83dp5dppfyk5yk6osy",
|
||||
Short: "Download compliance export file",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: withClient(complianceExportDownloadCmdF),
|
||||
}
|
||||
|
||||
var ComplianceExportCreateCmd = &cobra.Command{
|
||||
Use: "create [complianceExportType] --date \"2025-03-27 -0400\"",
|
||||
Example: "compliance-export create csv --date \"2025-03-27 -0400\"",
|
||||
Long: "Create a compliance export job, of type 'csv' or 'actiance' or 'globalrelay'. If --date is set, the job will run for one day, from 12am to 12am (minus one millisecond) inclusively, in the format with timezone offset: `\"YYYY-MM-DD -0000\"`. E.g., \"2024-10-21 -0400\" for Oct 21, 2024 EDT timezone. \"2023-11-01 +0000\" for Nov 01, 2024 UTC. If set, the 'start' and 'end' flags will be ignored.\n\n" +
|
||||
"Important: Running a compliance export job from mmctl will NOT affect the next scheduled job's batch_start_time. This means that if you run a compliance export job from mmctl, the next scheduled job will run from the batch_end_time of the previous scheduled job, as usual.",
|
||||
Short: "Create a compliance export job, of type 'csv' or 'actiance' or 'globalrelay'",
|
||||
Args: cobra.MinimumNArgs(1),
|
||||
RunE: withClient(complianceExportCreateCmdF),
|
||||
}
|
||||
|
||||
func init() {
|
||||
ComplianceExportListCmd.Flags().Int("page", 0, "Page number to fetch for the list of compliance export jobs")
|
||||
ComplianceExportListCmd.Flags().Int("per-page", DefaultPageSize, "Number of compliance export jobs to be fetched")
|
||||
@@ -57,11 +73,28 @@ func init() {
|
||||
|
||||
ComplianceExportDownloadCmd.Flags().Int("num-retries", 5, "Number of retries if the download fails")
|
||||
|
||||
ComplianceExportCreateCmd.Flags().String(
|
||||
"date",
|
||||
"",
|
||||
"Run the export for one day, from 12am to 12am (minus one millisecond) inclusively, in the format with timezone offset: `\"YYYY-MM-DD -0000\"`. E.g., `\"2024-10-21 -0400\"` for Oct 21, 2024 EDT timezone. `\"2023-11-01 +0000\"` for Nov 01, 2024 UTC. If set, the 'start' and 'end' flags will be ignored.",
|
||||
)
|
||||
ComplianceExportCreateCmd.Flags().Int(
|
||||
"start",
|
||||
0,
|
||||
"The start timestamp in unix milliseconds. Posts with updateAt >= start will be exported. If set, 'end' must be set as well. eg, `1743048000000` for 2025-03-27 EDT.",
|
||||
)
|
||||
ComplianceExportCreateCmd.Flags().Int(
|
||||
"end",
|
||||
0,
|
||||
"The end timestamp in unix milliseconds. Posts with updateAt <= end will be exported. If set, 'start' must be set as well. eg, `1743134400000` for 2025-03-28 EDT.",
|
||||
)
|
||||
|
||||
ComplianceExportCmd.AddCommand(
|
||||
ComplianceExportListCmd,
|
||||
ComplianceExportShowCmd,
|
||||
ComplianceExportCancelCmd,
|
||||
ComplianceExportDownloadCmd,
|
||||
ComplianceExportCreateCmd,
|
||||
)
|
||||
RootCmd.AddCommand(ComplianceExportCmd)
|
||||
}
|
||||
@@ -126,3 +159,85 @@ func complianceExportDownloadCmdF(c client.Client, command *cobra.Command, args
|
||||
printer.Print(fmt.Sprintf("Compliance export file downloaded to %q", path))
|
||||
return nil
|
||||
}
|
||||
|
||||
func complianceExportCreateCmdF(c client.Client, command *cobra.Command, args []string) error {
|
||||
exportType := args[0]
|
||||
if exportType != model.ComplianceExportTypeActiance &&
|
||||
exportType != model.ComplianceExportTypeCsv &&
|
||||
exportType != model.ComplianceExportTypeGlobalrelay {
|
||||
return fmt.Errorf("invalid export type: %s, must be one of: csv, actiance, globalrelay", exportType)
|
||||
}
|
||||
|
||||
dateStr, err := command.Flags().GetString("date")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
start, err := command.Flags().GetInt("start")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
end, err := command.Flags().GetInt("end")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
startTimestamp, endTimestamp, err := getStartAndEnd(dateStr, start, end)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
startTime := strconv.FormatInt(startTimestamp, 10)
|
||||
endTime := strconv.FormatInt(endTimestamp, 10)
|
||||
exportDir := path.Join(model.ComplianceExportPath, fmt.Sprintf("%s-%s-%s", time.Now().Format(model.ComplianceExportDirectoryFormat), startTime, endTime))
|
||||
|
||||
// If start and end are 0, we need to not set those keys in the job data.
|
||||
// This will make the job like a manual job (it will pick up where the previous job left off).
|
||||
data := model.StringMap{
|
||||
shared.JobDataInitiatedBy: "mmctl",
|
||||
shared.JobDataExportType: exportType,
|
||||
shared.JobDataBatchStartId: "",
|
||||
shared.JobDataJobStartId: "",
|
||||
}
|
||||
if startTimestamp != 0 && endTimestamp != 0 {
|
||||
data[shared.JobDataBatchStartTime] = startTime
|
||||
data[shared.JobDataJobStartTime] = startTime
|
||||
data[shared.JobDataJobEndTime] = endTime
|
||||
data[shared.JobDataExportDir] = exportDir
|
||||
}
|
||||
|
||||
job := &model.Job{
|
||||
Type: model.JobTypeMessageExport,
|
||||
Data: data,
|
||||
}
|
||||
|
||||
if job, _, err = c.CreateJob(context.TODO(), job); err != nil {
|
||||
return fmt.Errorf("failed to create compliance export job: %w", err)
|
||||
}
|
||||
|
||||
printer.Print(fmt.Sprintf("Compliance export job created with ID: %s", job.Id))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getStartAndEnd returns the start and end timestamps in unix milliseconds
|
||||
func getStartAndEnd(dateStr string, start int, end int) (int64, int64, error) {
|
||||
if dateStr == "" && start == 0 && end == 0 {
|
||||
// return 0 so that the job will be like a manual job
|
||||
return 0, 0, nil
|
||||
}
|
||||
|
||||
if dateStr != "" && (start > 0 || end > 0) {
|
||||
return 0, 0, errors.New("if date is used, start and end must not be set")
|
||||
}
|
||||
|
||||
if dateStr != "" {
|
||||
t, err := time.Parse("2006-01-02 -0700", dateStr)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("could not parse date string: %s, use the format with time zone offset: YYYY-MM-DD -0700, eg for EDT: `2024-12-24 -0400`, error details: %w", dateStr, err)
|
||||
}
|
||||
endTimestamp := t.AddDate(0, 0, 1).UnixMilli() - 1
|
||||
return t.UnixMilli(), endTimestamp, nil
|
||||
}
|
||||
if start <= 0 || end <= 0 || start >= end {
|
||||
return 0, 0, fmt.Errorf("if date is not used, start: %d and end: %d must both be > 0, and start must be < end", start, end)
|
||||
}
|
||||
return int64(start), int64(end), nil
|
||||
}
|
||||
|
||||
@@ -10,12 +10,14 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/v8"
|
||||
st "github.com/mattermost/mattermost/server/v8/channels/store/storetest"
|
||||
"github.com/mattermost/mattermost/server/v8/cmd/mmctl/client"
|
||||
"github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer"
|
||||
"github.com/mattermost/mattermost/server/v8/enterprise/message_export/shared"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
@@ -232,7 +234,7 @@ func (s *MmctlE2ETestSuite) TestComplianceExportCancelCmdE2E() {
|
||||
|
||||
cmd := makeCmd()
|
||||
err = complianceExportCancelCmdF(s.th.Client, cmd, []string{job.Id})
|
||||
s.Require().EqualError(err, "failed to get compliance export job: You do not have the appropriate permissions.")
|
||||
s.Require().EqualError(err, "failed to cancel compliance export job: You do not have the appropriate permissions.")
|
||||
s.Require().Empty(printer.GetLines())
|
||||
s.Require().Empty(printer.GetErrorLines())
|
||||
})
|
||||
@@ -242,7 +244,7 @@ func (s *MmctlE2ETestSuite) TestComplianceExportCancelCmdE2E() {
|
||||
|
||||
cmd := makeCmd()
|
||||
err := complianceExportCancelCmdF(c, cmd, []string{"non-existent-job-id"})
|
||||
s.Require().EqualError(err, "failed to get compliance export job: Sorry, we could not find the page., There doesn't appear to be an api call for the url='/api/v4/jobs/non-existent-job-id'. Typo? are you missing a team_id or user_id as part of the url?")
|
||||
s.Require().EqualError(err, "failed to cancel compliance export job: Sorry, we could not find the page., There doesn't appear to be an api call for the url='/api/v4/jobs/non-existent-job-id/cancel'. Typo? are you missing a team_id or user_id as part of the url?")
|
||||
s.Require().Empty(printer.GetLines())
|
||||
s.Require().Empty(printer.GetErrorLines())
|
||||
})
|
||||
@@ -585,3 +587,147 @@ func (s *MmctlE2ETestSuite) TestComplianceExportDownloadCmdE2E() {
|
||||
s.Require().True(foundExport2, "export2.zip not found in downloaded file")
|
||||
})
|
||||
}
|
||||
|
||||
func (s *MmctlE2ETestSuite) TestComplianceExportMmctlJobStartTimeE2E() {
|
||||
s.SetupMessageExportTestHelper()
|
||||
|
||||
s.RunForSystemAdminAndLocal("mmctl job uses batch_start_time from previous regular job", func(c client.Client) {
|
||||
// Ensure no jobs exist before we start
|
||||
jobs, _, err := s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 1000)
|
||||
s.Require().NoError(err)
|
||||
for _, job := range jobs {
|
||||
var result string
|
||||
result, err = s.th.App.Srv().Store().Job().Delete(job.Id)
|
||||
s.Require().NoError(err, "Failed to delete job (result: %v)", result)
|
||||
}
|
||||
|
||||
now := model.GetMillis()
|
||||
|
||||
// Create a regular (non-mmctl) export job
|
||||
regularStartTime := now - 10000
|
||||
regularEndTime := now - 5000
|
||||
regularJob := s.runJobForTest(map[string]string{
|
||||
shared.JobDataBatchStartTime: strconv.FormatInt(regularStartTime, 10),
|
||||
shared.JobDataJobEndTime: strconv.FormatInt(regularEndTime, 10),
|
||||
})
|
||||
|
||||
s.Require().Equal(model.JobStatusSuccess, regularJob.Status, "Regular job should complete successfully")
|
||||
s.Require().NotEmpty(regularJob.Data[shared.JobDataBatchStartTime], "Regular job should have a batch start time")
|
||||
regularJobBatchStartTime := regularJob.Data[shared.JobDataBatchStartTime]
|
||||
|
||||
// Run an mmctl-initiated export job
|
||||
cmd := &cobra.Command{}
|
||||
cmd.Flags().String("date", "", "")
|
||||
cmd.Flags().Int("start", 0, "")
|
||||
cmd.Flags().Int("end", 0, "")
|
||||
err = complianceExportCreateCmdF(c, cmd, []string{model.ComplianceExportTypeActiance})
|
||||
s.Require().NoError(err, "Should create mmctl job successfully")
|
||||
|
||||
// Find the mmctl job
|
||||
jobs, _, err = s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().True(len(jobs) > 1, "Should have at least 2 jobs")
|
||||
|
||||
// The most recent job should be the mmctl job
|
||||
mmctlJob := jobs[0]
|
||||
s.Require().Equal("mmctl", mmctlJob.Data[shared.JobDataInitiatedBy])
|
||||
|
||||
// Wait for the mmctl job to complete
|
||||
s.checkJobForStatus(mmctlJob.Id, model.JobStatusSuccess)
|
||||
mmctlJob = s.getMostRecentJobWithId(mmctlJob.Id)
|
||||
|
||||
// The job_start_time should match the batch_start_time from the previous regular job
|
||||
s.Require().Equal(regularJobBatchStartTime, mmctlJob.Data[shared.JobDataJobStartTime],
|
||||
"mmctl job should use batch_start_time from previous regular job as its job_start_time")
|
||||
|
||||
// Clean up jobs
|
||||
for _, job := range jobs {
|
||||
result, err := s.th.App.Srv().Store().Job().Delete(job.Id)
|
||||
s.Require().NoError(err, "Failed to delete job (result: %v)", result)
|
||||
}
|
||||
})
|
||||
|
||||
s.RunForSystemAdminAndLocal("mmctl job ignores previous mmctl jobs and uses regular job", func(c client.Client) {
|
||||
// Ensure no jobs exist before we start
|
||||
jobs, _, err := s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 1000)
|
||||
s.Require().NoError(err)
|
||||
for _, job := range jobs {
|
||||
var result string
|
||||
result, err = s.th.App.Srv().Store().Job().Delete(job.Id)
|
||||
s.Require().NoError(err, "Failed to delete job (result: %v)", result)
|
||||
}
|
||||
|
||||
now := model.GetMillis()
|
||||
|
||||
// Create a regular (non-mmctl) export job
|
||||
regularStartTime := now - 10000
|
||||
regularEndTime := now - 5000
|
||||
regularJob := s.runJobForTest(map[string]string{
|
||||
shared.JobDataBatchStartTime: strconv.FormatInt(regularStartTime, 10),
|
||||
shared.JobDataJobEndTime: strconv.FormatInt(regularEndTime, 10),
|
||||
})
|
||||
|
||||
s.Require().Equal(model.JobStatusSuccess, regularJob.Status, "Regular job should complete successfully")
|
||||
s.Require().NotEmpty(regularJob.Data[shared.JobDataBatchStartTime], "Regular job should have a batch start time")
|
||||
regularJobBatchStartTime := regularJob.Data[shared.JobDataBatchStartTime]
|
||||
|
||||
// Run an mmctl-initiated export job with an explicit start time (different from the regular job)
|
||||
cmd := &cobra.Command{}
|
||||
cmd.Flags().String("date", "", "")
|
||||
cmd.Flags().Int("start", int(now-2000), "")
|
||||
cmd.Flags().Int("end", int(now-1000), "")
|
||||
err = complianceExportCreateCmdF(c, cmd, []string{model.ComplianceExportTypeActiance})
|
||||
s.Require().NoError(err, "Should create first mmctl job successfully")
|
||||
|
||||
// Find the mmctl job
|
||||
jobs, _, err = s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().True(len(jobs) > 1, "Should have at least 2 jobs")
|
||||
|
||||
// The most recent job should be the mmctl job
|
||||
mmctlJob1 := jobs[0]
|
||||
s.Require().Equal("mmctl", mmctlJob1.Data[shared.JobDataInitiatedBy])
|
||||
|
||||
// Wait for the mmctl job to complete
|
||||
s.checkJobForStatus(mmctlJob1.Id, model.JobStatusSuccess)
|
||||
mmctlJob1 = s.getMostRecentJobWithId(mmctlJob1.Id)
|
||||
|
||||
// Verify this job has a different batch_start_time than the regular job
|
||||
s.Require().NotEqual(regularJobBatchStartTime, mmctlJob1.Data[shared.JobDataBatchStartTime],
|
||||
"First mmctl job should have a different batch_start_time than regular job")
|
||||
|
||||
// Run a second mmctl-initiated export job WITHOUT a specified start time
|
||||
cmd = &cobra.Command{}
|
||||
cmd.Flags().String("date", "", "")
|
||||
cmd.Flags().Int("start", 0, "")
|
||||
cmd.Flags().Int("end", 0, "")
|
||||
err = complianceExportCreateCmdF(c, cmd, []string{model.ComplianceExportTypeActiance})
|
||||
s.Require().NoError(err, "Should create second mmctl job successfully")
|
||||
|
||||
// Find the second mmctl job
|
||||
jobs, _, err = s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 10)
|
||||
s.Require().NoError(err)
|
||||
s.Require().True(len(jobs) > 2, "Should have at least 3 jobs")
|
||||
|
||||
// The most recent job should be the second mmctl job
|
||||
mmctlJob2 := jobs[0]
|
||||
s.Require().Equal("mmctl", mmctlJob2.Data[shared.JobDataInitiatedBy])
|
||||
|
||||
// Wait for the second mmctl job to complete
|
||||
s.checkJobForStatus(mmctlJob2.Id, model.JobStatusSuccess)
|
||||
mmctlJob2 = s.getMostRecentJobWithId(mmctlJob2.Id)
|
||||
|
||||
// The job_start_time of the second mmctl job should match the batch_start_time from the regular job,
|
||||
// not from the mmctl job that ran in between
|
||||
s.Require().Equal(regularJobBatchStartTime, mmctlJob2.Data[shared.JobDataJobStartTime],
|
||||
"Second mmctl job should use batch_start_time from previous regular job as its job_start_time, not from previous mmctl job")
|
||||
s.Require().NotEqual(mmctlJob1.Data[shared.JobDataBatchStartTime], mmctlJob2.Data[shared.JobDataJobStartTime],
|
||||
"Second mmctl job should not use batch_start_time from previous mmctl job as its job_start_time")
|
||||
|
||||
// Clean up jobs
|
||||
for _, job := range jobs {
|
||||
result, err := s.th.App.Srv().Store().Job().Delete(job.Id)
|
||||
s.Require().NoError(err, "Failed to delete job (result: %v)", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
gomock "github.com/golang/mock/gomock"
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
@@ -298,6 +299,121 @@ func (s *MmctlUnitTestSuite) TestComplianceExportDownloadCmdF() {
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetStartAndEnd(t *testing.T) {
|
||||
type args struct {
|
||||
dateStr string
|
||||
start int
|
||||
end int
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
expectedStart int64
|
||||
expectedEnd int64
|
||||
wantErr bool
|
||||
}{
|
||||
// check with: https://www.epochconverter.com/
|
||||
{
|
||||
name: "parse a date in EDT (-0400)",
|
||||
args: args{
|
||||
dateStr: "2024-10-21 -0400",
|
||||
},
|
||||
expectedStart: 1729483200000,
|
||||
expectedEnd: 1729569599999,
|
||||
},
|
||||
{
|
||||
name: "parse a date in UTC (+0)",
|
||||
args: args{
|
||||
dateStr: "2024-10-21 +0000",
|
||||
},
|
||||
expectedStart: 1729468800000,
|
||||
expectedEnd: 1729555199999,
|
||||
},
|
||||
{
|
||||
name: "parse a date in CDT (-0500)",
|
||||
args: args{
|
||||
dateStr: "2024-10-21 -0500",
|
||||
},
|
||||
expectedStart: 1729486800000,
|
||||
expectedEnd: 1729573199999,
|
||||
},
|
||||
{
|
||||
name: "bad format",
|
||||
args: args{
|
||||
dateStr: "2024-10-21 CT",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "bad format",
|
||||
args: args{
|
||||
dateStr: "2024-1-2 CDT",
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "it's ok to not have date, start, or end",
|
||||
args: args{},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "needs both start and end pt1",
|
||||
args: args{
|
||||
start: 12345,
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "needs both start and end pt2",
|
||||
args: args{
|
||||
end: 12345,
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "start and end",
|
||||
args: args{
|
||||
start: 12345,
|
||||
end: 678912,
|
||||
},
|
||||
expectedStart: 12345,
|
||||
expectedEnd: 678912,
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "date and start",
|
||||
args: args{
|
||||
dateStr: "2024-10-21 -0400",
|
||||
start: 12345,
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "date and end",
|
||||
args: args{
|
||||
dateStr: "2024-10-21 -0400",
|
||||
end: 678912,
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
gotStart, gotEnd, err := getStartAndEnd(tt.args.dateStr, tt.args.start, tt.args.end)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("getStartAndEnd() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if gotStart != tt.expectedStart {
|
||||
t.Errorf("getStartAndEnd() got = %v, want %v", gotStart, tt.expectedStart)
|
||||
}
|
||||
if gotEnd != tt.expectedEnd {
|
||||
t.Errorf("getStartAndEnd() got1 = %v, want %v", gotEnd, tt.expectedEnd)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func makeCmd() *cobra.Command {
|
||||
cmd := &cobra.Command{}
|
||||
cmd.Flags().Int("page", 0, "")
|
||||
|
||||
@@ -244,7 +244,7 @@ func exportDownloadCmdF(c client.Client, command *cobra.Command, args []string)
|
||||
return nil
|
||||
}
|
||||
|
||||
// downloadFile handles the common logic for downloading files in export and compliance_export commands
|
||||
// downloadFile handles the common logic for downloading files in export and compliance-export commands
|
||||
func downloadFile(path string, downloadFn func(*os.File) (string, error), retries int, fileType string) (string, error) {
|
||||
var outFile *os.File
|
||||
info, err := os.Stat(path)
|
||||
|
||||
@@ -4,6 +4,10 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/api4"
|
||||
@@ -81,6 +85,8 @@ func (s *MmctlE2ETestSuite) SetupMessageExportTestHelper() *api4.TestHelper {
|
||||
s.th.App.Srv().Jobs.RegisterJobType(model.JobTypeMessageExport, messageExportImpl.MakeWorker(), messageExportImpl.MakeScheduler())
|
||||
s.th.App.UpdateConfig(func(cfg *model.Config) {
|
||||
*cfg.MessageExportSettings.DownloadExportResults = true
|
||||
*cfg.MessageExportSettings.EnableExport = true
|
||||
*cfg.MessageExportSettings.ExportFormat = model.ComplianceExportTypeActiance
|
||||
})
|
||||
|
||||
err := s.th.App.Srv().Jobs.StartWorkers()
|
||||
@@ -125,3 +131,47 @@ func (s *MmctlE2ETestSuite) RunForAllClients(testName string, fn func(client.Cli
|
||||
func (s *MmctlE2ETestSuite) CheckErrorID(err error, errorId string) {
|
||||
api4.CheckErrorID(s.T(), err, errorId)
|
||||
}
|
||||
|
||||
// Helper functions for compliance export job testing
|
||||
|
||||
// getMostRecentJobWithId gets the most recent job with the specified ID
|
||||
func (s *MmctlE2ETestSuite) getMostRecentJobWithId(id string) *model.Job {
|
||||
list, _, err := s.th.SystemAdminClient.GetJobsByType(context.Background(), model.JobTypeMessageExport, 0, 1)
|
||||
s.Require().NoError(err)
|
||||
s.Require().Len(list, 1)
|
||||
s.Require().Equal(id, list[0].Id)
|
||||
return list[0]
|
||||
}
|
||||
|
||||
// checkJobForStatus polls until the job with the specified ID reaches the expected status
|
||||
func (s *MmctlE2ETestSuite) checkJobForStatus(id string, status string) {
|
||||
doneChan := make(chan bool)
|
||||
var job *model.Job
|
||||
go func() {
|
||||
defer close(doneChan)
|
||||
for {
|
||||
job = s.getMostRecentJobWithId(id)
|
||||
if job.Status == status {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
s.Require().Equal(status, job.Status)
|
||||
}()
|
||||
select {
|
||||
case <-doneChan:
|
||||
case <-time.After(15 * time.Second):
|
||||
s.Require().Fail(fmt.Sprintf("expected job's status to be %s, got %s", status, job.Status))
|
||||
}
|
||||
}
|
||||
|
||||
// runJobForTest creates a job and waits for it to complete
|
||||
func (s *MmctlE2ETestSuite) runJobForTest(jobData map[string]string) *model.Job {
|
||||
job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(),
|
||||
&model.Job{Type: model.JobTypeMessageExport, Data: jobData})
|
||||
s.Require().NoError(err)
|
||||
// poll until completion
|
||||
s.checkJobForStatus(job.Id, model.JobStatusSuccess)
|
||||
job = s.getMostRecentJobWithId(job.Id)
|
||||
return job
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ SEE ALSO
|
||||
|
||||
* `mmctl <mmctl.rst>`_ - Remote client for the Open Source, self-hosted Slack-alternative
|
||||
* `mmctl compliance-export cancel <mmctl_compliance-export_cancel.rst>`_ - Cancel compliance export job
|
||||
* `mmctl compliance-export create <mmctl_compliance-export_create.rst>`_ - Create a compliance export job, of type 'csv' or 'actiance' or 'globalrelay'
|
||||
* `mmctl compliance-export download <mmctl_compliance-export_download.rst>`_ - Download compliance export file
|
||||
* `mmctl compliance-export list <mmctl_compliance-export_list.rst>`_ - List compliance export jobs, sorted by creation date descending (newest first)
|
||||
* `mmctl compliance-export show <mmctl_compliance-export_show.rst>`_ - Show compliance export job
|
||||
|
||||
56
server/cmd/mmctl/docs/mmctl_compliance-export_create.rst
Обычный файл
56
server/cmd/mmctl/docs/mmctl_compliance-export_create.rst
Обычный файл
@@ -0,0 +1,56 @@
|
||||
.. _mmctl_compliance-export_create:
|
||||
|
||||
mmctl compliance-export create
|
||||
------------------------------
|
||||
|
||||
Create a compliance export job, of type 'csv' or 'actiance' or 'globalrelay'
|
||||
|
||||
Synopsis
|
||||
~~~~~~~~
|
||||
|
||||
|
||||
Create a compliance export job, of type 'csv' or 'actiance' or 'globalrelay'. If --date is set, the job will run for one day, from 12am to 12am (minus one millisecond) inclusively, in the format with timezone offset: `"YYYY-MM-DD -0000"`. E.g., "2024-10-21 -0400" for Oct 21, 2024 EDT timezone. "2023-11-01 +0000" for Nov 01, 2024 UTC. If set, the 'start' and 'end' flags will be ignored.
|
||||
|
||||
Important: Running a compliance export job from mmctl will NOT affect the next scheduled job's batch_start_time. This means that if you run a compliance export job from mmctl, the next scheduled job will run from the batch_end_time of the previous scheduled job, as usual.
|
||||
|
||||
::
|
||||
|
||||
mmctl compliance-export create [complianceExportType] --date "2025-03-27 -0400" [flags]
|
||||
|
||||
Examples
|
||||
~~~~~~~~
|
||||
|
||||
::
|
||||
|
||||
compliance-export create csv --date "2025-03-27 -0400"
|
||||
|
||||
Options
|
||||
~~~~~~~
|
||||
|
||||
::
|
||||
|
||||
--date "YYYY-MM-DD -0000" Run the export for one day, from 12am to 12am (minus one millisecond) inclusively, in the format with timezone offset: "YYYY-MM-DD -0000". E.g., `"2024-10-21 -0400"` for Oct 21, 2024 EDT timezone. `"2023-11-01 +0000"` for Nov 01, 2024 UTC. If set, the 'start' and 'end' flags will be ignored.
|
||||
--end 1743134400000 The end timestamp in unix milliseconds. Posts with updateAt <= end will be exported. If set, 'start' must be set as well. eg, 1743134400000 for 2025-03-28 EDT.
|
||||
-h, --help help for create
|
||||
--start 1743048000000 The start timestamp in unix milliseconds. Posts with updateAt >= start will be exported. If set, 'end' must be set as well. eg, 1743048000000 for 2025-03-27 EDT.
|
||||
|
||||
Options inherited from parent commands
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
::
|
||||
|
||||
--config string path to the configuration file (default "$XDG_CONFIG_HOME/mmctl/config")
|
||||
--disable-pager disables paged output
|
||||
--insecure-sha1-intermediate allows to use insecure TLS protocols, such as SHA-1
|
||||
--insecure-tls-version allows to use TLS versions 1.0 and 1.1
|
||||
--json the output format will be in json format
|
||||
--local allows communicating with the server through a unix socket
|
||||
--quiet prevent mmctl to generate output for the commands
|
||||
--strict will only run commands if the mmctl version matches the server one
|
||||
--suppress-warnings disables printing warning messages
|
||||
|
||||
SEE ALSO
|
||||
~~~~~~~~
|
||||
|
||||
* `mmctl compliance-export <mmctl_compliance-export.rst>`_ - Management of compliance exports
|
||||
|
||||
@@ -20,7 +20,7 @@ Examples
|
||||
|
||||
::
|
||||
|
||||
compliance_export download o98rj3ur83dp5dppfyk5yk6osy
|
||||
compliance-export download o98rj3ur83dp5dppfyk5yk6osy
|
||||
|
||||
Options
|
||||
~~~~~~~
|
||||
|
||||
Ссылка в новой задаче
Block a user