From b33a7e362f7727bdb41cc716c8784a10fbc2f145 Mon Sep 17 00:00:00 2001 From: Christopher Poile Date: Tue, 24 Jun 2025 16:27:54 -0400 Subject: [PATCH] [MM-63556] mmctl: Add compliance export download cmd (#30576) * add mmctl compliance export download command and tests - Introduced `ComplianceExportDownloadCmd` to facilitate downloading compliance export files. - Implemented the `DownloadComplianceExport` method in the Client interface for handling file downloads. - Added unit tests for the download command, covering successful downloads, error handling for non-existent jobs, and retries on failure. - Included end-to-end tests to validate the command's functionality. - Updated documentation to include usage examples and options for the new command. * don't know why this was left out * PR comments * adjust test for new retry logic * refactored download fn for compliance_export and export * fix test due to fixed logic * docs --- server/channels/api4/job_local.go | 1 + server/cmd/mmctl/client/client.go | 1 + .../cmd/mmctl/commands/compliance_export.go | 51 ++++ .../commands/compliance_export_e2e_test.go | 284 ++++++++++++++++++ .../mmctl/commands/compliance_export_test.go | 74 +++++ server/cmd/mmctl/commands/export.go | 49 +-- server/cmd/mmctl/commands/export_e2e_test.go | 11 +- server/cmd/mmctl/commands/mmctl_test.go | 3 + .../mmctl/docs/mmctl_compliance-export.rst | 1 + .../docs/mmctl_compliance-export_download.rst | 52 ++++ server/cmd/mmctl/mocks/client_mock.go | 15 + server/public/model/client4.go | 28 +- 12 files changed, 548 insertions(+), 22 deletions(-) create mode 100644 server/cmd/mmctl/docs/mmctl_compliance-export_download.rst diff --git a/server/channels/api4/job_local.go b/server/channels/api4/job_local.go index bf19aa9fef..048092ef7c 100644 --- a/server/channels/api4/job_local.go +++ b/server/channels/api4/job_local.go @@ -9,6 +9,7 @@ func (api *API) InitJobLocal() { api.BaseRoutes.Jobs.Handle("", api.APILocal(getJobs)).Methods(http.MethodGet) api.BaseRoutes.Jobs.Handle("", api.APILocal(createJob)).Methods(http.MethodPost) api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}", api.APILocal(getJob)).Methods(http.MethodGet) + api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}/download", api.APILocal(downloadJob)).Methods(http.MethodGet) api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}/cancel", api.APILocal(cancelJob)).Methods(http.MethodPost) api.BaseRoutes.Jobs.Handle("/type/{job_type:[A-Za-z0-9_-]+}", api.APILocal(getJobsByType)).Methods(http.MethodGet) api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}/status", api.APILocal(updateJobStatus)).Methods(http.MethodPatch) diff --git a/server/cmd/mmctl/client/client.go b/server/cmd/mmctl/client/client.go index 4660afca19..914b6aa656 100644 --- a/server/cmd/mmctl/client/client.go +++ b/server/cmd/mmctl/client/client.go @@ -152,6 +152,7 @@ type Client interface { ListExports(ctx context.Context) ([]string, *model.Response, error) DeleteExport(ctx context.Context, name string) (*model.Response, error) DownloadExport(ctx context.Context, name string, wr io.Writer, offset int64) (int64, *model.Response, error) + DownloadComplianceExport(ctx context.Context, jobID string, wr io.Writer) (string, error) GeneratePresignedURL(ctx context.Context, name string) (*model.PresignURLResponse, *model.Response, error) ResetSamlAuthDataToEmail(ctx context.Context, includeDeleted bool, dryRun bool, userIDs []string) (int64, *model.Response, error) GenerateSupportPacket(ctx context.Context) (io.ReadCloser, string, *model.Response, error) diff --git a/server/cmd/mmctl/commands/compliance_export.go b/server/cmd/mmctl/commands/compliance_export.go index 01b5349659..563c66b689 100644 --- a/server/cmd/mmctl/commands/compliance_export.go +++ b/server/cmd/mmctl/commands/compliance_export.go @@ -6,8 +6,10 @@ package commands import ( "context" "fmt" + "os" "github.com/mattermost/mattermost/server/v8/cmd/mmctl/client" + "github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer" "github.com/spf13/cobra" ) @@ -40,15 +42,26 @@ var ComplianceExportCancelCmd = &cobra.Command{ RunE: withClient(complianceExportCancelCmdF), } +var ComplianceExportDownloadCmd = &cobra.Command{ + Use: "download [complianceExportJobID] [output filepath (optional)]", + Example: " compliance_export download o98rj3ur83dp5dppfyk5yk6osy", + Short: "Download compliance export file", + Args: cobra.MinimumNArgs(1), + RunE: withClient(complianceExportDownloadCmdF), +} + func init() { ComplianceExportListCmd.Flags().Int("page", 0, "Page number to fetch for the list of compliance export jobs") ComplianceExportListCmd.Flags().Int("per-page", DefaultPageSize, "Number of compliance export jobs to be fetched") ComplianceExportListCmd.Flags().Bool("all", false, "Fetch all compliance export jobs. --page flag will be ignored if provided") + ComplianceExportDownloadCmd.Flags().Int("num-retries", 5, "Number of retries if the download fails") + ComplianceExportCmd.AddCommand( ComplianceExportListCmd, ComplianceExportShowCmd, ComplianceExportCancelCmd, + ComplianceExportDownloadCmd, ) RootCmd.AddCommand(ComplianceExportCmd) } @@ -75,3 +88,41 @@ func complianceExportCancelCmdF(c client.Client, command *cobra.Command, args [] return nil } + +func complianceExportDownloadCmdF(c client.Client, command *cobra.Command, args []string) error { + jobID := args[0] + var path string + if len(args) > 1 { + path = args[1] + } else { + path = jobID + ".zip" + } + + retries, _ := command.Flags().GetInt("num-retries") + + downloadFn := func(outFile *os.File) (string, error) { + return c.DownloadComplianceExport(context.TODO(), jobID, outFile) + } + + suggestedFilename, err := downloadFile(path, downloadFn, retries, "compliance export") + if err != nil { + return err + } + + // If we didn't provide a path and got a suggested filename, rename the file + if len(args) == 1 && suggestedFilename != "" && suggestedFilename != path { + // If the suggested name already exists, don't overwrite + if _, err := os.Stat(suggestedFilename); err == nil { + printer.PrintWarning(fmt.Sprintf("File with the server's suggested name %q already exists, keeping %q", suggestedFilename, path)) + } else { + if err := os.Rename(path, suggestedFilename); err != nil { + printer.PrintWarning(fmt.Sprintf("Could not rename file to the server's suggested name %q: %v", suggestedFilename, err)) + } else { + path = suggestedFilename + } + } + } + + printer.Print(fmt.Sprintf("Compliance export file downloaded to %q", path)) + return nil +} diff --git a/server/cmd/mmctl/commands/compliance_export_e2e_test.go b/server/cmd/mmctl/commands/compliance_export_e2e_test.go index fa971a782f..6df88d7a9b 100644 --- a/server/cmd/mmctl/commands/compliance_export_e2e_test.go +++ b/server/cmd/mmctl/commands/compliance_export_e2e_test.go @@ -4,12 +4,19 @@ package commands import ( + "archive/zip" "context" + "fmt" + "io" + "os" + "path/filepath" "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8" st "github.com/mattermost/mattermost/server/v8/channels/store/storetest" "github.com/mattermost/mattermost/server/v8/cmd/mmctl/client" "github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer" + "github.com/spf13/cobra" ) func (s *MmctlE2ETestSuite) TestComplianceExportListCmdE2E() { @@ -301,3 +308,280 @@ func (s *MmctlE2ETestSuite) TestComplianceExportCancelCmdE2E() { s.Require().Empty(printer.GetErrorLines()) }) } + +func (s *MmctlE2ETestSuite) TestComplianceExportDownloadCmdE2E() { + s.SetupMessageExportTestHelper() + + s.Run("no permissions", func() { + printer.Clean() + + now := model.GetMillis() + // Create a job + job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ + Id: st.NewTestID(), + CreateAt: now - 1000, + Status: model.JobStatusSuccess, + Type: model.JobTypeMessageExport, + StartAt: now - 1000, + LastActivityAt: now - 1000, + }) + s.Require().NoError(err) + defer func() { + // Ensure job is deleted from the database + var result string + result, err = s.th.App.Srv().Store().Job().Delete(job.Id) + s.Require().NoError(err, "Failed to delete job (result: %v)", result) + }() + + cmd := makeCmd() + cmd.Flags().Int("num-retries", 0, "") + err = complianceExportDownloadCmdF(s.th.Client, cmd, []string{job.Id}) + s.Require().EqualError(err, "failed to download compliance export file: You do not have the appropriate permissions.") + s.Require().Empty(printer.GetLines()) + s.Require().Empty(printer.GetErrorLines()) + }) + + s.RunForSystemAdminAndLocal("Download non-existent job", func(c client.Client) { + printer.Clean() + + cmd := makeCmd() + cmd.Flags().Int("num-retries", 0, "") + err := complianceExportDownloadCmdF(c, cmd, []string{"non-existent-job-id"}) + s.Require().EqualError(err, "failed to download compliance export file: Sorry, we could not find the page., There doesn't appear to be an api call for the url='/api/v4/jobs/non-existent-job-id/download'. Typo? are you missing a team_id or user_id as part of the url?") + s.Require().Empty(printer.GetLines()) + s.Require().Empty(printer.GetErrorLines()) + }) + + s.RunForSystemAdminAndLocal("existing, non empty compliance export", func(c client.Client) { + printer.Clean() + + importFilePath := filepath.Join(server.GetPackagePath(), "test.zip") + f, err := os.Create(importFilePath) + s.Require().Nil(err) + _, err = f.WriteString("test data") + s.Require().Nil(err) + _ = f.Close() + + defer func() { + _ = os.Remove(importFilePath) + }() + + cmd := &cobra.Command{} + cmd.Flags().Int("num-retries", 0, "") + + err = complianceExportDownloadCmdF(c, cmd, []string{"jobId", importFilePath}) + s.Require().EqualError(err, "compliance export file already exists") + s.Require().Empty(printer.GetLines()) + s.Require().Empty(printer.GetErrorLines()) + }) + + s.RunForSystemAdminAndLocal("download with explicit path", func(c client.Client) { + printer.Clean() + + downloadPath := "explicit_path.zip" + defer os.Remove(downloadPath) + + serverDataDir, err := filepath.Abs(*s.th.App.Config().FileSettings.Directory) + s.Require().Nil(err) + + exportDir := "job_test_export" + // Create a compliance export with two zip files + exportFilePath := filepath.Join(serverDataDir, exportDir) + err = os.Mkdir(exportFilePath, 0755) + s.Require().Nil(err) + defer func() { + _ = os.RemoveAll(exportFilePath) + }() + + // Create first zip file + zipPath1 := exportFilePath + "/export1.zip" + f1, err := os.Create(zipPath1) + s.Require().Nil(err) + _, err = f1.WriteString("test data 1") + s.Require().Nil(err) + _ = f1.Close() + + // Create second zip file + zipPath2 := exportFilePath + "/export2.zip" + f2, err := os.Create(zipPath2) + s.Require().Nil(err) + _, err = f2.WriteString("test data 2") + s.Require().Nil(err) + _ = f2.Close() + + defer func() { + _ = os.RemoveAll(exportFilePath) + }() + + now := model.GetMillis() + // Create a job + job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ + Id: st.NewTestID(), + CreateAt: now - 1000, + Status: model.JobStatusSuccess, + Type: model.JobTypeMessageExport, + StartAt: now - 1000, + LastActivityAt: now - 1000, + Data: model.StringMap{"export_dir": exportDir, "is_downloadable": "true"}, + }) + s.Require().NoError(err) + defer func() { + // Ensure job is deleted from the database + var result string + result, err = s.th.App.Srv().Store().Job().Delete(job.Id) + s.Require().NoError(err, "Failed to delete job (result: %v)", result) + }() + + cmd := makeCmd() + cmd.Flags().Int("num-retries", 0, "") + + err = complianceExportDownloadCmdF(c, cmd, []string{job.Id, downloadPath}) + s.Require().NoError(err) + s.Require().Contains(printer.GetLines()[0], fmt.Sprintf("Compliance export file downloaded to %q", downloadPath)) + s.Require().Empty(printer.GetErrorLines()) + + // Verify the file was downloaded + _, err = os.Stat(downloadPath) + s.Require().Nil(err) + defer os.Remove(downloadPath) + + // Verify the file is a zip with a directory with two zip files + zipReader, err := zip.OpenReader(downloadPath) + s.Require().NoError(err) + defer zipReader.Close() + + // Check that we have the expected files in the zip + foundExport1 := false + foundExport2 := false + for _, file := range zipReader.File { + if file.Name == "export1.zip" { + foundExport1 = true + // Verify contents of export1.zip + rc, err := file.Open() + s.Require().NoError(err) + content, err := io.ReadAll(rc) + s.Require().NoError(err) + s.Require().Equal("test data 1", string(content)) + rc.Close() + } else if file.Name == "export2.zip" { + foundExport2 = true + // Verify contents of export2.zip + rc, err := file.Open() + s.Require().NoError(err) + content, err := io.ReadAll(rc) + s.Require().NoError(err) + s.Require().Equal("test data 2", string(content)) + rc.Close() + } else { + s.Failf("unexpected file found in downloaded zip", file.Name) + } + } + s.Require().True(foundExport1, "export1.zip not found in downloaded file") + s.Require().True(foundExport2, "export2.zip not found in downloaded file") + }) + + s.RunForSystemAdminAndLocal("download with explicit path", func(c client.Client) { + printer.Clean() + + serverDataDir, err := filepath.Abs(*s.th.App.Config().FileSettings.Directory) + s.Require().Nil(err) + + exportDir := "job_test_export" + expectedDownloadPath := exportDir + ".zip" + // Create a compliance export with two zip files + exportFilePath := filepath.Join(serverDataDir, exportDir) + err = os.Mkdir(exportFilePath, 0755) + s.Require().Nil(err) + defer func() { + _ = os.RemoveAll(exportFilePath) + + // also remove the downloaded file + defer os.Remove(expectedDownloadPath) + }() + + // Create first zip file + zipPath1 := exportFilePath + "/export1.zip" + f1, err := os.Create(zipPath1) + s.Require().Nil(err) + _, err = f1.WriteString("test data 1") + s.Require().Nil(err) + _ = f1.Close() + + // Create second zip file + zipPath2 := exportFilePath + "/export2.zip" + f2, err := os.Create(zipPath2) + s.Require().Nil(err) + _, err = f2.WriteString("test data 2") + s.Require().Nil(err) + _ = f2.Close() + + defer func() { + _ = os.RemoveAll(exportFilePath) + }() + + now := model.GetMillis() + // Create a job + job, _, err := s.th.SystemAdminClient.CreateJob(context.Background(), &model.Job{ + Id: st.NewTestID(), + CreateAt: now - 1000, + Status: model.JobStatusSuccess, + Type: model.JobTypeMessageExport, + StartAt: now - 1000, + LastActivityAt: now - 1000, + Data: model.StringMap{"export_dir": exportDir, "is_downloadable": "true"}, + }) + s.Require().NoError(err) + defer func() { + // Ensure job is deleted from the database + var result string + result, err = s.th.App.Srv().Store().Job().Delete(job.Id) + s.Require().NoError(err, "Failed to delete job (result: %v)", result) + }() + + cmd := makeCmd() + cmd.Flags().Int("num-retries", 0, "") + + err = complianceExportDownloadCmdF(c, cmd, []string{job.Id}) + s.Require().NoError(err) + s.Require().Contains(printer.GetLines()[0], fmt.Sprintf("Compliance export file downloaded to %q", expectedDownloadPath)) + s.Require().Empty(printer.GetErrorLines()) + + // Verify the file was downloaded + _, err = os.Stat(expectedDownloadPath) + s.Require().Nil(err) + + // Verify the file is a zip with a directory with two zip files + zipReader, err := zip.OpenReader(expectedDownloadPath) + s.Require().NoError(err) + defer zipReader.Close() + + // Check that we have the expected files in the zip + foundExport1 := false + foundExport2 := false + for _, file := range zipReader.File { + if file.Name == "export1.zip" { + foundExport1 = true + // Verify contents of export1.zip + rc, err := file.Open() + s.Require().NoError(err) + content, err := io.ReadAll(rc) + s.Require().NoError(err) + s.Require().Equal("test data 1", string(content)) + rc.Close() + } else if file.Name == "export2.zip" { + foundExport2 = true + // Verify contents of export2.zip + rc, err := file.Open() + s.Require().NoError(err) + content, err := io.ReadAll(rc) + s.Require().NoError(err) + s.Require().Equal("test data 2", string(content)) + rc.Close() + } else { + s.Failf("unexpected file found in downloaded zip", file.Name) + } + } + s.Require().True(foundExport1, "export1.zip not found in downloaded file") + s.Require().True(foundExport2, "export2.zip not found in downloaded file") + }) +} diff --git a/server/cmd/mmctl/commands/compliance_export_test.go b/server/cmd/mmctl/commands/compliance_export_test.go index acf0540cd0..9b135439df 100644 --- a/server/cmd/mmctl/commands/compliance_export_test.go +++ b/server/cmd/mmctl/commands/compliance_export_test.go @@ -5,7 +5,10 @@ package commands import ( "context" + "fmt" + "os" + gomock "github.com/golang/mock/gomock" "github.com/mattermost/mattermost/server/public/model" "github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer" "github.com/spf13/cobra" @@ -224,6 +227,77 @@ func (s *MmctlUnitTestSuite) TestComplianceExportCancelCmdF() { }) } +func (s *MmctlUnitTestSuite) TestComplianceExportDownloadCmdF() { + mockJob := &model.Job{ + Id: model.NewId(), + CreateAt: model.GetMillis(), + Type: model.JobTypeMessageExport, + } + + s.Run("download job file successfully", func() { + printer.Clean() + defer func() { + _ = os.Remove("suggested-filename.zip") + }() + + s.client. + EXPECT(). + DownloadComplianceExport(gomock.Any(), mockJob.Id, gomock.Any()). + Return("suggested-filename.zip", nil). + Times(1) + + cmd := makeCmd() + cmd.Flags().Int("num-retries", 5, "") + err := complianceExportDownloadCmdF(s.client, cmd, []string{mockJob.Id}) + s.Require().Nil(err) + s.Len(printer.GetLines(), 1) + s.Len(printer.GetErrorLines(), 0) + s.Equal(fmt.Sprintf("Compliance export file downloaded to %q", "suggested-filename.zip"), printer.GetLines()[0]) + }) + + s.Run("download job file with explicit path", func() { + printer.Clean() + defer func() { + _ = os.Remove("custom-path.zip") + }() + + s.client. + EXPECT(). + DownloadComplianceExport(context.TODO(), mockJob.Id, gomock.Any()). + Return("", nil). + Times(1) + + cmd := makeCmd() + cmd.Flags().Int("num-retries", 5, "") + err := complianceExportDownloadCmdF(s.client, cmd, []string{mockJob.Id, "custom-path.zip"}) + s.Require().Nil(err) + s.Len(printer.GetLines(), 1) + s.Len(printer.GetErrorLines(), 0) + s.Equal(fmt.Sprintf("Compliance export file downloaded to %q", "custom-path.zip"), printer.GetLines()[0]) + }) + + s.Run("download job with error", func() { + printer.Clean() + mockError := &model.AppError{ + Message: "failed to download file", + } + + s.client. + EXPECT(). + DownloadComplianceExport(context.TODO(), mockJob.Id, gomock.Any()). + Return("", mockError). + Times(6) // Initial attempt + 5 retries + + cmd := makeCmd() + cmd.Flags().Int("num-retries", 5, "") + err := complianceExportDownloadCmdF(s.client, cmd, []string{mockJob.Id}) + s.Require().NotNil(err) + s.EqualError(err, "failed to download compliance export after 5 retries: failed to download file") + s.Len(printer.GetLines(), 0) + s.Len(printer.GetErrorLines(), 0) + }) +} + func makeCmd() *cobra.Command { cmd := &cobra.Command{} cmd.Flags().Int("page", 0, "") diff --git a/server/cmd/mmctl/commands/export.go b/server/cmd/mmctl/commands/export.go index 077d738aee..2e19cdd568 100644 --- a/server/cmd/mmctl/commands/export.go +++ b/server/cmd/mmctl/commands/export.go @@ -225,15 +225,36 @@ func exportDownloadCmdF(c client.Client, command *cobra.Command, args []string) retries, _ := command.Flags().GetInt("num-retries") + downloadFn := func(outFile *os.File) (string, error) { + off, err := outFile.Seek(0, io.SeekEnd) + if err != nil { + return "", fmt.Errorf("failed to seek file: %w", err) + } + + _, _, err = c.DownloadExport(context.TODO(), name, outFile, off) + return "", err + } + + _, err := downloadFile(path, downloadFn, retries, "export") + if err != nil { + return err + } + + printer.Print(fmt.Sprintf("Export file downloaded to %q", path)) + return nil +} + +// downloadFile handles the common logic for downloading files in export and compliance_export commands +func downloadFile(path string, downloadFn func(*os.File) (string, error), retries int, fileType string) (string, error) { var outFile *os.File info, err := os.Stat(path) switch { case err != nil && !os.IsNotExist(err): // some error occurred and not because file doesn't exist - return fmt.Errorf("failed to stat export file: %w", err) + return "", fmt.Errorf("failed to stat %s file: %w", fileType, err) case err == nil && info.Size() > 0: // we exit to avoid overwriting an existing non-empty file - return fmt.Errorf("export file already exists") + return "", fmt.Errorf("%s file already exists", fileType) case err != nil: // file does not exist, we create it outFile, err = os.Create(path) @@ -243,30 +264,24 @@ func exportDownloadCmdF(c client.Client, command *cobra.Command, args []string) } if err != nil { - return fmt.Errorf("failed to create/open export file: %w", err) + return "", fmt.Errorf("failed to create/open %s file: %w", fileType, err) } defer outFile.Close() - i := 0 - for i < retries+1 { - off, err := outFile.Seek(0, io.SeekEnd) + var suggestedFilename string + for i := range retries + 1 { // need to include the first attempt + suggestedFilename, err = downloadFn(outFile) if err != nil { - return fmt.Errorf("failed to seek export file: %w", err) - } - - if _, _, err := c.DownloadExport(context.TODO(), name, outFile, off); err != nil { - printer.PrintWarning(fmt.Sprintf("failed to download export file: %v. Retrying...", err)) - i++ + if i >= retries { + return "", fmt.Errorf("failed to download %s after %d retries: %w", fileType, retries, err) + } + printer.PrintWarning(fmt.Sprintf("Download attempt %d/%d failed. Retrying...", i+1, retries+1)) continue } break } - if retries != 0 && i == retries+1 { - return fmt.Errorf("failed to download export after %d retries", retries) - } - - return nil + return suggestedFilename, nil } func exportJobListCmdF(c client.Client, command *cobra.Command, args []string) error { diff --git a/server/cmd/mmctl/commands/export_e2e_test.go b/server/cmd/mmctl/commands/export_e2e_test.go index ee8a24bf8d..67a2ad4b21 100644 --- a/server/cmd/mmctl/commands/export_e2e_test.go +++ b/server/cmd/mmctl/commands/export_e2e_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/mattermost/mattermost/server/v8" @@ -193,7 +194,7 @@ func (s *MmctlE2ETestSuite) TestExportDownloadCmdF() { cmd.Flags().Int("num-retries", 5, "") err := exportDownloadCmdF(s.th.Client, cmd, []string{exportName}) - s.Require().EqualError(err, "failed to download export after 5 retries") + s.Require().EqualError(err, "failed to download export after 5 retries: You do not have the appropriate permissions.") s.Require().Empty(printer.GetLines()) s.Require().Empty(printer.GetErrorLines()) }) @@ -227,7 +228,7 @@ func (s *MmctlE2ETestSuite) TestExportDownloadCmdF() { defer os.Remove(downloadPath) err = exportDownloadCmdF(c, cmd, []string{exportName, downloadPath}) - s.Require().EqualError(err, "failed to download export after 5 retries") + s.Require().EqualError(err, "failed to download export after 5 retries: Unable to find export file.") s.Require().Empty(printer.GetLines()) s.Require().Empty(printer.GetErrorLines()) }) @@ -252,7 +253,8 @@ func (s *MmctlE2ETestSuite) TestExportDownloadCmdF() { err = exportDownloadCmdF(c, cmd, []string{exportName, downloadPath}) s.Require().Nil(err) - s.Require().Empty(printer.GetLines()) + s.Require().Len(printer.GetLines(), 1) + s.Require().True(strings.HasPrefix(printer.GetLines()[0].(string), "Export file downloaded to ")) s.Require().Empty(printer.GetErrorLines()) }) @@ -273,7 +275,8 @@ func (s *MmctlE2ETestSuite) TestExportDownloadCmdF() { err = exportDownloadCmdF(c, cmd, []string{exportName, downloadPath}) s.Require().Nil(err) - s.Require().Empty(printer.GetLines()) + s.Require().Len(printer.GetLines(), 1) + s.Require().True(strings.HasPrefix(printer.GetLines()[0].(string), "Export file downloaded to ")) s.Require().Empty(printer.GetErrorLines()) expected, err := os.ReadFile(exportFilePath) diff --git a/server/cmd/mmctl/commands/mmctl_test.go b/server/cmd/mmctl/commands/mmctl_test.go index c5bc832d22..2c35517c2c 100644 --- a/server/cmd/mmctl/commands/mmctl_test.go +++ b/server/cmd/mmctl/commands/mmctl_test.go @@ -79,6 +79,9 @@ func (s *MmctlE2ETestSuite) SetupMessageExportTestHelper() *api4.TestHelper { s.th.App.Srv().SetLicense(model.NewTestLicense("message_export")) messageExportImpl := message_export.MessageExportJobInterfaceImpl{Server: s.th.App.Srv()} s.th.App.Srv().Jobs.RegisterJobType(model.JobTypeMessageExport, messageExportImpl.MakeWorker(), messageExportImpl.MakeScheduler()) + s.th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.MessageExportSettings.DownloadExportResults = true + }) err := s.th.App.Srv().Jobs.StartWorkers() require.NoError(s.T(), err) diff --git a/server/cmd/mmctl/docs/mmctl_compliance-export.rst b/server/cmd/mmctl/docs/mmctl_compliance-export.rst index 7d6c91b156..f01019bb46 100644 --- a/server/cmd/mmctl/docs/mmctl_compliance-export.rst +++ b/server/cmd/mmctl/docs/mmctl_compliance-export.rst @@ -38,6 +38,7 @@ SEE ALSO * `mmctl `_ - Remote client for the Open Source, self-hosted Slack-alternative * `mmctl compliance-export cancel `_ - Cancel compliance export job +* `mmctl compliance-export download `_ - Download compliance export file * `mmctl compliance-export list `_ - List compliance export jobs, sorted by creation date descending (newest first) * `mmctl compliance-export show `_ - Show compliance export job diff --git a/server/cmd/mmctl/docs/mmctl_compliance-export_download.rst b/server/cmd/mmctl/docs/mmctl_compliance-export_download.rst new file mode 100644 index 0000000000..0d893b3dab --- /dev/null +++ b/server/cmd/mmctl/docs/mmctl_compliance-export_download.rst @@ -0,0 +1,52 @@ +.. _mmctl_compliance-export_download: + +mmctl compliance-export download +-------------------------------- + +Download compliance export file + +Synopsis +~~~~~~~~ + + +Download compliance export file + +:: + + mmctl compliance-export download [complianceExportJobID] [output filepath (optional)] [flags] + +Examples +~~~~~~~~ + +:: + + compliance_export download o98rj3ur83dp5dppfyk5yk6osy + +Options +~~~~~~~ + +:: + + -h, --help help for download + --num-retries int Number of retries if the download fails (default 5) + +Options inherited from parent commands +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + --config string path to the configuration file (default "$XDG_CONFIG_HOME/mmctl/config") + --disable-pager disables paged output + --insecure-sha1-intermediate allows to use insecure TLS protocols, such as SHA-1 + --insecure-tls-version allows to use TLS versions 1.0 and 1.1 + --json the output format will be in json format + --local allows communicating with the server through a unix socket + --quiet prevent mmctl to generate output for the commands + --strict will only run commands if the mmctl version matches the server one + --suppress-warnings disables printing warning messages + +SEE ALSO +~~~~~~~~ + +* `mmctl compliance-export `_ - Management of compliance exports + diff --git a/server/cmd/mmctl/mocks/client_mock.go b/server/cmd/mmctl/mocks/client_mock.go index d9366fef7a..394ec0a746 100644 --- a/server/cmd/mmctl/mocks/client_mock.go +++ b/server/cmd/mmctl/mocks/client_mock.go @@ -523,6 +523,21 @@ func (mr *MockClientMockRecorder) DoAPIPost(arg0, arg1, arg2 interface{}) *gomoc return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DoAPIPost", reflect.TypeOf((*MockClient)(nil).DoAPIPost), arg0, arg1, arg2) } +// DownloadComplianceExport mocks base method. +func (m *MockClient) DownloadComplianceExport(arg0 context.Context, arg1 string, arg2 io.Writer) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DownloadComplianceExport", arg0, arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DownloadComplianceExport indicates an expected call of DownloadComplianceExport. +func (mr *MockClientMockRecorder) DownloadComplianceExport(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DownloadComplianceExport", reflect.TypeOf((*MockClient)(nil).DownloadComplianceExport), arg0, arg1, arg2) +} + // DownloadExport mocks base method. func (m *MockClient) DownloadExport(arg0 context.Context, arg1 string, arg2 io.Writer, arg3 int64) (int64, *model.Response, error) { m.ctrl.T.Helper() diff --git a/server/public/model/client4.go b/server/public/model/client4.go index 82cc559055..659d67dfaf 100644 --- a/server/public/model/client4.go +++ b/server/public/model/client4.go @@ -1512,6 +1512,7 @@ func (c *Client4) GetUsersByIdsWithOptions(ctx context.Context, userIds []string return nil, BuildResponse(r), err } defer closeBody(r) + var list []*User if err := json.NewDecoder(r.Body).Decode(&list); err != nil { return nil, nil, NewAppError("GetUsersByIdsWithOptions", "api.unmarshal_error", nil, "", http.StatusInternalServerError).Wrap(err) @@ -3207,7 +3208,7 @@ func (c *Client4) PatchChannel(ctx context.Context, channelId string, patch *Cha var ch *Channel err = json.NewDecoder(r.Body).Decode(&ch) if err != nil { - return nil, BuildResponse(r), NewAppError("PatchChannel", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + return nil, BuildResponse(r), NewAppError("PatchChannel", "api.unmarshal_error", nil, "", http.StatusInternalServerError).Wrap(err) } return ch, BuildResponse(r), nil } @@ -8927,6 +8928,31 @@ func (c *Client4) GetUserThreads(ctx context.Context, userId, teamId string, opt return &threads, BuildResponse(r), nil } +func (c *Client4) DownloadComplianceExport(ctx context.Context, jobId string, wr io.Writer) (string, error) { + r, err := c.DoAPIGet(ctx, c.jobsRoute()+fmt.Sprintf("/%s/download", jobId), "") + if err != nil { + return "", err + } + defer closeBody(r) + + // Try to get the filename from the Content-Disposition header + var filename string + if cd := r.Header.Get("Content-Disposition"); cd != "" { + var params map[string]string + if _, params, err = mime.ParseMediaType(cd); err == nil { + if params["filename"] != "" { + filename = params["filename"] + } + } + } + + _, err = io.Copy(wr, r.Body) + if err != nil { + return filename, NewAppError("DownloadComplianceExport", "model.client.copy.app_error", nil, "", r.StatusCode).Wrap(err) + } + return filename, nil +} + func (c *Client4) GetUserThread(ctx context.Context, userId, teamId, threadId string, extended bool) (*ThreadResponse, *Response, error) { url := c.userThreadRoute(userId, teamId, threadId) if extended {