MM-8810: Add CSV Compliance export (#8966)

* MM-8810: Add CSV Compliance export

* Only allowing to schedule actiances export throught the cli

* De-duplicating some code

* Fixes on texts

* Fixes on translations

cmd/mattermost/commands/message_export.go

@@ -18,18 +18,45 @@ var MessageExportCmd = &cobra.Command{
 	Use:   "export",
 	Short: "Export data from Mattermost",
 	Long:  "Export data from Mattermost in a format suitable for import into a third-party application",
-	Example: "export --format=actiance --exportFrom=12345",
+}
-	RunE:    messageExportCmdF,
+
+var ScheduleExportCmd = &cobra.Command{
+	Use:     "schedule",
+	Short:   "Schedule an export data job in Mattermost",
+	Long:    "Schedule an export data job in Mattermost (this will run asynchronously via a background worker)",
+	Example: "export schedule --format=actiance --exportFrom=12345 --timeoutSeconds=12345",
+	RunE:    scheduleExportCmdF,
+}
+
+var CsvExportCmd = &cobra.Command{
+	Use:     "csv",
+	Short:   "Export data from Mattermost in CSV format",
+	Long:    "Export data from Mattermost in CSV format",
+	Example: "export csv --exportFrom=12345",
+	RunE:    buildExportCmdF("csv"),
+}
+
+var ActianceExportCmd = &cobra.Command{
+	Use:     "actiance",
+	Short:   "Export data from Mattermost in Actiance format",
+	Long:    "Export data from Mattermost in Actiance format",
+	Example: "export actiance --exportFrom=12345",
+	RunE:    buildExportCmdF("actiance"),
 }
 
 func init() {
-	MessageExportCmd.Flags().String("format", "actiance", "The format to export data in")
+	ScheduleExportCmd.Flags().String("format", "actiance", "The format to export data")
-	MessageExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
+	ScheduleExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
-	MessageExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.")
+	ScheduleExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.")
+	CsvExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
+	ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
+	MessageExportCmd.AddCommand(ScheduleExportCmd)
+	MessageExportCmd.AddCommand(CsvExportCmd)
+	MessageExportCmd.AddCommand(ActianceExportCmd)
 	RootCmd.AddCommand(MessageExportCmd)
 }
 
-func messageExportCmdF(command *cobra.Command, args []string) error {
+func scheduleExportCmdF(command *cobra.Command, args []string) error {
 	a, err := InitDBCommandContextCobra(command)
 	if err != nil {
 		return err
@@ -79,3 +106,32 @@ func messageExportCmdF(command *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func buildExportCmdF(format string) func(command *cobra.Command, args []string) error {
+	return func(command *cobra.Command, args []string) error {
+		a, err := InitDBCommandContextCobra(command)
+		if err != nil {
+			return err
+		}
+		defer a.Shutdown()
+
+		startTime, err := command.Flags().GetInt64("exportFrom")
+		if err != nil {
+			return errors.New("exportFrom flag error")
+		} else if startTime < 0 {
+			return errors.New("exportFrom must be a positive integer")
+		}
+
+		if a.MessageExport == nil {
+			CommandPrettyPrintln("MessageExport feature not available")
+		}
+
+		err2 := a.MessageExport.RunExport(format, startTime)
+		if err2 != nil {
+			return err2
+		}
+		CommandPrettyPrintln("SUCCESS: Your data was exported.")
+
+		return nil
+	}
+}

cmd/mattermost/commands/message_export_test.go

@@ -24,7 +24,7 @@ func TestMessageExportNotEnabled(t *testing.T) {
 	defer os.RemoveAll(filepath.Dir(configPath))
 
 	// should fail fast because the feature isn't enabled
-	require.Error(t, RunCommand(t, "--config", configPath, "export"))
+	require.Error(t, RunCommand(t, "--config", configPath, "export", "schedule"))
 }
 
 func TestMessageExportInvalidFormat(t *testing.T) {
@@ -32,7 +32,7 @@ func TestMessageExportInvalidFormat(t *testing.T) {
 	defer os.RemoveAll(filepath.Dir(configPath))
 
 	// should fail fast because format isn't supported
-	require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export"))
+	require.Error(t, RunCommand(t, "--config", configPath, "--format", "not_actiance", "export", "schedule"))
 }
 
 func TestMessageExportNegativeExportFrom(t *testing.T) {
@@ -40,7 +40,7 @@ func TestMessageExportNegativeExportFrom(t *testing.T) {
 	defer os.RemoveAll(filepath.Dir(configPath))
 
 	// should fail fast because export from must be a valid timestamp
-	require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export"))
+	require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export", "schedule"))
 }
 
 func TestMessageExportNegativeTimeoutSeconds(t *testing.T) {
@@ -48,7 +48,7 @@ func TestMessageExportNegativeTimeoutSeconds(t *testing.T) {
 	defer os.RemoveAll(filepath.Dir(configPath))
 
 	// should fail fast because timeout seconds must be a positive int
-	require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export"))
+	require.Error(t, RunCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export", "schedule"))
 }
 
 func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string {

cmd/mattermost/commands/sampledata.go

@@ -56,7 +56,7 @@ func sliceIncludes(vs []string, t string) bool {
 func randomPastTime(seconds int) int64 {
 	now := time.Now()
 	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.FixedZone("UTC", 0))
-	return today.Unix() - int64(rand.Intn(seconds*1000))
+	return (today.Unix() * 1000) - int64(rand.Intn(seconds*1000))
 }
 
 func randomEmoji() string {

einterfaces/message_export.go

@@ -11,4 +11,5 @@ import (
 
 type MessageExportInterface interface {
 	StartSynchronizeJob(ctx context.Context, exportFromTimestamp int64) (*model.Job, *model.AppError)
+	RunExport(format string, since int64) *model.AppError
 }

i18n/en.json

@@ -1046,11 +1046,19 @@
   },
   {
     "id": "api.file.read_file.reading_local.app_error",
-    "translation": "Encountered an error reading from local server storage"
+    "translation": "Encountered an error reading from local server file storage"
   },
   {
     "id": "api.file.read_file.s3.app_error",
-    "translation": ""
+    "translation": "Encountered an error reading from S3 storage"
+  },
+  {
+    "id": "api.file.reader.reading_local.app_error",
+    "translation": "Encountered an error opening a reader from local server file storage"
+  },
+  {
+    "id": "api.file.reader.s3.app_error",
+    "translation": "Encountered an error opening a reader from S3 storage"
   },
   {
     "id": "api.file.test_connection.local.connection.app_error",
@@ -3580,27 +3588,27 @@
   },
   {
     "id": "ent.compliance.bad_export_type.appError",
-    "translation": ""
+    "translation": "Unknown output format {{.ExportType}}"
   },
   {
     "id": "ent.compliance.csv.attachment.copy.appError",
-    "translation": ""
+    "translation": "Unable to copy the attachment into the zip file."
   },
   {
     "id": "ent.compliance.csv.attachment.export.appError",
-    "translation": ""
+    "translation": "Unable to add attachment to the CSV export."
   },
   {
     "id": "ent.compliance.csv.file.creation.appError",
-    "translation": ""
+    "translation": "Unable to create temporary CSV export file."
   },
   {
     "id": "ent.compliance.csv.header.export.appError",
-    "translation": ""
+    "translation": "Unable to add header to the CSV export."
   },
   {
     "id": "ent.compliance.csv.metadata.export.appError",
-    "translation": ""
+    "translation": "Unable to add metadata file to the zip file."
   },
   {
     "id": "ent.compliance.csv.metadata.json.marshalling.appError",
@@ -4444,7 +4452,7 @@
   },
   {
     "id": "model.config.is_valid.message_export.export_type.app_error",
-    "translation": "Message export job ExportFormat must be one of either 'actiance' or 'globalrelay'"
+    "translation": "Message export job ExportFormat must be one of 'actiance', 'csv' or 'globalrelay'"
   },
   {
     "id": "model.config.is_valid.message_export.global_relay.config_missing.app_error",

model/config.go

@@ -156,6 +156,7 @@ const (
 
 	TIMEZONE_SETTINGS_DEFAULT_SUPPORTED_TIMEZONES_PATH = "timezones.json"
 
+	COMPLIANCE_EXPORT_TYPE_CSV         = "csv"
 	COMPLIANCE_EXPORT_TYPE_ACTIANCE    = "actiance"
 	COMPLIANCE_EXPORT_TYPE_GLOBALRELAY = "globalrelay"
 	GLOBALRELAY_CUSTOMER_TYPE_A9       = "A9"
@@ -2366,7 +2367,7 @@ func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest)
 		} else if mes.BatchSize == nil || *mes.BatchSize < 0 {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest)
-		} else if mes.ExportFormat == nil || (*mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_ACTIANCE && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_GLOBALRELAY) {
+		} else if mes.ExportFormat == nil || (*mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_ACTIANCE && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_GLOBALRELAY && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_CSV) {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_type.app_error", nil, "", http.StatusBadRequest)
 		}
 

model/message_export.go

@@ -4,7 +4,12 @@
 package model
 
 type MessageExport struct {
+	TeamId          *string
+	TeamName        *string
+	TeamDisplayName *string
+
 	ChannelId          *string
+	ChannelName        *string
 	ChannelDisplayName *string
 	ChannelType        *string
 
@@ -16,5 +21,6 @@ type MessageExport struct {
 	PostCreateAt   *int64
 	PostMessage    *string
 	PostType       *string
+	PostOriginalId *string
 	PostFileIds    StringArray
 }

store/sqlstore/compliance_store.go

@@ -223,13 +223,18 @@ func (s SqlComplianceStore) MessageExport(after int64, limit int) store.StoreCha
 				Posts.CreateAt AS PostCreateAt,
 				Posts.Message AS PostMessage,
 				Posts.Type AS PostType,
+				Posts.OriginalId AS PostOriginalId,
 				Posts.FileIds AS PostFileIds,
+				Teams.Id AS TeamId,
+				Teams.Name AS TeamName,
+				Teams.DisplayName AS TeamDisplayName,
 				Channels.Id AS ChannelId,
 				CASE
 					WHEN Channels.Type = 'D' THEN 'Direct Message'
 					WHEN Channels.Type = 'G' THEN 'Group Message'
 					ELSE Channels.DisplayName
 				END AS ChannelDisplayName,
+				Channels.Name AS ChannelName,
 				Channels.Type AS ChannelType,
 				Users.Id AS UserId,
 				Users.Email AS UserEmail,
@@ -237,6 +242,7 @@ func (s SqlComplianceStore) MessageExport(after int64, limit int) store.StoreCha
 			FROM
 				Posts
 				LEFT OUTER JOIN Channels ON Posts.ChannelId = Channels.Id
+				LEFT OUTER JOIN Teams ON Channels.TeamId = Teams.Id
 				LEFT OUTER JOIN Users ON Posts.UserId = Users.Id
 			WHERE
 				Posts.CreateAt > :StartTime AND

utils/file_backend.go

@@ -13,6 +13,7 @@ import (
 type FileBackend interface {
 	TestConnection() *model.AppError
 
+	Reader(path string) (io.ReadCloser, *model.AppError)
 	ReadFile(path string) ([]byte, *model.AppError)
 	CopyFile(oldPath, newPath string) *model.AppError
 	MoveFile(oldPath, newPath string) *model.AppError

utils/file_backend_local.go

@@ -33,6 +33,14 @@ func (b *LocalFileBackend) TestConnection() *model.AppError {
 	return nil
 }
 
+func (b *LocalFileBackend) Reader(path string) (io.ReadCloser, *model.AppError) {
+	if f, err := os.Open(filepath.Join(b.directory, path)); err != nil {
+		return nil, model.NewAppError("Reader", "api.file.reader.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError)
+	} else {
+		return f, nil
+	}
+}
+
 func (b *LocalFileBackend) ReadFile(path string) ([]byte, *model.AppError) {
 	if f, err := ioutil.ReadFile(filepath.Join(b.directory, path)); err != nil {
 		return nil, model.NewAppError("ReadFile", "api.file.read_file.reading_local.app_error", nil, err.Error(), http.StatusInternalServerError)

utils/file_backend_s3.go

@@ -82,6 +82,18 @@ func (b *S3FileBackend) TestConnection() *model.AppError {
 	return nil
 }
 
+func (b *S3FileBackend) Reader(path string) (io.ReadCloser, *model.AppError) {
+	s3Clnt, err := b.s3New()
+	if err != nil {
+		return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+	minioObject, err := s3Clnt.GetObject(b.bucket, path, s3.GetObjectOptions{})
+	if err != nil {
+		return nil, model.NewAppError("Reader", "api.file.reader.s3.app_error", nil, err.Error(), http.StatusInternalServerError)
+	}
+	return minioObject, nil
+}
+
 func (b *S3FileBackend) ReadFile(path string) ([]byte, *model.AppError) {
 	s3Clnt, err := b.s3New()
 	if err != nil {