PLT-8314: Test Message Export Against S3 Bucket (#7957)

* Removed export directory config setting, in favour of hard-coding it to an 'export' directory under the local file directory. Improved the local file backend copy implementation to implicitly create the destination directory if it's missing

* Fixed the tests

model/config.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"path/filepath"
 	"strings"
 	"time"
 )
@@ -25,6 +24,10 @@ const (
 	DATABASE_DRIVER_MYSQL    = "mysql"
 	DATABASE_DRIVER_POSTGRES = "postgres"
 
+	MINIO_ACCESS_KEY = "minioaccesskey"
+	MINIO_SECRET_KEY = "miniosecretkey"
+	MINIO_BUCKET     = "mattermost-test"
+
 	PASSWORD_MAXIMUM_LENGTH = 64
 	PASSWORD_MINIMUM_LENGTH = 5
 
@@ -1523,7 +1526,6 @@ type MessageExportSettings struct {
 	EnableExport        *bool
 	DailyRunTime        *string
 	ExportFromTimestamp *int64
-	FileLocation        *string
 	BatchSize           *int
 }
 
@@ -1532,10 +1534,6 @@ func (s *MessageExportSettings) SetDefaults() {
 		s.EnableExport = NewBool(false)
 	}
 
-	if s.FileLocation == nil {
-		s.FileLocation = NewString("export")
-	}
-
 	if s.DailyRunTime == nil {
 		s.DailyRunTime = NewString("01:00")
 	}
@@ -2064,19 +2062,8 @@ func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, "", http.StatusBadRequest)
 		} else if _, err := time.Parse("15:04", *mes.DailyRunTime); err != nil {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest)
-		} else if mes.FileLocation == nil {
-			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.app_error", nil, "", http.StatusBadRequest)
 		} else if mes.BatchSize == nil || *mes.BatchSize < 0 {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest)
-		} else if *fs.DriverName != IMAGE_DRIVER_LOCAL {
-			if absFileDir, err := filepath.Abs(fs.Directory); err != nil {
-				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest)
-			} else if absMessageExportDir, err := filepath.Abs(*mes.FileLocation); err != nil {
-				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, err.Error(), http.StatusBadRequest)
-			} else if !strings.HasPrefix(absMessageExportDir, absFileDir) {
-				// configured export directory must be relative to data directory
-				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.file_location.relative", nil, "", http.StatusBadRequest)
-			}
 		}
 	}
 	return nil

model/config_test.go

@@ -6,8 +6,6 @@ package model
 import (
 	"testing"
 
-	"os"
-
 	"github.com/stretchr/testify/require"
 )
 
@@ -100,34 +98,12 @@ func TestMessageExportSettingsIsValidBatchSizeInvalid(t *testing.T) {
 		EnableExport:        NewBool(true),
 		ExportFromTimestamp: NewInt64(0),
 		DailyRunTime:        NewString("15:04"),
-		FileLocation:        NewString("foo"),
 	}
 
 	// should fail fast because batch size isn't set
 	require.Error(t, mes.isValid(*fs))
 }
 
-func TestMessageExportSettingsIsValidFileLocationInvalid(t *testing.T) {
-	fs := &FileSettings{}
-	mes := &MessageExportSettings{
-		EnableExport:        NewBool(true),
-		ExportFromTimestamp: NewInt64(0),
-		DailyRunTime:        NewString("15:04"),
-		BatchSize:           NewInt(100),
-	}
-
-	// should fail fast because FileLocation isn't set
-	require.Error(t, mes.isValid(*fs))
-
-	// if using the local file driver, there are more rules for FileLocation
-	fs.DriverName = NewString(IMAGE_DRIVER_LOCAL)
-	fs.Directory, _ = os.Getwd()
-	mes.FileLocation = NewString("")
-
-	// should fail fast because file location is not relative to basepath
-	require.Error(t, mes.isValid(*fs))
-}
-
 func TestMessageExportSettingsIsValid(t *testing.T) {
 	fs := &FileSettings{
 		DriverName: NewString("foo"), // bypass file location check
@@ -136,7 +112,6 @@ func TestMessageExportSettingsIsValid(t *testing.T) {
 		EnableExport:        NewBool(true),
 		ExportFromTimestamp: NewInt64(0),
 		DailyRunTime:        NewString("15:04"),
-		FileLocation:        NewString("foo"),
 		BatchSize:           NewInt(100),
 	}
 
@@ -149,7 +124,6 @@ func TestMessageExportSetDefaults(t *testing.T) {
 	mes.SetDefaults()
 
 	require.False(t, *mes.EnableExport)
-	require.Equal(t, "export", *mes.FileLocation)
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.Equal(t, int64(0), *mes.ExportFromTimestamp)
 	require.Equal(t, 10000, *mes.BatchSize)
@@ -162,7 +136,6 @@ func TestMessageExportSetDefaultsExportEnabledExportFromTimestampNil(t *testing.
 	mes.SetDefaults()
 
 	require.True(t, *mes.EnableExport)
-	require.Equal(t, "export", *mes.FileLocation)
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.NotEqual(t, int64(0), *mes.ExportFromTimestamp)
 	require.True(t, *mes.ExportFromTimestamp <= GetMillis())
@@ -177,7 +150,6 @@ func TestMessageExportSetDefaultsExportEnabledExportFromTimestampZero(t *testing
 	mes.SetDefaults()
 
 	require.True(t, *mes.EnableExport)
-	require.Equal(t, "export", *mes.FileLocation)
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.NotEqual(t, int64(0), *mes.ExportFromTimestamp)
 	require.True(t, *mes.ExportFromTimestamp <= GetMillis())
@@ -192,7 +164,6 @@ func TestMessageExportSetDefaultsExportEnabledExportFromTimestampNonZero(t *test
 	mes.SetDefaults()
 
 	require.True(t, *mes.EnableExport)
-	require.Equal(t, "export", *mes.FileLocation)
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.Equal(t, int64(12345), *mes.ExportFromTimestamp)
 	require.Equal(t, 10000, *mes.BatchSize)
@@ -205,7 +176,6 @@ func TestMessageExportSetDefaultsExportDisabledExportFromTimestampNil(t *testing
 	mes.SetDefaults()
 
 	require.False(t, *mes.EnableExport)
-	require.Equal(t, "export", *mes.FileLocation)
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.Equal(t, int64(0), *mes.ExportFromTimestamp)
 	require.Equal(t, 10000, *mes.BatchSize)
@@ -219,7 +189,6 @@ func TestMessageExportSetDefaultsExportDisabledExportFromTimestampZero(t *testin
 	mes.SetDefaults()
 
 	require.False(t, *mes.EnableExport)
-	require.Equal(t, "export", *mes.FileLocation)
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.Equal(t, int64(0), *mes.ExportFromTimestamp)
 	require.Equal(t, 10000, *mes.BatchSize)
@@ -233,7 +202,6 @@ func TestMessageExportSetDefaultsExportDisabledExportFromTimestampNonZero(t *tes
 	mes.SetDefaults()
 
 	require.False(t, *mes.EnableExport)
-	require.Equal(t, "export", *mes.FileLocation)
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.Equal(t, int64(0), *mes.ExportFromTimestamp)
 	require.Equal(t, 10000, *mes.BatchSize)

utils/file.go

@@ -21,6 +21,9 @@ func CopyFile(src, dst string) (err error) {
 	}
 	defer in.Close()
 
+	if err = os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
+		return
+	}
 	out, err := os.Create(dst)
 	if err != nil {
 		return

utils/file_backend_test.go

@@ -51,9 +51,9 @@ func TestS3FileBackendTestSuite(t *testing.T) {
 	suite.Run(t, &FileBackendTestSuite{
 		settings: model.FileSettings{
 			DriverName:              model.NewString(model.IMAGE_DRIVER_S3),
-			AmazonS3AccessKeyId:     "minioaccesskey",
+			AmazonS3AccessKeyId:     model.MINIO_ACCESS_KEY,
-			AmazonS3SecretAccessKey: "miniosecretkey",
+			AmazonS3SecretAccessKey: model.MINIO_SECRET_KEY,
-			AmazonS3Bucket:          "mattermost-test",
+			AmazonS3Bucket:          model.MINIO_BUCKET,
 			AmazonS3Endpoint:        s3Endpoint,
 			AmazonS3SSL:             model.NewBool(false),
 		},
@@ -106,6 +106,26 @@ func (s *FileBackendTestSuite) TestCopyFile() {
 	s.Nil(err)
 }
 
+func (s *FileBackendTestSuite) TestCopyFileToDirectoryThatDoesntExist() {
+	b := []byte("test")
+	path1 := "tests/" + model.NewId()
+	path2 := "tests/newdirectory/" + model.NewId()
+
+	err := s.backend.WriteFile(b, path1)
+	s.Nil(err)
+	defer s.backend.RemoveFile(path1)
+
+	err = s.backend.CopyFile(path1, path2)
+	s.Nil(err)
+	defer s.backend.RemoveFile(path2)
+
+	_, err = s.backend.ReadFile(path1)
+	s.Nil(err)
+
+	_, err = s.backend.ReadFile(path2)
+	s.Nil(err)
+}
+
 func (s *FileBackendTestSuite) TestMoveFile() {
 	b := []byte("test")
 	path1 := "tests/" + model.NewId()