Support attachments in post and replies - Bulk import (#9124)

* 9006 - process the attachments of the post * 9006 enabling the import of attachments in the reply post * 9006 assert if the post and files are linked * 9006 fixed the typo
2018-07-20 10:49:49 +02:00
--- a/app/import.go
+++ b/app/import.go
@@ -20,6 +20,7 @@ import (
 	"github.com/mattermost/mattermost-server/mlog"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/store"
+	"github.com/mattermost/mattermost-server/utils"
 )

 // Import Data Models
@@ -132,8 +133,9 @@ type ReplyImportData struct {
 	Message  *string `json:"message"`
 	CreateAt *int64  `json:"create_at"`

-	FlaggedBy *[]string             `json:"flagged_by"`
-	Reactions *[]ReactionImportData `json:"reactions"`
+	FlaggedBy   *[]string               `json:"flagged_by"`
+	Reactions   *[]ReactionImportData   `json:"reactions"`
+	Attachments *[]AttachmentImportData `json:"attachments"`
 }

 type PostImportData struct {
@@ -144,9 +146,10 @@ type PostImportData struct {
 	Message  *string `json:"message"`
 	CreateAt *int64  `json:"create_at"`

-	FlaggedBy *[]string             `json:"flagged_by"`
-	Reactions *[]ReactionImportData `json:"reactions"`
-	Replies   *[]ReplyImportData    `json:"replies"`
+	FlaggedBy   *[]string               `json:"flagged_by"`
+	Reactions   *[]ReactionImportData   `json:"reactions"`
+	Replies     *[]ReplyImportData      `json:"replies"`
+	Attachments *[]AttachmentImportData `json:"attachments"`
 }

 type DirectChannelImportData struct {
@@ -196,6 +199,10 @@ type LineImportWorkerError struct {
 	LineNumber int
 }

+type AttachmentImportData struct {
+	Path *string `json:"path"`
+}
+
 //
 // -- Bulk Import Functions --
 // These functions import data directly into the database. Security and permission checks are bypassed but validity is
@@ -1391,7 +1398,7 @@ func (a *App) ImportReaction(data *ReactionImportData, post *model.Post, dryRun
 	return nil
 }

-func (a *App) ImportReply(data *ReplyImportData, post *model.Post, dryRun bool) *model.AppError {
+func (a *App) ImportReply(data *ReplyImportData, post *model.Post, teamId string, dryRun bool) *model.AppError {
 	if err := validateReplyImportData(data, post.CreateAt, a.MaxPostSize()); err != nil {
 		return err
 	}
@@ -1429,6 +1436,14 @@ func (a *App) ImportReply(data *ReplyImportData, post *model.Post, dryRun bool)
 	reply.Message = *data.Message
 	reply.CreateAt = *data.CreateAt

+	if data.Attachments != nil {
+		fileIds, err := a.uploadAttachments(data.Attachments, reply, teamId, dryRun)
+		if err != nil {
+			return err
+		}
+		reply.FileIds = fileIds
+	}
+
 	if reply.Id == "" {
 		if result := <-a.Srv.Store.Post().Save(reply); result.Err != nil {
 			return result.Err
@@ -1438,9 +1453,36 @@ func (a *App) ImportReply(data *ReplyImportData, post *model.Post, dryRun bool)
 			return result.Err
 		}
 	}
+
+	a.UpdateFileInfoWithPostId(reply)
+
 	return nil
 }

+func (a *App) ImportAttachment(data *AttachmentImportData, post *model.Post, teamId string, dryRun bool) (*model.FileInfo, *model.AppError) {
+	fileUploadError := model.NewAppError("BulkImport", "app.import.attachment.file_upload.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest)
+	file, err := os.Open(*data.Path)
+	if err != nil {
+		return nil, model.NewAppError("BulkImport", "app.import.attachment.bad_file.error", map[string]interface{}{"FilePath": *data.Path}, "", http.StatusBadRequest)
+	}
+	if file != nil {
+		timestamp := utils.TimeFromMillis(post.CreateAt)
+		buf := bytes.NewBuffer(nil)
+		io.Copy(buf, file)
+
+		fileInfo, err := a.DoUploadFile(timestamp, teamId, post.ChannelId, post.UserId, file.Name(), buf.Bytes())
+
+		if err != nil {
+			fmt.Print(err)
+			return nil, fileUploadError
+		}
+
+		mlog.Info(fmt.Sprintf("uploading file with name %s", file.Name()))
+		return fileInfo, nil
+	}
+	return nil, fileUploadError
+}
+
 func (a *App) ImportPost(data *PostImportData, dryRun bool) *model.AppError {
 	if err := validatePostImportData(data, a.MaxPostSize()); err != nil {
 		return err
@@ -1499,6 +1541,14 @@ func (a *App) ImportPost(data *PostImportData, dryRun bool) *model.AppError {

 	post.Hashtags, _ = model.ParseHashtags(post.Message)

+	if data.Attachments != nil {
+		fileIds, err := a.uploadAttachments(data.Attachments, post, team.Id, dryRun)
+		if err != nil {
+			return err
+		}
+		post.FileIds = fileIds
+	}
+
 	if post.Id == "" {
 		if result := <-a.Srv.Store.Post().Save(post); result.Err != nil {
 			return result.Err
@@ -1546,15 +1596,35 @@ func (a *App) ImportPost(data *PostImportData, dryRun bool) *model.AppError {

 	if data.Replies != nil {
 		for _, reply := range *data.Replies {
-			if err := a.ImportReply(&reply, post, dryRun); err != nil {
+			if err := a.ImportReply(&reply, post, team.Id, dryRun); err != nil {
 				return err
 			}
 		}
 	}

+	a.UpdateFileInfoWithPostId(post)
 	return nil
 }

+func (a *App) uploadAttachments(attachments *[]AttachmentImportData, post *model.Post, teamId string, dryRun bool) ([]string, *model.AppError) {
+	fileIds := []string{}
+	for _, attachment := range *attachments {
+		fileInfo, err := a.ImportAttachment(&attachment, post, teamId, dryRun)
+		if err != nil {
+			return nil, err
+		}
+		fileIds = append(fileIds, fileInfo.Id)
+	}
+	return fileIds, nil
+}
+
+func (a *App) UpdateFileInfoWithPostId(post *model.Post) {
+	for _, fileId := range post.FileIds {
+		if result := <-a.Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil {
+			mlog.Error(fmt.Sprintf("Error attaching files to post. postId=%v, fileIds=%v, message=%v", post.Id, post.FileIds, result.Err), mlog.String("post_id", post.Id))
+		}
+	}
+}
 func validateReactionImportData(data *ReactionImportData, parentCreateAt int64) *model.AppError {
 	if data.User == nil {
 		return model.NewAppError("BulkImport", "app.import.validate_reaction_import_data.user_missing.error", nil, "", http.StatusBadRequest)
@@ -1869,7 +1939,7 @@ func (a *App) ImportDirectPost(data *DirectPostImportData, dryRun bool) *model.A

 	if data.Replies != nil {
 		for _, reply := range *data.Replies {
-			if err := a.ImportReply(&reply, post, dryRun); err != nil {
+			if err := a.ImportReply(&reply, post, "noteam", dryRun); err != nil {
 				return err
 			}
 		}
--- a/app/import_test.go
+++ b/app/import_test.go
@@ -3792,7 +3792,7 @@ func TestImportBulkImport(t *testing.T) {
 {"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}}
 {"type": "user", "user": {"username": "` + username2 + `", "email": "` + username2 + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}}
 {"type": "user", "user": {"username": "` + username3 + `", "email": "` + username3 + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}}
-{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012}}
+{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012, "attachments":[{"path": "` + testImage + `"}]}}
 {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username2 + `"]}}
 {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username2 + `", "` + username3 + `"]}}
 {"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username2 + `"], "user": "` + username + `", "message": "Hello Direct Channel", "create_at": 123456789013}}
@@ -3911,3 +3911,189 @@ func TestImportImportEmoji(t *testing.T) {
 	err = th.App.ImportEmoji(&data, false)
 	assert.Nil(t, err, "Second run should have succeeded apply mode")
 }
+
+func TestImportAttachment(t *testing.T) {
+	th := Setup()
+	defer th.TearDown()
+
+	testsDir, _ := utils.FindDir("tests")
+	testImage := filepath.Join(testsDir, "test.png")
+	invalidPath := "some-invalid-path"
+
+	userId := model.NewId()
+	data := AttachmentImportData{Path: &testImage}
+	_, err := th.App.ImportAttachment(&data, &model.Post{UserId: userId, ChannelId: "some-channel"}, "some-team", true)
+	assert.Nil(t, err, "sample run without errors")
+
+	attachments := GetAttachments(userId, th, t)
+	assert.Equal(t, len(attachments), 1)
+
+	data = AttachmentImportData{Path: &invalidPath}
+	_, err = th.App.ImportAttachment(&data, &model.Post{UserId: model.NewId(), ChannelId: "some-channel"}, "some-team", true)
+	assert.NotNil(t, err, "should have failed when opening the file")
+	assert.Equal(t, err.Id, "app.import.attachment.bad_file.error")
+}
+
+func TestImportPostAndRepliesWithAttachments(t *testing.T) {
+
+	th := Setup()
+	defer th.TearDown()
+
+	// Create a Team.
+	teamName := model.NewId()
+	th.App.ImportTeam(&TeamImportData{
+		Name:        &teamName,
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+	}, false)
+	team, err := th.App.GetTeamByName(teamName)
+	if err != nil {
+		t.Fatalf("Failed to get team from database.")
+	}
+
+	// Create a Channel.
+	channelName := model.NewId()
+	th.App.ImportChannel(&ChannelImportData{
+		Team:        &teamName,
+		Name:        &channelName,
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+	}, false)
+	_, err = th.App.GetChannelByName(channelName, team.Id)
+	if err != nil {
+		t.Fatalf("Failed to get channel from database.")
+	}
+
+	// Create a user3.
+	username := model.NewId()
+	th.App.ImportUser(&UserImportData{
+		Username: &username,
+		Email:    ptrStr(model.NewId() + "@example.com"),
+	}, false)
+	user3, err := th.App.GetUserByUsername(username)
+	if err != nil {
+		t.Fatalf("Failed to get user3 from database.")
+	}
+
+	username2 := model.NewId()
+	th.App.ImportUser(&UserImportData{
+		Username: &username2,
+		Email:    ptrStr(model.NewId() + "@example.com"),
+	}, false)
+	user4, err := th.App.GetUserByUsername(username2)
+	if err != nil {
+		t.Fatalf("Failed to get user3 from database.")
+	}
+
+	// Post with attachments.
+	time := model.GetMillis()
+	attachmentsPostTime := time
+	attachmentsReplyTime := time + 1
+	testsDir, _ := utils.FindDir("tests")
+	testImage := filepath.Join(testsDir, "test.png")
+	testMarkDown := filepath.Join(testsDir, "test-attachments.md")
+	data := &PostImportData{
+		Team:        &teamName,
+		Channel:     &channelName,
+		User:        &username,
+		Message:     ptrStr("Message with reply"),
+		CreateAt:    &attachmentsPostTime,
+		Attachments: &[]AttachmentImportData{{Path: &testImage}, {Path: &testMarkDown}},
+		Replies: &[]ReplyImportData{{
+			User:        &user4.Username,
+			Message:     ptrStr("Message reply"),
+			CreateAt:    &attachmentsReplyTime,
+			Attachments: &[]AttachmentImportData{{Path: &testImage}},
+		}},
+	}
+
+	if err := th.App.ImportPost(data, false); err != nil {
+		t.Fatalf("Expected success.")
+	}
+
+	attachments := GetAttachments(user3.Id, th, t)
+	assert.Equal(t, len(attachments), 2)
+	assert.Contains(t, attachments[0].Path, team.Id)
+	assert.Contains(t, attachments[1].Path, team.Id)
+	AssertFileIdsInPost(attachments, th, t)
+
+	attachments = GetAttachments(user4.Id, th, t)
+	assert.Equal(t, len(attachments), 1)
+	assert.Contains(t, attachments[0].Path, team.Id)
+	AssertFileIdsInPost(attachments, th, t)
+
+	// Reply with Attachments in Direct Post
+
+	// Create direct post users.
+
+	username3 := model.NewId()
+	th.App.ImportUser(&UserImportData{
+		Username: &username3,
+		Email:    ptrStr(model.NewId() + "@example.com"),
+	}, false)
+	user3, err = th.App.GetUserByUsername(username3)
+	if err != nil {
+		t.Fatalf("Failed to get user3 from database.")
+	}
+
+	username4 := model.NewId()
+	th.App.ImportUser(&UserImportData{
+		Username: &username4,
+		Email:    ptrStr(model.NewId() + "@example.com"),
+	}, false)
+
+	user4, err = th.App.GetUserByUsername(username4)
+	if err != nil {
+		t.Fatalf("Failed to get user3 from database.")
+	}
+
+	directImportData := &DirectPostImportData{
+		ChannelMembers: &[]string{
+			user3.Username,
+			user4.Username,
+		},
+		User:     &user3.Username,
+		Message:  ptrStr("Message with Replies"),
+		CreateAt: ptrInt64(model.GetMillis()),
+		Replies: &[]ReplyImportData{{
+			User:        &user4.Username,
+			Message:     ptrStr("Message reply with attachment"),
+			CreateAt:    ptrInt64(model.GetMillis()),
+			Attachments: &[]AttachmentImportData{{Path: &testImage}},
+		}},
+	}
+
+	if err := th.App.ImportDirectPost(directImportData, false); err != nil {
+		t.Fatalf("Expected success.")
+	}
+
+	attachments = GetAttachments(user4.Id, th, t)
+	assert.Equal(t, len(attachments), 1)
+	assert.Contains(t, attachments[0].Path, "noteam")
+	AssertFileIdsInPost(attachments, th, t)
+
+}
+
+func GetAttachments(userId string, th *TestHelper, t *testing.T) []*model.FileInfo {
+	if result := <-th.App.Srv.Store.FileInfo().GetForUser(userId); result.Err != nil {
+		t.Fatal(result.Err.Error())
+	} else {
+		return result.Data.([]*model.FileInfo)
+	}
+	return nil
+}
+
+func AssertFileIdsInPost(files []*model.FileInfo, th *TestHelper, t *testing.T) {
+	postId := files[0].PostId
+	assert.NotNil(t, postId)
+
+	if result := <-th.App.Srv.Store.Post().GetPostsByIds([]string{postId}); result.Err != nil {
+		t.Fatal(result.Err.Error())
+	} else {
+		posts := result.Data.([]*model.Post)
+		assert.Equal(t, len(posts), 1)
+		for _, file := range files {
+			assert.Contains(t, posts[0].FileIds, file.Id)
+		}
+	}
+}
--- a/i18n/en.json
+++ b/i18n/en.json
@@ -2926,6 +2926,14 @@
    "id": "app.import.validate_user_teams_import_data.team_name_missing.error",
    "translation": "Team name missing from User's Team Membership."
  },
+  {
+    "id": "app.import.attachment.bad_file.error",
+    "translation": "Error reading the file at: \"{{.FilePath}}\""
+  },
+  {
+    "id": "app.import.attachment.file_upload.error",
+    "translation": "Error uploading the file: \"{{.FilePath}}\""
+  },
  {
    "id": "app.notification.body.intro.direct.full",
    "translation": "You have a new Direct Message."