Add support for threads import from Slack (fixes #10203) (#10732)

Also add remainders to thread in long message handling and fix file handling.

app/slackimport.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -54,6 +55,7 @@ type SlackPost struct {
 	BotUsername string                   `json:"username"`
 	Text        string                   `json:"text"`
 	TimeStamp   string                   `json:"ts"`
+	ThreadTS    string                   `json:"thread_ts"`
 	Type        string                   `json:"type"`
 	SubType     string                   `json:"subtype"`
 	Comment     *SlackComment            `json:"comment"`
@@ -221,6 +223,10 @@ func (a *App) SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
 }
 
 func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
+	sort.Slice(posts, func(i, j int) bool {
+		return SlackConvertTimeStamp(posts[i].TimeStamp) < SlackConvertTimeStamp(posts[j].TimeStamp)
+	})
+	threads := make(map[string]string)
 	for _, sPost := range posts {
 		switch {
 		case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
@@ -241,16 +247,18 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack
 			if sPost.Upload {
 				if fileInfo, ok := a.SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok {
 					newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
-					newPost.Message = sPost.File.Title
 				}
 			}
-			a.OldImportPost(&newPost)
+			// If post in thread
-			for _, fileId := range newPost.FileIds {
+			if sPost.ThreadTS != "" && sPost.ThreadTS != sPost.TimeStamp {
-				if result := <-a.Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id, newPost.UserId); result.Err != nil {
+				newPost.RootId = threads[sPost.ThreadTS]
-					mlog.Error(fmt.Sprintf("Slack Import: An error occurred when attaching files to a message, post_id=%s, file_ids=%v, err=%v.", newPost.Id, newPost.FileIds, result.Err))
+				newPost.ParentId = threads[sPost.ThreadTS]
-				}
+			}
+			postId := a.OldImportPost(&newPost)
+			// If post is thread starter
+			if sPost.ThreadTS == sPost.TimeStamp {
+				threads[sPost.ThreadTS] = postId
 			}
-
 		case sPost.Type == "message" && sPost.SubType == "file_comment":
 			if sPost.Comment == nil {
 				mlog.Debug("Slack Import: Unable to import the message as it has no comments.")
@@ -295,7 +303,11 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack
 				Type:      model.POST_SLACK_ATTACHMENT,
 			}
 
-			a.OldImportIncomingWebhookPost(post, props)
+			postId := a.OldImportIncomingWebhookPost(post, props)
+			// If post is thread starter
+			if sPost.ThreadTS == sPost.TimeStamp {
+				threads[sPost.ThreadTS] = postId
+			}
 		case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
 			if sPost.User == "" {
 				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
@@ -339,7 +351,11 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack
 				Message:   "*" + sPost.Text + "*",
 				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
 			}
-			a.OldImportPost(&newPost)
+			postId := a.OldImportPost(&newPost)
+			// If post is thread starter
+			if sPost.ThreadTS == sPost.TimeStamp {
+				threads[sPost.ThreadTS] = postId
+			}
 		case sPost.Type == "message" && sPost.SubType == "channel_topic":
 			if sPost.User == "" {
 				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
@@ -700,12 +716,15 @@ func (a *App) SlackImport(fileData multipart.File, fileSize int64, teamID string
 // some of the usual checks. (IsValid is still run)
 //
 
-func (a *App) OldImportPost(post *model.Post) {
+func (a *App) OldImportPost(post *model.Post) string {
 	// Workaround for empty messages, which may be the case if they are webhook posts.
 	firstIteration := true
+	firstPostId := ""
+	if post.ParentId != "" {
+		firstPostId = post.ParentId
+	}
 	maxPostSize := a.MaxPostSize()
 	for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) {
-		firstIteration = false
 		var remainder string
 		if messageRuneCount > maxPostSize {
 			remainder = string(([]rune(post.Message))[maxPostSize:])
@@ -716,20 +735,31 @@ func (a *App) OldImportPost(post *model.Post) {
 
 		post.Hashtags, _ = model.ParseHashtags(post.Message)
 
+		post.RootId = firstPostId
+		post.ParentId = firstPostId
+
 		if result := <-a.Srv.Store.Post().Save(post); result.Err != nil {
 			mlog.Debug(fmt.Sprintf("Error saving post. user=%v, message=%v", post.UserId, post.Message))
 		}
 
-		for _, fileId := range post.FileIds {
+		if firstIteration {
-			if result := <-a.Srv.Store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); result.Err != nil {
+			if firstPostId == "" {
-				mlog.Error(fmt.Sprintf("Error attaching files to post. postId=%v, fileIds=%v, message=%v", post.Id, post.FileIds, result.Err), mlog.String("post_id", post.Id))
+				firstPostId = post.Id
 			}
+			for _, fileId := range post.FileIds {
+				if result := <-a.Srv.Store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); result.Err != nil {
+					mlog.Error(fmt.Sprintf("Error attaching files to post. postId=%v, fileIds=%v, message=%v", post.Id, post.FileIds, result.Err), mlog.String("post_id", post.Id))
+				}
+			}
+			post.FileIds = nil
 		}
 
 		post.Id = ""
 		post.CreateAt++
 		post.Message = remainder
+		firstIteration = false
 	}
+	return firstPostId
 }
 
 func (a *App) OldImportUser(team *model.Team, user *model.User) *model.User {
@@ -786,7 +816,7 @@ func (a *App) OldImportFile(timestamp time.Time, file io.Reader, teamId string,
 	return fileInfo, nil
 }
 
-func (a *App) OldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) {
+func (a *App) OldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) string {
 	linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`)
 	post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})")
 
@@ -808,5 +838,5 @@ func (a *App) OldImportIncomingWebhookPost(post *model.Post, props model.StringI
 		}
 	}
 
-	a.OldImportPost(post)
+	return a.OldImportPost(post)
 }