From 788c130774f064184ce50027a2de729dbde458e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 12 Aug 2020 18:29:36 +0200 Subject: [PATCH] Moving slack importer into a service (#14860) * Moving slack importer into a service * Adding missed license headers * Fixing tests * Adding license header --- app/app_iface.go | 6 - app/opentracing/opentracing_layer.go | 84 --- app/slack.go | 23 + services/slackimport/converters.go | 150 +++++ services/slackimport/main_test.go | 31 + services/slackimport/parsers.go | 49 ++ {app => services/slackimport}/slackimport.go | 541 +++++++----------- .../slackimport}/slackimport_test.go | 119 ++-- 8 files changed, 525 insertions(+), 478 deletions(-) create mode 100644 services/slackimport/converters.go create mode 100644 services/slackimport/main_test.go create mode 100644 services/slackimport/parsers.go rename {app => services/slackimport}/slackimport.go (65%) rename {app => services/slackimport}/slackimport_test.go (75%) diff --git a/app/app_iface.go b/app/app_iface.go index 7326cc1ce2..582787cb7b 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -7,7 +7,6 @@ package app import ( - "archive/zip" "bytes" "context" "crypto/ecdsa" @@ -905,12 +904,7 @@ type AppIface interface { SetTeamIconFromFile(team *model.Team, file io.Reader) *model.AppError SetTeamIconFromMultiPartFile(teamId string, file multipart.File) *model.AppError SetUserAgent(s string) - SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User - SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel - SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) - SlackAddUsers(teamId string, slackusers []SlackUser, importerLog *bytes.Buffer) map[string]*model.User SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) - SlackUploadFile(slackPostFile *SlackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) SoftDeleteTeam(teamId string) *model.AppError Srv() *Server SubmitInteractiveDialog(request model.SubmitDialogRequest) (*model.SubmitDialogResponse, *model.AppError) diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index bfd7c30707..e5693aff10 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -7,7 +7,6 @@ package opentracing import ( - "archive/zip" "bytes" "context" "crypto/ecdsa" @@ -13389,72 +13388,6 @@ func (a *OpenTracingAppLayer) SetTeamIconFromMultiPartFile(teamId string, file m return resultVar0 } -func (a *OpenTracingAppLayer) SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SlackAddBotUser") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0 := a.app.SlackAddBotUser(teamId, log) - - return resultVar0 -} - -func (a *OpenTracingAppLayer) SlackAddChannels(teamId string, slackchannels []app.SlackChannel, posts map[string][]app.SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SlackAddChannels") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0 := a.app.SlackAddChannels(teamId, slackchannels, posts, users, uploads, botUser, importerLog) - - return resultVar0 -} - -func (a *OpenTracingAppLayer) SlackAddPosts(teamId string, channel *model.Channel, posts []app.SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SlackAddPosts") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - a.app.SlackAddPosts(teamId, channel, posts, users, uploads, botUser) -} - -func (a *OpenTracingAppLayer) SlackAddUsers(teamId string, slackusers []app.SlackUser, importerLog *bytes.Buffer) map[string]*model.User { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SlackAddUsers") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0 := a.app.SlackAddUsers(teamId, slackusers, importerLog) - - return resultVar0 -} - func (a *OpenTracingAppLayer) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SlackImport") @@ -13477,23 +13410,6 @@ func (a *OpenTracingAppLayer) SlackImport(fileData multipart.File, fileSize int6 return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) SlackUploadFile(slackPostFile *app.SlackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) { - origCtx := a.ctx - span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SlackUploadFile") - - a.ctx = newCtx - a.app.Srv().Store.SetContext(newCtx) - defer func() { - a.app.Srv().Store.SetContext(origCtx) - a.ctx = origCtx - }() - - defer span.Finish() - resultVar0, resultVar1 := a.app.SlackUploadFile(slackPostFile, uploads, teamId, channelId, userId, slackTimestamp) - - return resultVar0, resultVar1 -} - func (a *OpenTracingAppLayer) SoftDeleteTeam(teamId string) *model.AppError { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SoftDeleteTeam") diff --git a/app/slack.go b/app/slack.go index d6b2d3948d..de19d75f13 100644 --- a/app/slack.go +++ b/app/slack.go @@ -4,15 +4,38 @@ package app import ( + "bytes" + "mime/multipart" "regexp" "fmt" "strings" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/slackimport" "github.com/mattermost/mattermost-server/v5/store" ) +func (a *App) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { + actions := slackimport.Actions{ + UpdateActive: a.UpdateActive, + AddUserToChannel: a.AddUserToChannel, + JoinUserToTeam: a.JoinUserToTeam, + CreateDirectChannel: a.createDirectChannel, + CreateGroupChannel: a.createGroupChannel, + CreateChannel: a.CreateChannel, + DoUploadFile: a.DoUploadFile, + GenerateThumbnailImage: a.generateThumbnailImage, + GeneratePreviewImage: a.generatePreviewImage, + InvalidateAllCaches: func() { a.srv.InvalidateAllCaches() }, + MaxPostSize: func() int { return a.srv.MaxPostSize() }, + PrepareImage: prepareImage, + } + + importer := slackimport.New(a.srv.Store, actions, a.Config()) + return importer.SlackImport(fileData, fileSize, teamID) +} + func (a *App) ProcessSlackText(text string) string { text = expandAnnouncement(text) text = replaceUserIds(a.Srv().Store.User(), text) diff --git a/services/slackimport/converters.go b/services/slackimport/converters.go new file mode 100644 index 0000000000..09f8838344 --- /dev/null +++ b/services/slackimport/converters.go @@ -0,0 +1,150 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package slackimport + +import ( + "regexp" + "strconv" + "strings" + + "github.com/mattermost/mattermost-server/v5/mlog" +) + +func slackConvertTimeStamp(ts string) int64 { + timeString := strings.SplitN(ts, ".", 2)[0] + + timeStamp, err := strconv.ParseInt(timeString, 10, 64) + if err != nil { + mlog.Warn("Slack Import: Bad timestamp detected.") + return 1 + } + return timeStamp * 1000 // Convert to milliseconds +} + +func slackConvertChannelName(channelName string, channelId string) string { + newName := strings.Trim(channelName, "_-") + if len(newName) == 1 { + return "slack-channel-" + newName + } + + if isValidChannelNameCharacters(newName) { + return newName + } + return strings.ToLower(channelId) +} + +func slackConvertUserMentions(users []slackUser, posts map[string][]slackPost) map[string][]slackPost { + var regexes = make(map[string]*regexp.Regexp, len(users)) + for _, user := range users { + r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>") + if err != nil { + mlog.Warn("Slack Import: Unable to compile the @mention, matching regular expression for the Slack user.", mlog.String("user_name", user.Username), mlog.String("user_id", user.Id)) + continue + } + regexes["@"+user.Username] = r + } + + // Special cases. + regexes["@here"], _ = regexp.Compile(``) + regexes["@channel"], _ = regexp.Compile("") + regexes["@all"], _ = regexp.Compile("") + + for channelName, channelPosts := range posts { + for postIdx, post := range channelPosts { + for mention, r := range regexes { + post.Text = r.ReplaceAllString(post.Text, mention) + posts[channelName][postIdx] = post + } + } + } + + return posts +} + +func slackConvertChannelMentions(channels []slackChannel, posts map[string][]slackPost) map[string][]slackPost { + var regexes = make(map[string]*regexp.Regexp, len(channels)) + for _, channel := range channels { + r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>") + if err != nil { + mlog.Warn("Slack Import: Unable to compile the !channel, matching regular expression for the Slack channel.", mlog.String("channel_id", channel.Id), mlog.String("channel_name", channel.Name)) + continue + } + regexes["~"+channel.Name] = r + } + + for channelName, channelPosts := range posts { + for postIdx, post := range channelPosts { + for channelReplace, r := range regexes { + post.Text = r.ReplaceAllString(post.Text, channelReplace) + posts[channelName][postIdx] = post + } + } + } + + return posts +} + +func slackConvertPostsMarkup(posts map[string][]slackPost) map[string][]slackPost { + regexReplaceAllString := []struct { + regex *regexp.Regexp + rpl string + }{ + // URL + { + regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`), + "[$2]($1)", + }, + // bold + { + regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`), + "$1**$2**", + }, + // strikethrough + { + regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`), + "$1~~$2~~", + }, + // single paragraph blockquote + // Slack converts > character to > + { + regexp.MustCompile(`(?sm)^>`), + ">", + }, + } + + regexReplaceAllStringFunc := []struct { + regex *regexp.Regexp + fn func(string) string + }{ + // multiple paragraphs blockquotes + { + regexp.MustCompile(`(?sm)^>>>(.+)$`), + func(src string) string { + // remove >>> prefix, might have leading \n + prefixRegexp := regexp.MustCompile(`^([\n])?>>>(.*)`) + src = prefixRegexp.ReplaceAllString(src, "$1$2") + // append > to start of line + appendRegexp := regexp.MustCompile(`(?m)^`) + return appendRegexp.ReplaceAllString(src, ">$0") + }, + }, + } + + for channelName, channelPosts := range posts { + for postIdx, post := range channelPosts { + result := post.Text + + for _, rule := range regexReplaceAllString { + result = rule.regex.ReplaceAllString(result, rule.rpl) + } + + for _, rule := range regexReplaceAllStringFunc { + result = rule.regex.ReplaceAllStringFunc(result, rule.fn) + } + posts[channelName][postIdx].Text = result + } + } + + return posts +} diff --git a/services/slackimport/main_test.go b/services/slackimport/main_test.go new file mode 100644 index 0000000000..28e21ea7ea --- /dev/null +++ b/services/slackimport/main_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package slackimport + +import ( + "fmt" + "os" + "testing" +) + +func TestMain(m *testing.M) { + prevDir, err := os.Getwd() + if err != nil { + panic("Failed to get current working directory: " + err.Error()) + } + + err = os.Chdir("../..") + if err != nil { + panic(fmt.Sprintf("Failed to set current working directory to %s: %s", "../..", err.Error())) + } + + defer func() { + err := os.Chdir(prevDir) + if err != nil { + panic(fmt.Sprintf("Failed to restore current working directory to %s: %s", prevDir, err.Error())) + } + }() + + m.Run() +} diff --git a/services/slackimport/parsers.go b/services/slackimport/parsers.go new file mode 100644 index 0000000000..a21dd26614 --- /dev/null +++ b/services/slackimport/parsers.go @@ -0,0 +1,49 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package slackimport + +import ( + "encoding/json" + "io" + + "github.com/mattermost/mattermost-server/v5/mlog" +) + +func slackParseChannels(data io.Reader, channelType string) ([]slackChannel, error) { + decoder := json.NewDecoder(data) + + var channels []slackChannel + if err := decoder.Decode(&channels); err != nil { + mlog.Warn("Slack Import: Error occurred when parsing some Slack channels. Import may work anyway.") + return channels, err + } + + for i := range channels { + channels[i].Type = channelType + } + + return channels, nil +} + +func slackParseUsers(data io.Reader) ([]slackUser, error) { + decoder := json.NewDecoder(data) + + var users []slackUser + err := decoder.Decode(&users) + // This actually returns errors that are ignored. + // In this case it is erroring because of a null that Slack + // introduced. So we just return the users here. + return users, err +} + +func slackParsePosts(data io.Reader) ([]slackPost, error) { + decoder := json.NewDecoder(data) + + var posts []slackPost + if err := decoder.Decode(&posts); err != nil { + mlog.Warn("Slack Import: Error occurred when parsing some Slack posts. Import may work anyway.") + return posts, err + } + return posts, nil +} diff --git a/app/slackimport.go b/services/slackimport/slackimport.go similarity index 65% rename from app/slackimport.go rename to services/slackimport/slackimport.go index eed160cba8..a454bab20c 100644 --- a/app/slackimport.go +++ b/services/slackimport/slackimport.go @@ -1,60 +1,60 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package app +package slackimport import ( "archive/zip" "bytes" - "encoding/json" + "image" "io" "mime/multipart" "net/http" "path/filepath" "regexp" "sort" - "strconv" "strings" "time" "unicode/utf8" "github.com/mattermost/mattermost-server/v5/mlog" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/store" "github.com/mattermost/mattermost-server/v5/utils" ) -type SlackChannel struct { +type slackChannel struct { Id string `json:"id"` Name string `json:"name"` Creator string `json:"creator"` Members []string `json:"members"` - Purpose SlackChannelSub `json:"purpose"` - Topic SlackChannelSub `json:"topic"` + Purpose slackChannelSub `json:"purpose"` + Topic slackChannelSub `json:"topic"` Type string } -type SlackChannelSub struct { +type slackChannelSub struct { Value string `json:"value"` } -type SlackProfile struct { +type slackProfile struct { FirstName string `json:"first_name"` LastName string `json:"last_name"` Email string `json:"email"` } -type SlackUser struct { +type slackUser struct { Id string `json:"id"` Username string `json:"name"` - Profile SlackProfile `json:"profile"` + Profile slackProfile `json:"profile"` } -type SlackFile struct { +type slackFile struct { Id string `json:"id"` Title string `json:"title"` } -type SlackPost struct { +type slackPost struct { User string `json:"user"` BotId string `json:"bot_id"` BotUsername string `json:"username"` @@ -63,22 +63,139 @@ type SlackPost struct { ThreadTS string `json:"thread_ts"` Type string `json:"type"` SubType string `json:"subtype"` - Comment *SlackComment `json:"comment"` + Comment *slackComment `json:"comment"` Upload bool `json:"upload"` - File *SlackFile `json:"file"` - Files []*SlackFile `json:"files"` + File *slackFile `json:"file"` + Files []*slackFile `json:"files"` Attachments []*model.SlackAttachment `json:"attachments"` } var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString -const SLACK_IMPORT_MAX_FILE_SIZE = 1024 * 1024 * 70 +const slackImportMaxFileSize = 1024 * 1024 * 70 -type SlackComment struct { +type slackComment struct { User string `json:"user"` Comment string `json:"comment"` } +// Actions provides the actions that needs to be used for import slack data +type Actions struct { + UpdateActive func(*model.User, bool) (*model.User, *model.AppError) + AddUserToChannel func(*model.User, *model.Channel) (*model.ChannelMember, *model.AppError) + JoinUserToTeam func(*model.Team, *model.User, string) *model.AppError + CreateDirectChannel func(string, string) (*model.Channel, *model.AppError) + CreateGroupChannel func([]string, string) (*model.Channel, *model.AppError) + CreateChannel func(*model.Channel, bool) (*model.Channel, *model.AppError) + DoUploadFile func(time.Time, string, string, string, string, []byte) (*model.FileInfo, *model.AppError) + GenerateThumbnailImage func(image.Image, string, int, int) + GeneratePreviewImage func(image.Image, string, int) + InvalidateAllCaches func() + MaxPostSize func() int + PrepareImage func(fileData []byte) (image.Image, int, int) +} + +// SlackImporter is a service that allows to import slack dumps into mattermost +type SlackImporter struct { + store store.Store + actions Actions + config *model.Config +} + +// New creates a new SlackImporter service instance. It receive a store, a set of actions and the current config. +// It is expected to be used right away and discarded after that +func New(store store.Store, actions Actions, config *model.Config) *SlackImporter { + return &SlackImporter{ + store: store, + actions: actions, + config: config, + } +} + +func (si *SlackImporter) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { + // Create log file + log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log")) + + zipreader, err := zip.NewReader(fileData, fileSize) + if err != nil || zipreader.File == nil { + log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error")) + return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error(), http.StatusBadRequest), log + } + + var channels []slackChannel + var publicChannels []slackChannel + var privateChannels []slackChannel + var groupChannels []slackChannel + var directChannels []slackChannel + + var users []slackUser + posts := make(map[string][]slackPost) + uploads := make(map[string]*zip.File) + for _, file := range zipreader.File { + if file.UncompressedSize64 > slackImportMaxFileSize { + log.WriteString(utils.T("api.slackimport.slack_import.zip.file_too_large", map[string]interface{}{"Filename": file.Name})) + continue + } + reader, err := file.Open() + if err != nil { + log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name})) + return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error(), http.StatusInternalServerError), log + } + if file.Name == "channels.json" { + publicChannels, _ = slackParseChannels(reader, model.CHANNEL_OPEN) + channels = append(channels, publicChannels...) + } else if file.Name == "dms.json" { + directChannels, _ = slackParseChannels(reader, model.CHANNEL_DIRECT) + channels = append(channels, directChannels...) + } else if file.Name == "groups.json" { + privateChannels, _ = slackParseChannels(reader, model.CHANNEL_PRIVATE) + channels = append(channels, privateChannels...) + } else if file.Name == "mpims.json" { + groupChannels, _ = slackParseChannels(reader, model.CHANNEL_GROUP) + channels = append(channels, groupChannels...) + } else if file.Name == "users.json" { + users, _ = slackParseUsers(reader) + } else { + spl := strings.Split(file.Name, "/") + if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { + newposts, _ := slackParsePosts(reader) + channel := spl[0] + if _, ok := posts[channel]; !ok { + posts[channel] = newposts + } else { + posts[channel] = append(posts[channel], newposts...) + } + } else if len(spl) == 3 && spl[0] == "__uploads" { + uploads[spl[1]] = file + } + } + } + + posts = slackConvertUserMentions(users, posts) + posts = slackConvertChannelMentions(channels, posts) + posts = slackConvertPostsMarkup(posts) + + addedUsers := si.slackAddUsers(teamID, users, log) + botUser := si.slackAddBotUser(teamID, log) + + si.slackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log) + + if botUser != nil { + si.deactivateSlackBotUser(botUser) + } + + si.actions.InvalidateAllCaches() + + log.WriteString(utils.T("api.slackimport.slack_import.notes")) + log.WriteString("=======\r\n\r\n") + + log.WriteString(utils.T("api.slackimport.slack_import.note1")) + log.WriteString(utils.T("api.slackimport.slack_import.note2")) + log.WriteString(utils.T("api.slackimport.slack_import.note3")) + + return nil, log +} + func truncateRunes(s string, i int) string { runes := []rune(s) if len(runes) > i { @@ -87,68 +204,7 @@ func truncateRunes(s string, i int) string { return s } -func SlackConvertTimeStamp(ts string) int64 { - timeString := strings.SplitN(ts, ".", 2)[0] - - timeStamp, err := strconv.ParseInt(timeString, 10, 64) - if err != nil { - mlog.Warn("Slack Import: Bad timestamp detected.") - return 1 - } - return timeStamp * 1000 // Convert to milliseconds -} - -func SlackConvertChannelName(channelName string, channelId string) string { - newName := strings.Trim(channelName, "_-") - if len(newName) == 1 { - return "slack-channel-" + newName - } - - if isValidChannelNameCharacters(newName) { - return newName - } - return strings.ToLower(channelId) -} - -func SlackParseChannels(data io.Reader, channelType string) ([]SlackChannel, error) { - decoder := json.NewDecoder(data) - - var channels []SlackChannel - if err := decoder.Decode(&channels); err != nil { - mlog.Warn("Slack Import: Error occurred when parsing some Slack channels. Import may work anyway.") - return channels, err - } - - for i := range channels { - channels[i].Type = channelType - } - - return channels, nil -} - -func SlackParseUsers(data io.Reader) ([]SlackUser, error) { - decoder := json.NewDecoder(data) - - var users []SlackUser - err := decoder.Decode(&users) - // This actually returns errors that are ignored. - // In this case it is erroring because of a null that Slack - // introduced. So we just return the users here. - return users, err -} - -func SlackParsePosts(data io.Reader) ([]SlackPost, error) { - decoder := json.NewDecoder(data) - - var posts []SlackPost - if err := decoder.Decode(&posts); err != nil { - mlog.Warn("Slack Import: Error occurred when parsing some Slack posts. Import may work anyway.") - return posts, err - } - return posts, nil -} - -func (a *App) SlackAddUsers(teamId string, slackusers []SlackUser, importerLog *bytes.Buffer) map[string]*model.User { +func (si *SlackImporter) slackAddUsers(teamId string, slackusers []slackUser, importerLog *bytes.Buffer) map[string]*model.User { // Log header importerLog.WriteString(utils.T("api.slackimport.slack_add_users.created")) importerLog.WriteString("===============\r\n\r\n") @@ -156,7 +212,7 @@ func (a *App) SlackAddUsers(teamId string, slackusers []SlackUser, importerLog * addedUsers := make(map[string]*model.User) // Need the team - team, err := a.Srv().Store.Team().Get(teamId) + team, err := si.store.Team().Get(teamId) if err != nil { importerLog.WriteString(utils.T("api.slackimport.slack_import.team_fail")) return addedUsers @@ -175,9 +231,9 @@ func (a *App) SlackAddUsers(teamId string, slackusers []SlackUser, importerLog * password := model.NewId() // Check for email conflict and use existing user if found - if existingUser, err := a.Srv().Store.User().GetByEmail(email); err == nil { + if existingUser, err := si.store.User().GetByEmail(email); err == nil { addedUsers[sUser.Id] = existingUser - if err := a.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil { + if err := si.actions.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil { importerLog.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) } else { importerLog.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) @@ -194,7 +250,7 @@ func (a *App) SlackAddUsers(teamId string, slackusers []SlackUser, importerLog * Password: password, } - mUser := a.oldImportUser(team, &newUser) + mUser := si.oldImportUser(team, &newUser) if mUser == nil { importerLog.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username})) continue @@ -206,8 +262,8 @@ func (a *App) SlackAddUsers(teamId string, slackusers []SlackUser, importerLog * return addedUsers } -func (a *App) SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User { - team, err := a.Srv().Store.Team().Get(teamId) +func (si *SlackImporter) slackAddBotUser(teamId string, log *bytes.Buffer) *model.User { + team, err := si.store.Team().Get(teamId) if err != nil { log.WriteString(utils.T("api.slackimport.slack_import.team_fail")) return nil @@ -225,7 +281,7 @@ func (a *App) SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User { Password: password, } - mUser := a.oldImportUser(team, &botUser) + mUser := si.oldImportUser(team, &botUser) if mUser == nil { log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username})) return nil @@ -235,9 +291,9 @@ func (a *App) SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User { return mUser } -func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { +func (si *SlackImporter) slackAddPosts(teamId string, channel *model.Channel, posts []slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { sort.Slice(posts, func(i, j int) bool { - return SlackConvertTimeStamp(posts[i].TimeStamp) < SlackConvertTimeStamp(posts[j].TimeStamp) + return slackConvertTimeStamp(posts[i].TimeStamp) < slackConvertTimeStamp(posts[j].TimeStamp) }) threads := make(map[string]string) for _, sPost := range posts { @@ -255,16 +311,16 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack UserId: users[sPost.User].Id, ChannelId: channel.Id, Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + CreateAt: slackConvertTimeStamp(sPost.TimeStamp), } if sPost.Upload { if sPost.File != nil { - if fileInfo, ok := a.SlackUploadFile(sPost.File, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { + if fileInfo, ok := si.slackUploadFile(sPost.File, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { newPost.FileIds = append(newPost.FileIds, fileInfo.Id) } } else if sPost.Files != nil { for _, file := range sPost.Files { - if fileInfo, ok := a.SlackUploadFile(file, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { + if fileInfo, ok := si.slackUploadFile(file, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { newPost.FileIds = append(newPost.FileIds, fileInfo.Id) } } @@ -275,7 +331,7 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack newPost.RootId = threads[sPost.ThreadTS] newPost.ParentId = threads[sPost.ThreadTS] } - postId := a.oldImportPost(&newPost) + postId := si.oldImportPost(&newPost) // If post is thread starter if sPost.ThreadTS == sPost.TimeStamp { threads[sPost.ThreadTS] = postId @@ -297,9 +353,9 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack UserId: users[sPost.Comment.User].Id, ChannelId: channel.Id, Message: sPost.Comment.Comment, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + CreateAt: slackConvertTimeStamp(sPost.TimeStamp), } - a.oldImportPost(&newPost) + si.oldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "bot_message": if botUser == nil { mlog.Warn("Slack Import: Unable to import the bot message as the bot user does not exist.") @@ -319,12 +375,12 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack post := &model.Post{ UserId: botUser.Id, ChannelId: channel.Id, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + CreateAt: slackConvertTimeStamp(sPost.TimeStamp), Message: sPost.Text, Type: model.POST_SLACK_ATTACHMENT, } - postId := a.oldImportIncomingWebhookPost(post, props) + postId := si.oldImportIncomingWebhookPost(post, props) // If post is thread starter if sPost.ThreadTS == sPost.TimeStamp { threads[sPost.ThreadTS] = postId @@ -350,13 +406,13 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack UserId: users[sPost.User].Id, ChannelId: channel.Id, Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + CreateAt: slackConvertTimeStamp(sPost.TimeStamp), Type: postType, Props: model.StringInterface{ "username": users[sPost.User].Username, }, } - a.oldImportPost(&newPost) + si.oldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "me_message": if sPost.User == "" { mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") @@ -370,9 +426,9 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack UserId: users[sPost.User].Id, ChannelId: channel.Id, Message: "*" + sPost.Text + "*", - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + CreateAt: slackConvertTimeStamp(sPost.TimeStamp), } - postId := a.oldImportPost(&newPost) + postId := si.oldImportPost(&newPost) // If post is thread starter if sPost.ThreadTS == sPost.TimeStamp { threads[sPost.ThreadTS] = postId @@ -390,10 +446,10 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack UserId: users[sPost.User].Id, ChannelId: channel.Id, Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + CreateAt: slackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_HEADER_CHANGE, } - a.oldImportPost(&newPost) + si.oldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "channel_purpose": if sPost.User == "" { mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") @@ -407,10 +463,10 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack UserId: users[sPost.User].Id, ChannelId: channel.Id, Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + CreateAt: slackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_PURPOSE_CHANGE, } - a.oldImportPost(&newPost) + si.oldImportPost(&newPost) case sPost.Type == "message" && sPost.SubType == "channel_name": if sPost.User == "" { mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") @@ -424,10 +480,10 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack UserId: users[sPost.User].Id, ChannelId: channel.Id, Message: sPost.Text, - CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), + CreateAt: slackConvertTimeStamp(sPost.TimeStamp), Type: model.POST_DISPLAYNAME_CHANGE, } - a.oldImportPost(&newPost) + si.oldImportPost(&newPost) default: mlog.Warn( "Slack Import: Unable to import the message as its type is not supported", @@ -438,7 +494,7 @@ func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []Slack } } -func (a *App) SlackUploadFile(slackPostFile *SlackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) { +func (si *SlackImporter) slackUploadFile(slackPostFile *slackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) { if slackPostFile == nil { mlog.Warn("Slack Import: Unable to attach the file to the post as the latter has no file section present in Slack export.") return nil, false @@ -455,8 +511,8 @@ func (a *App) SlackUploadFile(slackPostFile *SlackFile, uploads map[string]*zip. } defer openFile.Close() - timestamp := utils.TimeFromMillis(SlackConvertTimeStamp(slackTimestamp)) - uploadedFile, err := a.oldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name)) + timestamp := utils.TimeFromMillis(slackConvertTimeStamp(slackTimestamp)) + uploadedFile, err := si.oldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name)) if err != nil { mlog.Warn("Slack Import: An error occurred when uploading file.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err)) return nil, false @@ -465,26 +521,26 @@ func (a *App) SlackUploadFile(slackPostFile *SlackFile, uploads map[string]*zip. return uploadedFile, true } -func (a *App) deactivateSlackBotUser(user *model.User) { - if _, err := a.UpdateActive(user, false); err != nil { +func (si *SlackImporter) deactivateSlackBotUser(user *model.User) { + if _, err := si.actions.UpdateActive(user, false); err != nil { mlog.Warn("Slack Import: Unable to deactivate the user account used for the bot.") } } -func (a *App) addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { +func (si *SlackImporter) addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { for _, member := range members { user, ok := users[member] if !ok { log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"})) continue } - if _, err := a.AddUserToChannel(user, channel); err != nil { + if _, err := si.actions.AddUserToChannel(user, channel); err != nil { log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username})) } } } -func SlackSanitiseChannelProperties(channel model.Channel) model.Channel { +func slackSanitiseChannelProperties(channel model.Channel) model.Channel { if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES { mlog.Warn("Slack Import: Channel display name exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES) @@ -508,7 +564,7 @@ func SlackSanitiseChannelProperties(channel model.Channel) model.Channel { return channel } -func (a *App) SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel { +func (si *SlackImporter) slackAddChannels(teamId string, slackchannels []slackChannel, posts map[string][]slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel { // Write Header importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.added")) importerLog.WriteString("=================\r\n\r\n") @@ -519,7 +575,7 @@ func (a *App) SlackAddChannels(teamId string, slackchannels []SlackChannel, post TeamId: teamId, Type: sChannel.Type, DisplayName: sChannel.Name, - Name: SlackConvertChannelName(sChannel.Name, sChannel.Id), + Name: slackConvertChannelName(sChannel.Name, sChannel.Id), Purpose: sChannel.Purpose.Value, Header: sChannel.Topic.Value, } @@ -529,22 +585,22 @@ func (a *App) SlackAddChannels(teamId string, slackchannels []SlackChannel, post sChannel.Name = sChannel.Id } - newChannel = SlackSanitiseChannelProperties(newChannel) + newChannel = slackSanitiseChannelProperties(newChannel) var mChannel *model.Channel var err error - if mChannel, err = a.Srv().Store.Channel().GetByName(teamId, sChannel.Name, true); err == nil { + if mChannel, err = si.store.Channel().GetByName(teamId, sChannel.Name, true); err == nil { // The channel already exists as an active channel. Merge with the existing one. importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName})) - } else if _, nErr := a.Srv().Store.Channel().GetDeletedByName(teamId, sChannel.Name); nErr == nil { + } else if _, nErr := si.store.Channel().GetDeletedByName(teamId, sChannel.Name); nErr == nil { // The channel already exists but has been deleted. Generate a random string for the handle instead. newChannel.Name = model.NewId() - newChannel = SlackSanitiseChannelProperties(newChannel) + newChannel = slackSanitiseChannelProperties(newChannel) } if mChannel == nil { // Haven't found an existing channel to merge with. Try importing it as a new one. - mChannel = a.oldImportChannel(&newChannel, sChannel, users) + mChannel = si.oldImportChannel(&newChannel, sChannel, users) if mChannel == nil { mlog.Warn("Slack Import: Unable to import Slack channel.", mlog.String("channel_display_name", newChannel.DisplayName)) importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName})) @@ -554,229 +610,30 @@ func (a *App) SlackAddChannels(teamId string, slackchannels []SlackChannel, post // Members for direct and group channels are added during the creation of the channel in the oldImportChannel function if sChannel.Type == model.CHANNEL_OPEN || sChannel.Type == model.CHANNEL_PRIVATE { - a.addSlackUsersToChannel(sChannel.Members, users, mChannel, importerLog) + si.addSlackUsersToChannel(sChannel.Members, users, mChannel, importerLog) } importerLog.WriteString(newChannel.DisplayName + "\r\n") addedChannels[sChannel.Id] = mChannel - a.SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser) + si.slackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser) } return addedChannels } -func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost { - var regexes = make(map[string]*regexp.Regexp, len(users)) - for _, user := range users { - r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>") - if err != nil { - mlog.Warn("Slack Import: Unable to compile the @mention, matching regular expression for the Slack user.", mlog.String("user_name", user.Username), mlog.String("user_id", user.Id)) - continue - } - regexes["@"+user.Username] = r - } - - // Special cases. - regexes["@here"], _ = regexp.Compile(``) - regexes["@channel"], _ = regexp.Compile("") - regexes["@all"], _ = regexp.Compile("") - - for channelName, channelPosts := range posts { - for postIdx, post := range channelPosts { - for mention, r := range regexes { - post.Text = r.ReplaceAllString(post.Text, mention) - posts[channelName][postIdx] = post - } - } - } - - return posts -} - -func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost { - var regexes = make(map[string]*regexp.Regexp, len(channels)) - for _, channel := range channels { - r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>") - if err != nil { - mlog.Warn("Slack Import: Unable to compile the !channel, matching regular expression for the Slack channel.", mlog.String("channel_id", channel.Id), mlog.String("channel_name", channel.Name)) - continue - } - regexes["~"+channel.Name] = r - } - - for channelName, channelPosts := range posts { - for postIdx, post := range channelPosts { - for channelReplace, r := range regexes { - post.Text = r.ReplaceAllString(post.Text, channelReplace) - posts[channelName][postIdx] = post - } - } - } - - return posts -} - -func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost { - regexReplaceAllString := []struct { - regex *regexp.Regexp - rpl string - }{ - // URL - { - regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`), - "[$2]($1)", - }, - // bold - { - regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`), - "$1**$2**", - }, - // strikethrough - { - regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`), - "$1~~$2~~", - }, - // single paragraph blockquote - // Slack converts > character to > - { - regexp.MustCompile(`(?sm)^>`), - ">", - }, - } - - regexReplaceAllStringFunc := []struct { - regex *regexp.Regexp - fn func(string) string - }{ - // multiple paragraphs blockquotes - { - regexp.MustCompile(`(?sm)^>>>(.+)$`), - func(src string) string { - // remove >>> prefix, might have leading \n - prefixRegexp := regexp.MustCompile(`^([\n])?>>>(.*)`) - src = prefixRegexp.ReplaceAllString(src, "$1$2") - // append > to start of line - appendRegexp := regexp.MustCompile(`(?m)^`) - return appendRegexp.ReplaceAllString(src, ">$0") - }, - }, - } - - for channelName, channelPosts := range posts { - for postIdx, post := range channelPosts { - result := post.Text - - for _, rule := range regexReplaceAllString { - result = rule.regex.ReplaceAllString(result, rule.rpl) - } - - for _, rule := range regexReplaceAllStringFunc { - result = rule.regex.ReplaceAllStringFunc(result, rule.fn) - } - posts[channelName][postIdx].Text = result - } - } - - return posts -} - -func (a *App) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { - // Create log file - log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log")) - - zipreader, err := zip.NewReader(fileData, fileSize) - if err != nil || zipreader.File == nil { - log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error")) - return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error(), http.StatusBadRequest), log - } - - var channels []SlackChannel - var publicChannels []SlackChannel - var privateChannels []SlackChannel - var groupChannels []SlackChannel - var directChannels []SlackChannel - - var users []SlackUser - posts := make(map[string][]SlackPost) - uploads := make(map[string]*zip.File) - for _, file := range zipreader.File { - if file.UncompressedSize64 > SLACK_IMPORT_MAX_FILE_SIZE { - log.WriteString(utils.T("api.slackimport.slack_import.zip.file_too_large", map[string]interface{}{"Filename": file.Name})) - continue - } - reader, err := file.Open() - if err != nil { - log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name})) - return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error(), http.StatusInternalServerError), log - } - if file.Name == "channels.json" { - publicChannels, _ = SlackParseChannels(reader, model.CHANNEL_OPEN) - channels = append(channels, publicChannels...) - } else if file.Name == "dms.json" { - directChannels, _ = SlackParseChannels(reader, model.CHANNEL_DIRECT) - channels = append(channels, directChannels...) - } else if file.Name == "groups.json" { - privateChannels, _ = SlackParseChannels(reader, model.CHANNEL_PRIVATE) - channels = append(channels, privateChannels...) - } else if file.Name == "mpims.json" { - groupChannels, _ = SlackParseChannels(reader, model.CHANNEL_GROUP) - channels = append(channels, groupChannels...) - } else if file.Name == "users.json" { - users, _ = SlackParseUsers(reader) - } else { - spl := strings.Split(file.Name, "/") - if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { - newposts, _ := SlackParsePosts(reader) - channel := spl[0] - if _, ok := posts[channel]; !ok { - posts[channel] = newposts - } else { - posts[channel] = append(posts[channel], newposts...) - } - } else if len(spl) == 3 && spl[0] == "__uploads" { - uploads[spl[1]] = file - } - } - } - - posts = SlackConvertUserMentions(users, posts) - posts = SlackConvertChannelMentions(channels, posts) - posts = SlackConvertPostsMarkup(posts) - - addedUsers := a.SlackAddUsers(teamID, users, log) - botUser := a.SlackAddBotUser(teamID, log) - - a.SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log) - - if botUser != nil { - a.deactivateSlackBotUser(botUser) - } - - a.Srv().InvalidateAllCaches() - - log.WriteString(utils.T("api.slackimport.slack_import.notes")) - log.WriteString("=======\r\n\r\n") - - log.WriteString(utils.T("api.slackimport.slack_import.note1")) - log.WriteString(utils.T("api.slackimport.slack_import.note2")) - log.WriteString(utils.T("api.slackimport.slack_import.note3")) - - return nil, log -} - // // -- Old SlackImport Functions -- // Import functions are suitable for entering posts and users into the database without // some of the usual checks. (IsValid is still run) // -func (a *App) oldImportPost(post *model.Post) string { +func (si *SlackImporter) oldImportPost(post *model.Post) string { // Workaround for empty messages, which may be the case if they are webhook posts. firstIteration := true firstPostId := "" if post.ParentId != "" { firstPostId = post.ParentId } - maxPostSize := a.MaxPostSize() + maxPostSize := si.actions.MaxPostSize() for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { var remainder string if messageRuneCount > maxPostSize { @@ -791,7 +648,7 @@ func (a *App) oldImportPost(post *model.Post) string { post.RootId = firstPostId post.ParentId = firstPostId - _, err := a.Srv().Store.Post().Save(post) + _, err := si.store.Post().Save(post) if err != nil { mlog.Debug("Error saving post.", mlog.String("user_id", post.UserId), mlog.String("message", post.Message)) } @@ -801,7 +658,7 @@ func (a *App) oldImportPost(post *model.Post) string { firstPostId = post.Id } for _, fileId := range post.FileIds { - if err := a.Srv().Store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); err != nil { + if err := si.store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); err != nil { mlog.Error( "Error attaching files to post.", mlog.String("post_id", post.Id), @@ -822,29 +679,29 @@ func (a *App) oldImportPost(post *model.Post) string { return firstPostId } -func (a *App) oldImportUser(team *model.Team, user *model.User) *model.User { +func (si *SlackImporter) oldImportUser(team *model.Team, user *model.User) *model.User { user.MakeNonNil() user.Roles = model.SYSTEM_USER_ROLE_ID - ruser, err := a.Srv().Store.User().Save(user) + ruser, err := si.store.User().Save(user) if err != nil { mlog.Error("Error saving user.", mlog.Err(err)) return nil } - if _, err = a.Srv().Store.User().VerifyEmail(ruser.Id, ruser.Email); err != nil { + if _, err = si.store.User().VerifyEmail(ruser.Id, ruser.Email); err != nil { mlog.Error("Failed to set email verified.", mlog.Err(err)) } - if err = a.JoinUserToTeam(team, user, ""); err != nil { + if err = si.actions.JoinUserToTeam(team, user, ""); err != nil { mlog.Error("Failed to join team when importing.", mlog.Err(err)) } return ruser } -func (a *App) oldImportChannel(channel *model.Channel, sChannel SlackChannel, users map[string]*model.User) *model.Channel { +func (si *SlackImporter) oldImportChannel(channel *model.Channel, sChannel slackChannel, users map[string]*model.User) *model.Channel { switch { case channel.Type == model.CHANNEL_DIRECT: if len(sChannel.Members) < 2 { @@ -856,7 +713,7 @@ func (a *App) oldImportChannel(channel *model.Channel, sChannel SlackChannel, us mlog.Warn("Either or both of user ids not found in users.json. Ignoring.", mlog.String("id1", sChannel.Members[0]), mlog.String("id2", sChannel.Members[1])) return nil } - sc, err := a.createDirectChannel(u1.Id, u2.Id) + sc, err := si.actions.CreateDirectChannel(u1.Id, u2.Id) if err != nil { return nil } @@ -879,7 +736,7 @@ func (a *App) oldImportChannel(channel *model.Channel, sChannel SlackChannel, us if creator == nil { return nil } - sc, err := a.createGroupChannel(members, creator.Id) + sc, err := si.actions.CreateGroupChannel(members, creator.Id) if err != nil { return nil } @@ -887,7 +744,7 @@ func (a *App) oldImportChannel(channel *model.Channel, sChannel SlackChannel, us return sc case channel.Type == model.CHANNEL_GROUP: channel.Type = model.CHANNEL_PRIVATE - sc, err := a.CreateChannel(channel, false) + sc, err := si.actions.CreateChannel(channel, false) if err != nil { return nil } @@ -895,7 +752,7 @@ func (a *App) oldImportChannel(channel *model.Channel, sChannel SlackChannel, us return sc } - sc, err := a.Srv().Store.Channel().Save(channel, *a.Config().TeamSettings.MaxChannelsPerTeam) + sc, err := si.store.Channel().Save(channel, *si.config.TeamSettings.MaxChannelsPerTeam) if err != nil { return nil } @@ -903,28 +760,28 @@ func (a *App) oldImportChannel(channel *model.Channel, sChannel SlackChannel, us return sc } -func (a *App) oldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { +func (si *SlackImporter) oldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { buf := bytes.NewBuffer(nil) io.Copy(buf, file) data := buf.Bytes() - fileInfo, err := a.DoUploadFile(timestamp, teamId, channelId, userId, fileName, data) + fileInfo, err := si.actions.DoUploadFile(timestamp, teamId, channelId, userId, fileName, data) if err != nil { return nil, err } if fileInfo.IsImage() && fileInfo.MimeType != "image/svg+xml" { - img, width, height := prepareImage(data) + img, width, height := si.actions.PrepareImage(data) if img != nil { - a.generateThumbnailImage(img, fileInfo.ThumbnailPath, width, height) - a.generatePreviewImage(img, fileInfo.PreviewPath, width) + si.actions.GenerateThumbnailImage(img, fileInfo.ThumbnailPath, width, height) + si.actions.GeneratePreviewImage(img, fileInfo.PreviewPath, width) } } return fileInfo, nil } -func (a *App) oldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) string { +func (si *SlackImporter) oldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) string { linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") @@ -946,5 +803,5 @@ func (a *App) oldImportIncomingWebhookPost(post *model.Post, props model.StringI } } - return a.oldImportPost(post) + return si.oldImportPost(post) } diff --git a/app/slackimport_test.go b/services/slackimport/slackimport_test.go similarity index 75% rename from app/slackimport_test.go rename to services/slackimport/slackimport_test.go index 48f2859f6a..9eac1fb3d0 100644 --- a/app/slackimport_test.go +++ b/services/slackimport/slackimport_test.go @@ -1,7 +1,7 @@ // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See LICENSE.txt for license information. -package app +package slackimport import ( "os" @@ -12,10 +12,11 @@ import ( "github.com/stretchr/testify/require" "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/store/storetest/mocks" ) func TestSlackConvertTimeStamp(t *testing.T) { - assert.EqualValues(t, SlackConvertTimeStamp("1469785419.000033"), 1469785419000) + assert.EqualValues(t, slackConvertTimeStamp("1469785419.000033"), 1469785419000) } func TestSlackConvertChannelName(t *testing.T) { @@ -31,17 +32,17 @@ func TestSlackConvertChannelName(t *testing.T) { {"a", "C0G05DLQH", "slack-channel-a"}, {"случайный", "C0G05DLQD", "c0g05dlqd"}, } { - assert.Equal(t, SlackConvertChannelName(tc.nameInput, tc.idInput), tc.output, "nameInput = %v", tc.nameInput) + assert.Equal(t, slackConvertChannelName(tc.nameInput, tc.idInput), tc.output, "nameInput = %v", tc.nameInput) } } func TestSlackConvertUserMentions(t *testing.T) { - users := []SlackUser{ + users := []slackUser{ {Id: "U00000A0A", Username: "firstuser"}, {Id: "U00000B1B", Username: "seconduser"}, } - posts := map[string][]SlackPost{ + posts := map[string][]slackPost{ "test-channel": { { Text: ": Hi guys.", @@ -58,7 +59,7 @@ func TestSlackConvertUserMentions(t *testing.T) { }, } - expectedPosts := map[string][]SlackPost{ + expectedPosts := map[string][]slackPost{ "test-channel": { { Text: "@channel: Hi guys.", @@ -75,16 +76,16 @@ func TestSlackConvertUserMentions(t *testing.T) { }, } - assert.Equal(t, expectedPosts, SlackConvertUserMentions(users, posts)) + assert.Equal(t, expectedPosts, slackConvertUserMentions(users, posts)) } func TestSlackConvertChannelMentions(t *testing.T) { - channels := []SlackChannel{ + channels := []slackChannel{ {Id: "C000AA00A", Name: "one"}, {Id: "C000BB11B", Name: "two"}, } - posts := map[string][]SlackPost{ + posts := map[string][]slackPost{ "test-channel": { { Text: "Go to <#C000AA00A>.", @@ -96,7 +97,7 @@ func TestSlackConvertChannelMentions(t *testing.T) { }, } - expectedPosts := map[string][]SlackPost{ + expectedPosts := map[string][]slackPost{ "test-channel": { { Text: "Go to ~one.", @@ -108,7 +109,7 @@ func TestSlackConvertChannelMentions(t *testing.T) { }, } - assert.Equal(t, expectedPosts, SlackConvertChannelMentions(channels, posts)) + assert.Equal(t, expectedPosts, slackConvertChannelMentions(channels, posts)) } func TestSlackParseChannels(t *testing.T) { @@ -116,7 +117,7 @@ func TestSlackParseChannels(t *testing.T) { require.NoError(t, err) defer file.Close() - channels, err := SlackParseChannels(file, "O") + channels, err := slackParseChannels(file, "O") require.NoError(t, err) assert.Equal(t, 6, len(channels)) } @@ -126,7 +127,7 @@ func TestSlackParseDirectMessages(t *testing.T) { require.NoError(t, err) defer file.Close() - channels, err := SlackParseChannels(file, "D") + channels, err := slackParseChannels(file, "D") require.NoError(t, err) assert.Equal(t, 4, len(channels)) } @@ -136,7 +137,7 @@ func TestSlackParsePrivateChannels(t *testing.T) { require.NoError(t, err) defer file.Close() - channels, err := SlackParseChannels(file, "P") + channels, err := slackParseChannels(file, "P") require.NoError(t, err) assert.Equal(t, 1, len(channels)) } @@ -146,7 +147,7 @@ func TestSlackParseGroupDirectMessages(t *testing.T) { require.NoError(t, err) defer file.Close() - channels, err := SlackParseChannels(file, "G") + channels, err := slackParseChannels(file, "G") require.NoError(t, err) assert.Equal(t, 3, len(channels)) } @@ -156,7 +157,7 @@ func TestSlackParseUsers(t *testing.T) { require.NoError(t, err) defer file.Close() - users, err := SlackParseUsers(file) + users, err := slackParseUsers(file) require.NoError(t, err) assert.Equal(t, 11, len(users)) } @@ -166,7 +167,7 @@ func TestSlackParsePosts(t *testing.T) { require.NoError(t, err) defer file.Close() - posts, err := SlackParsePosts(file) + posts, err := slackParsePosts(file) require.NoError(t, err) assert.Equal(t, 9, len(posts)) } @@ -176,7 +177,7 @@ func TestSlackParseMultipleAttachments(t *testing.T) { require.NoError(t, err) defer file.Close() - posts, err := SlackParsePosts(file) + posts, err := slackParsePosts(file) require.NoError(t, err) assert.Equal(t, 2, len(posts[8].Files)) } @@ -189,7 +190,7 @@ func TestSlackSanitiseChannelProperties(t *testing.T) { Header: "The channel header", } - c1s := SlackSanitiseChannelProperties(c1) + c1s := slackSanitiseChannelProperties(c1) assert.Equal(t, c1, c1s) c2 := model.Channel{ @@ -199,7 +200,7 @@ func TestSlackSanitiseChannelProperties(t *testing.T) { Header: strings.Repeat("0123456789", 120), } - c2s := SlackSanitiseChannelProperties(c2) + c2s := slackSanitiseChannelProperties(c2) assert.Equal(t, model.Channel{ DisplayName: strings.Repeat("abcdefghij", 6) + "abcd", Name: strings.Repeat("abcdefghij", 6) + "abcd", @@ -209,8 +210,8 @@ func TestSlackSanitiseChannelProperties(t *testing.T) { } func TestSlackConvertPostsMarkup(t *testing.T) { - input := make(map[string][]SlackPost) - input["test"] = []SlackPost{ + input := make(map[string][]slackPost) + input["test"] = []slackPost{ { Text: "This message contains a link to .", }, @@ -259,8 +260,8 @@ third`, }, } - expectedOutput := make(map[string][]SlackPost) - expectedOutput["test"] = []SlackPost{ + expectedOutput := make(map[string][]slackPost) + expectedOutput["test"] = []slackPost{ { Text: "This message contains a link to [Google](https://google.com).", }, @@ -309,53 +310,79 @@ in this~.`, }, } - assert.Equal(t, expectedOutput, SlackConvertPostsMarkup(input)) + assert.Equal(t, expectedOutput, slackConvertPostsMarkup(input)) } func TestOldImportChannel(t *testing.T) { - th := Setup(t).InitBasic() - defer th.TearDown() + u1 := &model.User{ + Id: model.NewId(), + Username: "test-user-1", + } + u2 := &model.User{ + Id: model.NewId(), + Username: "test-user-2", + } + store := &mocks.Store{} + config := &model.Config{} + config.SetDefaults() - u1 := th.CreateUser() - u2 := th.CreateUser() t.Run("No panic on direct channel", func(t *testing.T) { - ch := th.CreateDmChannel(u1) - users := map[string]*model.User{ - th.BasicUser.Id: th.BasicUser, + //ch := th.CreateDmChannel(u1) + ch := &model.Channel{ + Type: model.CHANNEL_DIRECT, + Name: "test-channel", } - sCh := SlackChannel{ + users := map[string]*model.User{ + u2.Id: u2, + } + sCh := slackChannel{ Id: "someid", Members: []string{u1.Id, "randomID"}, Creator: "randomID2", } - _ = th.App.oldImportChannel(ch, sCh, users) + actions := Actions{} + + importer := New(store, actions, config) + _ = importer.oldImportChannel(ch, sCh, users) }) t.Run("No panic on direct channel with 1 member", func(t *testing.T) { - ch := th.CreateDmChannel(u1) - users := map[string]*model.User{ - th.BasicUser.Id: th.BasicUser, + ch := &model.Channel{ + Type: model.CHANNEL_DIRECT, + Name: "test-channel", } - sCh := SlackChannel{ + users := map[string]*model.User{ + u1.Id: u1, + } + sCh := slackChannel{ Id: "someid", - Members: []string{th.BasicUser.Id}, + Members: []string{u1.Id}, Creator: "randomID2", } - _ = th.App.oldImportChannel(ch, sCh, users) + actions := Actions{} + + importer := New(store, actions, config) + _ = importer.oldImportChannel(ch, sCh, users) }) t.Run("No panic on group channel", func(t *testing.T) { - ch := th.CreateGroupChannel(u1, u2) - users := map[string]*model.User{ - th.BasicUser.Id: th.BasicUser, + ch := &model.Channel{ + Type: model.CHANNEL_GROUP, + Name: "test-channel", } - sCh := SlackChannel{ + users := map[string]*model.User{ + u1.Id: u1, + } + sCh := slackChannel{ Id: "someid", - Members: []string{th.BasicUser.Id}, + Members: []string{u1.Id}, Creator: "randomID2", } - _ = th.App.oldImportChannel(ch, sCh, users) + actions := Actions{} + + importer := New(store, actions, config) + _ = importer.oldImportChannel(ch, sCh, users) }) }