8c8bc7111c
* Slack Import unit tests (for some functions). Covers the completely self contained functions in the Slack Importer that are straightforward to unit test. JSON processing functions are not yet unit tested. * Unit tests for Slack Import JSON parsing. This tests the 3 functions for parsing JSON, using a sample set of Slack data for posts, channels, and users.
438 строки
14 KiB
Go
438 строки
14 KiB
Go
// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
|
|
// See License.txt for license information.
|
|
|
|
package api
|
|
|
|
import (
|
|
"archive/zip"
|
|
"bytes"
|
|
"encoding/json"
|
|
l4g "github.com/alecthomas/log4go"
|
|
"github.com/mattermost/platform/model"
|
|
"github.com/mattermost/platform/utils"
|
|
"io"
|
|
"mime/multipart"
|
|
"path/filepath"
|
|
"regexp"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
type SlackChannel struct {
|
|
Id string `json:"id"`
|
|
Name string `json:"name"`
|
|
Members []string `json:"members"`
|
|
Topic map[string]string `json:"topic"`
|
|
Purpose map[string]string `json:"purpose"`
|
|
}
|
|
|
|
type SlackUser struct {
|
|
Id string `json:"id"`
|
|
Username string `json:"name"`
|
|
Profile map[string]string `json:"profile"`
|
|
}
|
|
|
|
type SlackFile struct {
|
|
Id string `json:"id"`
|
|
Title string `json:"title"`
|
|
}
|
|
|
|
type SlackPost struct {
|
|
User string `json:"user"`
|
|
BotId string `json:"bot_id"`
|
|
BotUsername string `json:"username"`
|
|
Text string `json:"text"`
|
|
TimeStamp string `json:"ts"`
|
|
Type string `json:"type"`
|
|
SubType string `json:"subtype"`
|
|
Comment *SlackComment `json:"comment"`
|
|
Upload bool `json:"upload"`
|
|
File *SlackFile `json:"file"`
|
|
}
|
|
|
|
type SlackComment struct {
|
|
User string `json:"user"`
|
|
Comment string `json:"comment"`
|
|
}
|
|
|
|
func SlackConvertTimeStamp(ts string) int64 {
|
|
timeString := strings.SplitN(ts, ".", 2)[0]
|
|
|
|
timeStamp, err := strconv.ParseInt(timeString, 10, 64)
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_convert_timestamp.bad.warn"))
|
|
return 1
|
|
}
|
|
return timeStamp * 1000 // Convert to milliseconds
|
|
}
|
|
|
|
func SlackConvertChannelName(channelName string) string {
|
|
newName := strings.Trim(channelName, "_-")
|
|
if len(newName) == 1 {
|
|
return "slack-channel-" + newName
|
|
}
|
|
|
|
return newName
|
|
}
|
|
|
|
func SlackParseChannels(data io.Reader) ([]SlackChannel, error) {
|
|
decoder := json.NewDecoder(data)
|
|
|
|
var channels []SlackChannel
|
|
if err := decoder.Decode(&channels); err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_parse_channels.error"))
|
|
return channels, err
|
|
}
|
|
return channels, nil
|
|
}
|
|
|
|
func SlackParseUsers(data io.Reader) ([]SlackUser, error) {
|
|
decoder := json.NewDecoder(data)
|
|
|
|
var users []SlackUser
|
|
if err := decoder.Decode(&users); err != nil {
|
|
// This actually returns errors that are ignored.
|
|
// In this case it is erroring because of a null that Slack
|
|
// introduced. So we just return the users here.
|
|
return users, err
|
|
}
|
|
return users, nil
|
|
}
|
|
|
|
func SlackParsePosts(data io.Reader) ([]SlackPost, error) {
|
|
decoder := json.NewDecoder(data)
|
|
|
|
var posts []SlackPost
|
|
if err := decoder.Decode(&posts); err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_parse_posts.error"))
|
|
return posts, err
|
|
}
|
|
return posts, nil
|
|
}
|
|
|
|
func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User {
|
|
// Log header
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.created"))
|
|
log.WriteString("===============\r\n\r\n")
|
|
|
|
addedUsers := make(map[string]*model.User)
|
|
|
|
// Need the team
|
|
var team *model.Team
|
|
if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
|
|
return addedUsers
|
|
} else {
|
|
team = result.Data.(*model.Team)
|
|
}
|
|
|
|
for _, sUser := range slackusers {
|
|
firstName := ""
|
|
lastName := ""
|
|
if name, ok := sUser.Profile["first_name"]; ok {
|
|
firstName = name
|
|
}
|
|
if name, ok := sUser.Profile["last_name"]; ok {
|
|
lastName = name
|
|
}
|
|
|
|
email := sUser.Profile["email"]
|
|
|
|
password := model.NewId()
|
|
|
|
// Check for email conflict and use existing user if found
|
|
if result := <-Srv.Store.User().GetByEmail(email); result.Err == nil {
|
|
existingUser := result.Data.(*model.User)
|
|
addedUsers[sUser.Id] = existingUser
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
|
|
continue
|
|
}
|
|
|
|
newUser := model.User{
|
|
Username: sUser.Username,
|
|
FirstName: firstName,
|
|
LastName: lastName,
|
|
Email: email,
|
|
Password: password,
|
|
}
|
|
|
|
if mUser := ImportUser(team, &newUser); mUser != nil {
|
|
addedUsers[sUser.Id] = mUser
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password}))
|
|
} else {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username}))
|
|
}
|
|
}
|
|
|
|
return addedUsers
|
|
}
|
|
|
|
func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File) {
|
|
for _, sPost := range posts {
|
|
switch {
|
|
case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Text,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
}
|
|
if sPost.Upload {
|
|
if fileInfo, ok := SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok == true {
|
|
newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
|
|
newPost.Message = sPost.File.Title
|
|
}
|
|
}
|
|
ImportPost(&newPost)
|
|
for _, fileId := range newPost.FileIds {
|
|
if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
|
|
l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err)
|
|
}
|
|
}
|
|
|
|
case sPost.Type == "message" && sPost.SubType == "file_comment":
|
|
if sPost.Comment == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_comment.debug"))
|
|
continue
|
|
} else if sPost.Comment.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
|
|
continue
|
|
} else if users[sPost.Comment.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.Comment.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Comment.Comment,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
}
|
|
ImportPost(&newPost)
|
|
case sPost.Type == "message" && sPost.SubType == "bot_message":
|
|
continue
|
|
case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: sPost.Text,
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
Type: model.POST_JOIN_LEAVE,
|
|
}
|
|
ImportPost(&newPost)
|
|
case sPost.Type == "message" && sPost.SubType == "me_message":
|
|
if sPost.User == "" {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
|
|
continue
|
|
} else if users[sPost.User] == nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
|
|
continue
|
|
}
|
|
newPost := model.Post{
|
|
UserId: users[sPost.User].Id,
|
|
ChannelId: channel.Id,
|
|
Message: "*" + sPost.Text + "*",
|
|
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
|
}
|
|
ImportPost(&newPost)
|
|
default:
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType)
|
|
}
|
|
}
|
|
}
|
|
|
|
func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) {
|
|
if sPost.File != nil {
|
|
if file, ok := uploads[sPost.File.Id]; ok == true {
|
|
openFile, err := file.Open()
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_open_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
|
|
return nil, false
|
|
}
|
|
defer openFile.Close()
|
|
|
|
uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name))
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
|
|
return nil, false
|
|
}
|
|
|
|
return uploadedFile, true
|
|
} else {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_found.warn", map[string]interface{}{"FileId": sPost.File.Id}))
|
|
return nil, false
|
|
}
|
|
} else {
|
|
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_in_json.warn"))
|
|
return nil, false
|
|
}
|
|
}
|
|
|
|
func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
|
|
for _, member := range members {
|
|
if user, ok := users[member]; !ok {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"}))
|
|
} else {
|
|
if _, err := AddUserToChannel(user, channel); err != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username}))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, log *bytes.Buffer) map[string]*model.Channel {
|
|
// Write Header
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.added"))
|
|
log.WriteString("=================\r\n\r\n")
|
|
|
|
addedChannels := make(map[string]*model.Channel)
|
|
for _, sChannel := range slackchannels {
|
|
newChannel := model.Channel{
|
|
TeamId: teamId,
|
|
Type: model.CHANNEL_OPEN,
|
|
DisplayName: sChannel.Name,
|
|
Name: SlackConvertChannelName(sChannel.Name),
|
|
Purpose: sChannel.Purpose["value"],
|
|
Header: sChannel.Topic["value"],
|
|
}
|
|
mChannel := ImportChannel(&newChannel)
|
|
if mChannel == nil {
|
|
// Maybe it already exists?
|
|
if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
|
|
l4g.Debug(utils.T("api.slackimport.slack_add_channels.import_failed.debug"), newChannel.DisplayName)
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
|
|
continue
|
|
} else {
|
|
mChannel = result.Data.(*model.Channel)
|
|
log.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
|
|
}
|
|
}
|
|
addSlackUsersToChannel(sChannel.Members, users, mChannel, log)
|
|
log.WriteString(newChannel.DisplayName + "\r\n")
|
|
addedChannels[sChannel.Id] = mChannel
|
|
SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads)
|
|
}
|
|
|
|
return addedChannels
|
|
}
|
|
|
|
func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost {
|
|
var regexes = make(map[string]*regexp.Regexp, len(users))
|
|
for _, user := range users {
|
|
r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>")
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_convert_user_mentions.compile_regexp_failed.warn"), user.Id, user.Username)
|
|
continue
|
|
}
|
|
regexes["@"+user.Username] = r
|
|
}
|
|
|
|
// Special cases.
|
|
regexes["@here"], _ = regexp.Compile(`<!here\|@here>`)
|
|
regexes["@channel"], _ = regexp.Compile("<!channel>")
|
|
regexes["@all"], _ = regexp.Compile("<!everyone>")
|
|
|
|
for channelName, channelPosts := range posts {
|
|
for postIdx, post := range channelPosts {
|
|
for mention, r := range regexes {
|
|
post.Text = r.ReplaceAllString(post.Text, mention)
|
|
posts[channelName][postIdx] = post
|
|
}
|
|
}
|
|
}
|
|
|
|
return posts
|
|
}
|
|
|
|
func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost {
|
|
var regexes = make(map[string]*regexp.Regexp, len(channels))
|
|
for _, channel := range channels {
|
|
r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>")
|
|
if err != nil {
|
|
l4g.Warn(utils.T("api.slackimport.slack_convert_channel_mentions.compile_regexp_failed.warn"), channel.Id, channel.Name)
|
|
continue
|
|
}
|
|
regexes["!"+channel.Name] = r
|
|
}
|
|
|
|
for channelName, channelPosts := range posts {
|
|
for postIdx, post := range channelPosts {
|
|
for channelReplace, r := range regexes {
|
|
post.Text = r.ReplaceAllString(post.Text, channelReplace)
|
|
posts[channelName][postIdx] = post
|
|
}
|
|
}
|
|
}
|
|
|
|
return posts
|
|
}
|
|
|
|
func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
|
|
// Create log file
|
|
log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log"))
|
|
|
|
zipreader, err := zip.NewReader(fileData, fileSize)
|
|
if err != nil || zipreader.File == nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error"))
|
|
return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error()), log
|
|
}
|
|
|
|
var channels []SlackChannel
|
|
var users []SlackUser
|
|
posts := make(map[string][]SlackPost)
|
|
uploads := make(map[string]*zip.File)
|
|
for _, file := range zipreader.File {
|
|
reader, err := file.Open()
|
|
if err != nil {
|
|
log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}))
|
|
return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error()), log
|
|
}
|
|
if file.Name == "channels.json" {
|
|
channels, _ = SlackParseChannels(reader)
|
|
} else if file.Name == "users.json" {
|
|
users, _ = SlackParseUsers(reader)
|
|
} else {
|
|
spl := strings.Split(file.Name, "/")
|
|
if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
|
|
newposts, _ := SlackParsePosts(reader)
|
|
channel := spl[0]
|
|
if _, ok := posts[channel]; ok == false {
|
|
posts[channel] = newposts
|
|
} else {
|
|
posts[channel] = append(posts[channel], newposts...)
|
|
}
|
|
} else if len(spl) == 3 && spl[0] == "__uploads" {
|
|
uploads[spl[1]] = file
|
|
}
|
|
}
|
|
}
|
|
|
|
posts = SlackConvertUserMentions(users, posts)
|
|
posts = SlackConvertChannelMentions(channels, posts)
|
|
|
|
addedUsers := SlackAddUsers(teamID, users, log)
|
|
SlackAddChannels(teamID, channels, posts, addedUsers, uploads, log)
|
|
|
|
log.WriteString(utils.T("api.slackimport.slack_import.notes"))
|
|
log.WriteString("=======\r\n\r\n")
|
|
|
|
log.WriteString(utils.T("api.slackimport.slack_import.note1"))
|
|
log.WriteString(utils.T("api.slackimport.slack_import.note2"))
|
|
log.WriteString(utils.T("api.slackimport.slack_import.note3"))
|
|
|
|
return nil, log
|
|
}
|