Move Slack Import to App Layer. (#5135)

app/file.go

@@ -5,11 +5,17 @@ package app
 
 import (
 	"bytes"
+	_ "image/gif"
 	"crypto/sha256"
 	"encoding/base64"
 	"fmt"
+	"image"
+	"image/color"
+	"image/draw"
+	"image/jpeg"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -20,8 +26,33 @@ import (
 	l4g "github.com/alecthomas/log4go"
 	"github.com/mattermost/platform/model"
 	"github.com/mattermost/platform/utils"
-
+	"github.com/disintegration/imaging"
 	s3 "github.com/minio/minio-go"
+	"github.com/rwcarlsen/goexif/exif"
+	_ "golang.org/x/image/bmp"
+)
+
+const (
+	/*
+	  EXIF Image Orientations
+	  1        2       3      4         5            6           7          8
+
+	  888888  888888      88  88      8888888888  88                  88  8888888888
+	  88          88      88  88      88  88      88  88          88  88      88  88
+	  8888      8888    8888  8888    88          8888888888  8888888888          88
+	  88          88      88  88
+	  88          88  888888  888888
+	*/
+	Upright            = 1
+	UprightMirrored    = 2
+	UpsideDown         = 3
+	UpsideDownMirrored = 4
+	RotatedCWMirrored  = 5
+	RotatedCCW         = 6
+	RotatedCCWMirrored = 7
+	RotatedCW          = 8
+
+	MaxImageSize = 6048 * 4032 // 24 megapixels, roughly 36MB as a raw image
 )
 
 func ReadFile(path string) ([]byte, *model.AppError) {
@@ -338,3 +369,161 @@ func GeneratePublicLinkHash(fileId, salt string) string {
 
 	return base64.RawURLEncoding.EncodeToString(hash.Sum(nil))
 }
+
+func DoUploadFile(teamId string, channelId string, userId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) {
+	filename := filepath.Base(rawFilename)
+
+	info, err := model.GetInfoForBytes(filename, data)
+	if err != nil {
+		err.StatusCode = http.StatusBadRequest
+		return nil, err
+	}
+
+	info.Id = model.NewId()
+	info.CreatorId = userId
+
+	pathPrefix := "teams/" + teamId + "/channels/" + channelId + "/users/" + userId + "/" + info.Id + "/"
+	info.Path = pathPrefix + filename
+
+	if info.IsImage() {
+		// Check dimensions before loading the whole thing into memory later on
+		if info.Width*info.Height > MaxImageSize {
+			err := model.NewLocAppError("uploadFile", "api.file.upload_file.large_image.app_error", map[string]interface{}{"Filename": filename}, "")
+			err.StatusCode = http.StatusBadRequest
+			return nil, err
+		}
+
+		nameWithoutExtension := filename[:strings.LastIndex(filename, ".")]
+		info.PreviewPath = pathPrefix + nameWithoutExtension + "_preview.jpg"
+		info.ThumbnailPath = pathPrefix + nameWithoutExtension + "_thumb.jpg"
+	}
+
+	if err := WriteFile(data, info.Path); err != nil {
+		return nil, err
+	}
+
+	if result := <-Srv.Store.FileInfo().Save(info); result.Err != nil {
+		return nil, result.Err
+	}
+
+	return info, nil
+}
+
+func HandleImages(previewPathList []string, thumbnailPathList []string, fileData [][]byte) {
+	for i, data := range fileData {
+		go func(i int, data []byte) {
+			img, width, height := prepareImage(fileData[i])
+			if img != nil {
+				go generateThumbnailImage(*img, thumbnailPathList[i], width, height)
+				go generatePreviewImage(*img, previewPathList[i], width)
+			}
+		}(i, data)
+	}
+}
+
+func prepareImage(fileData []byte) (*image.Image, int, int) {
+	// Decode image bytes into Image object
+	img, imgType, err := image.Decode(bytes.NewReader(fileData))
+	if err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.decode.error"), err)
+		return nil, 0, 0
+	}
+
+	width := img.Bounds().Dx()
+	height := img.Bounds().Dy()
+
+	// Fill in the background of a potentially-transparent png file as white
+	if imgType == "png" {
+		dst := image.NewRGBA(img.Bounds())
+		draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
+		draw.Draw(dst, dst.Bounds(), img, img.Bounds().Min, draw.Over)
+		img = dst
+	}
+
+	// Flip the image to be upright
+	orientation, _ := getImageOrientation(fileData)
+
+	switch orientation {
+	case UprightMirrored:
+		img = imaging.FlipH(img)
+	case UpsideDown:
+		img = imaging.Rotate180(img)
+	case UpsideDownMirrored:
+		img = imaging.FlipV(img)
+	case RotatedCWMirrored:
+		img = imaging.Transpose(img)
+	case RotatedCCW:
+		img = imaging.Rotate270(img)
+	case RotatedCCWMirrored:
+		img = imaging.Transverse(img)
+	case RotatedCW:
+		img = imaging.Rotate90(img)
+	}
+
+	return &img, width, height
+}
+
+func getImageOrientation(imageData []byte) (int, error) {
+	if exifData, err := exif.Decode(bytes.NewReader(imageData)); err != nil {
+		return Upright, err
+	} else {
+		if tag, err := exifData.Get("Orientation"); err != nil {
+			return Upright, err
+		} else {
+			orientation, err := tag.Int(0)
+			if err != nil {
+				return Upright, err
+			} else {
+				return orientation, nil
+			}
+		}
+	}
+}
+
+func generateThumbnailImage(img image.Image, thumbnailPath string, width int, height int) {
+	thumbWidth := float64(utils.Cfg.FileSettings.ThumbnailWidth)
+	thumbHeight := float64(utils.Cfg.FileSettings.ThumbnailHeight)
+	imgWidth := float64(width)
+	imgHeight := float64(height)
+
+	var thumbnail image.Image
+	if imgHeight < thumbHeight && imgWidth < thumbWidth {
+		thumbnail = img
+	} else if imgHeight/imgWidth < thumbHeight/thumbWidth {
+		thumbnail = imaging.Resize(img, 0, utils.Cfg.FileSettings.ThumbnailHeight, imaging.Lanczos)
+	} else {
+		thumbnail = imaging.Resize(img, utils.Cfg.FileSettings.ThumbnailWidth, 0, imaging.Lanczos)
+	}
+
+	buf := new(bytes.Buffer)
+	if err := jpeg.Encode(buf, thumbnail, &jpeg.Options{Quality: 90}); err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.encode_jpeg.error"), thumbnailPath, err)
+		return
+	}
+
+	if err := WriteFile(buf.Bytes(), thumbnailPath); err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.upload_thumb.error"), thumbnailPath, err)
+		return
+	}
+}
+
+func generatePreviewImage(img image.Image, previewPath string, width int) {
+	var preview image.Image
+	if width > int(utils.Cfg.FileSettings.PreviewWidth) {
+		preview = imaging.Resize(img, utils.Cfg.FileSettings.PreviewWidth, utils.Cfg.FileSettings.PreviewHeight, imaging.Lanczos)
+	} else {
+		preview = img
+	}
+
+	buf := new(bytes.Buffer)
+
+	if err := jpeg.Encode(buf, preview, &jpeg.Options{Quality: 90}); err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.encode_preview.error"), previewPath, err)
+		return
+	}
+
+	if err := WriteFile(buf.Bytes(), previewPath); err != nil {
+		l4g.Error(utils.T("api.file.handle_images_forget.upload_preview.error"), previewPath, err)
+		return
+	}
+}

app/import.go

@@ -0,0 +1,157 @@
+// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"bytes"
+	"io"
+	"regexp"
+	"unicode/utf8"
+
+	l4g "github.com/alecthomas/log4go"
+	"github.com/mattermost/platform/model"
+	"github.com/mattermost/platform/utils"
+)
+
+//
+// Import functions are sutible for entering posts and users into the database without
+// some of the usual checks. (IsValid is still run)
+//
+
+func ImportPost(post *model.Post) {
+	// Workaround for empty messages, which may be the case if they are webhook posts.
+	firstIteration := true
+	for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) {
+		firstIteration = false
+		var remainder string
+		if messageRuneCount > model.POST_MESSAGE_MAX_RUNES {
+			remainder = string(([]rune(post.Message))[model.POST_MESSAGE_MAX_RUNES:])
+			post.Message = truncateRunes(post.Message, model.POST_MESSAGE_MAX_RUNES)
+		} else {
+			remainder = ""
+		}
+
+		post.Hashtags, _ = model.ParseHashtags(post.Message)
+
+		if result := <-Srv.Store.Post().Save(post); result.Err != nil {
+			l4g.Debug(utils.T("api.import.import_post.saving.debug"), post.UserId, post.Message)
+		}
+
+		for _, fileId := range post.FileIds {
+			if result := <-Srv.Store.FileInfo().AttachToPost(fileId, post.Id); result.Err != nil {
+				l4g.Error(utils.T("api.import.import_post.attach_files.error"), post.Id, post.FileIds, result.Err)
+			}
+		}
+
+		post.Id = ""
+		post.CreateAt++
+		post.Message = remainder
+	}
+}
+
+func ImportUser(team *model.Team, user *model.User) *model.User {
+	user.MakeNonNil()
+
+	user.Roles = model.ROLE_SYSTEM_USER.Id
+
+	if result := <-Srv.Store.User().Save(user); result.Err != nil {
+		l4g.Error(utils.T("api.import.import_user.saving.error"), result.Err)
+		return nil
+	} else {
+		ruser := result.Data.(*model.User)
+
+		if cresult := <-Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil {
+			l4g.Error(utils.T("api.import.import_user.set_email.error"), cresult.Err)
+		}
+
+		if err := JoinUserToTeam(team, user); err != nil {
+			l4g.Error(utils.T("api.import.import_user.join_team.error"), err)
+		}
+
+		return ruser
+	}
+}
+
+func ImportChannel(channel *model.Channel) *model.Channel {
+	if result := <-Srv.Store.Channel().Save(channel); result.Err != nil {
+		return nil
+	} else {
+		sc := result.Data.(*model.Channel)
+
+		return sc
+	}
+}
+
+func ImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
+	buf := bytes.NewBuffer(nil)
+	io.Copy(buf, file)
+	data := buf.Bytes()
+
+	fileInfo, err := DoUploadFile(teamId, channelId, userId, fileName, data)
+	if err != nil {
+		return nil, err
+	}
+
+	img, width, height := prepareImage(data)
+	if img != nil {
+		generateThumbnailImage(*img, fileInfo.ThumbnailPath, width, height)
+		generatePreviewImage(*img, fileInfo.PreviewPath, width)
+	}
+
+	return fileInfo, nil
+}
+
+func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) {
+	linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`)
+	post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})")
+
+	post.AddProp("from_webhook", "true")
+
+	if _, ok := props["override_username"]; !ok {
+		post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME)
+	}
+
+	if len(props) > 0 {
+		for key, val := range props {
+			if key == "attachments" {
+				if list, success := val.([]interface{}); success {
+					// parse attachment links into Markdown format
+					for i, aInt := range list {
+						attachment := aInt.(map[string]interface{})
+						if aText, ok := attachment["text"].(string); ok {
+							aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})")
+							attachment["text"] = aText
+							list[i] = attachment
+						}
+						if aText, ok := attachment["pretext"].(string); ok {
+							aText = linkWithTextRegex.ReplaceAllString(aText, "[${2}](${1})")
+							attachment["pretext"] = aText
+							list[i] = attachment
+						}
+						if fVal, ok := attachment["fields"]; ok {
+							if fields, ok := fVal.([]interface{}); ok {
+								// parse attachment field links into Markdown format
+								for j, fInt := range fields {
+									field := fInt.(map[string]interface{})
+									if fValue, ok := field["value"].(string); ok {
+										fValue = linkWithTextRegex.ReplaceAllString(fValue, "[${2}](${1})")
+										field["value"] = fValue
+										fields[j] = field
+									}
+								}
+								attachment["fields"] = fields
+								list[i] = attachment
+							}
+						}
+					}
+					post.AddProp(key, list)
+				}
+			} else if key != "from_webhook" {
+				post.AddProp(key, val)
+			}
+		}
+	}
+
+	ImportPost(post)
+}

app/slackimport.go

@@ -0,0 +1,621 @@
+// Copyright (c) 2015 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/json"
+	"io"
+	"mime/multipart"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	l4g "github.com/alecthomas/log4go"
+	"github.com/mattermost/platform/model"
+	"github.com/mattermost/platform/utils"
+)
+
+type SlackChannel struct {
+	Id      string            `json:"id"`
+	Name    string            `json:"name"`
+	Members []string          `json:"members"`
+	Topic   map[string]string `json:"topic"`
+	Purpose map[string]string `json:"purpose"`
+}
+
+type SlackUser struct {
+	Id       string            `json:"id"`
+	Username string            `json:"name"`
+	Profile  map[string]string `json:"profile"`
+}
+
+type SlackFile struct {
+	Id    string `json:"id"`
+	Title string `json:"title"`
+}
+
+type SlackPost struct {
+	User        string            `json:"user"`
+	BotId       string            `json:"bot_id"`
+	BotUsername string            `json:"username"`
+	Text        string            `json:"text"`
+	TimeStamp   string            `json:"ts"`
+	Type        string            `json:"type"`
+	SubType     string            `json:"subtype"`
+	Comment     *SlackComment     `json:"comment"`
+	Upload      bool              `json:"upload"`
+	File        *SlackFile        `json:"file"`
+	Attachments []SlackAttachment `json:"attachments"`
+}
+
+type SlackComment struct {
+	User    string `json:"user"`
+	Comment string `json:"comment"`
+}
+
+type SlackAttachment struct {
+	Id      int                      `json:"id"`
+	Text    string                   `json:"text"`
+	Pretext string                   `json:"pretext"`
+	Fields  []map[string]interface{} `json:"fields"`
+}
+
+func truncateRunes(s string, i int) string {
+	runes := []rune(s)
+	if len(runes) > i {
+		return string(runes[:i])
+	}
+	return s
+}
+
+func SlackConvertTimeStamp(ts string) int64 {
+	timeString := strings.SplitN(ts, ".", 2)[0]
+
+	timeStamp, err := strconv.ParseInt(timeString, 10, 64)
+	if err != nil {
+		l4g.Warn(utils.T("api.slackimport.slack_convert_timestamp.bad.warn"))
+		return 1
+	}
+	return timeStamp * 1000 // Convert to milliseconds
+}
+
+func SlackConvertChannelName(channelName string) string {
+	newName := strings.Trim(channelName, "_-")
+	if len(newName) == 1 {
+		return "slack-channel-" + newName
+	}
+
+	return newName
+}
+
+func SlackParseChannels(data io.Reader) ([]SlackChannel, error) {
+	decoder := json.NewDecoder(data)
+
+	var channels []SlackChannel
+	if err := decoder.Decode(&channels); err != nil {
+		l4g.Warn(utils.T("api.slackimport.slack_parse_channels.error"))
+		return channels, err
+	}
+	return channels, nil
+}
+
+func SlackParseUsers(data io.Reader) ([]SlackUser, error) {
+	decoder := json.NewDecoder(data)
+
+	var users []SlackUser
+	if err := decoder.Decode(&users); err != nil {
+		// This actually returns errors that are ignored.
+		// In this case it is erroring because of a null that Slack
+		// introduced. So we just return the users here.
+		return users, err
+	}
+	return users, nil
+}
+
+func SlackParsePosts(data io.Reader) ([]SlackPost, error) {
+	decoder := json.NewDecoder(data)
+
+	var posts []SlackPost
+	if err := decoder.Decode(&posts); err != nil {
+		l4g.Warn(utils.T("api.slackimport.slack_parse_posts.error"))
+		return posts, err
+	}
+	return posts, nil
+}
+
+func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User {
+	// Log header
+	log.WriteString(utils.T("api.slackimport.slack_add_users.created"))
+	log.WriteString("===============\r\n\r\n")
+
+	addedUsers := make(map[string]*model.User)
+
+	// Need the team
+	var team *model.Team
+	if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
+		log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
+		return addedUsers
+	} else {
+		team = result.Data.(*model.Team)
+	}
+
+	for _, sUser := range slackusers {
+		firstName := ""
+		lastName := ""
+		if name, ok := sUser.Profile["first_name"]; ok {
+			firstName = name
+		}
+		if name, ok := sUser.Profile["last_name"]; ok {
+			lastName = name
+		}
+
+		email := sUser.Profile["email"]
+
+		password := model.NewId()
+
+		// Check for email conflict and use existing user if found
+		if result := <-Srv.Store.User().GetByEmail(email); result.Err == nil {
+			existingUser := result.Data.(*model.User)
+			addedUsers[sUser.Id] = existingUser
+			if err := JoinUserToTeam(team, addedUsers[sUser.Id]); err != nil {
+				log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
+			} else {
+				log.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
+			}
+			continue
+		}
+
+		newUser := model.User{
+			Username:  sUser.Username,
+			FirstName: firstName,
+			LastName:  lastName,
+			Email:     email,
+			Password:  password,
+		}
+
+		if mUser := ImportUser(team, &newUser); mUser != nil {
+			addedUsers[sUser.Id] = mUser
+			log.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password}))
+		} else {
+			log.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username}))
+		}
+	}
+
+	return addedUsers
+}
+
+func SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
+	var team *model.Team
+	if result := <-Srv.Store.Team().Get(teamId); result.Err != nil {
+		log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
+		return nil
+	} else {
+		team = result.Data.(*model.Team)
+	}
+
+	password := model.NewId()
+	username := "slackimportuser_" + model.NewId()
+	email := username + "@localhost"
+
+	botUser := model.User{
+		Username:  username,
+		FirstName: "",
+		LastName:  "",
+		Email:     email,
+		Password:  password,
+	}
+
+	if mUser := ImportUser(team, &botUser); mUser != nil {
+		log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password}))
+		return mUser
+	} else {
+		log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username}))
+		return nil
+	}
+}
+
+func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
+	for _, sPost := range posts {
+		switch {
+		case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
+			if sPost.User == "" {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
+				continue
+			} else if users[sPost.User] == nil {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+			}
+			if sPost.Upload {
+				if fileInfo, ok := SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok == true {
+					newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
+					newPost.Message = sPost.File.Title
+				}
+			}
+			ImportPost(&newPost)
+			for _, fileId := range newPost.FileIds {
+				if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
+					l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err)
+				}
+			}
+
+		case sPost.Type == "message" && sPost.SubType == "file_comment":
+			if sPost.Comment == nil {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_comment.debug"))
+				continue
+			} else if sPost.Comment.User == "" {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
+				continue
+			} else if users[sPost.Comment.User] == nil {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.Comment.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Comment.Comment,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+			}
+			ImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "bot_message":
+			if botUser == nil {
+				l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn"))
+				continue
+			} else if sPost.BotId == "" {
+				l4g.Warn(utils.T("api.slackimport.slack_add_posts.no_bot_id.warn"))
+				continue
+			}
+
+			props := make(model.StringInterface)
+			props["override_username"] = sPost.BotUsername
+			if len(sPost.Attachments) > 0 {
+				var mAttachments []interface{}
+				for _, attachment := range sPost.Attachments {
+					mAttachments = append(mAttachments, map[string]interface{}{
+						"text":    attachment.Text,
+						"pretext": attachment.Pretext,
+						"fields":  attachment.Fields,
+					})
+				}
+				props["attachments"] = mAttachments
+			}
+
+			post := &model.Post{
+				UserId:    botUser.Id,
+				ChannelId: channel.Id,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+				Message:   sPost.Text,
+				Type:      model.POST_SLACK_ATTACHMENT,
+			}
+
+			ImportIncomingWebhookPost(post, props)
+		case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
+			if sPost.User == "" {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
+				continue
+			} else if users[sPost.User] == nil {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+				Type:      model.POST_JOIN_LEAVE,
+			}
+			ImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "me_message":
+			if sPost.User == "" {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
+				continue
+			} else if users[sPost.User] == nil {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   "*" + sPost.Text + "*",
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+			}
+			ImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "channel_topic":
+			if sPost.User == "" {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
+				continue
+			} else if users[sPost.User] == nil {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+				Type:      model.POST_HEADER_CHANGE,
+			}
+			ImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "channel_purpose":
+			if sPost.User == "" {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
+				continue
+			} else if users[sPost.User] == nil {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+				Type:      model.POST_PURPOSE_CHANGE,
+			}
+			ImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "channel_name":
+			if sPost.User == "" {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
+				continue
+			} else if users[sPost.User] == nil {
+				l4g.Debug(utils.T("api.slackimport.slack_add_posts.user_no_exists.debug"), sPost.User)
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+				Type:      model.POST_DISPLAYNAME_CHANGE,
+			}
+			ImportPost(&newPost)
+		default:
+			l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType)
+		}
+	}
+}
+
+func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) {
+	if sPost.File != nil {
+		if file, ok := uploads[sPost.File.Id]; ok == true {
+			openFile, err := file.Open()
+			if err != nil {
+				l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_open_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
+				return nil, false
+			}
+			defer openFile.Close()
+
+			uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name))
+			if err != nil {
+				l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
+				return nil, false
+			}
+
+			return uploadedFile, true
+		} else {
+			l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_found.warn", map[string]interface{}{"FileId": sPost.File.Id}))
+			return nil, false
+		}
+	} else {
+		l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_not_in_json.warn"))
+		return nil, false
+	}
+}
+
+func deactivateSlackBotUser(user *model.User) {
+	_, err := UpdateActive(user, false)
+	if err != nil {
+		l4g.Warn(utils.T("api.slackimport.slack_deactivate_bot_user.failed_to_deactivate", err))
+	}
+}
+
+func addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
+	for _, member := range members {
+		if user, ok := users[member]; !ok {
+			log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"}))
+		} else {
+			if _, err := AddUserToChannel(user, channel); err != nil {
+				log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username}))
+			}
+		}
+	}
+}
+
+func SlackSanitiseChannelProperties(channel model.Channel) model.Channel {
+	if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES {
+		l4g.Warn("api.slackimport.slack_sanitise_channel_properties.display_name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
+		channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES)
+	}
+
+	if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH {
+		l4g.Warn("api.slackimport.slack_sanitise_channel_properties.name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
+		channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH]
+	}
+
+	if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES {
+		l4g.Warn("api.slackimport.slack_sanitise_channel_properties.purpose_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
+		channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES)
+	}
+
+	if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES {
+		l4g.Warn("api.slackimport.slack_sanitise_channel_properties.header_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName})
+		channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES)
+	}
+
+	return channel
+}
+
+func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, log *bytes.Buffer) map[string]*model.Channel {
+	// Write Header
+	log.WriteString(utils.T("api.slackimport.slack_add_channels.added"))
+	log.WriteString("=================\r\n\r\n")
+
+	addedChannels := make(map[string]*model.Channel)
+	for _, sChannel := range slackchannels {
+		newChannel := model.Channel{
+			TeamId:      teamId,
+			Type:        model.CHANNEL_OPEN,
+			DisplayName: sChannel.Name,
+			Name:        SlackConvertChannelName(sChannel.Name),
+			Purpose:     sChannel.Purpose["value"],
+			Header:      sChannel.Topic["value"],
+		}
+		newChannel = SlackSanitiseChannelProperties(newChannel)
+		mChannel := ImportChannel(&newChannel)
+		if mChannel == nil {
+			// Maybe it already exists?
+			if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
+				l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName)
+				log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
+				continue
+			} else {
+				mChannel = result.Data.(*model.Channel)
+				log.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
+			}
+		}
+		addSlackUsersToChannel(sChannel.Members, users, mChannel, log)
+		log.WriteString(newChannel.DisplayName + "\r\n")
+		addedChannels[sChannel.Id] = mChannel
+		SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser)
+	}
+
+	return addedChannels
+}
+
+func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost {
+	var regexes = make(map[string]*regexp.Regexp, len(users))
+	for _, user := range users {
+		r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>")
+		if err != nil {
+			l4g.Warn(utils.T("api.slackimport.slack_convert_user_mentions.compile_regexp_failed.warn"), user.Id, user.Username)
+			continue
+		}
+		regexes["@"+user.Username] = r
+	}
+
+	// Special cases.
+	regexes["@here"], _ = regexp.Compile(`<!here\|@here>`)
+	regexes["@channel"], _ = regexp.Compile("<!channel>")
+	regexes["@all"], _ = regexp.Compile("<!everyone>")
+
+	for channelName, channelPosts := range posts {
+		for postIdx, post := range channelPosts {
+			for mention, r := range regexes {
+				post.Text = r.ReplaceAllString(post.Text, mention)
+				posts[channelName][postIdx] = post
+			}
+		}
+	}
+
+	return posts
+}
+
+func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost {
+	var regexes = make(map[string]*regexp.Regexp, len(channels))
+	for _, channel := range channels {
+		r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>")
+		if err != nil {
+			l4g.Warn(utils.T("api.slackimport.slack_convert_channel_mentions.compile_regexp_failed.warn"), channel.Id, channel.Name)
+			continue
+		}
+		regexes["~"+channel.Name] = r
+	}
+
+	for channelName, channelPosts := range posts {
+		for postIdx, post := range channelPosts {
+			for channelReplace, r := range regexes {
+				post.Text = r.ReplaceAllString(post.Text, channelReplace)
+				posts[channelName][postIdx] = post
+			}
+		}
+	}
+
+	return posts
+}
+
+func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost {
+	// Convert URLs in Slack's format to Markdown format.
+	regex := regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`)
+
+	for channelName, channelPosts := range posts {
+		for postIdx, post := range channelPosts {
+			posts[channelName][postIdx].Text = regex.ReplaceAllString(post.Text, "[$2]($1)")
+		}
+	}
+
+	return posts
+}
+
+func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
+	// Create log file
+	log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log"))
+
+	zipreader, err := zip.NewReader(fileData, fileSize)
+	if err != nil || zipreader.File == nil {
+		log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error"))
+		return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error()), log
+	}
+
+	var channels []SlackChannel
+	var users []SlackUser
+	posts := make(map[string][]SlackPost)
+	uploads := make(map[string]*zip.File)
+	for _, file := range zipreader.File {
+		reader, err := file.Open()
+		if err != nil {
+			log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}))
+			return model.NewLocAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error()), log
+		}
+		if file.Name == "channels.json" {
+			channels, _ = SlackParseChannels(reader)
+		} else if file.Name == "users.json" {
+			users, _ = SlackParseUsers(reader)
+		} else {
+			spl := strings.Split(file.Name, "/")
+			if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
+				newposts, _ := SlackParsePosts(reader)
+				channel := spl[0]
+				if _, ok := posts[channel]; ok == false {
+					posts[channel] = newposts
+				} else {
+					posts[channel] = append(posts[channel], newposts...)
+				}
+			} else if len(spl) == 3 && spl[0] == "__uploads" {
+				uploads[spl[1]] = file
+			}
+		}
+	}
+
+	posts = SlackConvertUserMentions(users, posts)
+	posts = SlackConvertChannelMentions(channels, posts)
+	posts = SlackConvertPostsMarkup(posts)
+
+	addedUsers := SlackAddUsers(teamID, users, log)
+	botUser := SlackAddBotUser(teamID, log)
+
+	SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log)
+
+	if botUser != nil {
+		deactivateSlackBotUser(botUser)
+	}
+
+	InvalidateAllCaches()
+
+	log.WriteString(utils.T("api.slackimport.slack_import.notes"))
+	log.WriteString("=======\r\n\r\n")
+
+	log.WriteString(utils.T("api.slackimport.slack_import.note1"))
+	log.WriteString(utils.T("api.slackimport.slack_import.note2"))
+	log.WriteString(utils.T("api.slackimport.slack_import.note3"))
+
+	return nil, log
+}

app/slackimport_test.go

@@ -0,0 +1,240 @@
+// Copyright (c) 2016 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"github.com/mattermost/platform/model"
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestSlackConvertTimeStamp(t *testing.T) {
+
+	testTimeStamp := "1469785419.000033"
+
+	result := SlackConvertTimeStamp(testTimeStamp)
+
+	if result != 1469785419000 {
+		t.Fatalf("Unexpected timestamp value %v returned.", result)
+	}
+}
+
+func TestSlackConvertChannelName(t *testing.T) {
+	var testData = []struct {
+		input  string
+		output string
+	}{
+		{"test-channel", "test-channel"},
+		{"_test_channel_", "test_channel"},
+		{"__test", "test"},
+		{"-t", "slack-channel-t"},
+		{"a", "slack-channel-a"},
+	}
+
+	for _, td := range testData {
+		if td.output != SlackConvertChannelName(td.input) {
+			t.Fatalf("Did not convert channel name correctly: %v", td.input)
+		}
+	}
+}
+
+func TestSlackConvertUserMentions(t *testing.T) {
+	users := []SlackUser{
+		{Id: "U00000A0A", Username: "firstuser"},
+		{Id: "U00000B1B", Username: "seconduser"},
+	}
+
+	posts := map[string][]SlackPost{
+		"test-channel": {
+			{
+				Text: "<!channel>: Hi guys.",
+			},
+			{
+				Text: "Calling <!here|@here>.",
+			},
+			{
+				Text: "Yo <!everyone>.",
+			},
+			{
+				Text: "Regular user test <@U00000B1B|seconduser> and <@U00000A0A>.",
+			},
+		},
+	}
+
+	expectedPosts := map[string][]SlackPost{
+		"test-channel": {
+			{
+				Text: "@channel: Hi guys.",
+			},
+			{
+				Text: "Calling @here.",
+			},
+			{
+				Text: "Yo @all.",
+			},
+			{
+				Text: "Regular user test @seconduser and @firstuser.",
+			},
+		},
+	}
+
+	convertedPosts := SlackConvertUserMentions(users, posts)
+
+	for channelName, channelPosts := range convertedPosts {
+		for postIdx, post := range channelPosts {
+			if post.Text != expectedPosts[channelName][postIdx].Text {
+				t.Fatalf("Converted post text not as expected: %v", post.Text)
+			}
+		}
+	}
+}
+
+func TestSlackConvertChannelMentions(t *testing.T) {
+	channels := []SlackChannel{
+		{Id: "C000AA00A", Name: "one"},
+		{Id: "C000BB11B", Name: "two"},
+	}
+
+	posts := map[string][]SlackPost{
+		"test-channel": {
+			{
+				Text: "Go to <#C000AA00A>.",
+			},
+			{
+				User: "U00000A0A",
+				Text: "Try <#C000BB11B|two> for this.",
+			},
+		},
+	}
+
+	expectedPosts := map[string][]SlackPost{
+		"test-channel": {
+			{
+				Text: "Go to ~one.",
+			},
+			{
+				Text: "Try ~two for this.",
+			},
+		},
+	}
+
+	convertedPosts := SlackConvertChannelMentions(channels, posts)
+
+	for channelName, channelPosts := range convertedPosts {
+		for postIdx, post := range channelPosts {
+			if post.Text != expectedPosts[channelName][postIdx].Text {
+				t.Fatalf("Converted post text not as expected: %v", post.Text)
+			}
+		}
+	}
+
+}
+
+func TestSlackParseChannels(t *testing.T) {
+	file, err := os.Open("../tests/slack-import-test-channels.json")
+	if err != nil {
+		t.Fatalf("Failed to open data file: %v", err)
+	}
+
+	channels, err := SlackParseChannels(file)
+	if err != nil {
+		t.Fatalf("Error occurred parsing channels: %v", err)
+	}
+
+	if len(channels) != 6 {
+		t.Fatalf("Unexpected number of channels: %v", len(channels))
+	}
+}
+
+func TestSlackParseUsers(t *testing.T) {
+	file, err := os.Open("../tests/slack-import-test-users.json")
+	if err != nil {
+		t.Fatalf("Failed to open data file: %v", err)
+	}
+
+	users, err := SlackParseUsers(file)
+	if err != nil {
+		t.Fatalf("Error occurred parsing users: %v", err)
+	}
+
+	if len(users) != 11 {
+		t.Fatalf("Unexpected number of users: %v", len(users))
+	}
+}
+
+func TestSlackParsePosts(t *testing.T) {
+	file, err := os.Open("../tests/slack-import-test-posts.json")
+	if err != nil {
+		t.Fatalf("Failed to open data file: %v", err)
+	}
+
+	posts, err := SlackParsePosts(file)
+	if err != nil {
+		t.Fatalf("Error occurred parsing posts: %v", err)
+	}
+
+	if len(posts) != 8 {
+		t.Fatalf("Unexpected number of posts: %v", len(posts))
+	}
+}
+
+func TestSlackSanitiseChannelProperties(t *testing.T) {
+	c1 := model.Channel{
+		DisplayName: "display-name",
+		Name:        "name",
+		Purpose:     "The channel purpose",
+		Header:      "The channel header",
+	}
+
+	c1s := SlackSanitiseChannelProperties(c1)
+	if c1.DisplayName != c1s.DisplayName || c1.Name != c1s.Name || c1.Purpose != c1s.Purpose || c1.Header != c1s.Header {
+		t.Fatalf("Unexpected alterations to the channel properties.")
+	}
+
+	c2 := model.Channel{
+		DisplayName: strings.Repeat("abcdefghij", 7),
+		Name:        strings.Repeat("abcdefghij", 7),
+		Purpose:     strings.Repeat("0123456789", 30),
+		Header:      strings.Repeat("0123456789", 120),
+	}
+
+	c2s := SlackSanitiseChannelProperties(c2)
+	if c2s.DisplayName != strings.Repeat("abcdefghij", 6)+"abcd" {
+		t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.DisplayName)
+	}
+
+	if c2s.Name != strings.Repeat("abcdefghij", 6)+"abcd" {
+		t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Name)
+	}
+
+	if c2s.Purpose != strings.Repeat("0123456789", 25) {
+		t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Purpose)
+	}
+
+	if c2s.Header != strings.Repeat("0123456789", 102)+"0123" {
+		t.Fatalf("Unexpected alterations to the channel properties: %v", c2s.Header)
+	}
+}
+
+func TestSlackConvertPostsMarkup(t *testing.T) {
+	input := make(map[string][]SlackPost)
+	input["test"] = []SlackPost{
+		{
+			Text: "This message contains a link to <https://google.com|Google>.",
+		},
+		{
+			Text: "This message contains a mailto link to <mailto:me@example.com|me@example.com> in it.",
+		},
+	}
+
+	output := SlackConvertPostsMarkup(input)
+
+	if output["test"][0].Text != "This message contains a link to [Google](https://google.com)." {
+		t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text)
+	}
+	if output["test"][1].Text != "This message contains a mailto link to [me@example.com](mailto:me@example.com) in it." {
+		t.Fatalf("Unexpected message after markup translation: %v", output["test"][0].Text)
+	}
+}