Implemention of slack import feature.

2015-07-07 09:16:13 -04:00
--- a/api/channel.go
+++ b/api/channel.go
@@ -397,7 +397,7 @@ func JoinChannel(c *Context, channelId string, role string) {
 	}
 }

-func JoinDefaultChannels(c *Context, user *model.User, channelRole string) *model.AppError {
+func JoinDefaultChannels(user *model.User, channelRole string) *model.AppError {
 	// We don't call JoinChannel here since c.Session is not populated on user creation

 	var err *model.AppError = nil
--- a/api/import.go
+++ b/api/import.go
@@ -0,0 +1,57 @@
+// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package api
+
+import (
+	l4g "code.google.com/p/log4go"
+	"github.com/mattermost/platform/model"
+	"github.com/mattermost/platform/utils"
+)
+
+//
+// Import functions are sutible for entering posts and users into the database without
+// some of the usual checks. (IsValid is still run)
+//
+
+func ImportPost(post *model.Post) {
+	post.Hashtags, _ = model.ParseHashtags(post.Message)
+
+	if result := <-Srv.Store.Post().Save(post); result.Err != nil {
+		l4g.Debug("Error saving post. user=" + post.UserId + ", message=" + post.Message)
+	}
+}
+
+func ImportUser(user *model.User) *model.User {
+	user.MakeNonNil()
+	if len(user.Props["theme"]) == 0 {
+		user.AddProp("theme", utils.Cfg.TeamSettings.DefaultThemeColor)
+	}
+
+	if result := <-Srv.Store.User().Save(user); result.Err != nil {
+		l4g.Error("Error saving user. err=%v", result.Err)
+		return nil
+	} else {
+		ruser := result.Data.(*model.User)
+
+		if err := JoinDefaultChannels(ruser, ""); err != nil {
+			l4g.Error("Encountered an issue joining default channels user_id=%s, team_id=%s, err=%v", ruser.Id, ruser.TeamId, err)
+		}
+
+		if cresult := <-Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil {
+			l4g.Error("Failed to set email verified err=%v", cresult.Err)
+		}
+
+		return ruser
+	}
+}
+
+func ImportChannel(channel *model.Channel) *model.Channel {
+	if result := <-Srv.Store.Channel().Save(channel); result.Err != nil {
+		return nil
+	} else {
+		sc := result.Data.(*model.Channel)
+
+		return sc
+	}
+}
--- a/api/slackimport.go
+++ b/api/slackimport.go
@@ -0,0 +1,246 @@
+// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package api
+
+import (
+	"archive/zip"
+	"bytes"
+	l4g "code.google.com/p/log4go"
+	"encoding/json"
+	"github.com/mattermost/platform/model"
+	"io"
+	"mime/multipart"
+	"strconv"
+	"strings"
+)
+
+type SlackChannel struct {
+	Id      string            `json:"id"`
+	Name    string            `json:"name"`
+	Members []string          `json:"members"`
+	Topic   map[string]string `json:"topic"`
+}
+
+type SlackUser struct {
+	Id       string            `json:"id"`
+	UserName string            `json:"name"`
+	Profile  map[string]string `json:"profile"`
+}
+
+type SlackPost struct {
+	User        string            `json:"user"`
+	BotId       string            `json:"bot_id"`
+	BotUsername string            `json:"username"`
+	Text        string            `json:"text"`
+	TimeStamp   string            `json:"ts"`
+	Type        string            `json:"type"`
+	SubType     string            `json:"subtype"`
+	Comment     map[string]string `json:"comment"`
+}
+
+func SlackConvertTimeStamp(ts string) int64 {
+	timeString := strings.SplitN(ts, ".", 2)[0]
+
+	timeStamp, err := strconv.ParseInt(timeString, 10, 64)
+	if err != nil {
+		l4g.Warn("Bad timestamp detected")
+		return 1
+	}
+	return timeStamp * 1000 // Convert to milliseconds
+}
+
+func SlackParseChannels(data io.Reader) []SlackChannel {
+	decoder := json.NewDecoder(data)
+
+	var channels []SlackChannel
+	if err := decoder.Decode(&channels); err != nil {
+		return make([]SlackChannel, 0)
+	}
+	return channels
+}
+
+func SlackParseUsers(data io.Reader) []SlackUser {
+	decoder := json.NewDecoder(data)
+
+	var users []SlackUser
+	if err := decoder.Decode(&users); err != nil {
+		return make([]SlackUser, 0)
+	}
+	return users
+}
+
+func SlackParsePosts(data io.Reader) []SlackPost {
+	decoder := json.NewDecoder(data)
+
+	var posts []SlackPost
+	if err := decoder.Decode(&posts); err != nil {
+		return make([]SlackPost, 0)
+	}
+	return posts
+}
+
+func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User {
+	// Log header
+	log.WriteString("\n Users Created\n")
+	log.WriteString("===============\n\n")
+
+	// Add users
+	addedUsers := make(map[string]*model.User)
+	for _, sUser := range slackusers {
+		firstName := ""
+		lastName := ""
+		if name, ok := sUser.Profile["first_name"]; ok {
+			firstName = name
+		}
+		if name, ok := sUser.Profile["last_name"]; ok {
+			lastName = name
+		}
+
+		password := model.NewId()
+
+		newUser := model.User{
+			TeamId:    teamId,
+			Username:  sUser.UserName,
+			FirstName: firstName,
+			LastName:  lastName,
+			Email:     sUser.Profile["email"],
+			Password:  password,
+		}
+
+		if mUser := ImportUser(&newUser); mUser != nil {
+			addedUsers[sUser.Id] = mUser
+			log.WriteString("Email, Password: " + newUser.Email + ", " + password + "\n")
+		} else {
+			log.WriteString("Unable to import user: " + sUser.UserName)
+		}
+	}
+
+	return addedUsers
+}
+
+func SlackAddPosts(channel *model.Channel, posts []SlackPost, users map[string]*model.User) {
+	for _, sPost := range posts {
+		switch {
+		case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
+			if sPost.User == "" {
+				l4g.Debug("Message without user")
+				continue
+			} else if users[sPost.User] == nil {
+				l4g.Debug("User: " + sPost.User + " does not exist!")
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+			}
+			ImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "file_comment":
+			if sPost.Comment["user"] == "" {
+				l4g.Debug("Message without user")
+				continue
+			} else if users[sPost.Comment["user"]] == nil {
+				l4g.Debug("User: " + sPost.User + " does not exist!")
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.Comment["user"]].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Comment["comment"],
+				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
+			}
+			ImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "bot_message":
+			// In the future this will use the "Action Post" spec to post
+			// a message without using a username. For now we just warn that we don't handle this case
+			l4g.Warn("Slack bot posts are not imported yet")
+		default:
+			l4g.Warn("Unsupported post type: " + sPost.Type + ", " + sPost.SubType)
+		}
+	}
+}
+
+func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, log *bytes.Buffer) map[string]*model.Channel {
+	// Write Header
+	log.WriteString("\n Channels Added \n")
+	log.WriteString("=================\n\n")
+
+	// Add channels
+	addedChannels := make(map[string]*model.Channel)
+	for _, sChannel := range slackchannels {
+		newChannel := model.Channel{
+			TeamId:      teamId,
+			Type:        model.CHANNEL_OPEN,
+			DisplayName: sChannel.Name,
+			Name:        sChannel.Name,
+			Description: sChannel.Topic["value"],
+		}
+		mChannel := ImportChannel(&newChannel)
+		if mChannel == nil {
+			// Maybe it already exists?
+			if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
+				l4g.Debug("Failed to import: %s", newChannel.DisplayName)
+				log.WriteString("Failed to import: " + newChannel.DisplayName + "\n")
+				continue
+			} else {
+				mChannel = result.Data.(*model.Channel)
+				log.WriteString("Merged with existing channel: " + newChannel.DisplayName + "\n")
+			}
+		}
+		log.WriteString(newChannel.DisplayName + "\n")
+		addedChannels[sChannel.Id] = mChannel
+		SlackAddPosts(mChannel, posts[sChannel.Name], users)
+	}
+
+	return addedChannels
+}
+
+func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
+	zipreader, err := zip.NewReader(fileData, fileSize)
+	if err != nil || zipreader.File == nil {
+		return model.NewAppError("SlackImport", "Unable to open zip file", err.Error()), nil
+	}
+
+	// Create log file
+	log := bytes.NewBufferString("Mattermost Slack Import Log\n")
+
+	var channels []SlackChannel
+	var users []SlackUser
+	posts := make(map[string][]SlackPost)
+	for _, file := range zipreader.File {
+		reader, err := file.Open()
+		if err != nil {
+			return model.NewAppError("SlackImport", "Unable to open: "+file.Name, err.Error()), log
+		}
+		if file.Name == "channels.json" {
+			channels = SlackParseChannels(reader)
+		} else if file.Name == "users.json" {
+			users = SlackParseUsers(reader)
+		} else {
+			spl := strings.Split(file.Name, "/")
+			if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
+				newposts := SlackParsePosts(reader)
+				channel := spl[0]
+				if _, ok := posts[channel]; ok == false {
+					posts[channel] = newposts
+				} else {
+					posts[channel] = append(posts[channel], newposts...)
+				}
+			}
+
+		}
+	}
+
+	addedUsers := SlackAddUsers(teamID, users, log)
+	SlackAddChannels(teamID, channels, posts, addedUsers, log)
+
+	log.WriteString("\n Notes \n")
+	log.WriteString("=======\n\n")
+
+	log.WriteString("- Some posts may not have been imported because they where not supported by this importer.\n")
+	log.WriteString("- Slack bot posts are currently not supported.\n")
+
+	return nil, log
+}
--- a/api/team.go
+++ b/api/team.go
@@ -4,6 +4,7 @@
 package api

 import (
+	"bytes"
 	l4g "code.google.com/p/log4go"
 	"fmt"
 	"github.com/gorilla/mux"
@@ -13,6 +14,7 @@ import (
 	"net/url"
 	"strconv"
 	"strings"
+	"time"
 )

 func InitTeam(r *mux.Router) {
@@ -29,6 +31,7 @@ func InitTeam(r *mux.Router) {
 	sr.Handle("/update_name", ApiUserRequired(updateTeamDisplayName)).Methods("POST")
 	sr.Handle("/update_valet_feature", ApiUserRequired(updateValetFeature)).Methods("POST")
 	sr.Handle("/me", ApiUserRequired(getMyTeam)).Methods("GET")
+	sr.Handle("/import_team", ApiUserRequired(importTeam)).Methods("POST")
 }

 func signupTeam(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -489,3 +492,71 @@ func getMyTeam(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 }
+
+func importTeam(c *Context, w http.ResponseWriter, r *http.Request) {
+
+	if !c.HasPermissionsToTeam(c.Session.TeamId, "import") {
+		c.Err = model.NewAppError("importTeam", "Only a team admin can import data.", "userId="+c.Session.UserId)
+		c.Err.StatusCode = http.StatusForbidden
+		return
+	}
+
+	if err := r.ParseMultipartForm(10000000); err != nil {
+		c.Err = model.NewAppError("importTeam", "Could not parse multipart form", err.Error())
+		return
+	}
+
+	importFromArray, ok := r.MultipartForm.Value["importFrom"]
+	importFrom := importFromArray[0]
+
+	fileSizeStr, ok := r.MultipartForm.Value["filesize"]
+	if !ok {
+		c.Err = model.NewAppError("importTeam", "Filesize unavilable", "")
+		c.Err.StatusCode = http.StatusBadRequest
+		return
+	}
+
+	fileSize, err := strconv.ParseInt(fileSizeStr[0], 10, 64)
+	if err != nil {
+		c.Err = model.NewAppError("importTeam", "Filesize not an integer", "")
+		c.Err.StatusCode = http.StatusBadRequest
+		return
+	}
+
+	fileInfoArray, ok := r.MultipartForm.File["file"]
+	if !ok {
+		c.Err = model.NewAppError("importTeam", "No file under 'file' in request", "")
+		c.Err.StatusCode = http.StatusBadRequest
+		return
+	}
+
+	if len(fileInfoArray) <= 0 {
+		c.Err = model.NewAppError("importTeam", "Empty array under 'file' in request", "")
+		c.Err.StatusCode = http.StatusBadRequest
+		return
+	}
+
+	fileInfo := fileInfoArray[0]
+
+	fileData, err := fileInfo.Open()
+	defer fileData.Close()
+	if err != nil {
+		c.Err = model.NewAppError("importTeam", "Could not open file", err.Error())
+		c.Err.StatusCode = http.StatusBadRequest
+		return
+	}
+
+	var log *bytes.Buffer
+	switch importFrom {
+	case "slack":
+		var err *model.AppError
+		if err, log = SlackImport(fileData, fileSize, c.Session.TeamId); err != nil {
+			c.Err = err
+			c.Err.StatusCode = http.StatusBadRequest
+		}
+	}
+
+	w.Header().Set("Content-Disposition", "attachment; filename=MattermostImportLog.txt")
+	w.Header().Set("Content-Type", "application/octet-stream")
+	http.ServeContent(w, r, "MattermostImportLog.txt", time.Now(), bytes.NewReader(log.Bytes()))
+}
--- a/api/user.go
+++ b/api/user.go
@@ -181,12 +181,13 @@ func CreateUser(c *Context, team *model.Team, user *model.User) *model.User {

 	if result := <-Srv.Store.User().Save(user); result.Err != nil {
 		c.Err = result.Err
+		l4g.Error("Filae err=%v", result.Err)
 		return nil
 	} else {
 		ruser := result.Data.(*model.User)

 		// Soft error if there is an issue joining the default channels
-		if err := JoinDefaultChannels(c, ruser, channelRole); err != nil {
+		if err := JoinDefaultChannels(ruser, channelRole); err != nil {
 			l4g.Error("Encountered an issue joining default channels user_id=%s, team_id=%s, err=%v", ruser.Id, ruser.TeamId, err)
 		}

--- a/model/post.go
+++ b/model/post.go
@@ -120,7 +120,9 @@ func (o *Post) PreSave() {

 	o.OriginalId = ""

-	o.CreateAt = GetMillis()
+	if o.CreateAt <= 0 {
+		o.CreateAt = GetMillis()
+	}
 	o.UpdateAt = o.CreateAt

 	if o.Props == nil {
--- a/web/react/components/setting_upload.jsx
+++ b/web/react/components/setting_upload.jsx
@@ -0,0 +1,79 @@
+// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+module.exports = React.createClass({
+    displayName: 'Setting Upload',
+    propTypes: {
+        title: React.PropTypes.string.isRequired,
+        submit: React.PropTypes.func.isRequired,
+        fileTypesAccepted: React.PropTypes.string.isRequired,
+        clientError: React.PropTypes.string,
+        serverError: React.PropTypes.string
+    },
+    getInitialState: function() {
+        return {
+            clientError: this.props.clientError,
+            serverError: this.props.serverError
+        };
+    },
+    componentWillReceiveProps: function() {
+        this.setState({
+            clientError: this.props.clientError,
+            serverError: this.props.serverError
+        });
+    },
+    doFileSelect: function(e) {
+        e.preventDefault();
+        this.setState({
+            clientError: '',
+            serverError: ''
+        });
+    },
+    doSubmit: function(e) {
+        e.preventDefault();
+        var inputnode = this.refs.uploadinput.getDOMNode();
+        if (inputnode.files && inputnode.files[0]) {
+            this.props.submit(inputnode.files[0]);
+        } else {
+            this.setState({clientError: 'No file selected.'});
+        }
+    },
+    doCancel: function(e) {
+        e.preventDefault();
+        this.refs.uploadinput.getDOMNode().value = '';
+        this.setState({
+            clientError: '',
+            serverError: ''
+        });
+    },
+    render: function() {
+        var clientError = null;
+        if (this.state.clientError) {
+            clientError = (
+                <div className='form-group has-error'><label className='control-label'>{this.state.clientError}</label></div>
+            );
+        }
+        var serverError = null;
+        if (this.state.serverError) {
+            serverError = (
+                <div className='form-group has-error'><label className='control-label'>{this.state.serverError}</label></div>
+            );
+        }
+        return (
+            <ul className='section-max'>
+                <li className='col-xs-12 section-title'>{this.props.title}</li>
+                <li className='col-xs-offset-3 col-xs-8'>
+                    <ul className='setting-list'>
+                        <li className='setting-list-item'>
+                            {serverError}
+                            {clientError}
+                            <span className='btn btn-sm btn-primary btn-file sel-btn'>SelectFile<input ref='uploadinput' accept={this.props.fileTypesAccepted} type='file' onChange={this.onFileSelect}/></span>
+                            <a className={'btn btn-sm btn-primary'} onClick={this.doSubmit}>Import</a>
+                            <a className='btn btn-sm theme' href='#' onClick={this.doCancel}>Cancel</a>
+                        </li>
+                    </ul>
+                </li>
+            </ul>
+        );
+    }
+});
--- a/web/react/components/team_settings.jsx
+++ b/web/react/components/team_settings.jsx
@@ -6,6 +6,7 @@ var TeamStore = require('../stores/team_store.jsx');
 var SettingItemMin = require('./setting_item_min.jsx');
 var SettingItemMax = require('./setting_item_max.jsx');
 var SettingPicture = require('./setting_picture.jsx');
+var SettingUpload = require('./setting_upload.jsx');
 var utils = require('../utils/utils.jsx');

 var client = require('../utils/client.jsx');
@@ -126,6 +127,69 @@ var FeatureTab = React.createClass({
    }
 });

+var ImportTab = React.createClass({
+    getInitialState: function() {
+        return {status: 'ready'};
+    },
+    onImportFailure: function() {
+        this.setState({status: 'fail'});
+    },
+    onImportSuccess: function(data) {
+        this.setState({status: 'done'});
+    },
+    doImportSlack: function(file) {
+        this.setState({status: 'in-progress'});
+        utils.importSlack(file, this.onImportSuccess, this.onImportFailure);
+    },
+    render: function() {
+
+		uploadSection = (
+			<SettingUpload
+				title="Import from Slack"
+                submit={this.doImportSlack}
+                fileTypesAccepted='.zip'/>
+		);
+
+        var messageSection;
+        switch (this.state.status) {
+            case 'ready':
+                messageSection = '';
+                break;
+            case 'in-progress':
+                messageSection = (
+                    <p>Importing...</p>
+                );
+                break;
+            case 'done':
+                messageSection = (
+                    <p>Import sucessfull: <a href={this.state.link} download='MattermostImportSummery.txt'>View Summery</a></p>
+                );
+                break;
+            case 'fail':
+                messageSection = (
+                    <p>Import failure: <a href={this.state.link} download='MattermostImportSummery.txt'>View Summery</a></p>
+                );
+                break;
+        }
+
+        return (
+            <div>
+                <div className="modal-header">
+                    <button type="button" className="close" data-dismiss="modal" aria-label="Close"><span aria-hidden="true">&times;</span></button>
+                    <h4 className="modal-title" ref="title"><i className="modal-back"></i>Import</h4>
+                </div>
+                <div ref="wrapper" className="user-settings">
+                    <h3 className="tab-header">Import</h3>
+                    <div className="divider-dark first"/>
+					{uploadSection}
+                    {messageSection}
+                    <div className="divider-dark"/>
+                </div>
+            </div>
+        );
+	}
+});
+
 module.exports = React.createClass({
    componentDidMount: function() {
        TeamStore.addChangeListener(this._onChange);
@@ -154,7 +218,13 @@ module.exports = React.createClass({
                    <FeatureTab team={this.state.team} activeSection={this.props.activeSection} updateSection={this.props.updateSection} />
                </div>
            );
-        } else {
+		} else if (this.props.activeTab === 'import') {
+			return (
+                <div>
+                    <ImportTab team={this.state.team} activeSection={this.props.activeSection} updateSection={this.props.updateSection} />
+                </div>
+			);
+		} else {
            return <div/>;
        }
    }
--- a/web/react/components/team_settings_modal.jsx
+++ b/web/react/components/team_settings_modal.jsx
@@ -27,6 +27,7 @@ module.exports = React.createClass({
    render: function() {
        var tabs = [];
        tabs.push({name: "feature", ui_name: "Features", icon: "glyphicon glyphicon-wrench"});
+        tabs.push({name: "import", ui_name: "Import", icon: "glyphicon glyphicon-upload"});

        return (
            <div className="modal fade" ref="modal" id="team_settings" role="dialog" tabIndex="-1" aria-hidden="true">
--- a/web/react/utils/client.jsx
+++ b/web/react/utils/client.jsx
@@ -862,6 +862,22 @@ module.exports.uploadProfileImage = function(imageData, success, error) {
    });
 };

+module.exports.importSlack = function(fileData, success, error) {
+    $.ajax({
+		url: "/api/v1/teams/import_team",
+        type: 'POST',
+        data: fileData,
+        cache: false,
+        contentType: false,
+        processData: false,
+        success: success,
+        error: function(xhr, status, err) {
+            e = handleError("importTeam", xhr, status, err);
+            error(e);
+        }
+    });
+}
+
 module.exports.getStatuses = function(success, error) {
    $.ajax({
        url: "/api/v1/users/status",
--- a/web/react/utils/utils.jsx
+++ b/web/react/utils/utils.jsx
@@ -982,3 +982,12 @@ module.exports.getUserIdFromChannelName = function(channel) {

    return otherUserId;
 };
+
+module.exports.importSlack = function(file, success, error) {
+	formData = new FormData();
+	formData.append('file', file, file.name);
+	formData.append('filesize', file.size);
+    formData.append('importFrom', 'slack');
+
+	client.importSlack(formData, success, error);
+};