Merge pull request #394 from mattermost/mm-1410
MM-1410 Initial implementaiton of import from Slack
Этот коммит содержится в:
Joram Wilander
@@ -397,7 +397,7 @@ func JoinChannel(c *Context, channelId string, role string) {
|
||||
}
|
||||
}
|
||||
|
||||
func JoinDefaultChannels(c *Context, user *model.User, channelRole string) *model.AppError {
|
||||
func JoinDefaultChannels(user *model.User, channelRole string) *model.AppError {
|
||||
// We don't call JoinChannel here since c.Session is not populated on user creation
|
||||
|
||||
var err *model.AppError = nil
|
||||
|
||||
57
api/import.go
Обычный файл
57
api/import.go
Обычный файл
@@ -0,0 +1,57 @@
|
||||
// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package api
|
||||
|
||||
import (
|
||||
l4g "code.google.com/p/log4go"
|
||||
"github.com/mattermost/platform/model"
|
||||
"github.com/mattermost/platform/utils"
|
||||
)
|
||||
|
||||
//
|
||||
// Import functions are sutible for entering posts and users into the database without
|
||||
// some of the usual checks. (IsValid is still run)
|
||||
//
|
||||
|
||||
func ImportPost(post *model.Post) {
|
||||
post.Hashtags, _ = model.ParseHashtags(post.Message)
|
||||
|
||||
if result := <-Srv.Store.Post().Save(post); result.Err != nil {
|
||||
l4g.Debug("Error saving post. user=" + post.UserId + ", message=" + post.Message)
|
||||
}
|
||||
}
|
||||
|
||||
func ImportUser(user *model.User) *model.User {
|
||||
user.MakeNonNil()
|
||||
if len(user.Props["theme"]) == 0 {
|
||||
user.AddProp("theme", utils.Cfg.TeamSettings.DefaultThemeColor)
|
||||
}
|
||||
|
||||
if result := <-Srv.Store.User().Save(user); result.Err != nil {
|
||||
l4g.Error("Error saving user. err=%v", result.Err)
|
||||
return nil
|
||||
} else {
|
||||
ruser := result.Data.(*model.User)
|
||||
|
||||
if err := JoinDefaultChannels(ruser, ""); err != nil {
|
||||
l4g.Error("Encountered an issue joining default channels user_id=%s, team_id=%s, err=%v", ruser.Id, ruser.TeamId, err)
|
||||
}
|
||||
|
||||
if cresult := <-Srv.Store.User().VerifyEmail(ruser.Id); cresult.Err != nil {
|
||||
l4g.Error("Failed to set email verified err=%v", cresult.Err)
|
||||
}
|
||||
|
||||
return ruser
|
||||
}
|
||||
}
|
||||
|
||||
func ImportChannel(channel *model.Channel) *model.Channel {
|
||||
if result := <-Srv.Store.Channel().Save(channel); result.Err != nil {
|
||||
return nil
|
||||
} else {
|
||||
sc := result.Data.(*model.Channel)
|
||||
|
||||
return sc
|
||||
}
|
||||
}
|
||||
244
api/slackimport.go
Обычный файл
244
api/slackimport.go
Обычный файл
@@ -0,0 +1,244 @@
|
||||
// Copyright (c) 2015 Spinpunch, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package api
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
l4g "code.google.com/p/log4go"
|
||||
"encoding/json"
|
||||
"github.com/mattermost/platform/model"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type SlackChannel struct {
|
||||
Id string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Members []string `json:"members"`
|
||||
Topic map[string]string `json:"topic"`
|
||||
}
|
||||
|
||||
type SlackUser struct {
|
||||
Id string `json:"id"`
|
||||
Username string `json:"name"`
|
||||
Profile map[string]string `json:"profile"`
|
||||
}
|
||||
|
||||
type SlackPost struct {
|
||||
User string `json:"user"`
|
||||
BotId string `json:"bot_id"`
|
||||
BotUsername string `json:"username"`
|
||||
Text string `json:"text"`
|
||||
TimeStamp string `json:"ts"`
|
||||
Type string `json:"type"`
|
||||
SubType string `json:"subtype"`
|
||||
Comment map[string]string `json:"comment"`
|
||||
}
|
||||
|
||||
func SlackConvertTimeStamp(ts string) int64 {
|
||||
timeString := strings.SplitN(ts, ".", 2)[0]
|
||||
|
||||
timeStamp, err := strconv.ParseInt(timeString, 10, 64)
|
||||
if err != nil {
|
||||
l4g.Warn("Bad timestamp detected")
|
||||
return 1
|
||||
}
|
||||
return timeStamp * 1000 // Convert to milliseconds
|
||||
}
|
||||
|
||||
func SlackParseChannels(data io.Reader) []SlackChannel {
|
||||
decoder := json.NewDecoder(data)
|
||||
|
||||
var channels []SlackChannel
|
||||
if err := decoder.Decode(&channels); err != nil {
|
||||
return make([]SlackChannel, 0)
|
||||
}
|
||||
return channels
|
||||
}
|
||||
|
||||
func SlackParseUsers(data io.Reader) []SlackUser {
|
||||
decoder := json.NewDecoder(data)
|
||||
|
||||
var users []SlackUser
|
||||
if err := decoder.Decode(&users); err != nil {
|
||||
return make([]SlackUser, 0)
|
||||
}
|
||||
return users
|
||||
}
|
||||
|
||||
func SlackParsePosts(data io.Reader) []SlackPost {
|
||||
decoder := json.NewDecoder(data)
|
||||
|
||||
var posts []SlackPost
|
||||
if err := decoder.Decode(&posts); err != nil {
|
||||
return make([]SlackPost, 0)
|
||||
}
|
||||
return posts
|
||||
}
|
||||
|
||||
func SlackAddUsers(teamId string, slackusers []SlackUser, log *bytes.Buffer) map[string]*model.User {
|
||||
// Log header
|
||||
log.WriteString("\n Users Created\n")
|
||||
log.WriteString("===============\n\n")
|
||||
|
||||
addedUsers := make(map[string]*model.User)
|
||||
for _, sUser := range slackusers {
|
||||
firstName := ""
|
||||
lastName := ""
|
||||
if name, ok := sUser.Profile["first_name"]; ok {
|
||||
firstName = name
|
||||
}
|
||||
if name, ok := sUser.Profile["last_name"]; ok {
|
||||
lastName = name
|
||||
}
|
||||
|
||||
password := model.NewId()
|
||||
|
||||
newUser := model.User{
|
||||
TeamId: teamId,
|
||||
Username: sUser.Username,
|
||||
FirstName: firstName,
|
||||
LastName: lastName,
|
||||
Email: sUser.Profile["email"],
|
||||
Password: password,
|
||||
}
|
||||
|
||||
if mUser := ImportUser(&newUser); mUser != nil {
|
||||
addedUsers[sUser.Id] = mUser
|
||||
log.WriteString("Email, Password: " + newUser.Email + ", " + password + "\n")
|
||||
} else {
|
||||
log.WriteString("Unable to import user: " + sUser.Username)
|
||||
}
|
||||
}
|
||||
|
||||
return addedUsers
|
||||
}
|
||||
|
||||
func SlackAddPosts(channel *model.Channel, posts []SlackPost, users map[string]*model.User) {
|
||||
for _, sPost := range posts {
|
||||
switch {
|
||||
case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
|
||||
if sPost.User == "" {
|
||||
l4g.Debug("Message without user")
|
||||
continue
|
||||
} else if users[sPost.User] == nil {
|
||||
l4g.Debug("User: " + sPost.User + " does not exist!")
|
||||
continue
|
||||
}
|
||||
newPost := model.Post{
|
||||
UserId: users[sPost.User].Id,
|
||||
ChannelId: channel.Id,
|
||||
Message: sPost.Text,
|
||||
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
||||
}
|
||||
ImportPost(&newPost)
|
||||
case sPost.Type == "message" && sPost.SubType == "file_comment":
|
||||
if sPost.Comment["user"] == "" {
|
||||
l4g.Debug("Message without user")
|
||||
continue
|
||||
} else if users[sPost.Comment["user"]] == nil {
|
||||
l4g.Debug("User: " + sPost.User + " does not exist!")
|
||||
continue
|
||||
}
|
||||
newPost := model.Post{
|
||||
UserId: users[sPost.Comment["user"]].Id,
|
||||
ChannelId: channel.Id,
|
||||
Message: sPost.Comment["comment"],
|
||||
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
|
||||
}
|
||||
ImportPost(&newPost)
|
||||
case sPost.Type == "message" && sPost.SubType == "bot_message":
|
||||
// In the future this will use the "Action Post" spec to post
|
||||
// a message without using a username. For now we just warn that we don't handle this case
|
||||
l4g.Warn("Slack bot posts are not imported yet")
|
||||
default:
|
||||
l4g.Warn("Unsupported post type: " + sPost.Type + ", " + sPost.SubType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, log *bytes.Buffer) map[string]*model.Channel {
|
||||
// Write Header
|
||||
log.WriteString("\n Channels Added \n")
|
||||
log.WriteString("=================\n\n")
|
||||
|
||||
addedChannels := make(map[string]*model.Channel)
|
||||
for _, sChannel := range slackchannels {
|
||||
newChannel := model.Channel{
|
||||
TeamId: teamId,
|
||||
Type: model.CHANNEL_OPEN,
|
||||
DisplayName: sChannel.Name,
|
||||
Name: sChannel.Name,
|
||||
Description: sChannel.Topic["value"],
|
||||
}
|
||||
mChannel := ImportChannel(&newChannel)
|
||||
if mChannel == nil {
|
||||
// Maybe it already exists?
|
||||
if result := <-Srv.Store.Channel().GetByName(teamId, sChannel.Name); result.Err != nil {
|
||||
l4g.Debug("Failed to import: %s", newChannel.DisplayName)
|
||||
log.WriteString("Failed to import: " + newChannel.DisplayName + "\n")
|
||||
continue
|
||||
} else {
|
||||
mChannel = result.Data.(*model.Channel)
|
||||
log.WriteString("Merged with existing channel: " + newChannel.DisplayName + "\n")
|
||||
}
|
||||
}
|
||||
log.WriteString(newChannel.DisplayName + "\n")
|
||||
addedChannels[sChannel.Id] = mChannel
|
||||
SlackAddPosts(mChannel, posts[sChannel.Name], users)
|
||||
}
|
||||
|
||||
return addedChannels
|
||||
}
|
||||
|
||||
func SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
|
||||
zipreader, err := zip.NewReader(fileData, fileSize)
|
||||
if err != nil || zipreader.File == nil {
|
||||
return model.NewAppError("SlackImport", "Unable to open zip file", err.Error()), nil
|
||||
}
|
||||
|
||||
// Create log file
|
||||
log := bytes.NewBufferString("Mattermost Slack Import Log\n")
|
||||
|
||||
var channels []SlackChannel
|
||||
var users []SlackUser
|
||||
posts := make(map[string][]SlackPost)
|
||||
for _, file := range zipreader.File {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
return model.NewAppError("SlackImport", "Unable to open: "+file.Name, err.Error()), log
|
||||
}
|
||||
if file.Name == "channels.json" {
|
||||
channels = SlackParseChannels(reader)
|
||||
} else if file.Name == "users.json" {
|
||||
users = SlackParseUsers(reader)
|
||||
} else {
|
||||
spl := strings.Split(file.Name, "/")
|
||||
if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
|
||||
newposts := SlackParsePosts(reader)
|
||||
channel := spl[0]
|
||||
if _, ok := posts[channel]; ok == false {
|
||||
posts[channel] = newposts
|
||||
} else {
|
||||
posts[channel] = append(posts[channel], newposts...)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
addedUsers := SlackAddUsers(teamID, users, log)
|
||||
SlackAddChannels(teamID, channels, posts, addedUsers, log)
|
||||
|
||||
log.WriteString("\n Notes \n")
|
||||
log.WriteString("=======\n\n")
|
||||
|
||||
log.WriteString("- Some posts may not have been imported because they where not supported by this importer.\n")
|
||||
log.WriteString("- Slack bot posts are currently not supported.\n")
|
||||
|
||||
return nil, log
|
||||
}
|
||||
70
api/team.go
70
api/team.go
@@ -4,6 +4,7 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
l4g "code.google.com/p/log4go"
|
||||
"fmt"
|
||||
"github.com/gorilla/mux"
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func InitTeam(r *mux.Router) {
|
||||
@@ -29,6 +31,7 @@ func InitTeam(r *mux.Router) {
|
||||
sr.Handle("/update_name", ApiUserRequired(updateTeamDisplayName)).Methods("POST")
|
||||
sr.Handle("/update_valet_feature", ApiUserRequired(updateValetFeature)).Methods("POST")
|
||||
sr.Handle("/me", ApiUserRequired(getMyTeam)).Methods("GET")
|
||||
sr.Handle("/import_team", ApiUserRequired(importTeam)).Methods("POST")
|
||||
}
|
||||
|
||||
func signupTeam(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
@@ -489,3 +492,70 @@ func getMyTeam(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func importTeam(c *Context, w http.ResponseWriter, r *http.Request) {
|
||||
if !c.HasPermissionsToTeam(c.Session.TeamId, "import") || !c.IsTeamAdmin(c.Session.UserId) {
|
||||
c.Err = model.NewAppError("importTeam", "Only a team admin can import data.", "userId="+c.Session.UserId)
|
||||
c.Err.StatusCode = http.StatusForbidden
|
||||
return
|
||||
}
|
||||
|
||||
if err := r.ParseMultipartForm(10000000); err != nil {
|
||||
c.Err = model.NewAppError("importTeam", "Could not parse multipart form", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
importFromArray, ok := r.MultipartForm.Value["importFrom"]
|
||||
importFrom := importFromArray[0]
|
||||
|
||||
fileSizeStr, ok := r.MultipartForm.Value["filesize"]
|
||||
if !ok {
|
||||
c.Err = model.NewAppError("importTeam", "Filesize unavilable", "")
|
||||
c.Err.StatusCode = http.StatusBadRequest
|
||||
return
|
||||
}
|
||||
|
||||
fileSize, err := strconv.ParseInt(fileSizeStr[0], 10, 64)
|
||||
if err != nil {
|
||||
c.Err = model.NewAppError("importTeam", "Filesize not an integer", "")
|
||||
c.Err.StatusCode = http.StatusBadRequest
|
||||
return
|
||||
}
|
||||
|
||||
fileInfoArray, ok := r.MultipartForm.File["file"]
|
||||
if !ok {
|
||||
c.Err = model.NewAppError("importTeam", "No file under 'file' in request", "")
|
||||
c.Err.StatusCode = http.StatusBadRequest
|
||||
return
|
||||
}
|
||||
|
||||
if len(fileInfoArray) <= 0 {
|
||||
c.Err = model.NewAppError("importTeam", "Empty array under 'file' in request", "")
|
||||
c.Err.StatusCode = http.StatusBadRequest
|
||||
return
|
||||
}
|
||||
|
||||
fileInfo := fileInfoArray[0]
|
||||
|
||||
fileData, err := fileInfo.Open()
|
||||
defer fileData.Close()
|
||||
if err != nil {
|
||||
c.Err = model.NewAppError("importTeam", "Could not open file", err.Error())
|
||||
c.Err.StatusCode = http.StatusBadRequest
|
||||
return
|
||||
}
|
||||
|
||||
var log *bytes.Buffer
|
||||
switch importFrom {
|
||||
case "slack":
|
||||
var err *model.AppError
|
||||
if err, log = SlackImport(fileData, fileSize, c.Session.TeamId); err != nil {
|
||||
c.Err = err
|
||||
c.Err.StatusCode = http.StatusBadRequest
|
||||
}
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Disposition", "attachment; filename=MattermostImportLog.txt")
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
http.ServeContent(w, r, "MattermostImportLog.txt", time.Now(), bytes.NewReader(log.Bytes()))
|
||||
}
|
||||
|
||||
@@ -181,12 +181,13 @@ func CreateUser(c *Context, team *model.Team, user *model.User) *model.User {
|
||||
|
||||
if result := <-Srv.Store.User().Save(user); result.Err != nil {
|
||||
c.Err = result.Err
|
||||
l4g.Error("Filae err=%v", result.Err)
|
||||
return nil
|
||||
} else {
|
||||
ruser := result.Data.(*model.User)
|
||||
|
||||
// Soft error if there is an issue joining the default channels
|
||||
if err := JoinDefaultChannels(c, ruser, channelRole); err != nil {
|
||||
if err := JoinDefaultChannels(ruser, channelRole); err != nil {
|
||||
l4g.Error("Encountered an issue joining default channels user_id=%s, team_id=%s, err=%v", ruser.Id, ruser.TeamId, err)
|
||||
}
|
||||
|
||||
|
||||
Ссылка в новой задаче
Block a user