mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

PLT-5367: Basic post bulk importing. (#5562)

Просмотр исходного кода
Этот коммит содержится в:
George Goldberg
2017-02-28 23:55:21 +00:00
коммит произвёл enahum
родитель 48d124ec9a
Коммит f7682ad11a
7 изменённых файлов: 731 добавлений и 16 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

125
app/import.go
Просмотреть файл

@@ -8,6 +8,7 @@ import (
"bytes"
"encoding/json"
"io"
"net/http"
"regexp"
"strings"
"unicode/utf8"
@@ -15,7 +16,6 @@ import (
l4g "github.com/alecthomas/log4go"
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
"net/http"
)
// Import Data Models
@@ -25,6 +25,7 @@ type LineImportData struct {
Team *TeamImportData `json:"team"`
Channel *ChannelImportData `json:"channel"`
User *UserImportData `json:"user"`
Post *PostImportData `json:"post"`
Version *int `json:"version"`
}
@@ -85,6 +86,15 @@ type UserChannelNotifyPropsImportData struct {
MarkUnread *string `json:"mark_unread"`
}
type PostImportData struct {
Team *string `json:"team"`
Channel *string `json:"channel"`
User *string `json:"user"`
Message *string `json:"message"`
CreateAt *int64 `json:"create_at"`
}
//
// -- Bulk Import Functions --
// These functions import data directly into the database. Security and permission checks are bypassed but validity is
@@ -152,6 +162,12 @@ func ImportLine(line LineImportData, dryRun bool) *model.AppError {
} else {
return ImportUser(line.User, dryRun)
}
case line.Type == "post":
if line.Post == nil {
return model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest)
} else {
return ImportPost(line.Post, dryRun)
}
default:
return model.NewLocAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "")
}
@@ -698,13 +714,112 @@ func validateUserChannelsImportData(data *[]UserChannelImportData) *model.AppErr
return nil
}
func ImportPost(data *PostImportData, dryRun bool) *model.AppError {
if err := validatePostImportData(data); err != nil {
return err
}
// If this is a Dry Run, do not continue any further.
if dryRun {
return nil
}
var team *model.Team
if result := <-Srv.Store.Team().GetByName(*data.Team); result.Err != nil {
return model.NewAppError("BulkImport", "app.import.import_post.team_not_found.error", map[string]interface{}{"TeamName": *data.Team}, "", http.StatusBadRequest)
} else {
team = result.Data.(*model.Team)
}
var channel *model.Channel
if result := <-Srv.Store.Channel().GetByName(team.Id, *data.Channel, false); result.Err != nil {
return model.NewAppError("BulkImport", "app.import.import_post.channel_not_found.error", map[string]interface{}{"ChannelName": *data.Channel}, "", http.StatusBadRequest)
} else {
channel = result.Data.(*model.Channel)
}
var user *model.User
if result := <-Srv.Store.User().GetByUsername(*data.User); result.Err != nil {
return model.NewAppError("BulkImport", "app.import.import_post.user_not_found.error", map[string]interface{}{"Username": *data.User}, "", http.StatusBadRequest)
} else {
user = result.Data.(*model.User)
}
// Check if this post already exists.
var posts []*model.Post
if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, *data.CreateAt); result.Err != nil {
return result.Err
} else {
posts = result.Data.([]*model.Post)
}
var post *model.Post
for _, p := range posts {
if p.Message == *data.Message {
post = p
break
}
}
if post == nil {
post = &model.Post{}
}
post.ChannelId = channel.Id
post.Message = *data.Message
post.UserId = user.Id
post.CreateAt = *data.CreateAt
post.Hashtags, _ = model.ParseHashtags(post.Message)
if post.Id == "" {
if result := <-Srv.Store.Post().Save(post); result.Err != nil {
return result.Err
}
} else {
if result := <-Srv.Store.Post().Overwrite(post); result.Err != nil {
return result.Err
}
}
return nil
}
func validatePostImportData(data *PostImportData) *model.AppError {
if data.Team == nil {
return model.NewAppError("BulkImport", "app.import.validate_post_import_data.team_missing.error", nil, "", http.StatusBadRequest)
}
if data.Channel == nil {
return model.NewAppError("BulkImport", "app.import.validate_post_import_data.channel_missing.error", nil, "", http.StatusBadRequest)
}
if data.User == nil {
return model.NewAppError("BulkImport", "app.import.validate_post_import_data.user_missing.error", nil, "", http.StatusBadRequest)
}
if data.Message == nil {
return model.NewAppError("BulkImport", "app.import.validate_post_import_data.message_missing.error", nil, "", http.StatusBadRequest)
} else if utf8.RuneCountInString(*data.Message) > model.POST_MESSAGE_MAX_RUNES {
return model.NewAppError("BulkImport", "app.import.validate_post_import_data.message_length.error", nil, "", http.StatusBadRequest)
}
if data.CreateAt == nil {
return model.NewAppError("BulkImport", "app.import.validate_post_import_data.create_at_missing.error", nil, "", http.StatusBadRequest)
} else if *data.CreateAt == 0 {
return model.NewAppError("BulkImport", "app.import.validate_post_import_data.create_at_zero.error", nil, "", http.StatusBadRequest)
}
return nil
}
//
// -- Old SlackImport Functions --
// Import functions are sutible for entering posts and users into the database without
// some of the usual checks. (IsValid is still run)
//
func ImportPost(post *model.Post) {
func OldImportPost(post *model.Post) {
// Workaround for empty messages, which may be the case if they are webhook posts.
firstIteration := true
for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) {
@@ -768,7 +883,7 @@ func OldImportChannel(channel *model.Channel) *model.Channel {
}
}
func ImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
func OldImportFile(file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
buf := bytes.NewBuffer(nil)
io.Copy(buf, file)
data := buf.Bytes()
@@ -787,7 +902,7 @@ func ImportFile(file io.Reader, teamId string, channelId string, userId string,
return fileInfo, nil
}
func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) {
func OldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) {
linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`)
post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})")
@@ -838,5 +953,5 @@ func ImportIncomingWebhookPost(post *model.Post, props model.StringInterface) {
}
}
ImportPost(post)
OldImportPost(post)
}

360
app/import_test.go
Просмотреть файл

@@ -6,6 +6,7 @@ package app
import (
"github.com/mattermost/platform/model"
"github.com/mattermost/platform/utils"
"runtime/debug"
"strings"
"testing"
)
@@ -454,6 +455,108 @@ func TestImportValidateUserChannelsImportData(t *testing.T) {
}
}
func TestImportValidatePostImportData(t *testing.T) {
// Test with minimum required valid properties.
data := PostImportData{
Team: ptrStr("teamname"),
Channel: ptrStr("channelname"),
User: ptrStr("username"),
Message: ptrStr("message"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := validatePostImportData(&data); err != nil {
t.Fatal("Validation failed but should have been valid.")
}
// Test with missing required properties.
data = PostImportData{
Channel: ptrStr("channelname"),
User: ptrStr("username"),
Message: ptrStr("message"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := validatePostImportData(&data); err == nil {
t.Fatal("Should have failed due to missing required property.")
}
data = PostImportData{
Team: ptrStr("teamname"),
User: ptrStr("username"),
Message: ptrStr("message"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := validatePostImportData(&data); err == nil {
t.Fatal("Should have failed due to missing required property.")
}
data = PostImportData{
Team: ptrStr("teamname"),
Channel: ptrStr("channelname"),
Message: ptrStr("message"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := validatePostImportData(&data); err == nil {
t.Fatal("Should have failed due to missing required property.")
}
data = PostImportData{
Team: ptrStr("teamname"),
Channel: ptrStr("channelname"),
User: ptrStr("username"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := validatePostImportData(&data); err == nil {
t.Fatal("Should have failed due to missing required property.")
}
data = PostImportData{
Team: ptrStr("teamname"),
Channel: ptrStr("channelname"),
User: ptrStr("username"),
Message: ptrStr("message"),
}
if err := validatePostImportData(&data); err == nil {
t.Fatal("Should have failed due to missing required property.")
}
// Test with invalid message.
data = PostImportData{
Team: ptrStr("teamname"),
Channel: ptrStr("channelname"),
User: ptrStr("username"),
Message: ptrStr(strings.Repeat("1234567890", 500)),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := validatePostImportData(&data); err == nil {
t.Fatal("Should have failed due to too long message.")
}
// Test with invalid CreateAt
data = PostImportData{
Team: ptrStr("teamname"),
Channel: ptrStr("channelname"),
User: ptrStr("username"),
Message: ptrStr("message"),
CreateAt: ptrInt64(0),
}
if err := validatePostImportData(&data); err == nil {
t.Fatal("Should have failed due to 0 create-at value.")
}
// Test with valid all optional parameters.
data = PostImportData{
Team: ptrStr("teamname"),
Channel: ptrStr("channelname"),
User: ptrStr("username"),
Message: ptrStr("message"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := validatePostImportData(&data); err != nil {
t.Fatal("Should have succeeded.")
}
}
func TestImportImportTeam(t *testing.T) {
_ = Setup()
@@ -1333,6 +1436,252 @@ func TestImportImportUser(t *testing.T) {
}
}
func AssertAllPostsCount(t *testing.T, initialCount int64, change int64, teamName string) {
if result := <-Srv.Store.Post().AnalyticsPostCount(teamName, false, false); result.Err != nil {
t.Fatal(result.Err)
} else {
if initialCount+change != result.Data.(int64) {
debug.PrintStack()
t.Fatalf("Did not find the expected number of posts.")
}
}
}
func TestImportImportPost(t *testing.T) {
_ = Setup()
// Create a Team.
teamName := model.NewId()
ImportTeam(&TeamImportData{
Name: &teamName,
DisplayName: ptrStr("Display Name"),
Type: ptrStr("O"),
}, false)
team, err := GetTeamByName(teamName)
if err != nil {
t.Fatalf("Failed to get team from database.")
}
// Create a Channel.
channelName := model.NewId()
ImportChannel(&ChannelImportData{
Team: &teamName,
Name: &channelName,
DisplayName: ptrStr("Display Name"),
Type: ptrStr("O"),
}, false)
channel, err := GetChannelByName(channelName, team.Id)
if err != nil {
t.Fatalf("Failed to get channel from database.")
}
// Create a user.
username := model.NewId()
ImportUser(&UserImportData{
Username: &username,
Email: ptrStr(model.NewId() + "@example.com"),
}, false)
user, err := GetUserByUsername(username)
if err != nil {
t.Fatalf("Failed to get user from database.")
}
// Count the number of posts in the testing team.
var initialPostCount int64
if result := <-Srv.Store.Post().AnalyticsPostCount(team.Id, false, false); result.Err != nil {
t.Fatal(result.Err)
} else {
initialPostCount = result.Data.(int64)
}
// Try adding an invalid post in dry run mode.
data := &PostImportData{
Team: &teamName,
Channel: &channelName,
User: &username,
}
if err := ImportPost(data, true); err == nil {
t.Fatalf("Expected error.")
}
AssertAllPostsCount(t, initialPostCount, 0, team.Id)
// Try adding a valid post in dry run mode.
data = &PostImportData{
Team: &teamName,
Channel: &channelName,
User: &username,
Message: ptrStr("Hello"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := ImportPost(data, true); err != nil {
t.Fatalf("Expected success.")
}
AssertAllPostsCount(t, initialPostCount, 0, team.Id)
// Try adding an invalid post in apply mode.
data = &PostImportData{
Team: &teamName,
Channel: &channelName,
User: &username,
CreateAt: ptrInt64(model.GetMillis()),
}
if err := ImportPost(data, false); err == nil {
t.Fatalf("Expected error.")
}
AssertAllPostsCount(t, initialPostCount, 0, team.Id)
// Try adding a valid post with invalid team in apply mode.
data = &PostImportData{
Team: ptrStr(model.NewId()),
Channel: &channelName,
User: &username,
Message: ptrStr("Message"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := ImportPost(data, false); err == nil {
t.Fatalf("Expected error.")
}
AssertAllPostsCount(t, initialPostCount, 0, team.Id)
// Try adding a valid post with invalid channel in apply mode.
data = &PostImportData{
Team: &teamName,
Channel: ptrStr(model.NewId()),
User: &username,
Message: ptrStr("Message"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := ImportPost(data, false); err == nil {
t.Fatalf("Expected error.")
}
AssertAllPostsCount(t, initialPostCount, 0, team.Id)
// Try adding a valid post with invalid user in apply mode.
data = &PostImportData{
Team: &teamName,
Channel: &channelName,
User: ptrStr(model.NewId()),
Message: ptrStr("Message"),
CreateAt: ptrInt64(model.GetMillis()),
}
if err := ImportPost(data, false); err == nil {
t.Fatalf("Expected error.")
}
AssertAllPostsCount(t, initialPostCount, 0, team.Id)
// Try adding a valid post in apply mode.
time := model.GetMillis()
data = &PostImportData{
Team: &teamName,
Channel: &channelName,
User: &username,
Message: ptrStr("Message"),
CreateAt: &time,
}
if err := ImportPost(data, false); err != nil {
t.Fatalf("Expected success.")
}
AssertAllPostsCount(t, initialPostCount, 1, team.Id)
// Check the post values.
if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, time); result.Err != nil {
t.Fatal(result.Err.Error())
} else {
posts := result.Data.([]*model.Post)
if len(posts) != 1 {
t.Fatal("Unexpected number of posts found.")
}
post := posts[0]
if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id {
t.Fatal("Post properties not as expected")
}
}
// Update the post.
data = &PostImportData{
Team: &teamName,
Channel: &channelName,
User: &username,
Message: ptrStr("Message"),
CreateAt: &time,
}
if err := ImportPost(data, false); err != nil {
t.Fatalf("Expected success.")
}
AssertAllPostsCount(t, initialPostCount, 1, team.Id)
// Check the post values.
if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, time); result.Err != nil {
t.Fatal(result.Err.Error())
} else {
posts := result.Data.([]*model.Post)
if len(posts) != 1 {
t.Fatal("Unexpected number of posts found.")
}
post := posts[0]
if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id {
t.Fatal("Post properties not as expected")
}
}
// Save the post with a different time.
newTime := time + 1
data = &PostImportData{
Team: &teamName,
Channel: &channelName,
User: &username,
Message: ptrStr("Message"),
CreateAt: &newTime,
}
if err := ImportPost(data, false); err != nil {
t.Fatalf("Expected success.")
}
AssertAllPostsCount(t, initialPostCount, 2, team.Id)
// Save the post with a different message.
data = &PostImportData{
Team: &teamName,
Channel: &channelName,
User: &username,
Message: ptrStr("Message 2"),
CreateAt: &time,
}
if err := ImportPost(data, false); err != nil {
t.Fatalf("Expected success.")
}
AssertAllPostsCount(t, initialPostCount, 3, team.Id)
// Test with hashtags
hashtagTime := time + 2
data = &PostImportData{
Team: &teamName,
Channel: &channelName,
User: &username,
Message: ptrStr("Message 2 #hashtagmashupcity"),
CreateAt: &hashtagTime,
}
if err := ImportPost(data, false); err != nil {
t.Fatalf("Expected success.")
}
AssertAllPostsCount(t, initialPostCount, 4, team.Id)
if result := <-Srv.Store.Post().GetPostsCreatedAt(channel.Id, hashtagTime); result.Err != nil {
t.Fatal(result.Err.Error())
} else {
posts := result.Data.([]*model.Post)
if len(posts) != 1 {
t.Fatal("Unexpected number of posts found.")
}
post := posts[0]
if post.Message != *data.Message || post.CreateAt != *data.CreateAt || post.UserId != user.Id {
t.Fatal("Post properties not as expected")
}
if post.Hashtags != "#hashtagmashupcity" {
t.Fatalf("Hashtags not as expected: %s", post.Hashtags)
}
}
}
func TestImportImportLine(t *testing.T) {
_ = Setup()
@@ -1362,6 +1711,12 @@ func TestImportImportLine(t *testing.T) {
if err := ImportLine(line, false); err == nil {
t.Fatalf("Expected an error when importing a line with type uesr with a nil user.")
}
// Try import line with post type but nil post.
line.Type = "post"
if err := ImportLine(line, false); err == nil {
t.Fatalf("Expected an error when importing a line with type post with a nil post.")
}
}
func TestImportBulkImport(t *testing.T) {
@@ -1369,13 +1724,14 @@ func TestImportBulkImport(t *testing.T) {
teamName := model.NewId()
channelName := model.NewId()
username := model.NewId()
// Run bulk import with a valid 1 of everything.
data1 := `{"type": "version", "version": 1}
{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}}
{"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}}
{"type": "user", "user": {"username": "kufjgnkxkrhhfgbrip6qxkfsaa", "email": "kufjgnkxkrhhfgbrip6qxkfsaa@example.com"}}
{"type": "user", "user": {"username": "bwshaim6qnc2ne7oqkd5b2s2rq", "email": "bwshaim6qnc2ne7oqkd5b2s2rq@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}}`
{"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}}
{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012}}`
if err, line := BulkImport(strings.NewReader(data1), false); err != nil || line != 0 {
t.Fatalf("BulkImport should have succeeded: %v, %v", err.Error(), line)

18
app/slackimport.go
Просмотреть файл

@@ -242,7 +242,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
newPost.Message = sPost.File.Title
}
}
ImportPost(&newPost)
OldImportPost(&newPost)
for _, fileId := range newPost.FileIds {
if result := <-Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
l4g.Error(utils.T("api.slackimport.slack_add_posts.attach_files.error"), newPost.Id, newPost.FileIds, result.Err)
@@ -266,7 +266,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
Message: sPost.Comment.Comment,
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
}
ImportPost(&newPost)
OldImportPost(&newPost)
case sPost.Type == "message" && sPost.SubType == "bot_message":
if botUser == nil {
l4g.Warn(utils.T("api.slackimport.slack_add_posts.bot_user_no_exists.warn"))
@@ -298,7 +298,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
Type: model.POST_SLACK_ATTACHMENT,
}
ImportIncomingWebhookPost(post, props)
OldImportIncomingWebhookPost(post, props)
case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
if sPost.User == "" {
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
@@ -325,7 +325,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
"username": users[sPost.User].Username,
},
}
ImportPost(&newPost)
OldImportPost(&newPost)
case sPost.Type == "message" && sPost.SubType == "me_message":
if sPost.User == "" {
l4g.Debug(utils.T("api.slackimport.slack_add_posts.without_user.debug"))
@@ -340,7 +340,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
Message: "*" + sPost.Text + "*",
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
}
ImportPost(&newPost)
OldImportPost(&newPost)
case sPost.Type == "message" && sPost.SubType == "channel_topic":
if sPost.User == "" {
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
@@ -356,7 +356,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
Type: model.POST_HEADER_CHANGE,
}
ImportPost(&newPost)
OldImportPost(&newPost)
case sPost.Type == "message" && sPost.SubType == "channel_purpose":
if sPost.User == "" {
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
@@ -372,7 +372,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
Type: model.POST_PURPOSE_CHANGE,
}
ImportPost(&newPost)
OldImportPost(&newPost)
case sPost.Type == "message" && sPost.SubType == "channel_name":
if sPost.User == "" {
l4g.Debug(utils.T("api.slackimport.slack_add_posts.msg_no_usr.debug"))
@@ -388,7 +388,7 @@ func SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, use
CreateAt: SlackConvertTimeStamp(sPost.TimeStamp),
Type: model.POST_DISPLAYNAME_CHANGE,
}
ImportPost(&newPost)
OldImportPost(&newPost)
default:
l4g.Warn(utils.T("api.slackimport.slack_add_posts.unsupported.warn"), sPost.Type, sPost.SubType)
}
@@ -405,7 +405,7 @@ func SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId strin
}
defer openFile.Close()
uploadedFile, err := ImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name))
uploadedFile, err := OldImportFile(openFile, teamId, channelId, userId, filepath.Base(file.Name))
if err != nil {
l4g.Warn(utils.T("api.slackimport.slack_add_posts.upload_file_upload_failed.warn", map[string]interface{}{"FileId": sPost.File.Id, "Error": err.Error()}))
return nil, false