[MM 12464] Include DM/GM Channels and Their Posts in the Bulk Export (#10421)
* transplant the existing PR into the working tree * start addressing review comments * move existing direct channel export code into this branch * modify channel exporter to use squirell and populate members in two steps * use squirrel to build sql queries for channel and dm/gm export methods * remove debug helpers and use Username instead of UserId * unit test for DM Channel exporter * add more unit tests for channel export * add test for DM/GM post export * checkpoint with failing test for postgres * use getQueryBuilder to make sure squirrel uses the correct formatting for each database * add a test for post export * fix shadowed vars that broke the build * address review comments and add tests to support it * address review comments and add a mlog call * s/Info/Debug/ * address review comments in post_store * address review comments in channel_store * address review comments in export * address review comment in post_store: drop GroupBy * address review comment on supplier: move getQueryBuilder to sqlstore * address review comments: explicit TearDown * address review comments: improve test coverage * address review comments: make sure public and private channels are excluded * address review comments: improve test coverage * address review comments: make sure Channels table gets truncated after each test * more cleanups and better assertions * wrap PostStore in a StoreTestWithSqlSupplier * last minute changes: improve post export test coverage and check members * address review comments: make sure all posts have their channel members set * address review comments: make sure all posts have their ChannelMembers exported correctly * gofmt fix * sort channels so it's possible to assert on index
Этот коммит содержится в:
Fernando Correa Neto
коммит произвёл
Jesús Espino
Jesús Espino
родитель
eb49713c96
Коммит
9abd4dd7dc
@@ -11,6 +11,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mattermost/mattermost-server/mlog"
|
||||
"github.com/mattermost/mattermost-server/model"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
@@ -70,10 +71,19 @@ func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, d
|
||||
if err := a.ExportAllPosts(writer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := a.ExportCustomEmoji(writer, file, pathToEmojiDir, dirNameToExportEmoji); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := a.ExportAllDirectChannels(writer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := a.ExportAllDirectPosts(writer); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -503,3 +513,83 @@ func (a *App) copyEmojiImages(emojiId string, emojiImagePath string, pathToDir s
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) ExportAllDirectChannels(writer io.Writer) *model.AppError {
|
||||
afterId := strings.Repeat("0", 26)
|
||||
for {
|
||||
result := <-a.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, afterId)
|
||||
if result.Err != nil {
|
||||
return result.Err
|
||||
}
|
||||
|
||||
channels := result.Data.([]*model.DirectChannelForExport)
|
||||
if len(channels) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, channel := range channels {
|
||||
afterId = channel.Id
|
||||
|
||||
// Skip deleted.
|
||||
if channel.DeleteAt != 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// There's no import support for single member channels yet.
|
||||
if len(*channel.Members) == 1 {
|
||||
mlog.Debug("Bulk export for direct channels containing a single member is not supported.")
|
||||
continue
|
||||
}
|
||||
|
||||
channelLine := ImportLineFromDirectChannel(channel)
|
||||
if err := a.ExportWriteLine(writer, channelLine); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) ExportAllDirectPosts(writer io.Writer) *model.AppError {
|
||||
afterId := strings.Repeat("0", 26)
|
||||
for {
|
||||
result := <-a.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, afterId)
|
||||
if result.Err != nil {
|
||||
return result.Err
|
||||
}
|
||||
|
||||
posts := result.Data.([]*model.DirectPostForExport)
|
||||
if len(posts) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, post := range posts {
|
||||
afterId = post.Id
|
||||
|
||||
// Skip deleted.
|
||||
if post.DeleteAt != 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// There's no import support for single member channels yet.
|
||||
if len(*post.ChannelMembers) == 1 {
|
||||
mlog.Debug("Bulk export for posts containing a single member is not supported.")
|
||||
continue
|
||||
}
|
||||
|
||||
// Do the Replies.
|
||||
replies, err := a.buildPostReplies(post.Id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
postLine := ImportLineForDirectPost(post)
|
||||
postLine.DirectPost.Replies = replies
|
||||
if err := a.ExportWriteLine(writer, postLine); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -38,6 +38,16 @@ func ImportLineFromChannel(channel *model.ChannelForExport) *LineImportData {
|
||||
}
|
||||
}
|
||||
|
||||
func ImportLineFromDirectChannel(channel *model.DirectChannelForExport) *LineImportData {
|
||||
return &LineImportData{
|
||||
Type: "direct_channel",
|
||||
DirectChannel: &DirectChannelImportData{
|
||||
Header: &channel.Header,
|
||||
Members: channel.Members,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ImportLineFromUser(user *model.User, exportedPrefs map[string]*string) *LineImportData {
|
||||
// Bulk Importer doesn't accept "empty string" for AuthService.
|
||||
var authService *string
|
||||
@@ -141,6 +151,18 @@ func ImportLineForPost(post *model.PostForExport) *LineImportData {
|
||||
}
|
||||
}
|
||||
|
||||
func ImportLineForDirectPost(post *model.DirectPostForExport) *LineImportData {
|
||||
return &LineImportData{
|
||||
Type: "direct_post",
|
||||
DirectPost: &DirectPostImportData{
|
||||
ChannelMembers: post.ChannelMembers,
|
||||
User: &post.User,
|
||||
Message: &post.Message,
|
||||
CreateAt: &post.CreateAt,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ImportReplyFromPost(post *model.ReplyForExport) *ReplyImportData {
|
||||
return &ReplyImportData{
|
||||
User: &post.Username,
|
||||
|
||||
@@ -3,6 +3,7 @@ package app
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
@@ -201,16 +202,252 @@ func TestExportAllUsers(t *testing.T) {
|
||||
// Checking whether deactivated users were included in bulk export
|
||||
deletedUsers1, err := th1.App.GetUsers(&model.UserGetOptions{
|
||||
Inactive: true,
|
||||
Page: 0,
|
||||
PerPage: 10,
|
||||
Page: 0,
|
||||
PerPage: 10,
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
deletedUsers2, err := th1.App.GetUsers(&model.UserGetOptions{
|
||||
Inactive: true,
|
||||
Page: 0,
|
||||
PerPage: 10,
|
||||
Page: 0,
|
||||
PerPage: 10,
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, len(deletedUsers1), len(deletedUsers2))
|
||||
assert.ElementsMatch(t, deletedUsers1, deletedUsers2)
|
||||
}
|
||||
|
||||
func TestExportDMChannel(t *testing.T) {
|
||||
th1 := Setup(t).InitBasic()
|
||||
|
||||
// DM Channel
|
||||
th1.CreateDmChannel(th1.BasicUser2)
|
||||
|
||||
var b bytes.Buffer
|
||||
err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
|
||||
require.Nil(t, err)
|
||||
|
||||
result := <-th1.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels := result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 1, len(channels))
|
||||
|
||||
th1.TearDown()
|
||||
|
||||
th2 := Setup(t)
|
||||
defer th2.TearDown()
|
||||
|
||||
result = <-th2.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels = result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 0, len(channels))
|
||||
|
||||
// import the exported channel
|
||||
err, i := th2.App.BulkImport(&b, false, 5)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, i)
|
||||
|
||||
// Ensure the Members of the imported DM channel is the same was from the exported
|
||||
result = <-th2.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels = result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 1, len(channels))
|
||||
assert.ElementsMatch(t, []string{th1.BasicUser.Username, th1.BasicUser2.Username}, *channels[0].Members)
|
||||
}
|
||||
|
||||
func TestExportDMChannelToSelf(t *testing.T) {
|
||||
th1 := Setup(t).InitBasic()
|
||||
defer th1.TearDown()
|
||||
|
||||
// DM Channel with self (me channel)
|
||||
th1.CreateDmChannel(th1.BasicUser)
|
||||
|
||||
var b bytes.Buffer
|
||||
err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
|
||||
require.Nil(t, err)
|
||||
|
||||
result := <-th1.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels := result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 1, len(channels))
|
||||
|
||||
th2 := Setup(t)
|
||||
defer th2.TearDown()
|
||||
|
||||
result = <-th2.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels = result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 0, len(channels))
|
||||
|
||||
// import the exported channel
|
||||
err, i := th2.App.BulkImport(&b, false, 5)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, i)
|
||||
|
||||
// Ensure no channels were imported
|
||||
result = <-th2.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels = result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 0, len(channels))
|
||||
}
|
||||
|
||||
func TestExportGMChannel(t *testing.T) {
|
||||
th1 := Setup(t).InitBasic()
|
||||
|
||||
user1 := th1.CreateUser()
|
||||
th1.LinkUserToTeam(user1, th1.BasicTeam)
|
||||
user2 := th1.CreateUser()
|
||||
th1.LinkUserToTeam(user2, th1.BasicTeam)
|
||||
|
||||
// GM Channel
|
||||
th1.CreateGroupChannel(user1, user2)
|
||||
|
||||
var b bytes.Buffer
|
||||
err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
|
||||
require.Nil(t, err)
|
||||
|
||||
result := <-th1.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels := result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 1, len(channels))
|
||||
|
||||
th1.TearDown()
|
||||
|
||||
th2 := Setup(t)
|
||||
defer th2.TearDown()
|
||||
|
||||
result = <-th2.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels = result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 0, len(channels))
|
||||
}
|
||||
|
||||
func TestExportGMandDMChannels(t *testing.T) {
|
||||
th1 := Setup(t).InitBasic()
|
||||
|
||||
// DM Channel
|
||||
th1.CreateDmChannel(th1.BasicUser2)
|
||||
|
||||
user1 := th1.CreateUser()
|
||||
th1.LinkUserToTeam(user1, th1.BasicTeam)
|
||||
user2 := th1.CreateUser()
|
||||
th1.LinkUserToTeam(user2, th1.BasicTeam)
|
||||
|
||||
// GM Channel
|
||||
th1.CreateGroupChannel(user1, user2)
|
||||
|
||||
var b bytes.Buffer
|
||||
err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
|
||||
require.Nil(t, err)
|
||||
|
||||
result := <-th1.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels := result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 2, len(channels))
|
||||
|
||||
th1.TearDown()
|
||||
|
||||
th2 := Setup(t)
|
||||
defer th2.TearDown()
|
||||
|
||||
result = <-th2.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels = result.Data.([]*model.DirectChannelForExport)
|
||||
assert.Equal(t, 0, len(channels))
|
||||
|
||||
// import the exported channel
|
||||
err, i := th2.App.BulkImport(&b, false, 5)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, i)
|
||||
|
||||
// Ensure the Members of the imported GM channel is the same was from the exported
|
||||
result = <-th2.App.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, "00000000")
|
||||
channels = result.Data.([]*model.DirectChannelForExport)
|
||||
|
||||
// Adding some deteminism so its possible to assert on slice index
|
||||
sort.Slice(channels, func(i, j int) bool { return channels[i].CreateAt > channels[j].CreateAt })
|
||||
assert.Equal(t, 2, len(channels))
|
||||
assert.ElementsMatch(t, []string{th1.BasicUser.Username, user1.Username, user2.Username}, *channels[0].Members)
|
||||
assert.ElementsMatch(t, []string{th1.BasicUser.Username, th1.BasicUser2.Username}, *channels[1].Members)
|
||||
}
|
||||
|
||||
func TestExportDMandGMPost(t *testing.T) {
|
||||
th1 := Setup(t).InitBasic()
|
||||
|
||||
// DM Channel
|
||||
dmChannel := th1.CreateDmChannel(th1.BasicUser2)
|
||||
dmMembers := []string{th1.BasicUser.Username, th1.BasicUser2.Username}
|
||||
|
||||
user1 := th1.CreateUser()
|
||||
th1.LinkUserToTeam(user1, th1.BasicTeam)
|
||||
user2 := th1.CreateUser()
|
||||
th1.LinkUserToTeam(user2, th1.BasicTeam)
|
||||
|
||||
// GM Channel
|
||||
gmChannel := th1.CreateGroupChannel(user1, user2)
|
||||
gmMembers := []string{th1.BasicUser.Username, user1.Username, user2.Username}
|
||||
|
||||
// DM posts
|
||||
th1.CreatePost(dmChannel)
|
||||
th1.CreatePost(dmChannel)
|
||||
// GM posts
|
||||
th1.CreatePost(gmChannel)
|
||||
th1.CreatePost(gmChannel)
|
||||
|
||||
result := <-th1.App.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000")
|
||||
posts := result.Data.([]*model.DirectPostForExport)
|
||||
assert.Equal(t, 4, len(posts))
|
||||
|
||||
var b bytes.Buffer
|
||||
err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
|
||||
require.Nil(t, err)
|
||||
|
||||
th1.TearDown()
|
||||
|
||||
th2 := Setup(t)
|
||||
defer th2.TearDown()
|
||||
|
||||
result = <-th2.App.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000")
|
||||
posts = result.Data.([]*model.DirectPostForExport)
|
||||
assert.Equal(t, 0, len(posts))
|
||||
|
||||
// import the exported posts
|
||||
err, i := th2.App.BulkImport(&b, false, 5)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, i)
|
||||
|
||||
result = <-th2.App.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000")
|
||||
posts = result.Data.([]*model.DirectPostForExport)
|
||||
|
||||
// Adding some deteminism so its possible to assert on slice index
|
||||
sort.Slice(posts, func(i, j int) bool { return posts[i].CreateAt > posts[j].CreateAt })
|
||||
assert.Equal(t, 4, len(posts))
|
||||
assert.ElementsMatch(t, gmMembers, *posts[0].ChannelMembers)
|
||||
assert.ElementsMatch(t, gmMembers, *posts[1].ChannelMembers)
|
||||
assert.ElementsMatch(t, dmMembers, *posts[2].ChannelMembers)
|
||||
assert.ElementsMatch(t, dmMembers, *posts[3].ChannelMembers)
|
||||
}
|
||||
|
||||
func TestExportDMPostWithSelf(t *testing.T) {
|
||||
th1 := Setup(t).InitBasic()
|
||||
|
||||
// DM Channel with self (me channel)
|
||||
dmChannel := th1.CreateDmChannel(th1.BasicUser)
|
||||
|
||||
th1.CreatePost(dmChannel)
|
||||
|
||||
var b bytes.Buffer
|
||||
err := th1.App.BulkExport(&b, "somefile", "somePath", "someDir")
|
||||
require.Nil(t, err)
|
||||
|
||||
result := <-th1.App.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000")
|
||||
posts := result.Data.([]*model.DirectPostForExport)
|
||||
assert.Equal(t, 1, len(posts))
|
||||
|
||||
th1.TearDown()
|
||||
|
||||
th2 := Setup(t)
|
||||
defer th2.TearDown()
|
||||
|
||||
result = <-th2.App.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000")
|
||||
posts = result.Data.([]*model.DirectPostForExport)
|
||||
assert.Equal(t, 0, len(posts))
|
||||
|
||||
// import the exported posts
|
||||
err, i := th2.App.BulkImport(&b, false, 5)
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, i)
|
||||
|
||||
result = <-th2.App.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, "0000000")
|
||||
posts = result.Data.([]*model.DirectPostForExport)
|
||||
assert.Equal(t, 0, len(posts))
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user