MM-11781: Basic Data Export Command Line. (#9296)

* MM-11781: Basic Data Export Command Line.

* ChannelStore new unit tests.

* TeamStore new unit tests.

* Unit test for new UserStore function.

* Unit tests for post store new methods.

* Review fixes.

* Fix duplicate command name.
Этот коммит содержится в:
George Goldberg
2018-09-17 15:51:26 +01:00
коммит произвёл GitHub
родитель 5786b0d6d5
Коммит ab99f0656f
24 изменённых файлов: 1106 добавлений и 46 удалений

279
app/export.go Обычный файл
Просмотреть файл

@@ -0,0 +1,279 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package app
import (
"encoding/json"
"io"
"net/http"
"strings"
"github.com/mattermost/mattermost-server/model"
)
func (a *App) BulkExport(writer io.Writer) *model.AppError {
if err := a.ExportVersion(writer); err != nil {
return err
}
if err := a.ExportAllTeams(writer); err != nil {
return err
}
if err := a.ExportAllChannels(writer); err != nil {
return err
}
if err := a.ExportAllUsers(writer); err != nil {
return err
}
if err := a.ExportAllPosts(writer); err != nil {
return err
}
return nil
}
func (a *App) ExportWriteLine(writer io.Writer, line *LineImportData) *model.AppError {
b, err := json.Marshal(line)
if err != nil {
return model.NewAppError("BulkExport", "app.export.export_write_line.json_marshall.error", nil, "err="+err.Error(), http.StatusBadRequest)
}
if _, err := writer.Write(append(b, '\n')); err != nil {
return model.NewAppError("BulkExport", "app.export.export_write_line.io_writer.error", nil, "err="+err.Error(), http.StatusBadRequest)
}
return nil
}
func (a *App) ExportVersion(writer io.Writer) *model.AppError {
version := 1
versionLine := &LineImportData{
Type: "version",
Version: &version,
}
return a.ExportWriteLine(writer, versionLine)
}
func (a *App) ExportAllTeams(writer io.Writer) *model.AppError {
afterId := strings.Repeat("0", 26)
for {
result := <-a.Srv.Store.Team().GetAllForExportAfter(1000, afterId)
if result.Err != nil {
return result.Err
}
teams := result.Data.([]*model.TeamForExport)
if len(teams) == 0 {
break
}
for _, team := range teams {
afterId = team.Id
// Skip deleted.
if team.DeleteAt != 0 {
continue
}
teamLine := ImportLineFromTeam(team)
if err := a.ExportWriteLine(writer, teamLine); err != nil {
return err
}
}
}
return nil
}
func (a *App) ExportAllChannels(writer io.Writer) *model.AppError {
afterId := strings.Repeat("0", 26)
for {
result := <-a.Srv.Store.Channel().GetAllChannelsForExportAfter(1000, afterId)
if result.Err != nil {
return result.Err
}
channels := result.Data.([]*model.ChannelForExport)
if len(channels) == 0 {
break
}
for _, channel := range channels {
afterId = channel.Id
// Skip deleted.
if channel.DeleteAt != 0 {
continue
}
channelLine := ImportLineFromChannel(channel)
if err := a.ExportWriteLine(writer, channelLine); err != nil {
return err
}
}
}
return nil
}
func (a *App) ExportAllUsers(writer io.Writer) *model.AppError {
afterId := strings.Repeat("0", 26)
for {
result := <-a.Srv.Store.User().GetAllAfter(1000, afterId)
if result.Err != nil {
return result.Err
}
users := result.Data.([]*model.User)
if len(users) == 0 {
break
}
for _, user := range users {
afterId = user.Id
// Skip deleted.
if user.DeleteAt != 0 {
continue
}
userLine := ImportLineFromUser(user)
// Do the Team Memberships.
members, err := a.buildUserTeamAndChannelMemberships(user.Id)
if err != nil {
return err
}
userLine.User.Teams = members
if err := a.ExportWriteLine(writer, userLine); err != nil {
return err
}
}
}
return nil
}
func (a *App) buildUserTeamAndChannelMemberships(userId string) (*[]UserTeamImportData, *model.AppError) {
var memberships []UserTeamImportData
result := <-a.Srv.Store.Team().GetTeamMembersForExport(userId)
if result.Err != nil {
return nil, result.Err
}
members := result.Data.([]*model.TeamMemberForExport)
for _, member := range members {
// Skip deleted.
if member.DeleteAt != 0 {
continue
}
memberData := ImportUserTeamDataFromTeamMember(member)
// Do the Channel Memberships.
channelMembers, err := a.buildUserChannelMemberships(userId, member.TeamId)
if err != nil {
return nil, err
}
memberData.Channels = channelMembers
memberships = append(memberships, *memberData)
}
return &memberships, nil
}
func (a *App) buildUserChannelMemberships(userId string, teamId string) (*[]UserChannelImportData, *model.AppError) {
var memberships []UserChannelImportData
result := <-a.Srv.Store.Channel().GetChannelMembersForExport(userId, teamId)
if result.Err != nil {
return nil, result.Err
}
members := result.Data.([]*model.ChannelMemberForExport)
for _, member := range members {
memberships = append(memberships, *ImportUserChannelDataFromChannelMember(member))
}
return &memberships, nil
}
func (a *App) ExportAllPosts(writer io.Writer) *model.AppError {
afterId := strings.Repeat("0", 26)
for {
result := <-a.Srv.Store.Post().GetParentsForExportAfter(1000, afterId)
if result.Err != nil {
return result.Err
}
posts := result.Data.([]*model.PostForExport)
if len(posts) == 0 {
break
}
for _, post := range posts {
afterId = post.Id
// Skip deleted.
if post.DeleteAt != 0 {
continue
}
postLine := ImportLineForPost(post)
// Do the Replies.
replies, err := a.buildPostReplies(post.Id)
if err != nil {
return err
}
postLine.Post.Replies = replies
if err := a.ExportWriteLine(writer, postLine); err != nil {
return err
}
}
}
return nil
}
func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) {
var replies []ReplyImportData
result := <-a.Srv.Store.Post().GetRepliesForExport(postId)
if result.Err != nil {
return nil, result.Err
}
replyPosts := result.Data.([]*model.ReplyForExport)
for _, reply := range replyPosts {
replies = append(replies, *ImportReplyFromPost(reply))
}
return &replies, nil
}

113
app/export_converters.go Обычный файл
Просмотреть файл

@@ -0,0 +1,113 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package app
import (
"github.com/mattermost/mattermost-server/model"
"strings"
)
func ImportLineFromTeam(team *model.TeamForExport) *LineImportData {
return &LineImportData{
Type: "team",
Team: &TeamImportData{
Name: &team.Name,
DisplayName: &team.DisplayName,
Type: &team.Type,
Description: &team.Description,
AllowOpenInvite: &team.AllowOpenInvite,
Scheme: team.SchemeName,
},
}
}
func ImportLineFromChannel(channel *model.ChannelForExport) *LineImportData {
return &LineImportData{
Type: "channel",
Channel: &ChannelImportData{
Team: &channel.TeamName,
Name: &channel.Name,
DisplayName: &channel.DisplayName,
Type: &channel.Type,
Header: &channel.Header,
Purpose: &channel.Purpose,
Scheme: channel.SchemeName,
},
}
}
func ImportLineFromUser(user *model.User) *LineImportData {
// Bulk Importer doesn't accept "empty string" for AuthService.
var authService *string
if user.AuthService != "" {
authService = &user.AuthService
}
return &LineImportData{
Type: "user",
User: &UserImportData{
Username: &user.Username,
Email: &user.Email,
AuthService: authService,
AuthData: user.AuthData,
Nickname: &user.Nickname,
FirstName: &user.FirstName,
LastName: &user.LastName,
Position: &user.Position,
Roles: &user.Roles,
Locale: &user.Locale,
},
}
}
func ImportUserTeamDataFromTeamMember(member *model.TeamMemberForExport) *UserTeamImportData {
rolesList := strings.Fields(member.Roles)
if member.SchemeAdmin {
rolesList = append(rolesList, model.TEAM_ADMIN_ROLE_ID)
}
if member.SchemeUser {
rolesList = append(rolesList, model.TEAM_USER_ROLE_ID)
}
roles := strings.Join(rolesList, " ")
return &UserTeamImportData{
Name: &member.TeamName,
Roles: &roles,
}
}
func ImportUserChannelDataFromChannelMember(member *model.ChannelMemberForExport) *UserChannelImportData {
rolesList := strings.Fields(member.Roles)
if member.SchemeAdmin {
rolesList = append(rolesList, model.CHANNEL_ADMIN_ROLE_ID)
}
if member.SchemeUser {
rolesList = append(rolesList, model.CHANNEL_USER_ROLE_ID)
}
roles := strings.Join(rolesList, " ")
return &UserChannelImportData{
Name: &member.ChannelName,
Roles: &roles,
}
}
func ImportLineForPost(post *model.PostForExport) *LineImportData {
return &LineImportData{
Type: "post",
Post: &PostImportData{
Team: &post.TeamName,
Channel: &post.ChannelName,
User: &post.Username,
Message: &post.Message,
CreateAt: &post.CreateAt,
},
}
}
func ImportReplyFromPost(post *model.ReplyForExport) *ReplyImportData {
return &ReplyImportData{
User: &post.Username,
Message: &post.Message,
CreateAt: &post.CreateAt,
}
}

Просмотреть файл

@@ -9,24 +9,24 @@ import "github.com/mattermost/mattermost-server/model"
type LineImportData struct {
Type string `json:"type"`
Scheme *SchemeImportData `json:"scheme"`
Team *TeamImportData `json:"team"`
Channel *ChannelImportData `json:"channel"`
User *UserImportData `json:"user"`
Post *PostImportData `json:"post"`
DirectChannel *DirectChannelImportData `json:"direct_channel"`
DirectPost *DirectPostImportData `json:"direct_post"`
Emoji *EmojiImportData `json:"emoji"`
Version *int `json:"version"`
Scheme *SchemeImportData `json:"scheme,omitempty"`
Team *TeamImportData `json:"team,omitempty"`
Channel *ChannelImportData `json:"channel,omitempty"`
User *UserImportData `json:"user,omitempty"`
Post *PostImportData `json:"post,omitempty"`
DirectChannel *DirectChannelImportData `json:"direct_channel,omitempty"`
DirectPost *DirectPostImportData `json:"direct_post,omitempty"`
Emoji *EmojiImportData `json:"emoji,omitempty"`
Version *int `json:"version,omitempty"`
}
type TeamImportData struct {
Name *string `json:"name"`
DisplayName *string `json:"display_name"`
Type *string `json:"type"`
Description *string `json:"description"`
AllowOpenInvite *bool `json:"allow_open_invite"`
Scheme *string `json:"scheme"`
Description *string `json:"description,omitempty"`
AllowOpenInvite *bool `json:"allow_open_invite,omitempty"`
Scheme *string `json:"scheme,omitempty"`
}
type ChannelImportData struct {
@@ -34,38 +34,38 @@ type ChannelImportData struct {
Name *string `json:"name"`
DisplayName *string `json:"display_name"`
Type *string `json:"type"`
Header *string `json:"header"`
Purpose *string `json:"purpose"`
Scheme *string `json:"scheme"`
Header *string `json:"header,omitempty"`
Purpose *string `json:"purpose,omitempty"`
Scheme *string `json:"scheme,omitempty"`
}
type UserImportData struct {
ProfileImage *string `json:"profile_image"`
ProfileImage *string `json:"profile_image,omitempty"`
Username *string `json:"username"`
Email *string `json:"email"`
AuthService *string `json:"auth_service"`
AuthData *string `json:"auth_data"`
Password *string `json:"password"`
AuthData *string `json:"auth_data,omitempty"`
Password *string `json:"password,omitempty"`
Nickname *string `json:"nickname"`
FirstName *string `json:"first_name"`
LastName *string `json:"last_name"`
Position *string `json:"position"`
Roles *string `json:"roles"`
Locale *string `json:"locale"`
UseMarkdownPreview *string `json:"feature_enabled_markdown_preview"`
UseFormatting *string `json:"formatting"`
ShowUnreadSection *string `json:"show_unread_section"`
UseMarkdownPreview *string `json:"feature_enabled_markdown_preview,omitempty"`
UseFormatting *string `json:"formatting,omitempty"`
ShowUnreadSection *string `json:"show_unread_section,omitempty"`
Teams *[]UserTeamImportData `json:"teams"`
Teams *[]UserTeamImportData `json:"teams,omitempty"`
Theme *string `json:"theme"`
UseMilitaryTime *string `json:"military_time"`
CollapsePreviews *string `json:"link_previews"`
MessageDisplay *string `json:"message_display"`
ChannelDisplayMode *string `json:"channel_display_mode"`
TutorialStep *string `json:"tutorial_step"`
Theme *string `json:"theme,omitempty"`
UseMilitaryTime *string `json:"military_time,omitempty"`
CollapsePreviews *string `json:"link_previews,omitempty"`
MessageDisplay *string `json:"message_display,omitempty"`
ChannelDisplayMode *string `json:"channel_display_mode,omitempty"`
TutorialStep *string `json:"tutorial_step,omitempty"`
NotifyProps *UserNotifyPropsImportData `json:"notify_props"`
NotifyProps *UserNotifyPropsImportData `json:"notify_props,omitempty"`
}
type UserNotifyPropsImportData struct {
@@ -85,15 +85,15 @@ type UserNotifyPropsImportData struct {
type UserTeamImportData struct {
Name *string `json:"name"`
Roles *string `json:"roles"`
Theme *string `json:"theme"`
Channels *[]UserChannelImportData `json:"channels"`
Theme *string `json:"theme,omitempty"`
Channels *[]UserChannelImportData `json:"channels,omitempty"`
}
type UserChannelImportData struct {
Name *string `json:"name"`
Roles *string `json:"roles"`
NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props"`
Favorite *bool `json:"favorite"`
NotifyProps *UserChannelNotifyPropsImportData `json:"notify_props,omitempty"`
Favorite *bool `json:"favorite,omitempty"`
}
type UserChannelNotifyPropsImportData struct {
@@ -119,9 +119,9 @@ type ReplyImportData struct {
Message *string `json:"message"`
CreateAt *int64 `json:"create_at"`
FlaggedBy *[]string `json:"flagged_by"`
Reactions *[]ReactionImportData `json:"reactions"`
Attachments *[]AttachmentImportData `json:"attachments"`
FlaggedBy *[]string `json:"flagged_by,omitempty"`
Reactions *[]ReactionImportData `json:"reactions,omitempty"`
Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
}
type PostImportData struct {
@@ -132,10 +132,10 @@ type PostImportData struct {
Message *string `json:"message"`
CreateAt *int64 `json:"create_at"`
FlaggedBy *[]string `json:"flagged_by"`
Reactions *[]ReactionImportData `json:"reactions"`
Replies *[]ReplyImportData `json:"replies"`
Attachments *[]AttachmentImportData `json:"attachments"`
FlaggedBy *[]string `json:"flagged_by,omitempty"`
Reactions *[]ReactionImportData `json:"reactions,omitempty"`
Replies *[]ReplyImportData `json:"replies,omitempty"`
Attachments *[]AttachmentImportData `json:"attachments,omitempty"`
}
type DirectChannelImportData struct {

Просмотреть файл

@@ -5,6 +5,7 @@ package commands
import (
"errors"
"os"
"context"
@@ -14,10 +15,10 @@ import (
"github.com/spf13/cobra"
)
var MessageExportCmd = &cobra.Command{
var ExportCmd = &cobra.Command{
Use: "export",
Short: "Export data from Mattermost",
Long: "Export data from Mattermost in a format suitable for import into a third-party application",
Long: "Export data from Mattermost in a format suitable for import into a third-party application or another Mattermost instance",
}
var ScheduleExportCmd = &cobra.Command{
@@ -44,16 +45,31 @@ var ActianceExportCmd = &cobra.Command{
RunE: buildExportCmdF("actiance"),
}
var BulkExportCmd = &cobra.Command{
Use: "bulk [file]",
Short: "Export bulk data.",
Long: "Export data to a file compatible with the Mattermost Bulk Import format.",
Example: " export bulk bulk_data.json",
RunE: bulkExportCmdF,
}
func init() {
ScheduleExportCmd.Flags().String("format", "actiance", "The format to export data")
ScheduleExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
ScheduleExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.")
CsvExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
ActianceExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
MessageExportCmd.AddCommand(ScheduleExportCmd)
MessageExportCmd.AddCommand(CsvExportCmd)
MessageExportCmd.AddCommand(ActianceExportCmd)
RootCmd.AddCommand(MessageExportCmd)
BulkExportCmd.Flags().Bool("all-teams", false, "Export all teams from the server.")
ExportCmd.AddCommand(ScheduleExportCmd)
ExportCmd.AddCommand(CsvExportCmd)
ExportCmd.AddCommand(ActianceExportCmd)
ExportCmd.AddCommand(BulkExportCmd)
RootCmd.AddCommand(ExportCmd)
}
func scheduleExportCmdF(command *cobra.Command, args []string) error {
@@ -140,3 +156,33 @@ func buildExportCmdF(format string) func(command *cobra.Command, args []string)
return nil
}
}
func bulkExportCmdF(command *cobra.Command, args []string) error {
a, err := InitDBCommandContextCobra(command)
if err != nil {
return err
}
defer a.Shutdown()
allTeams, err := command.Flags().GetBool("all-teams")
if err != nil {
return errors.New("Apply flag error")
}
if !allTeams {
return errors.New("Nothing to export. Please specify the --all-teams flag to export all teams.")
}
fileWriter, err := os.Create(args[0])
if err != nil {
return err
}
defer fileWriter.Close()
if err := a.BulkExport(fileWriter); err != nil {
CommandPrettyPrintln(err.Error())
return err
}
return nil
}

Просмотреть файл

@@ -179,6 +179,14 @@
"id": "api.channel.leave.last_member.app_error",
"translation": "You're the only member left, try removing the Private Channel instead of leaving."
},
{
"id": "app.export.export_write_line.json_marshall.error",
"translation": "An error occured marshalling the JSON data for export."
},
{
"id": "app.export.export_write_line.io_writer.error",
"translation": "An error occurred writing the export data."
},
{
"id": "api.channel.leave.left",
"translation": "%v left the channel."

Просмотреть файл

@@ -57,6 +57,12 @@ type ChannelPatch struct {
Purpose *string `json:"purpose"`
}
type ChannelForExport struct {
Channel
TeamName string
SchemeName *string
}
func (o *Channel) DeepCopy() *Channel {
copy := *o
if copy.SchemeId != nil {

Просмотреть файл

@@ -43,6 +43,11 @@ type ChannelMember struct {
type ChannelMembers []ChannelMember
type ChannelMemberForExport struct {
ChannelMember
ChannelName string
}
func (o *ChannelMembers) ToJson() string {
if b, err := json.Marshal(o); err != nil {
return "[]"

Просмотреть файл

@@ -110,6 +110,19 @@ func (o *PostPatch) WithRewrittenImageURLs(f func(string) string) *PostPatch {
return &copy
}
type PostForExport struct {
Post
TeamName string
ChannelName string
Username string
ReplyCount int
}
type ReplyForExport struct {
Post
Username string
}
type PostForIndexing struct {
Post
TeamId string `json:"team_id"`

Просмотреть файл

@@ -52,6 +52,11 @@ type TeamPatch struct {
AllowOpenInvite *bool `json:"allow_open_invite"`
}
type TeamForExport struct {
Team
SchemeName *string
}
type Invites struct {
Invites []map[string]string `json:"invites"`
}

Просмотреть файл

@@ -26,6 +26,11 @@ type TeamUnread struct {
MentionCount int64 `json:"mention_count"`
}
type TeamMemberForExport struct {
TeamMember
TeamName string
}
func (o *TeamMember) ToJson() string {
b, _ := json.Marshal(o)
return string(b)

Просмотреть файл

@@ -2016,3 +2016,57 @@ func (s SqlChannelStore) IsExperimentalPublicChannelsMaterializationEnabled() bo
// See SqlChannelStoreExperimental
return false
}
func (s SqlChannelStore) GetAllChannelsForExportAfter(limit int, afterId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var data []*model.ChannelForExport
if _, err := s.GetReplica().Select(&data, `
SELECT
Channels.*,
Teams.Name as TeamName,
Schemes.Name as SchemeName
FROM Channels
INNER JOIN
Teams ON Channels.TeamId = Teams.Id
LEFT JOIN
Schemes ON Channels.SchemeId = Schemes.Id
WHERE
Channels.Id > :AfterId
AND Channels.Type IN ('O', 'P')
ORDER BY
Id
LIMIT :Limit`,
map[string]interface{}{"AfterId": afterId, "Limit": limit}); err != nil {
result.Err = model.NewAppError("SqlTeamStore.GetAllChannelsForExportAfter", "store.sql_channel.get_all.app_error", nil, err.Error(), http.StatusInternalServerError)
return
}
result.Data = data
})
}
func (s SqlChannelStore) GetChannelMembersForExport(userId string, teamId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var members []*model.ChannelMemberForExport
_, err := s.GetReplica().Select(&members, `
SELECT
ChannelMembers.*,
Channels.Name as ChannelName
FROM
ChannelMembers
INNER JOIN
Channels ON ChannelMembers.ChannelId = Channels.Id
WHERE
ChannelMembers.UserId = :UserId
AND Channels.TeamId = :TeamId
AND Channels.DeleteAt = 0`,
map[string]interface{}{"TeamId": teamId, "UserId": userId})
if err != nil {
result.Err = model.NewAppError("SqlChannelStore.GetChannelMembersForExport", "store.sql_channel.get_members.app_error", nil, "teamId="+teamId+", userId="+userId+", err="+err.Error(), http.StatusInternalServerError)
return
}
result.Data = members
})
}

Просмотреть файл

@@ -1308,3 +1308,68 @@ func (s *SqlPostStore) GetMaxPostSize() store.StoreChannel {
result.Data = s.maxPostSizeCached
})
}
func (s *SqlPostStore) GetParentsForExportAfter(limit int, afterId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var posts []*model.PostForExport
_, err1 := s.GetSearchReplica().Select(&posts, `
SELECT
p1.*,
Users.Username as Username,
Teams.Name as TeamName,
Channels.Name as ChannelName
FROM
Posts p1
INNER JOIN
Channels ON p1.ChannelId = Channels.Id
INNER JOIN
Teams ON Channels.TeamId = Teams.Id
INNER JOIN
Users ON p1.UserId = Users.Id
WHERE
p1.Id > :AfterId
AND p1.ParentId = ''
AND p1.DeleteAt = 0
AND Channels.DeleteAt = 0
AND Teams.DeleteAt = 0
AND Users.DeleteAt = 0
ORDER BY
p1.Id
LIMIT
:Limit`,
map[string]interface{}{"Limit": limit, "AfterId": afterId})
if err1 != nil {
result.Err = model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error", nil, err1.Error(), http.StatusInternalServerError)
} else {
result.Data = posts
}
})
}
func (s *SqlPostStore) GetRepliesForExport(parentId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var posts []*model.ReplyForExport
_, err1 := s.GetSearchReplica().Select(&posts, `
SELECT
Posts.*,
Users.Username as Username
FROM
Posts
INNER JOIN
Users ON Posts.UserId = Users.Id
WHERE
Posts.ParentId = :ParentId
AND Posts.DeleteAt = 0
AND Users.DeleteAt = 0
ORDER BY
Posts.Id`,
map[string]interface{}{"ParentId": parentId})
if err1 != nil {
result.Err = model.NewAppError("SqlPostStore.GetAllAfterForExport", "store.sql_post.get_posts.app_error", nil, err1.Error(), http.StatusInternalServerError)
} else {
result.Data = posts
}
})
}

Просмотреть файл

@@ -924,3 +924,59 @@ func (s SqlTeamStore) AnalyticsGetTeamCountForScheme(schemeId string) store.Stor
result.Data = count
})
}
func (s SqlTeamStore) GetAllForExportAfter(limit int, afterId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var data []*model.TeamForExport
if _, err := s.GetReplica().Select(&data, `
SELECT
Teams.*,
Schemes.Name as SchemeName
FROM
Teams
LEFT JOIN
Schemes ON Teams.SchemeId = Schemes.Id
WHERE
Teams.Id > :AfterId
ORDER BY
Id
LIMIT
:Limit`,
map[string]interface{}{"AfterId": afterId, "Limit": limit}); err != nil {
result.Err = model.NewAppError("SqlTeamStore.GetAllTeams", "store.sql_team.get_all.app_error", nil, err.Error(), http.StatusInternalServerError)
return
}
for _, team := range data {
if len(team.InviteId) == 0 {
team.InviteId = team.Id
}
}
result.Data = data
})
}
func (s SqlTeamStore) GetTeamMembersForExport(userId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var members []*model.TeamMemberForExport
_, err := s.GetReplica().Select(&members, `
SELECT
TeamMembers.*,
Teams.Name as TeamName
FROM
TeamMembers
INNER JOIN
Teams ON TeamMembers.TeamId = Teams.Id
WHERE
TeamMembers.UserId = :UserId
AND Teams.DeleteAt = 0`,
map[string]interface{}{"UserId": userId})
if err != nil {
result.Err = model.NewAppError("SqlTeamStore.GetTeamMembersForExport", "store.sql_team.get_members.app_error", nil, "userId="+userId+" "+err.Error(), http.StatusInternalServerError)
return
}
result.Data = members
})
}

Просмотреть файл

@@ -332,6 +332,17 @@ func (us SqlUserStore) GetAll() store.StoreChannel {
})
}
func (us SqlUserStore) GetAllAfter(limit int, afterId string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
var data []*model.User
if _, err := us.GetReplica().Select(&data, "SELECT * FROM Users WHERE Id > :AfterId ORDER BY Id LIMIT :Limit", map[string]interface{}{"AfterId": afterId, "Limit": limit}); err != nil {
result.Err = model.NewAppError("SqlUserStore.GetAllAfter", "store.sql_user.get.app_error", nil, err.Error(), http.StatusInternalServerError)
}
result.Data = data
})
}
func (s SqlUserStore) GetEtagForAllProfiles() store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
updateAt, err := s.GetReplica().SelectInt("SELECT UpdateAt FROM Users ORDER BY UpdateAt DESC LIMIT 1")

Просмотреть файл

@@ -111,6 +111,8 @@ type TeamStore interface {
ResetAllTeamSchemes() StoreChannel
ClearAllCustomRoleAssignments() StoreChannel
AnalyticsGetTeamCountForScheme(schemeId string) StoreChannel
GetAllForExportAfter(limit int, afterId string) StoreChannel
GetTeamMembersForExport(userId string) StoreChannel
}
type ChannelStore interface {
@@ -179,6 +181,8 @@ type ChannelStore interface {
EnableExperimentalPublicChannelsMaterialization()
DisableExperimentalPublicChannelsMaterialization()
IsExperimentalPublicChannelsMaterializationEnabled() bool
GetAllChannelsForExportAfter(limit int, afterId string) StoreChannel
GetChannelMembersForExport(userId string, teamId string) StoreChannel
}
type ChannelMemberHistoryStore interface {
@@ -217,6 +221,8 @@ type PostStore interface {
PermanentDeleteBatch(endTime int64, limit int64) StoreChannel
GetOldest() StoreChannel
GetMaxPostSize() StoreChannel
GetParentsForExportAfter(limit int, afterId string) StoreChannel
GetRepliesForExport(parentId string) StoreChannel
}
type UserStore interface {
@@ -272,6 +278,7 @@ type UserStore interface {
GetEtagForProfilesNotInTeam(teamId string) StoreChannel
ClearAllCustomRoleAssignments() StoreChannel
InferSystemInstallDate() StoreChannel
GetAllAfter(limit int, afterId string) StoreChannel
}
type SessionStore interface {

Просмотреть файл

@@ -78,6 +78,8 @@ func TestChannelStore(t *testing.T, ss store.Store, s SqlSupplier) {
t.Run("ResetAllChannelSchemes", func(t *testing.T) { testResetAllChannelSchemes(t, ss) })
t.Run("ClearAllCustomRoleAssignments", func(t *testing.T) { testChannelStoreClearAllCustomRoleAssignments(t, ss) })
t.Run("MaterializedPublicChannels", func(t *testing.T) { testMaterializedPublicChannels(t, ss, s) })
t.Run("GetAllChannelsForExportAfter", func(t *testing.T) { testChannelStoreGetAllChannelsForExportAfter(t, ss) })
t.Run("GetChannelMembersForExport", func(t *testing.T) { testChannelStoreGetChannelMembersForExport(t, ss) })
})
}
}
@@ -2693,3 +2695,85 @@ func testMaterializedPublicChannels(t *testing.T, ss store.Store, s SqlSupplier)
require.Equal(t, &model.ChannelList{&o2, &o3, &o4}, result.Data.(*model.ChannelList))
})
}
func testChannelStoreGetAllChannelsForExportAfter(t *testing.T, ss store.Store) {
t1 := model.Team{}
t1.DisplayName = "Name"
t1.Name = model.NewId()
t1.Email = MakeEmail()
t1.Type = model.TEAM_OPEN
store.Must(ss.Team().Save(&t1))
c1 := model.Channel{}
c1.TeamId = t1.Id
c1.DisplayName = "Channel1"
c1.Name = "zz" + model.NewId() + "b"
c1.Type = model.CHANNEL_OPEN
store.Must(ss.Channel().Save(&c1, -1))
r1 := <-ss.Channel().GetAllChannelsForExportAfter(10000, strings.Repeat("0", 26))
assert.Nil(t, r1.Err)
d1 := r1.Data.([]*model.ChannelForExport)
found := false
for _, c := range d1 {
if c.Id == c1.Id {
found = true
assert.Equal(t, t1.Id, c.TeamId)
assert.Nil(t, c.SchemeId)
assert.Equal(t, t1.Name, c.TeamName)
}
}
assert.True(t, found)
}
func testChannelStoreGetChannelMembersForExport(t *testing.T, ss store.Store) {
t1 := model.Team{}
t1.DisplayName = "Name"
t1.Name = model.NewId()
t1.Email = MakeEmail()
t1.Type = model.TEAM_OPEN
store.Must(ss.Team().Save(&t1))
c1 := model.Channel{}
c1.TeamId = t1.Id
c1.DisplayName = "Channel1"
c1.Name = "zz" + model.NewId() + "b"
c1.Type = model.CHANNEL_OPEN
store.Must(ss.Channel().Save(&c1, -1))
c2 := model.Channel{}
c2.TeamId = model.NewId()
c2.DisplayName = "Channel2"
c2.Name = "zz" + model.NewId() + "b"
c2.Type = model.CHANNEL_OPEN
store.Must(ss.Channel().Save(&c2, -1))
u1 := model.User{}
u1.Email = MakeEmail()
u1.Nickname = model.NewId()
store.Must(ss.User().Save(&u1))
m1 := model.ChannelMember{}
m1.ChannelId = c1.Id
m1.UserId = u1.Id
m1.NotifyProps = model.GetDefaultChannelNotifyProps()
store.Must(ss.Channel().SaveMember(&m1))
m2 := model.ChannelMember{}
m2.ChannelId = c2.Id
m2.UserId = u1.Id
m2.NotifyProps = model.GetDefaultChannelNotifyProps()
store.Must(ss.Channel().SaveMember(&m2))
r1 := <-ss.Channel().GetChannelMembersForExport(u1.Id, t1.Id)
assert.Nil(t, r1.Err)
d1 := r1.Data.([]*model.ChannelMemberForExport)
assert.Len(t, d1, 1)
cmfe1 := d1[0]
assert.Equal(t, c1.Name, cmfe1.ChannelName)
assert.Equal(t, c1.Id, cmfe1.ChannelId)
assert.Equal(t, u1.Id, cmfe1.UserId)
}

Просмотреть файл

@@ -218,6 +218,22 @@ func (_m *ChannelStore) GetAllChannelMembersNotifyPropsForChannel(channelId stri
return r0
}
// GetAllChannelsForExportAfter provides a mock function with given fields: limit, afterId
func (_m *ChannelStore) GetAllChannelsForExportAfter(limit int, afterId string) store.StoreChannel {
ret := _m.Called(limit, afterId)
var r0 store.StoreChannel
if rf, ok := ret.Get(0).(func(int, string) store.StoreChannel); ok {
r0 = rf(limit, afterId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(store.StoreChannel)
}
}
return r0
}
// GetByName provides a mock function with given fields: team_id, name, allowFromCache
func (_m *ChannelStore) GetByName(team_id string, name string, allowFromCache bool) store.StoreChannel {
ret := _m.Called(team_id, name, allowFromCache)
@@ -282,6 +298,22 @@ func (_m *ChannelStore) GetChannelCounts(teamId string, userId string) store.Sto
return r0
}
// GetChannelMembersForExport provides a mock function with given fields: userId, teamId
func (_m *ChannelStore) GetChannelMembersForExport(userId string, teamId string) store.StoreChannel {
ret := _m.Called(userId, teamId)
var r0 store.StoreChannel
if rf, ok := ret.Get(0).(func(string, string) store.StoreChannel); ok {
r0 = rf(userId, teamId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(store.StoreChannel)
}
}
return r0
}
// GetChannelUnread provides a mock function with given fields: channelId, userId
func (_m *ChannelStore) GetChannelUnread(channelId string, userId string) store.StoreChannel {
ret := _m.Called(channelId, userId)

Просмотреть файл

@@ -194,6 +194,22 @@ func (_m *PostStore) GetOldest() store.StoreChannel {
return r0
}
// GetParentsForExportAfter provides a mock function with given fields: limit, afterId
func (_m *PostStore) GetParentsForExportAfter(limit int, afterId string) store.StoreChannel {
ret := _m.Called(limit, afterId)
var r0 store.StoreChannel
if rf, ok := ret.Get(0).(func(int, string) store.StoreChannel); ok {
r0 = rf(limit, afterId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(store.StoreChannel)
}
}
return r0
}
// GetPosts provides a mock function with given fields: channelId, offset, limit, allowFromCache
func (_m *PostStore) GetPosts(channelId string, offset int, limit int, allowFromCache bool) store.StoreChannel {
ret := _m.Called(channelId, offset, limit, allowFromCache)
@@ -306,6 +322,22 @@ func (_m *PostStore) GetPostsSince(channelId string, time int64, allowFromCache
return r0
}
// GetRepliesForExport provides a mock function with given fields: parentId
func (_m *PostStore) GetRepliesForExport(parentId string) store.StoreChannel {
ret := _m.Called(parentId)
var r0 store.StoreChannel
if rf, ok := ret.Get(0).(func(string) store.StoreChannel); ok {
r0 = rf(parentId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(store.StoreChannel)
}
}
return r0
}
// GetSingle provides a mock function with given fields: id
func (_m *PostStore) GetSingle(id string) store.StoreChannel {
ret := _m.Called(id)

Просмотреть файл

@@ -109,6 +109,22 @@ func (_m *TeamStore) GetAll() store.StoreChannel {
return r0
}
// GetAllForExportAfter provides a mock function with given fields: limit, afterId
func (_m *TeamStore) GetAllForExportAfter(limit int, afterId string) store.StoreChannel {
ret := _m.Called(limit, afterId)
var r0 store.StoreChannel
if rf, ok := ret.Get(0).(func(int, string) store.StoreChannel); ok {
r0 = rf(limit, afterId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(store.StoreChannel)
}
}
return r0
}
// GetAllPage provides a mock function with given fields: offset, limit
func (_m *TeamStore) GetAllPage(offset int, limit int) store.StoreChannel {
ret := _m.Called(offset, limit)
@@ -269,6 +285,22 @@ func (_m *TeamStore) GetMembersByIds(teamId string, userIds []string) store.Stor
return r0
}
// GetTeamMembersForExport provides a mock function with given fields: userId
func (_m *TeamStore) GetTeamMembersForExport(userId string) store.StoreChannel {
ret := _m.Called(userId)
var r0 store.StoreChannel
if rf, ok := ret.Get(0).(func(string) store.StoreChannel); ok {
r0 = rf(userId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(store.StoreChannel)
}
}
return r0
}
// GetTeamsByScheme provides a mock function with given fields: schemeId, offset, limit
func (_m *TeamStore) GetTeamsByScheme(schemeId string, offset int, limit int) store.StoreChannel {
ret := _m.Called(schemeId, offset, limit)

Просмотреть файл

@@ -130,6 +130,22 @@ func (_m *UserStore) GetAll() store.StoreChannel {
return r0
}
// GetAllAfter provides a mock function with given fields: limit, afterId
func (_m *UserStore) GetAllAfter(limit int, afterId string) store.StoreChannel {
ret := _m.Called(limit, afterId)
var r0 store.StoreChannel
if rf, ok := ret.Get(0).(func(int, string) store.StoreChannel); ok {
r0 = rf(limit, afterId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(store.StoreChannel)
}
}
return r0
}
// GetAllProfiles provides a mock function with given fields: offset, limit
func (_m *UserStore) GetAllProfiles(offset int, limit int) store.StoreChannel {
ret := _m.Called(offset, limit)

Просмотреть файл

@@ -44,6 +44,8 @@ func TestPostStore(t *testing.T, ss store.Store) {
t.Run("PermanentDeleteBatch", func(t *testing.T) { testPostStorePermanentDeleteBatch(t, ss) })
t.Run("GetOldest", func(t *testing.T) { testPostStoreGetOldest(t, ss) })
t.Run("TestGetMaxPostSize", func(t *testing.T) { testGetMaxPostSize(t, ss) })
t.Run("GetParentsForExportAfter", func(t *testing.T) { testPostStoreGetParentsForExportAfter(t, ss) })
t.Run("GetRepliesForExport", func(t *testing.T) { testPostStoreGetRepliesForExport(t, ss) })
}
func testPostStoreSave(t *testing.T, ss store.Store) {
@@ -1814,3 +1816,97 @@ func testGetMaxPostSize(t *testing.T, ss store.Store) {
assert.Equal(t, model.POST_MESSAGE_MAX_RUNES_V2, (<-ss.Post().GetMaxPostSize()).Data.(int))
assert.Equal(t, model.POST_MESSAGE_MAX_RUNES_V2, (<-ss.Post().GetMaxPostSize()).Data.(int))
}
func testPostStoreGetParentsForExportAfter(t *testing.T, ss store.Store) {
t1 := model.Team{}
t1.DisplayName = "Name"
t1.Name = model.NewId()
t1.Email = MakeEmail()
t1.Type = model.TEAM_OPEN
store.Must(ss.Team().Save(&t1))
c1 := model.Channel{}
c1.TeamId = t1.Id
c1.DisplayName = "Channel1"
c1.Name = "zz" + model.NewId() + "b"
c1.Type = model.CHANNEL_OPEN
store.Must(ss.Channel().Save(&c1, -1))
u1 := model.User{}
u1.Username = model.NewId()
u1.Email = MakeEmail()
u1.Nickname = model.NewId()
store.Must(ss.User().Save(&u1))
p1 := &model.Post{}
p1.ChannelId = c1.Id
p1.UserId = u1.Id
p1.Message = "zz" + model.NewId() + "AAAAAAAAAAA"
p1.CreateAt = 1000
p1 = (<-ss.Post().Save(p1)).Data.(*model.Post)
r1 := <-ss.Post().GetParentsForExportAfter(10000, strings.Repeat("0", 26))
assert.Nil(t, r1.Err)
d1 := r1.Data.([]*model.PostForExport)
found := false
for _, p := range d1 {
if p.Id == p1.Id {
found = true
assert.Equal(t, p.Id, p1.Id)
assert.Equal(t, p.Message, p1.Message)
assert.Equal(t, p.Username, u1.Username)
assert.Equal(t, p.TeamName, t1.Name)
assert.Equal(t, p.ChannelName, c1.Name)
}
}
assert.True(t, found)
}
func testPostStoreGetRepliesForExport(t *testing.T, ss store.Store) {
t1 := model.Team{}
t1.DisplayName = "Name"
t1.Name = model.NewId()
t1.Email = MakeEmail()
t1.Type = model.TEAM_OPEN
store.Must(ss.Team().Save(&t1))
c1 := model.Channel{}
c1.TeamId = t1.Id
c1.DisplayName = "Channel1"
c1.Name = "zz" + model.NewId() + "b"
c1.Type = model.CHANNEL_OPEN
store.Must(ss.Channel().Save(&c1, -1))
u1 := model.User{}
u1.Email = MakeEmail()
u1.Nickname = model.NewId()
store.Must(ss.User().Save(&u1))
p1 := &model.Post{}
p1.ChannelId = c1.Id
p1.UserId = u1.Id
p1.Message = "zz" + model.NewId() + "AAAAAAAAAAA"
p1.CreateAt = 1000
p1 = (<-ss.Post().Save(p1)).Data.(*model.Post)
p2 := &model.Post{}
p2.ChannelId = c1.Id
p2.UserId = u1.Id
p2.Message = "zz" + model.NewId() + "AAAAAAAAAAA"
p2.CreateAt = 1001
p2.ParentId = p1.Id
p2.RootId = p1.Id
p2 = (<-ss.Post().Save(p2)).Data.(*model.Post)
r1 := <-ss.Post().GetRepliesForExport(p1.Id)
assert.Nil(t, r1.Err)
d1 := r1.Data.([]*model.ReplyForExport)
assert.Len(t, d1, 1)
reply1 := d1[0]
assert.Equal(t, reply1.Id, p2.Id)
assert.Equal(t, reply1.Message, p2.Message)
assert.Equal(t, reply1.Username, u1.Username)
}

Просмотреть файл

@@ -45,6 +45,8 @@ func TestTeamStore(t *testing.T, ss store.Store) {
t.Run("ResetAllTeamSchemes", func(t *testing.T) { testResetAllTeamSchemes(t, ss) })
t.Run("ClearAllCustomRoleAssignments", func(t *testing.T) { testTeamStoreClearAllCustomRoleAssignments(t, ss) })
t.Run("AnalyticsGetTeamCountForScheme", func(t *testing.T) { testTeamStoreAnalyticsGetTeamCountForScheme(t, ss) })
t.Run("GetAllForExportAfter", func(t *testing.T) { testTeamStoreGetAllForExportAfter(t, ss) })
t.Run("GetTeamMembersForExport", func(t *testing.T) { testTeamStoreGetTeamMembersForExport(t, ss) })
}
func testTeamStoreSave(t *testing.T, ss store.Store) {
@@ -1327,3 +1329,63 @@ func testTeamStoreAnalyticsGetTeamCountForScheme(t *testing.T, ss store.Store) {
count5 := (<-ss.Team().AnalyticsGetTeamCountForScheme(s1.Id)).Data.(int64)
assert.Equal(t, int64(2), count5)
}
func testTeamStoreGetAllForExportAfter(t *testing.T, ss store.Store) {
t1 := model.Team{}
t1.DisplayName = "Name"
t1.Name = model.NewId()
t1.Email = MakeEmail()
t1.Type = model.TEAM_OPEN
store.Must(ss.Team().Save(&t1))
r1 := <-ss.Team().GetAllForExportAfter(10000, strings.Repeat("0", 26))
assert.Nil(t, r1.Err)
d1 := r1.Data.([]*model.TeamForExport)
found := false
for _, team := range d1 {
if team.Id == t1.Id {
found = true
assert.Equal(t, t1.Id, team.Id)
assert.Nil(t, team.SchemeId)
assert.Equal(t, t1.Name, team.Name)
}
}
assert.True(t, found)
}
func testTeamStoreGetTeamMembersForExport(t *testing.T, ss store.Store) {
t1 := model.Team{}
t1.DisplayName = "Name"
t1.Name = model.NewId()
t1.Email = MakeEmail()
t1.Type = model.TEAM_OPEN
store.Must(ss.Team().Save(&t1))
u1 := model.User{}
u1.Email = MakeEmail()
u1.Nickname = model.NewId()
store.Must(ss.User().Save(&u1))
u2 := model.User{}
u2.Email = MakeEmail()
u2.Nickname = model.NewId()
store.Must(ss.User().Save(&u2))
m1 := &model.TeamMember{TeamId: t1.Id, UserId: u1.Id}
store.Must(ss.Team().SaveMember(m1, -1))
m2 := &model.TeamMember{TeamId: t1.Id, UserId: u2.Id}
store.Must(ss.Team().SaveMember(m2, -1))
r1 := <-ss.Team().GetTeamMembersForExport(u1.Id)
assert.Nil(t, r1.Err)
d1 := r1.Data.([]*model.TeamMemberForExport)
assert.Len(t, d1, 1)
tmfe1 := d1[0]
assert.Equal(t, t1.Id, tmfe1.TeamId)
assert.Equal(t, u1.Id, tmfe1.UserId)
assert.Equal(t, t1.Name, tmfe1.TeamName)
}

Просмотреть файл

@@ -51,6 +51,7 @@ func TestUserStore(t *testing.T, ss store.Store) {
t.Run("AnalyticsGetSystemAdminCount", func(t *testing.T) { testUserStoreAnalyticsGetSystemAdminCount(t, ss) })
t.Run("GetProfilesNotInTeam", func(t *testing.T) { testUserStoreGetProfilesNotInTeam(t, ss) })
t.Run("ClearAllCustomRoleAssignments", func(t *testing.T) { testUserStoreClearAllCustomRoleAssignments(t, ss) })
t.Run("GetAllAfter", func(t *testing.T) { testUserStoreGetAllAfter(t, ss) })
}
func testUserStoreSave(t *testing.T, ss store.Store) {
@@ -2164,3 +2165,35 @@ func testUserStoreClearAllCustomRoleAssignments(t *testing.T, ss store.Store) {
require.Nil(t, r4.Err)
assert.Equal(t, "", r4.Data.(*model.User).Roles)
}
func testUserStoreGetAllAfter(t *testing.T, ss store.Store) {
u1 := model.User{
Email: MakeEmail(),
Username: model.NewId(),
Roles: "system_user system_admin system_post_all",
}
store.Must(ss.User().Save(&u1))
r1 := <-ss.User().GetAllAfter(10000, strings.Repeat("0", 26))
require.Nil(t, r1.Err)
d1 := r1.Data.([]*model.User)
found := false
for _, u := range d1 {
if u.Id == u1.Id {
found = true
assert.Equal(t, u1.Id, u.Id)
assert.Equal(t, u1.Email, u.Email)
}
}
assert.True(t, found)
r2 := <-ss.User().GetAllAfter(10000, u1.Id)
require.Nil(t, r2.Err)
d2 := r2.Data.([]*model.User)
for _, u := range d2 {
assert.NotEqual(t, u1.Id, u.Id)
}
}