MM-35133 trigger sync user immediately after change (#17579)
- ensure changes to user profile sync immediately - refactor sync send
Этот коммит содержится в:
Doug Lauder
@@ -15,23 +15,6 @@ import (
|
||||
"github.com/mattermost/mattermost-server/v5/shared/mlog"
|
||||
)
|
||||
|
||||
// postToAttachments returns the file attachments for a post that need to be synchronized.
|
||||
func (scs *Service) postToAttachments(post *model.Post, rc *model.RemoteCluster) ([]*model.FileInfo, error) {
|
||||
infos := make([]*model.FileInfo, 0)
|
||||
|
||||
fis, err := scs.server.GetStore().FileInfo().GetForPost(post.Id, false, true, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not get file info for attachment: %w", err)
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
if scs.shouldSyncAttachment(fi, rc) {
|
||||
infos = append(infos, fi)
|
||||
}
|
||||
}
|
||||
return infos, nil
|
||||
}
|
||||
|
||||
// postsToAttachments returns the file attachments for a slice of posts that need to be synchronized.
|
||||
func (scs *Service) shouldSyncAttachment(fi *model.FileInfo, rc *model.RemoteCluster) bool {
|
||||
sca, err := scs.server.GetStore().SharedChannel().GetAttachment(fi.Id, rc.RemoteId)
|
||||
|
||||
@@ -1,83 +0,0 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package sharedchannel
|
||||
|
||||
import (
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
"github.com/mattermost/mattermost-server/v5/shared/mlog"
|
||||
)
|
||||
|
||||
type sinceResult struct {
|
||||
posts []*model.Post
|
||||
hasMore bool
|
||||
nextSince int64
|
||||
}
|
||||
|
||||
// getPostsSince fetches posts that need to be synchronized with a remote cluster.
|
||||
// There is a soft cap on the number of posts that will be synchronized in a single pass (MaxPostsPerSync).
|
||||
//
|
||||
// There is a special case where multiple posts have the same UpdateAt value. It is vital that this method
|
||||
// include all posts within that millisecond so that subsequent calls can use an incremented `since`. If this
|
||||
// method were to be called repeatedly with the same `since` value the same records would be returned each time
|
||||
// and the sync would never move forward.
|
||||
//
|
||||
// A boolean is also returned to indicate if there are more posts to be synchronized (true) or not (false).
|
||||
func (scs *Service) getPostsSince(channelId string, rc *model.RemoteCluster, since int64) (sinceResult, error) {
|
||||
opts := model.GetPostsSinceForSyncOptions{
|
||||
ChannelId: channelId,
|
||||
Since: since,
|
||||
IncludeDeleted: true,
|
||||
Limit: MaxPostsPerSync + 1, // ask for 1 more than needed to peek at first post in next batch
|
||||
}
|
||||
posts, err := scs.server.GetStore().Post().GetPostsSinceForSync(opts, true)
|
||||
if err != nil {
|
||||
return sinceResult{}, err
|
||||
}
|
||||
|
||||
if len(posts) == 0 {
|
||||
return sinceResult{nextSince: since}, nil
|
||||
}
|
||||
|
||||
var hasMore bool
|
||||
if len(posts) > MaxPostsPerSync {
|
||||
hasMore = true
|
||||
peekUpdateAt := posts[len(posts)-1].UpdateAt
|
||||
posts = posts[:MaxPostsPerSync] // trim the peeked at record
|
||||
|
||||
// If the last post to be synchronized has the same Update value as the first post in the next batch
|
||||
// then we need to grab the rest of the posts for that millisecond to ensure the next call can have an
|
||||
// incremented `since`.
|
||||
if peekUpdateAt == posts[len(posts)-1].UpdateAt {
|
||||
opts.Since = peekUpdateAt
|
||||
opts.Until = opts.Since
|
||||
opts.Limit = 1000
|
||||
opts.Offset = countPostsAtMillisecond(posts, peekUpdateAt)
|
||||
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "getPostsSince handling updateAt collision",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.Int64("update_at", peekUpdateAt),
|
||||
mlog.Int("offset", opts.Offset),
|
||||
)
|
||||
|
||||
morePosts, err := scs.server.GetStore().Post().GetPostsSinceForSync(opts, true)
|
||||
if err != nil {
|
||||
return sinceResult{}, err
|
||||
}
|
||||
posts = append(posts, morePosts...)
|
||||
}
|
||||
}
|
||||
return sinceResult{posts: posts, hasMore: hasMore, nextSince: posts[len(posts)-1].UpdateAt + 1}, nil
|
||||
}
|
||||
|
||||
func countPostsAtMillisecond(posts []*model.Post, milli int64) int {
|
||||
// walk backward through the slice until we find a post with UpdateAt that differs from milli.
|
||||
var count int
|
||||
for i := len(posts) - 1; i >= 0; i-- {
|
||||
if posts[i].UpdateAt != milli {
|
||||
return count
|
||||
}
|
||||
count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
@@ -4,28 +4,29 @@
|
||||
package sharedchannel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
"github.com/mattermost/mattermost-server/v5/services/remotecluster"
|
||||
"github.com/mattermost/mattermost-server/v5/shared/mlog"
|
||||
)
|
||||
|
||||
// syncMsg represents a change in content (post add/edit/delete, reaction add/remove, users).
|
||||
// It is sent to remote clusters as the payload of a `RemoteClusterMsg`.
|
||||
type syncMsg struct {
|
||||
ChannelId string `json:"channel_id"`
|
||||
PostId string `json:"post_id"`
|
||||
Post *model.Post `json:"post"`
|
||||
Users []*model.User `json:"users"`
|
||||
Reactions []*model.Reaction `json:"reactions"`
|
||||
Attachments []*model.FileInfo `json:"-"`
|
||||
Id string `json:"id"`
|
||||
ChannelId string `json:"channel_id"`
|
||||
Users map[string]*model.User `json:"users,omitempty"`
|
||||
Posts []*model.Post `json:"posts,omitempty"`
|
||||
Reactions []*model.Reaction `json:"reactions,omitempty"`
|
||||
}
|
||||
|
||||
func (sm syncMsg) ToJSON() ([]byte, error) {
|
||||
func newSyncMsg(channelID string) *syncMsg {
|
||||
return &syncMsg{
|
||||
Id: model.NewId(),
|
||||
ChannelId: channelID,
|
||||
}
|
||||
}
|
||||
|
||||
func (sm *syncMsg) ToJSON() ([]byte, error) {
|
||||
b, err := json.Marshal(sm)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -33,296 +34,10 @@ func (sm syncMsg) ToJSON() ([]byte, error) {
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (sm syncMsg) String() string {
|
||||
func (sm *syncMsg) String() string {
|
||||
json, err := sm.ToJSON()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(json)
|
||||
}
|
||||
|
||||
type userCache map[string]struct{}
|
||||
|
||||
func (u userCache) Has(id string) bool {
|
||||
_, ok := u[id]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (u userCache) Add(id string) {
|
||||
u[id] = struct{}{}
|
||||
}
|
||||
|
||||
// postsToSyncMessages takes a slice of posts and converts to a `RemoteClusterMsg` which can be
|
||||
// sent to a remote cluster.
|
||||
func (scs *Service) postsToSyncMessages(posts []*model.Post, channelID string, rc *model.RemoteCluster, nextSyncAt int64) ([]syncMsg, error) {
|
||||
syncMessages := make([]syncMsg, 0, len(posts))
|
||||
|
||||
var teamID string
|
||||
uCache := make(userCache)
|
||||
|
||||
for _, p := range posts {
|
||||
if p.IsSystemMessage() { // don't sync system messages
|
||||
continue
|
||||
}
|
||||
|
||||
// lookup team id once
|
||||
if teamID == "" {
|
||||
sc, err := scs.server.GetStore().SharedChannel().Get(p.ChannelId)
|
||||
if err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Could not get shared channel for post",
|
||||
mlog.String("post_id", p.Id),
|
||||
mlog.Err(err),
|
||||
)
|
||||
continue
|
||||
}
|
||||
teamID = sc.TeamId
|
||||
}
|
||||
|
||||
// any reactions originating from the remote cluster are filtered out
|
||||
reactions, err := scs.server.GetStore().Reaction().GetForPostSince(p.Id, nextSyncAt, rc.RemoteId, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
postSync := p
|
||||
|
||||
// Don't resend an existing post where only the reactions changed.
|
||||
// Posts we must send:
|
||||
// - new posts (EditAt == 0)
|
||||
// - edited posts (EditAt >= nextSyncAt)
|
||||
// - deleted posts (DeleteAt > 0)
|
||||
if p.EditAt > 0 && p.EditAt < nextSyncAt && p.DeleteAt == 0 {
|
||||
postSync = nil
|
||||
}
|
||||
|
||||
// Don't send a deleted post if it is just the original copy from an edit.
|
||||
if p.DeleteAt > 0 && p.OriginalId != "" {
|
||||
postSync = nil
|
||||
}
|
||||
|
||||
// don't sync a post back to the remote it came from.
|
||||
if p.RemoteId != nil && *p.RemoteId == rc.RemoteId {
|
||||
postSync = nil
|
||||
}
|
||||
|
||||
var attachments []*model.FileInfo
|
||||
if postSync != nil {
|
||||
// parse out all permalinks in the message.
|
||||
postSync.Message = scs.processPermalinkToRemote(postSync)
|
||||
|
||||
// get any file attachments
|
||||
attachments, err = scs.postToAttachments(postSync, rc)
|
||||
if err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Could not fetch attachments for post",
|
||||
mlog.String("post_id", postSync.Id),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// any users originating from the remote cluster are filtered out
|
||||
users := scs.usersForPost(postSync, reactions, channelID, teamID, rc, uCache)
|
||||
|
||||
// if everything was filtered out then don't send an empty message.
|
||||
if postSync == nil && len(reactions) == 0 && len(users) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
sm := syncMsg{
|
||||
ChannelId: p.ChannelId,
|
||||
PostId: p.Id,
|
||||
Post: postSync,
|
||||
Users: users,
|
||||
Reactions: reactions,
|
||||
Attachments: attachments,
|
||||
}
|
||||
syncMessages = append(syncMessages, sm)
|
||||
}
|
||||
return syncMessages, nil
|
||||
}
|
||||
|
||||
// usersForPost provides a list of Users associated with the post that need to be synchronized.
|
||||
// The user cache ensures the same user is not synchronized redundantly if they appear in multiple
|
||||
// posts for this sync batch.
|
||||
func (scs *Service) usersForPost(post *model.Post, reactions []*model.Reaction, channelID string, teamID string, rc *model.RemoteCluster, uCache userCache) []*model.User {
|
||||
userIds := make([]string, 0)
|
||||
var mentionMap model.UserMentionMap
|
||||
|
||||
if post != nil && !uCache.Has(post.UserId) {
|
||||
userIds = append(userIds, post.UserId)
|
||||
uCache.Add(post.UserId)
|
||||
}
|
||||
|
||||
for _, r := range reactions {
|
||||
if !uCache.Has(r.UserId) {
|
||||
userIds = append(userIds, r.UserId)
|
||||
uCache.Add(r.UserId)
|
||||
}
|
||||
}
|
||||
|
||||
// get mentions and userids for each mention
|
||||
if post != nil {
|
||||
mentionMap = scs.app.MentionsToTeamMembers(post.Message, teamID)
|
||||
for mention, id := range mentionMap {
|
||||
if !uCache.Has(id) {
|
||||
userIds = append(userIds, id)
|
||||
uCache.Add(id)
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Found mention",
|
||||
mlog.String("mention", mention),
|
||||
mlog.String("user_id", id),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
users := make([]*model.User, 0)
|
||||
|
||||
for _, id := range userIds {
|
||||
user, err := scs.server.GetStore().User().Get(context.Background(), id)
|
||||
if err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error checking if user should sync",
|
||||
mlog.String("user_id", id),
|
||||
mlog.Err(err),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
sync, syncImage, err2 := scs.shouldUserSync(user, channelID, rc)
|
||||
if err2 != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Could not find user for post",
|
||||
mlog.String("user_id", id),
|
||||
mlog.Err(err2),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if sync {
|
||||
users = append(users, sanitizeUserForSync(user))
|
||||
}
|
||||
|
||||
if syncImage {
|
||||
scs.syncProfileImage(user, channelID, rc)
|
||||
}
|
||||
|
||||
// if this was a mention then put the real username in place of the username+remotename, but only
|
||||
// when sending to the remote that the user belongs to.
|
||||
if user.RemoteId != nil && *user.RemoteId == rc.RemoteId {
|
||||
fixMention(post, mentionMap, user)
|
||||
}
|
||||
}
|
||||
return users
|
||||
}
|
||||
|
||||
// fixMention replaces any mentions in a post for the user with the user's real username.
|
||||
func fixMention(post *model.Post, mentionMap model.UserMentionMap, user *model.User) {
|
||||
if post == nil || len(mentionMap) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
realUsername, ok := user.GetProp(KeyRemoteUsername)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// there may be more than one mention for each user so we have to walk the whole map.
|
||||
for mention, id := range mentionMap {
|
||||
if id == user.Id && strings.Contains(mention, ":") {
|
||||
post.Message = strings.ReplaceAll(post.Message, "@"+mention, "@"+realUsername)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeUserForSync(user *model.User) *model.User {
|
||||
user.Password = model.NewId()
|
||||
user.AuthData = nil
|
||||
user.AuthService = ""
|
||||
user.Roles = "system_user"
|
||||
user.AllowMarketing = false
|
||||
user.NotifyProps = model.StringMap{}
|
||||
user.LastPasswordUpdate = 0
|
||||
user.LastPictureUpdate = 0
|
||||
user.FailedAttempts = 0
|
||||
user.MfaActive = false
|
||||
user.MfaSecret = ""
|
||||
|
||||
return user
|
||||
}
|
||||
|
||||
// shouldUserSync determines if a user needs to be synchronized.
|
||||
// User should be synchronized if it has no entry in the SharedChannelUsers table for the specified channel,
|
||||
// or there is an entry but the LastSyncAt is less than user.UpdateAt
|
||||
func (scs *Service) shouldUserSync(user *model.User, channelID string, rc *model.RemoteCluster) (sync bool, syncImage bool, err error) {
|
||||
// don't sync users with the remote they originated from.
|
||||
if user.RemoteId != nil && *user.RemoteId == rc.RemoteId {
|
||||
return false, false, nil
|
||||
}
|
||||
|
||||
scu, err := scs.server.GetStore().SharedChannel().GetUser(user.Id, channelID, rc.RemoteId)
|
||||
if err != nil {
|
||||
if _, ok := err.(errNotFound); !ok {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
// user not in the SharedChannelUsers table, so we must add them.
|
||||
scu = &model.SharedChannelUser{
|
||||
UserId: user.Id,
|
||||
RemoteId: rc.RemoteId,
|
||||
ChannelId: channelID,
|
||||
}
|
||||
if _, err = scs.server.GetStore().SharedChannel().SaveUser(scu); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error adding user to shared channel users",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.String("channel_id", user.Id),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
return true, true, nil
|
||||
}
|
||||
|
||||
return user.UpdateAt > scu.LastSyncAt, user.LastPictureUpdate > scu.LastSyncAt, nil
|
||||
}
|
||||
|
||||
func (scs *Service) syncProfileImage(user *model.User, channelID string, rc *model.RemoteCluster) {
|
||||
rcs := scs.server.GetRemoteClusterService()
|
||||
if rcs == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
|
||||
defer cancel()
|
||||
|
||||
rcs.SendProfileImage(ctx, user.Id, rc, scs.app, func(userId string, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
|
||||
if resp.IsSuccess() {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Users profile image synchronized",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
)
|
||||
|
||||
scu, err := scs.server.GetStore().SharedChannel().GetUser(user.Id, channelID, rc.RemoteId)
|
||||
if err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error fetching shared channel user while updating users LastSyncTime after profile image update",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.String("channel_id", channelID),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
|
||||
if err = scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(scu.Id, model.GetMillis()); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error updating users LastSyncTime after profile image update",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error synchronizing users profile image",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.String("Err", resp.Err),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -4,7 +4,13 @@
|
||||
package sharedchannel
|
||||
|
||||
type SyncResponse struct {
|
||||
LastSyncAt int64 `json:"last_sync_at"`
|
||||
PostErrors []string `json:"post_errors"`
|
||||
UsersSyncd []string `json:"users_syncd"`
|
||||
UsersLastUpdateAt int64 `json:"users_last_update_at"`
|
||||
UserErrors []string `json:"user_errors"`
|
||||
UsersSyncd []string `json:"users_syncd"`
|
||||
|
||||
PostsLastUpdateAt int64 `json:"posts_last_update_at"`
|
||||
PostErrors []string `json:"post_errors"`
|
||||
|
||||
ReactionsLastUpdateAt int64 `json:"reactions_last_update_at"`
|
||||
ReactionErrors []string `json:"reaction_errors"`
|
||||
}
|
||||
|
||||
@@ -24,9 +24,11 @@ const (
|
||||
TopicUploadCreate = "sharedchannel_upload"
|
||||
MaxRetries = 3
|
||||
MaxPostsPerSync = 12 // a bit more than one typical screenfull of posts
|
||||
MaxUsersPerSync = 25
|
||||
NotifyRemoteOfflineThreshold = time.Second * 10
|
||||
NotifyMinimumDelay = time.Second * 2
|
||||
MaxUpsertRetries = 25
|
||||
ProfileImageSyncTimeout = time.Second * 5
|
||||
KeyRemoteUsername = "RemoteUsername"
|
||||
KeyRemoteEmail = "RemoteEmail"
|
||||
)
|
||||
|
||||
@@ -33,136 +33,131 @@ func (scs *Service) onReceiveSyncMessage(msg model.RemoteClusterMsg, rc *model.R
|
||||
)
|
||||
}
|
||||
|
||||
var syncMessages []syncMsg
|
||||
var sm syncMsg
|
||||
|
||||
if err := json.Unmarshal(msg.Payload, &syncMessages); err != nil {
|
||||
if err := json.Unmarshal(msg.Payload, &sm); err != nil {
|
||||
return fmt.Errorf("invalid sync message: %w", err)
|
||||
}
|
||||
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Batch of sync messages received",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.Int("sync_msg_count", len(syncMessages)),
|
||||
)
|
||||
|
||||
return scs.processSyncMessages(syncMessages, rc, response)
|
||||
return scs.processSyncMessage(&sm, rc, response)
|
||||
}
|
||||
|
||||
func (scs *Service) processSyncMessages(syncMessages []syncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
|
||||
func (scs *Service) processSyncMessage(syncMsg *syncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
|
||||
var channel *model.Channel
|
||||
var team *model.Team
|
||||
|
||||
postErrors := make([]string, 0)
|
||||
usersSyncd := make([]string, 0)
|
||||
var lastSyncAt int64
|
||||
var err error
|
||||
syncResp := SyncResponse{
|
||||
UserErrors: make([]string, 0),
|
||||
UsersSyncd: make([]string, 0),
|
||||
PostErrors: make([]string, 0),
|
||||
ReactionErrors: make([]string, 0),
|
||||
}
|
||||
|
||||
for _, sm := range syncMessages {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Sync msg received",
|
||||
mlog.String("post_id", sm.PostId),
|
||||
mlog.String("channel_id", sm.ChannelId),
|
||||
mlog.Int("reaction_count", len(sm.Reactions)),
|
||||
mlog.Int("user_count", len(sm.Users)),
|
||||
mlog.Bool("has_post", sm.Post != nil),
|
||||
)
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Sync msg received",
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.String("channel_id", syncMsg.ChannelId),
|
||||
mlog.Int("user_count", len(syncMsg.Users)),
|
||||
mlog.Int("post_count", len(syncMsg.Posts)),
|
||||
mlog.Int("reaction_count", len(syncMsg.Reactions)),
|
||||
)
|
||||
|
||||
if channel == nil {
|
||||
if channel, err = scs.server.GetStore().Channel().Get(sm.ChannelId, true); err != nil {
|
||||
// if the channel doesn't exist then none of these sync messages are going to work.
|
||||
return fmt.Errorf("channel not found processing sync messages: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// add/update users before posts
|
||||
for _, user := range sm.Users {
|
||||
if userSaved, err := scs.upsertSyncUser(user, channel, rc); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync user",
|
||||
mlog.String("post_id", sm.PostId),
|
||||
mlog.String("channel_id", sm.ChannelId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.Err(err))
|
||||
} else {
|
||||
usersSyncd = append(usersSyncd, userSaved.Id)
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "User upserted via sync",
|
||||
mlog.String("post_id", sm.PostId),
|
||||
mlog.String("channel_id", sm.ChannelId),
|
||||
mlog.String("user_id", user.Id),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if sm.Post != nil {
|
||||
if sm.ChannelId != sm.Post.ChannelId {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "ChannelId mismatch",
|
||||
mlog.String("sm.ChannelId", sm.ChannelId),
|
||||
mlog.String("sm.Post.ChannelId", sm.Post.ChannelId),
|
||||
mlog.String("PostId", sm.Post.Id),
|
||||
)
|
||||
postErrors = append(postErrors, sm.Post.Id)
|
||||
continue
|
||||
}
|
||||
|
||||
if channel.Type != model.CHANNEL_DIRECT && team == nil {
|
||||
var err2 error
|
||||
team, err2 = scs.server.GetStore().Channel().GetTeamForChannel(sm.ChannelId)
|
||||
if err2 != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error getting Team for Channel",
|
||||
mlog.String("ChannelId", sm.Post.ChannelId),
|
||||
mlog.String("PostId", sm.Post.Id),
|
||||
mlog.Err(err2),
|
||||
)
|
||||
postErrors = append(postErrors, sm.Post.Id)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// process perma-links for remote
|
||||
if team != nil {
|
||||
sm.Post.Message = scs.processPermalinkFromRemote(sm.Post, team)
|
||||
}
|
||||
|
||||
// add/update post
|
||||
rpost, err := scs.upsertSyncPost(sm.Post, channel, rc)
|
||||
if err != nil {
|
||||
postErrors = append(postErrors, sm.Post.Id)
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync post",
|
||||
mlog.String("post_id", sm.Post.Id),
|
||||
mlog.String("channel_id", sm.Post.ChannelId),
|
||||
mlog.Err(err),
|
||||
)
|
||||
} else if lastSyncAt < rpost.UpdateAt {
|
||||
lastSyncAt = rpost.UpdateAt
|
||||
}
|
||||
}
|
||||
|
||||
// add/remove reactions
|
||||
for _, reaction := range sm.Reactions {
|
||||
if _, err := scs.upsertSyncReaction(reaction, rc); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync reaction",
|
||||
mlog.String("user_id", reaction.UserId),
|
||||
mlog.String("post_id", reaction.PostId),
|
||||
mlog.String("emoji", reaction.EmojiName),
|
||||
mlog.Int64("delete_at", reaction.DeleteAt),
|
||||
mlog.Err(err),
|
||||
)
|
||||
} else {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Reaction upserted via sync",
|
||||
mlog.String("user_id", reaction.UserId),
|
||||
mlog.String("post_id", reaction.PostId),
|
||||
mlog.String("emoji", reaction.EmojiName),
|
||||
mlog.Int64("delete_at", reaction.DeleteAt),
|
||||
)
|
||||
|
||||
if lastSyncAt < reaction.UpdateAt {
|
||||
lastSyncAt = reaction.UpdateAt
|
||||
}
|
||||
if channel, err = scs.server.GetStore().Channel().Get(syncMsg.ChannelId, true); err != nil {
|
||||
// if the channel doesn't exist then none of these sync items are going to work.
|
||||
return fmt.Errorf("channel not found processing sync message: %w", err)
|
||||
}
|
||||
|
||||
// add/update users before posts
|
||||
for _, user := range syncMsg.Users {
|
||||
if userSaved, err := scs.upsertSyncUser(user, channel, rc); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync user",
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.String("channel_id", syncMsg.ChannelId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.Err(err))
|
||||
} else {
|
||||
syncResp.UsersSyncd = append(syncResp.UsersSyncd, userSaved.Id)
|
||||
if syncResp.UsersLastUpdateAt < user.UpdateAt {
|
||||
syncResp.UsersLastUpdateAt = user.UpdateAt
|
||||
}
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "User upserted via sync",
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.String("channel_id", syncMsg.ChannelId),
|
||||
mlog.String("user_id", user.Id),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
syncResp := SyncResponse{
|
||||
LastSyncAt: lastSyncAt, // might be zero
|
||||
PostErrors: postErrors, // might be empty
|
||||
UsersSyncd: usersSyncd, // might be empty
|
||||
for _, post := range syncMsg.Posts {
|
||||
if syncMsg.ChannelId != post.ChannelId {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "ChannelId mismatch",
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.String("sm.ChannelId", syncMsg.ChannelId),
|
||||
mlog.String("sm.Post.ChannelId", post.ChannelId),
|
||||
mlog.String("PostId", post.Id),
|
||||
)
|
||||
syncResp.PostErrors = append(syncResp.PostErrors, post.Id)
|
||||
continue
|
||||
}
|
||||
|
||||
if channel.Type != model.CHANNEL_DIRECT && team == nil {
|
||||
var err2 error
|
||||
team, err2 = scs.server.GetStore().Channel().GetTeamForChannel(syncMsg.ChannelId)
|
||||
if err2 != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error getting Team for Channel",
|
||||
mlog.String("ChannelId", post.ChannelId),
|
||||
mlog.String("PostId", post.Id),
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.Err(err2),
|
||||
)
|
||||
syncResp.PostErrors = append(syncResp.PostErrors, post.Id)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// process perma-links for remote
|
||||
if team != nil {
|
||||
post.Message = scs.processPermalinkFromRemote(post, team)
|
||||
}
|
||||
|
||||
// add/update post
|
||||
rpost, err := scs.upsertSyncPost(post, channel, rc)
|
||||
if err != nil {
|
||||
syncResp.PostErrors = append(syncResp.PostErrors, post.Id)
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync post",
|
||||
mlog.String("post_id", post.Id),
|
||||
mlog.String("channel_id", post.ChannelId),
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.Err(err),
|
||||
)
|
||||
} else if syncResp.PostsLastUpdateAt < rpost.UpdateAt {
|
||||
syncResp.PostsLastUpdateAt = rpost.UpdateAt
|
||||
}
|
||||
}
|
||||
|
||||
// add/remove reactions
|
||||
for _, reaction := range syncMsg.Reactions {
|
||||
if _, err := scs.upsertSyncReaction(reaction, rc); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync reaction",
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.String("user_id", reaction.UserId),
|
||||
mlog.String("post_id", reaction.PostId),
|
||||
mlog.String("emoji", reaction.EmojiName),
|
||||
mlog.Int64("delete_at", reaction.DeleteAt),
|
||||
mlog.Err(err),
|
||||
)
|
||||
} else {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Reaction upserted via sync",
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.String("user_id", reaction.UserId),
|
||||
mlog.String("post_id", reaction.PostId),
|
||||
mlog.String("emoji", reaction.EmojiName),
|
||||
mlog.Int64("delete_at", reaction.DeleteAt),
|
||||
)
|
||||
|
||||
if syncResp.ReactionsLastUpdateAt < reaction.UpdateAt {
|
||||
syncResp.ReactionsLastUpdateAt = reaction.UpdateAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response.SetPayload(syncResp)
|
||||
@@ -345,24 +340,30 @@ func (scs *Service) upsertSyncPost(post *model.Post, channel *model.Channel, rc
|
||||
if rpost == nil {
|
||||
// post doesn't exist; create new one
|
||||
rpost, appErr = scs.app.CreatePost(request.EmptyContext(), post, channel, true, true)
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Created sync post",
|
||||
mlog.String("post_id", post.Id),
|
||||
mlog.String("channel_id", post.ChannelId),
|
||||
)
|
||||
if appErr == nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Created sync post",
|
||||
mlog.String("post_id", post.Id),
|
||||
mlog.String("channel_id", post.ChannelId),
|
||||
)
|
||||
}
|
||||
} else if post.DeleteAt > 0 {
|
||||
// delete post
|
||||
rpost, appErr = scs.app.DeletePost(post.Id, post.UserId)
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Deleted sync post",
|
||||
mlog.String("post_id", post.Id),
|
||||
mlog.String("channel_id", post.ChannelId),
|
||||
)
|
||||
if appErr == nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Deleted sync post",
|
||||
mlog.String("post_id", post.Id),
|
||||
mlog.String("channel_id", post.ChannelId),
|
||||
)
|
||||
}
|
||||
} else if post.EditAt > rpost.EditAt || post.Message != rpost.Message {
|
||||
// update post
|
||||
rpost, appErr = scs.app.UpdatePost(request.EmptyContext(), post, false)
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Updated sync post",
|
||||
mlog.String("post_id", post.Id),
|
||||
mlog.String("channel_id", post.ChannelId),
|
||||
)
|
||||
if appErr == nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Updated sync post",
|
||||
mlog.String("post_id", post.Id),
|
||||
mlog.String("channel_id", post.ChannelId),
|
||||
)
|
||||
}
|
||||
} else {
|
||||
// nothing to update
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Update to sync post ignored",
|
||||
|
||||
@@ -5,9 +5,7 @@ package sharedchannel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
@@ -18,25 +16,25 @@ import (
|
||||
|
||||
type syncTask struct {
|
||||
id string
|
||||
channelId string
|
||||
remoteId string
|
||||
channelID string
|
||||
remoteID string
|
||||
AddedAt time.Time
|
||||
retryCount int
|
||||
retryPost *model.Post
|
||||
retryMsg *syncMsg
|
||||
schedule time.Time
|
||||
}
|
||||
|
||||
func newSyncTask(channelId string, remoteId string, retryPost *model.Post) syncTask {
|
||||
var postId string
|
||||
if retryPost != nil {
|
||||
postId = retryPost.Id
|
||||
func newSyncTask(channelID string, remoteID string, retryMsg *syncMsg) syncTask {
|
||||
var retryID string
|
||||
if retryMsg != nil {
|
||||
retryID = retryMsg.Id
|
||||
}
|
||||
|
||||
return syncTask{
|
||||
id: channelId + remoteId + postId, // combination of ids to avoid duplicates
|
||||
channelId: channelId,
|
||||
remoteId: remoteId, // empty means update all remote clusters
|
||||
retryPost: retryPost,
|
||||
id: channelID + remoteID + retryID, // combination of ids to avoid duplicates
|
||||
channelID: channelID,
|
||||
remoteID: remoteID, // empty means update all remote clusters
|
||||
retryMsg: retryMsg,
|
||||
schedule: time.Now(),
|
||||
}
|
||||
}
|
||||
@@ -49,16 +47,52 @@ func (st *syncTask) incRetry() bool {
|
||||
|
||||
// NotifyChannelChanged is called to indicate that a shared channel has been modified,
|
||||
// thus triggering an update to all remote clusters.
|
||||
func (scs *Service) NotifyChannelChanged(channelId string) {
|
||||
func (scs *Service) NotifyChannelChanged(channelID string) {
|
||||
if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
|
||||
return
|
||||
}
|
||||
|
||||
task := newSyncTask(channelId, "", nil)
|
||||
task := newSyncTask(channelID, "", nil)
|
||||
task.schedule = time.Now().Add(NotifyMinimumDelay)
|
||||
scs.addTask(task)
|
||||
}
|
||||
|
||||
// NotifyUserProfileChanged is called to indicate that a user belonging to at least one
|
||||
// shared channel has modified their user profile (name, username, email, custom status, profile image)
|
||||
func (scs *Service) NotifyUserProfileChanged(userID string) {
|
||||
if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
|
||||
return
|
||||
}
|
||||
|
||||
scusers, err := scs.server.GetStore().SharedChannel().GetUsersForUser(userID)
|
||||
if err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Failed to fetch shared channel users",
|
||||
mlog.String("userID", userID),
|
||||
mlog.Err(err),
|
||||
)
|
||||
return
|
||||
}
|
||||
if len(scusers) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
notified := make(map[string]struct{})
|
||||
|
||||
for _, user := range scusers {
|
||||
// update every channel + remote combination they belong to.
|
||||
// Redundant updates (ie. to same remote for multiple channels) will be
|
||||
// filtered out.
|
||||
combo := user.ChannelId + user.RemoteId
|
||||
if _, ok := notified[combo]; ok {
|
||||
continue
|
||||
}
|
||||
notified[combo] = struct{}{}
|
||||
task := newSyncTask(user.ChannelId, user.RemoteId, nil)
|
||||
task.schedule = time.Now().Add(NotifyMinimumDelay)
|
||||
scs.addTask(task)
|
||||
}
|
||||
}
|
||||
|
||||
// ForceSyncForRemote causes all channels shared with the remote to be synchronized.
|
||||
func (scs *Service) ForceSyncForRemote(rc *model.RemoteCluster) {
|
||||
if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
|
||||
@@ -155,8 +189,8 @@ func (scs *Service) doSync() time.Duration {
|
||||
scs.addTask(task)
|
||||
} else {
|
||||
scs.server.GetLogger().Error("Failed to synchronize shared channel",
|
||||
mlog.String("channelId", task.channelId),
|
||||
mlog.String("remoteId", task.remoteId),
|
||||
mlog.String("channelId", task.channelID),
|
||||
mlog.String("remoteId", task.remoteID),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
@@ -204,9 +238,9 @@ func (scs *Service) processTask(task syncTask) error {
|
||||
var err error
|
||||
var remotes []*model.RemoteCluster
|
||||
|
||||
if task.remoteId == "" {
|
||||
if task.remoteID == "" {
|
||||
filter := model.RemoteClusterQueryFilter{
|
||||
InChannel: task.channelId,
|
||||
InChannel: task.channelID,
|
||||
OnlyConfirmed: true,
|
||||
}
|
||||
remotes, err = scs.server.GetStore().RemoteCluster().GetAll(filter)
|
||||
@@ -214,28 +248,27 @@ func (scs *Service) processTask(task syncTask) error {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
rc, err := scs.server.GetStore().RemoteCluster().Get(task.remoteId)
|
||||
rc, err := scs.server.GetStore().RemoteCluster().Get(task.remoteID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !rc.IsOnline() {
|
||||
return fmt.Errorf("Failed updating shared channel '%s' for offline remote cluster '%s'", task.channelId, rc.DisplayName)
|
||||
return fmt.Errorf("Failed updating shared channel '%s' for offline remote cluster '%s'", task.channelID, rc.DisplayName)
|
||||
}
|
||||
remotes = []*model.RemoteCluster{rc}
|
||||
}
|
||||
|
||||
for _, rc := range remotes {
|
||||
rtask := task
|
||||
rtask.remoteId = rc.RemoteId
|
||||
if err := scs.updateForRemote(rtask, rc); err != nil {
|
||||
rtask.remoteID = rc.RemoteId
|
||||
if err := scs.syncForRemote(rtask, rc); err != nil {
|
||||
// retry...
|
||||
if rtask.incRetry() {
|
||||
scs.addTask(rtask)
|
||||
} else {
|
||||
scs.server.GetLogger().Error("Failed to synchronize shared channel for remote cluster",
|
||||
mlog.String("channelId", rtask.channelId),
|
||||
mlog.String("channelId", rtask.channelID),
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("remoteId", rtask.remoteId),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
@@ -244,160 +277,8 @@ func (scs *Service) processTask(task syncTask) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// updateForRemote updates a remote cluster with any new posts/reactions for a specific
|
||||
// channel. If many changes are found, only the oldest X changes are sent and the channel
|
||||
// is re-added to the task map. This ensures no channels are starved for updates even if some
|
||||
// channels are very active.
|
||||
func (scs *Service) updateForRemote(task syncTask, rc *model.RemoteCluster) error {
|
||||
rcs := scs.server.GetRemoteClusterService()
|
||||
if rcs == nil {
|
||||
return fmt.Errorf("cannot update remote cluster for channel id %s; Remote Cluster Service not enabled", task.channelId)
|
||||
}
|
||||
|
||||
scr, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(task.channelId, rc.RemoteId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var posts []*model.Post
|
||||
var repeat bool
|
||||
nextSince := scr.NextSyncAt
|
||||
|
||||
if task.retryPost != nil {
|
||||
posts = []*model.Post{task.retryPost}
|
||||
} else {
|
||||
result, err2 := scs.getPostsSince(task.channelId, rc, scr.NextSyncAt)
|
||||
if err2 != nil {
|
||||
return err2
|
||||
}
|
||||
posts = result.posts
|
||||
repeat = result.hasMore
|
||||
nextSince = result.nextSince
|
||||
}
|
||||
|
||||
if len(posts) == 0 {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task found zero posts; skipping sync",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("channel_id", task.channelId),
|
||||
mlog.Int64("lastSyncAt", scr.NextSyncAt),
|
||||
mlog.Int64("nextSince", nextSince),
|
||||
mlog.Bool("repeat", repeat),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task found posts to sync",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("channel_id", task.channelId),
|
||||
mlog.Int64("lastSyncAt", scr.NextSyncAt),
|
||||
mlog.Int64("nextSince", nextSince),
|
||||
mlog.Int("count", len(posts)),
|
||||
mlog.Bool("repeat", repeat),
|
||||
)
|
||||
|
||||
if !rc.IsOnline() {
|
||||
scs.notifyRemoteOffline(posts, rc)
|
||||
return nil
|
||||
}
|
||||
|
||||
syncMessages, err := scs.postsToSyncMessages(posts, task.channelId, rc, scr.NextSyncAt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(syncMessages) == 0 {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task, all messages filtered out; skipping sync",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("channel_id", task.channelId),
|
||||
mlog.Bool("repeat", repeat),
|
||||
)
|
||||
|
||||
// All posts were filtered out, meaning no need to send them. Fast forward SharedChannelRemote's NextSyncAt.
|
||||
scs.updateNextSyncForRemote(scr.Id, rc, nextSince)
|
||||
|
||||
// if there are more posts eligible to sync then schedule another sync
|
||||
if repeat {
|
||||
scs.addTask(newSyncTask(task.channelId, task.remoteId, nil))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
scs.sendAttachments(syncMessages, rc)
|
||||
|
||||
b, err := json.Marshal(syncMessages)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
msg := model.NewRemoteClusterMsg(TopicSync, b)
|
||||
|
||||
if scs.server.GetLogger().IsLevelEnabled(mlog.LvlSharedChannelServiceMessagesOutbound) {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceMessagesOutbound, "outbound message",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.Int64("NextSyncAt", scr.NextSyncAt),
|
||||
mlog.String("msg", string(b)),
|
||||
)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
|
||||
defer cancel()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
err = rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
|
||||
defer wg.Done()
|
||||
if err != nil {
|
||||
return // this means the response could not be parsed; already logged
|
||||
}
|
||||
|
||||
var syncResp SyncResponse
|
||||
if err2 := json.Unmarshal(resp.Payload, &syncResp); err2 != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "invalid sync response after update shared channel",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.Err(err2),
|
||||
)
|
||||
}
|
||||
|
||||
// Any Post(s) that failed to save on remote side are included in an array of post ids in the Response payload.
|
||||
// Handle each error by retrying the post a fixed number of times before giving up.
|
||||
for _, p := range syncResp.PostErrors {
|
||||
scs.handlePostError(p, task, rc)
|
||||
}
|
||||
|
||||
// update NextSyncAt for all the users that were synchronized
|
||||
scs.updateSyncUsers(syncResp.UsersSyncd, task.channelId, rc, nextSince)
|
||||
})
|
||||
|
||||
wg.Wait()
|
||||
|
||||
if err == nil {
|
||||
// Optimistically update SharedChannelRemote's NextSyncAt; if any posts failed they will be retried.
|
||||
scs.updateNextSyncForRemote(scr.Id, rc, nextSince)
|
||||
}
|
||||
|
||||
if repeat {
|
||||
scs.addTask(newSyncTask(task.channelId, task.remoteId, nil))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (scs *Service) sendAttachments(syncMessages []syncMsg, rc *model.RemoteCluster) {
|
||||
for _, sm := range syncMessages {
|
||||
for _, fi := range sm.Attachments {
|
||||
if err := scs.sendAttachmentForRemote(fi, sm.Post, rc); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error syncing attachment for post",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("post_id", sm.Post.Id),
|
||||
mlog.String("file_id", fi.Id),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (scs *Service) handlePostError(postId string, task syncTask, rc *model.RemoteCluster) {
|
||||
if task.retryPost != nil && task.retryPost.Id == postId {
|
||||
if task.retryMsg != nil && len(task.retryMsg.Posts) == 1 && task.retryMsg.Posts[0].Id == postId {
|
||||
// this was a retry for specific post that failed previously. Try again if within MaxRetries.
|
||||
if task.incRetry() {
|
||||
scs.addTask(task)
|
||||
@@ -419,7 +300,11 @@ func (scs *Service) handlePostError(postId string, task syncTask, rc *model.Remo
|
||||
)
|
||||
return
|
||||
}
|
||||
scs.addTask(newSyncTask(task.channelId, task.remoteId, post))
|
||||
|
||||
syncMsg := newSyncMsg(task.channelID)
|
||||
syncMsg.Posts = []*model.Post{post}
|
||||
|
||||
scs.addTask(newSyncTask(task.channelID, task.remoteID, syncMsg))
|
||||
}
|
||||
|
||||
// notifyRemoteOffline creates an ephemeral post to the author for any posts created recently to remotes
|
||||
@@ -452,54 +337,22 @@ func (scs *Service) notifyRemoteOffline(posts []*model.Post, rc *model.RemoteClu
|
||||
}
|
||||
}
|
||||
|
||||
func (scs *Service) updateNextSyncForRemote(scrId string, rc *model.RemoteCluster, nextSyncAt int64) {
|
||||
if nextSyncAt == 0 {
|
||||
return
|
||||
}
|
||||
if err := scs.server.GetStore().SharedChannel().UpdateRemoteNextSyncAt(scrId, nextSyncAt); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error updating NextSyncAt for shared channel remote",
|
||||
func (scs *Service) updateCursorForRemote(scrId string, rc *model.RemoteCluster, cursor model.GetPostsSinceForSyncCursor) {
|
||||
if err := scs.server.GetStore().SharedChannel().UpdateRemoteCursor(scrId, cursor); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error updating cursor for shared channel remote",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.Err(err),
|
||||
)
|
||||
return
|
||||
}
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "updated NextSyncAt for remote",
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "updated cursor for remote",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.Int64("next_update_at", nextSyncAt),
|
||||
mlog.Int64("last_post_update_at", cursor.LastPostUpdateAt),
|
||||
mlog.String("last_post_id", cursor.LastPostId),
|
||||
)
|
||||
}
|
||||
|
||||
func (scs *Service) updateSyncUsers(userIds []string, channelID string, rc *model.RemoteCluster, lastSyncAt int64) {
|
||||
for _, uid := range userIds {
|
||||
scu, err := scs.server.GetStore().SharedChannel().GetUser(uid, channelID, rc.RemoteId)
|
||||
if err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error getting user for lastSyncAt update",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("user_id", uid),
|
||||
mlog.Err(err),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(scu.Id, lastSyncAt); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error updating lastSyncAt for user",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("user_id", uid),
|
||||
mlog.String("channel_id", channelID),
|
||||
mlog.Err(err),
|
||||
)
|
||||
} else {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "updated lastSyncAt for user",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("user_id", scu.UserId),
|
||||
mlog.String("channel_id", channelID),
|
||||
mlog.Int64("last_update_at", lastSyncAt),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (scs *Service) getUserTranslations(userId string) i18n.TranslateFunc {
|
||||
var locale string
|
||||
user, err := scs.server.GetStore().User().Get(context.Background(), userId)
|
||||
@@ -512,3 +365,72 @@ func (scs *Service) getUserTranslations(userId string) i18n.TranslateFunc {
|
||||
}
|
||||
return i18n.GetUserTranslations(locale)
|
||||
}
|
||||
|
||||
// shouldUserSync determines if a user needs to be synchronized.
|
||||
// User should be synchronized if it has no entry in the SharedChannelUsers table for the specified channel,
|
||||
// or there is an entry but the LastSyncAt is less than user.UpdateAt
|
||||
func (scs *Service) shouldUserSync(user *model.User, channelID string, rc *model.RemoteCluster) (sync bool, syncImage bool, err error) {
|
||||
// don't sync users with the remote they originated from.
|
||||
if user.RemoteId != nil && *user.RemoteId == rc.RemoteId {
|
||||
return false, false, nil
|
||||
}
|
||||
|
||||
scu, err := scs.server.GetStore().SharedChannel().GetSingleUser(user.Id, channelID, rc.RemoteId)
|
||||
if err != nil {
|
||||
if _, ok := err.(errNotFound); !ok {
|
||||
return false, false, err
|
||||
}
|
||||
|
||||
// user not in the SharedChannelUsers table, so we must add them.
|
||||
scu = &model.SharedChannelUser{
|
||||
UserId: user.Id,
|
||||
RemoteId: rc.RemoteId,
|
||||
ChannelId: channelID,
|
||||
}
|
||||
if _, err = scs.server.GetStore().SharedChannel().SaveUser(scu); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error adding user to shared channel users",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.String("channel_id", user.Id),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
return true, true, nil
|
||||
}
|
||||
|
||||
return user.UpdateAt > scu.LastSyncAt, user.LastPictureUpdate > scu.LastSyncAt, nil
|
||||
}
|
||||
|
||||
func (scs *Service) syncProfileImage(user *model.User, channelID string, rc *model.RemoteCluster) {
|
||||
rcs := scs.server.GetRemoteClusterService()
|
||||
if rcs == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), ProfileImageSyncTimeout)
|
||||
defer cancel()
|
||||
|
||||
rcs.SendProfileImage(ctx, user.Id, rc, scs.app, func(userId string, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
|
||||
if resp.IsSuccess() {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Users profile image synchronized",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
)
|
||||
|
||||
if err2 := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(user.Id, channelID, rc.RemoteId); err2 != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error updating users LastSyncTime after profile image update",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.Err(err2),
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error synchronizing users profile image",
|
||||
mlog.String("remote_id", rc.RemoteId),
|
||||
mlog.String("user_id", user.Id),
|
||||
mlog.Err(err),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
533
services/sharedchannel/sync_send_remote.go
Обычный файл
533
services/sharedchannel/sync_send_remote.go
Обычный файл
@@ -0,0 +1,533 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package sharedchannel
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/wiggin77/merror"
|
||||
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
"github.com/mattermost/mattermost-server/v5/services/remotecluster"
|
||||
"github.com/mattermost/mattermost-server/v5/shared/mlog"
|
||||
)
|
||||
|
||||
type sendSyncMsgResultFunc func(syncResp SyncResponse, err error)
|
||||
|
||||
type attachment struct {
|
||||
fi *model.FileInfo
|
||||
post *model.Post
|
||||
}
|
||||
|
||||
type syncData struct {
|
||||
task syncTask
|
||||
rc *model.RemoteCluster
|
||||
scr *model.SharedChannelRemote
|
||||
|
||||
users map[string]*model.User
|
||||
profileImages map[string]*model.User
|
||||
posts []*model.Post
|
||||
reactions []*model.Reaction
|
||||
attachments []attachment
|
||||
|
||||
resultRepeat bool
|
||||
resultNextCursor model.GetPostsSinceForSyncCursor
|
||||
}
|
||||
|
||||
func newSyncData(task syncTask, rc *model.RemoteCluster, scr *model.SharedChannelRemote) *syncData {
|
||||
return &syncData{
|
||||
task: task,
|
||||
rc: rc,
|
||||
scr: scr,
|
||||
users: make(map[string]*model.User),
|
||||
profileImages: make(map[string]*model.User),
|
||||
resultNextCursor: model.GetPostsSinceForSyncCursor{LastPostUpdateAt: scr.LastPostUpdateAt, LastPostId: scr.LastPostId},
|
||||
}
|
||||
}
|
||||
|
||||
func (sd *syncData) isEmpty() bool {
|
||||
return len(sd.users) == 0 && len(sd.profileImages) == 0 && len(sd.posts) == 0 && len(sd.reactions) == 0 && len(sd.attachments) == 0
|
||||
}
|
||||
|
||||
func (sd *syncData) isCursorChanged() bool {
|
||||
return sd.scr.LastPostUpdateAt != sd.resultNextCursor.LastPostUpdateAt || sd.scr.LastPostId != sd.resultNextCursor.LastPostId
|
||||
}
|
||||
|
||||
// syncForRemote updates a remote cluster with any new posts/reactions for a specific
|
||||
// channel. If many changes are found, only the oldest X changes are sent and the channel
|
||||
// is re-added to the task map. This ensures no channels are starved for updates even if some
|
||||
// channels are very active.
|
||||
// Returning an error forces a retry on the task.
|
||||
func (scs *Service) syncForRemote(task syncTask, rc *model.RemoteCluster) error {
|
||||
rcs := scs.server.GetRemoteClusterService()
|
||||
if rcs == nil {
|
||||
return fmt.Errorf("cannot update remote cluster %s for channel id %s; Remote Cluster Service not enabled", rc.Name, task.channelID)
|
||||
}
|
||||
|
||||
scr, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(task.channelID, rc.RemoteId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// if this is retrying a failed msg, just send it again.
|
||||
if task.retryMsg != nil {
|
||||
sd := newSyncData(task, rc, scr)
|
||||
sd.users = task.retryMsg.Users
|
||||
sd.posts = task.retryMsg.Posts
|
||||
sd.reactions = task.retryMsg.Reactions
|
||||
return scs.sendSyncData(sd)
|
||||
}
|
||||
|
||||
sd := newSyncData(task, rc, scr)
|
||||
|
||||
// schedule another sync if the repeat flag is set at some point.
|
||||
defer func(rpt *bool) {
|
||||
if *rpt {
|
||||
scs.addTask(newSyncTask(task.channelID, task.remoteID, nil))
|
||||
}
|
||||
}(&sd.resultRepeat)
|
||||
|
||||
// fetch new posts or retry post.
|
||||
if err := scs.fetchPostsForSync(sd); err != nil {
|
||||
return fmt.Errorf("cannot fetch posts for sync %v: %w", sd, err)
|
||||
}
|
||||
|
||||
if !rc.IsOnline() {
|
||||
if len(sd.posts) != 0 {
|
||||
scs.notifyRemoteOffline(sd.posts, rc)
|
||||
}
|
||||
sd.resultRepeat = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetch users that have updated their user profile or image.
|
||||
if err := scs.fetchUsersForSync(sd); err != nil {
|
||||
return fmt.Errorf("cannot fetch users for sync %v: %w", sd, err)
|
||||
}
|
||||
|
||||
// fetch reactions for posts
|
||||
if err := scs.fetchReactionsForSync(sd); err != nil {
|
||||
return fmt.Errorf("cannot fetch reactions for sync %v: %w", sd, err)
|
||||
}
|
||||
|
||||
// fetch users associated with posts & reactions
|
||||
if err := scs.fetchPostUsersForSync(sd); err != nil {
|
||||
return fmt.Errorf("cannot fetch post users for sync %v: %w", sd, err)
|
||||
}
|
||||
|
||||
// filter out any posts that don't need to be sent.
|
||||
scs.filterPostsForSync(sd)
|
||||
|
||||
// fetch attachments for posts
|
||||
if err := scs.fetchPostAttachmentsForSync(sd); err != nil {
|
||||
return fmt.Errorf("cannot fetch post attachments for sync %v: %w", sd, err)
|
||||
}
|
||||
|
||||
if sd.isEmpty() {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Not sending sync data; everything filtered out",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("channel_id", task.channelID),
|
||||
mlog.Bool("repeat", sd.resultRepeat),
|
||||
)
|
||||
if sd.isCursorChanged() {
|
||||
scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Sending sync data",
|
||||
mlog.String("remote", rc.DisplayName),
|
||||
mlog.String("channel_id", task.channelID),
|
||||
mlog.Bool("repeat", sd.resultRepeat),
|
||||
mlog.Int("users", len(sd.users)),
|
||||
mlog.Int("images", len(sd.profileImages)),
|
||||
mlog.Int("posts", len(sd.posts)),
|
||||
mlog.Int("reactions", len(sd.reactions)),
|
||||
mlog.Int("attachments", len(sd.attachments)),
|
||||
)
|
||||
|
||||
return scs.sendSyncData(sd)
|
||||
}
|
||||
|
||||
// fetchUsersForSync populates the sync data with any channel users who updated their user profile
|
||||
// since the last sync.
|
||||
func (scs *Service) fetchUsersForSync(sd *syncData) error {
|
||||
filter := model.GetUsersForSyncFilter{
|
||||
ChannelID: sd.task.channelID,
|
||||
Limit: MaxUsersPerSync,
|
||||
}
|
||||
users, err := scs.server.GetStore().SharedChannel().GetUsersForSync(filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, u := range users {
|
||||
if u.GetRemoteID() != sd.rc.RemoteId {
|
||||
sd.users[u.Id] = u
|
||||
}
|
||||
}
|
||||
|
||||
filter.CheckProfileImage = true
|
||||
usersImage, err := scs.server.GetStore().SharedChannel().GetUsersForSync(filter)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, u := range usersImage {
|
||||
if u.GetRemoteID() != sd.rc.RemoteId {
|
||||
sd.profileImages[u.Id] = u
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fetchPostsForSync populates the sync data with any new posts since the last sync.
|
||||
func (scs *Service) fetchPostsForSync(sd *syncData) error {
|
||||
options := model.GetPostsSinceForSyncOptions{
|
||||
ChannelId: sd.task.channelID,
|
||||
IncludeDeleted: true,
|
||||
}
|
||||
cursor := model.GetPostsSinceForSyncCursor{
|
||||
LastPostUpdateAt: sd.scr.LastPostUpdateAt,
|
||||
LastPostId: sd.scr.LastPostId,
|
||||
}
|
||||
|
||||
posts, nextCursor, err := scs.server.GetStore().Post().GetPostsSinceForSync(options, cursor, MaxPostsPerSync)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not fetch new posts for sync: %w", err)
|
||||
}
|
||||
|
||||
// Append the posts individually, checking for root posts that might appear later in the list.
|
||||
// This is due to the UpdateAt collision handling algorithm where the order of posts is not based
|
||||
// on UpdateAt or CreateAt when the posts have the same UpdateAt value. Here we are guarding
|
||||
// against a root post with the same UpdateAt (and probably the same CreateAt) appearing later
|
||||
// in the list and must be sync'd before the child post. This is and edge case that likely only
|
||||
// happens during load testing or bulk imports.
|
||||
for _, p := range posts {
|
||||
if p.RootId != "" {
|
||||
root, err := scs.server.GetStore().Post().GetSingle(p.RootId, true)
|
||||
if err == nil {
|
||||
if (root.CreateAt >= cursor.LastPostUpdateAt || root.UpdateAt >= cursor.LastPostUpdateAt) && !containsPost(sd.posts, root) {
|
||||
sd.posts = append(sd.posts, root)
|
||||
}
|
||||
}
|
||||
}
|
||||
sd.posts = append(sd.posts, p)
|
||||
}
|
||||
|
||||
sd.resultNextCursor = nextCursor
|
||||
sd.resultRepeat = len(posts) == MaxPostsPerSync
|
||||
return nil
|
||||
}
|
||||
|
||||
func containsPost(posts []*model.Post, post *model.Post) bool {
|
||||
for _, p := range posts {
|
||||
if p.Id == post.Id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// fetchReactionsForSync populates the sync data with any new reactions since the last sync.
|
||||
func (scs *Service) fetchReactionsForSync(sd *syncData) error {
|
||||
merr := merror.New()
|
||||
for _, post := range sd.posts {
|
||||
// any reactions originating from the remote cluster are filtered out
|
||||
reactions, err := scs.server.GetStore().Reaction().GetForPostSince(post.Id, sd.scr.LastPostUpdateAt, sd.rc.RemoteId, true)
|
||||
if err != nil {
|
||||
merr.Append(fmt.Errorf("could not get reactions for post %s: %w", post.Id, err))
|
||||
continue
|
||||
}
|
||||
sd.reactions = append(sd.reactions, reactions...)
|
||||
}
|
||||
return merr.ErrorOrNil()
|
||||
}
|
||||
|
||||
// fetchPostUsersForSync populates the sync data with all users associated with posts.
|
||||
func (scs *Service) fetchPostUsersForSync(sd *syncData) error {
|
||||
sc, err := scs.server.GetStore().SharedChannel().Get(sd.task.channelID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot determine teamID: %w", err)
|
||||
}
|
||||
|
||||
type p2mm struct {
|
||||
post *model.Post
|
||||
mentionMap model.UserMentionMap
|
||||
}
|
||||
|
||||
userIDs := make(map[string]p2mm)
|
||||
|
||||
for _, reaction := range sd.reactions {
|
||||
userIDs[reaction.UserId] = p2mm{}
|
||||
}
|
||||
|
||||
for _, post := range sd.posts {
|
||||
// add author
|
||||
userIDs[post.UserId] = p2mm{}
|
||||
|
||||
// get mentions and users for each mention
|
||||
mentionMap := scs.app.MentionsToTeamMembers(post.Message, sc.TeamId)
|
||||
for _, userID := range mentionMap {
|
||||
userIDs[userID] = p2mm{
|
||||
post: post,
|
||||
mentionMap: mentionMap,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
merr := merror.New()
|
||||
|
||||
for userID, v := range userIDs {
|
||||
user, err := scs.server.GetStore().User().Get(context.Background(), userID)
|
||||
if err != nil {
|
||||
merr.Append(fmt.Errorf("could not get user %s: %w", userID, err))
|
||||
continue
|
||||
}
|
||||
|
||||
sync, syncImage, err2 := scs.shouldUserSync(user, sd.task.channelID, sd.rc)
|
||||
if err2 != nil {
|
||||
merr.Append(fmt.Errorf("could not check should sync user %s: %w", userID, err))
|
||||
continue
|
||||
}
|
||||
|
||||
if sync {
|
||||
sd.users[user.Id] = sanitizeUserForSync(user)
|
||||
}
|
||||
|
||||
if syncImage {
|
||||
sd.profileImages[user.Id] = sanitizeUserForSync(user)
|
||||
}
|
||||
|
||||
// if this was a mention then put the real username in place of the username+remotename, but only
|
||||
// when sending to the remote that the user belongs to.
|
||||
if v.post != nil && user.RemoteId != nil && *user.RemoteId == sd.rc.RemoteId {
|
||||
fixMention(v.post, v.mentionMap, user)
|
||||
}
|
||||
}
|
||||
return merr.ErrorOrNil()
|
||||
}
|
||||
|
||||
// fetchPostAttachmentsForSync populates the sync data with any file attachments for new posts.
|
||||
func (scs *Service) fetchPostAttachmentsForSync(sd *syncData) error {
|
||||
merr := merror.New()
|
||||
for _, post := range sd.posts {
|
||||
fis, err := scs.server.GetStore().FileInfo().GetForPost(post.Id, false, true, true)
|
||||
if err != nil {
|
||||
merr.Append(fmt.Errorf("could not get file attachment info for post %s: %w", post.Id, err))
|
||||
continue
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
if scs.shouldSyncAttachment(fi, sd.rc) {
|
||||
sd.attachments = append(sd.attachments, attachment{fi: fi, post: post})
|
||||
}
|
||||
}
|
||||
}
|
||||
return merr.ErrorOrNil()
|
||||
}
|
||||
|
||||
// filterPostsforSync removes any posts that do not need to sync.
|
||||
func (scs *Service) filterPostsForSync(sd *syncData) {
|
||||
filtered := make([]*model.Post, 0, len(sd.posts))
|
||||
|
||||
for _, p := range sd.posts {
|
||||
// Don't resend an existing post where only the reactions changed.
|
||||
// Posts we must send:
|
||||
// - new posts (EditAt == 0)
|
||||
// - edited posts (EditAt >= LastPostUpdateAt)
|
||||
// - deleted posts (DeleteAt > 0)
|
||||
if p.EditAt > 0 && p.EditAt < sd.scr.LastPostUpdateAt && p.DeleteAt == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Don't send a deleted post if it is just the original copy from an edit.
|
||||
if p.DeleteAt > 0 && p.OriginalId != "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// don't sync a post back to the remote it came from.
|
||||
if p.GetRemoteID() == sd.rc.RemoteId {
|
||||
continue
|
||||
}
|
||||
|
||||
// parse out all permalinks in the message.
|
||||
p.Message = scs.processPermalinkToRemote(p)
|
||||
|
||||
filtered = append(filtered, p)
|
||||
}
|
||||
sd.posts = filtered
|
||||
}
|
||||
|
||||
// sendSyncData sends all the collected users, posts, reactions, images, and attachments to the
|
||||
// remote cluster.
|
||||
// The order of items sent is important: users -> attachments -> posts -> reactions -> profile images
|
||||
func (scs *Service) sendSyncData(sd *syncData) error {
|
||||
merr := merror.New()
|
||||
|
||||
// send users
|
||||
if len(sd.users) != 0 {
|
||||
if err := scs.sendUserSyncData(sd); err != nil {
|
||||
merr.Append(fmt.Errorf("cannot send user sync data: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
// send attachments
|
||||
if len(sd.attachments) != 0 {
|
||||
scs.sendAttachmentSyncData(sd)
|
||||
}
|
||||
|
||||
// send posts
|
||||
if len(sd.posts) != 0 {
|
||||
if err := scs.sendPostSyncData(sd); err != nil {
|
||||
merr.Append(fmt.Errorf("cannot send post sync data: %w", err))
|
||||
}
|
||||
} else if sd.isCursorChanged() {
|
||||
scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
|
||||
}
|
||||
|
||||
// send reactions
|
||||
if len(sd.reactions) != 0 {
|
||||
if err := scs.sendReactionSyncData(sd); err != nil {
|
||||
merr.Append(fmt.Errorf("cannot send reaction sync data: %w", err))
|
||||
}
|
||||
}
|
||||
|
||||
// send user profile images
|
||||
if len(sd.profileImages) != 0 {
|
||||
scs.sendProfileImageSyncData(sd)
|
||||
}
|
||||
|
||||
return merr.ErrorOrNil()
|
||||
}
|
||||
|
||||
// sendUserSyncData sends the collected user updates to the remote cluster.
|
||||
func (scs *Service) sendUserSyncData(sd *syncData) error {
|
||||
msg := newSyncMsg(sd.task.channelID)
|
||||
msg.Users = sd.users
|
||||
|
||||
err := scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
|
||||
for _, userID := range syncResp.UsersSyncd {
|
||||
if err := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(userID, sd.task.channelID, sd.rc.RemoteId); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Cannot update shared channel user LastSyncAt",
|
||||
mlog.String("user_id", userID),
|
||||
mlog.String("channel_id", sd.task.channelID),
|
||||
mlog.String("remote_id", sd.rc.RemoteId),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
}
|
||||
if len(syncResp.UserErrors) != 0 {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Response indicates error for user(s) sync",
|
||||
mlog.String("channel_id", sd.task.channelID),
|
||||
mlog.String("remote_id", sd.rc.RemoteId),
|
||||
mlog.Any("users", syncResp.UserErrors),
|
||||
)
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// sendAttachmentSyncData sends the collected post updates to the remote cluster.
|
||||
func (scs *Service) sendAttachmentSyncData(sd *syncData) {
|
||||
for _, a := range sd.attachments {
|
||||
if err := scs.sendAttachmentForRemote(a.fi, a.post, sd.rc); err != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Cannot sync post attachment",
|
||||
mlog.String("post_id", a.post.Id),
|
||||
mlog.String("channel_id", sd.task.channelID),
|
||||
mlog.String("remote_id", sd.rc.RemoteId),
|
||||
mlog.Err(err),
|
||||
)
|
||||
}
|
||||
// updating SharedChannelAttachments with LastSyncAt is already done.
|
||||
}
|
||||
}
|
||||
|
||||
// sendPostSyncData sends the collected post updates to the remote cluster.
|
||||
func (scs *Service) sendPostSyncData(sd *syncData) error {
|
||||
msg := newSyncMsg(sd.task.channelID)
|
||||
msg.Posts = sd.posts
|
||||
|
||||
return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
|
||||
if len(syncResp.PostErrors) != 0 {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Response indicates error for post(s) sync",
|
||||
mlog.String("channel_id", sd.task.channelID),
|
||||
mlog.String("remote_id", sd.rc.RemoteId),
|
||||
mlog.Any("posts", syncResp.PostErrors),
|
||||
)
|
||||
|
||||
for _, postID := range syncResp.PostErrors {
|
||||
scs.handlePostError(postID, sd.task, sd.rc)
|
||||
}
|
||||
}
|
||||
scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
|
||||
})
|
||||
}
|
||||
|
||||
// sendReactionSyncData sends the collected reaction updates to the remote cluster.
|
||||
func (scs *Service) sendReactionSyncData(sd *syncData) error {
|
||||
msg := newSyncMsg(sd.task.channelID)
|
||||
msg.Reactions = sd.reactions
|
||||
|
||||
return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
|
||||
if len(syncResp.ReactionErrors) != 0 {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Response indicates error for reactions(s) sync",
|
||||
mlog.String("channel_id", sd.task.channelID),
|
||||
mlog.String("remote_id", sd.rc.RemoteId),
|
||||
mlog.Any("reaction_posts", syncResp.ReactionErrors),
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// sendProfileImageSyncData sends the collected user profile image updates to the remote cluster.
|
||||
func (scs *Service) sendProfileImageSyncData(sd *syncData) {
|
||||
for _, user := range sd.profileImages {
|
||||
scs.syncProfileImage(user, sd.task.channelID, sd.rc)
|
||||
}
|
||||
}
|
||||
|
||||
// sendSyncMsgToRemote synchronously sends the sync message to the remote cluster.
|
||||
func (scs *Service) sendSyncMsgToRemote(msg *syncMsg, rc *model.RemoteCluster, f sendSyncMsgResultFunc) error {
|
||||
rcs := scs.server.GetRemoteClusterService()
|
||||
if rcs == nil {
|
||||
return fmt.Errorf("cannot update remote cluster %s for channel id %s; Remote Cluster Service not enabled", rc.Name, msg.ChannelId)
|
||||
}
|
||||
|
||||
b, err := json.Marshal(msg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rcMsg := model.NewRemoteClusterMsg(TopicSync, b)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
|
||||
defer cancel()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
err = rcs.SendMsg(ctx, rcMsg, rc, func(rcMsg model.RemoteClusterMsg, rc *model.RemoteCluster, rcResp *remotecluster.Response, errResp error) {
|
||||
defer wg.Done()
|
||||
|
||||
var syncResp SyncResponse
|
||||
if err2 := json.Unmarshal(rcResp.Payload, &syncResp); err2 != nil {
|
||||
scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Invalid sync msg response from remote cluster",
|
||||
mlog.String("remote", rc.Name),
|
||||
mlog.String("channel_id", msg.ChannelId),
|
||||
mlog.Err(err2),
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if f != nil {
|
||||
f(syncResp, errResp)
|
||||
}
|
||||
})
|
||||
|
||||
wg.Wait()
|
||||
return err
|
||||
}
|
||||
@@ -10,6 +10,41 @@ import (
|
||||
"github.com/mattermost/mattermost-server/v5/model"
|
||||
)
|
||||
|
||||
// fixMention replaces any mentions in a post for the user with the user's real username.
|
||||
func fixMention(post *model.Post, mentionMap model.UserMentionMap, user *model.User) {
|
||||
if post == nil || len(mentionMap) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
realUsername, ok := user.GetProp(KeyRemoteUsername)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// there may be more than one mention for each user so we have to walk the whole map.
|
||||
for mention, id := range mentionMap {
|
||||
if id == user.Id && strings.Contains(mention, ":") {
|
||||
post.Message = strings.ReplaceAll(post.Message, "@"+mention, "@"+realUsername)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func sanitizeUserForSync(user *model.User) *model.User {
|
||||
user.Password = model.NewId()
|
||||
user.AuthData = nil
|
||||
user.AuthService = ""
|
||||
user.Roles = "system_user"
|
||||
user.AllowMarketing = false
|
||||
user.NotifyProps = model.StringMap{}
|
||||
user.LastPasswordUpdate = 0
|
||||
user.LastPictureUpdate = 0
|
||||
user.FailedAttempts = 0
|
||||
user.MfaActive = false
|
||||
user.MfaSecret = ""
|
||||
|
||||
return user
|
||||
}
|
||||
|
||||
// mungUsername creates a new username by combining username and remote cluster name, plus
|
||||
// a suffix to create uniqueness. If the resulting username exceeds the max length then
|
||||
// it is truncated and ellipses added.
|
||||
|
||||
Ссылка в новой задаче
Block a user