Prep for MS Teams plugin API (#25565)

* - columns added to ShareChannelRemotes: lastpostcreateat, lastpostupdateat
- SyncMsg and SyncResponse moved to `model` package
- field added to RemoteCluster struct: PluginID

* sync new posts before updated posts to ensure post order in MS Teams

* add plugid to remoteclusters table and store

* don't sync history by default

server/platform/services/remotecluster/ping.go

@@ -106,6 +106,7 @@ func (rcs *Service) pingRemote(rc *model.RemoteCluster) error {
 	if err != nil {
 		return err
 	}
+	rc.LastPingAt = model.GetMillis()
 
 	ping := model.RemoteClusterPing{}
 	err = json.Unmarshal(resp, &ping)
@@ -120,7 +121,6 @@ func (rcs *Service) pingRemote(rc *model.RemoteCluster) error {
 			mlog.Err(err),
 		)
 	}
-	rc.LastPingAt = model.GetMillis()
 
 	if metrics := rcs.server.GetMetrics(); metrics != nil {
 		sentAt := time.Unix(0, ping.SentAt*int64(time.Millisecond))

server/platform/services/sharedchannel/channelinvite.go

@@ -87,6 +87,8 @@ func (scs *Service) SendChannelInvite(channel *model.Channel, userId string, rc
 			RemoteId:          rc.RemoteId,
 			IsInviteAccepted:  true,
 			IsInviteConfirmed: true,
+			LastPostCreateAt:  model.GetMillis(),
+			LastPostUpdateAt:  model.GetMillis(),
 		}
 		if _, err = scs.server.GetStore().SharedChannel().SaveRemote(scr); err != nil {
 			scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("Error confirming channel invite for %s: %v", rc.DisplayName, err))
@@ -169,6 +171,8 @@ func (scs *Service) onReceiveChannelInvite(msg model.RemoteClusterMsg, rc *model
 		IsInviteAccepted:  true,
 		IsInviteConfirmed: true,
 		RemoteId:          rc.RemoteId,
+		LastPostCreateAt:  model.GetMillis(),
+		LastPostUpdateAt:  model.GetMillis(),
 	}
 
 	if _, err := scs.server.GetStore().SharedChannel().SaveRemote(sharedChannelRemote); err != nil {

server/platform/services/sharedchannel/msg.go

@@ -1,43 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.txt for license information.
-
-package sharedchannel
-
-import (
-	"encoding/json"
-
-	"github.com/mattermost/mattermost/server/public/model"
-)
-
-// syncMsg represents a change in content (post add/edit/delete, reaction add/remove, users).
-// It is sent to remote clusters as the payload of a `RemoteClusterMsg`.
-type syncMsg struct {
-	Id        string                 `json:"id"`
-	ChannelId string                 `json:"channel_id"`
-	Users     map[string]*model.User `json:"users,omitempty"`
-	Posts     []*model.Post          `json:"posts,omitempty"`
-	Reactions []*model.Reaction      `json:"reactions,omitempty"`
-}
-
-func newSyncMsg(channelID string) *syncMsg {
-	return &syncMsg{
-		Id:        model.NewId(),
-		ChannelId: channelID,
-	}
-}
-
-func (sm *syncMsg) ToJSON() ([]byte, error) {
-	b, err := json.Marshal(sm)
-	if err != nil {
-		return nil, err
-	}
-	return b, nil
-}
-
-func (sm *syncMsg) String() string {
-	json, err := sm.ToJSON()
-	if err != nil {
-		return ""
-	}
-	return string(json)
-}

server/platform/services/sharedchannel/response.go

@@ -1,16 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.txt for license information.
-
-package sharedchannel
-
-type SyncResponse struct {
-	UsersLastUpdateAt int64    `json:"users_last_update_at"`
-	UserErrors        []string `json:"user_errors"`
-	UsersSyncd        []string `json:"users_syncd"`
-
-	PostsLastUpdateAt int64    `json:"posts_last_update_at"`
-	PostErrors        []string `json:"post_errors"`
-
-	ReactionsLastUpdateAt int64    `json:"reactions_last_update_at"`
-	ReactionErrors        []string `json:"reaction_errors"`
-}

server/platform/services/sharedchannel/service.go

@@ -23,7 +23,7 @@ const (
 	TopicChannelInvite           = "sharedchannel_invite"
 	TopicUploadCreate            = "sharedchannel_upload"
 	MaxRetries                   = 3
-	MaxPostsPerSync              = 12 // a bit more than one typical screenfull of posts
+	MaxPostsPerSync              = 50 // a bit more than 4 typical screenfulls of posts
 	MaxUsersPerSync              = 25
 	NotifyRemoteOfflineThreshold = time.Second * 10
 	NotifyMinimumDelay           = time.Second * 2

server/platform/services/sharedchannel/sync_recv.go

@@ -33,7 +33,7 @@ func (scs *Service) onReceiveSyncMessage(msg model.RemoteClusterMsg, rc *model.R
 		)
 	}
 
-	var sm syncMsg
+	var sm model.SyncMsg
 
 	if err := json.Unmarshal(msg.Payload, &sm); err != nil {
 		return fmt.Errorf("invalid sync message: %w", err)
@@ -41,12 +41,12 @@ func (scs *Service) onReceiveSyncMessage(msg model.RemoteClusterMsg, rc *model.R
 	return scs.processSyncMessage(request.EmptyContext(scs.server.Log()), &sm, rc, response)
 }
 
-func (scs *Service) processSyncMessage(c request.CTX, syncMsg *syncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+func (scs *Service) processSyncMessage(c request.CTX, syncMsg *model.SyncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
 	var channel *model.Channel
 	var team *model.Team
 
 	var err error
-	syncResp := SyncResponse{
+	syncResp := model.SyncResponse{
 		UserErrors:     make([]string, 0),
 		UsersSyncd:     make([]string, 0),
 		PostErrors:     make([]string, 0),

server/platform/services/sharedchannel/sync_send.go

@@ -21,11 +21,11 @@ type syncTask struct {
 	remoteID   string
 	AddedAt    time.Time
 	retryCount int
-	retryMsg   *syncMsg
+	retryMsg   *model.SyncMsg
 	schedule   time.Time
 }
 
-func newSyncTask(channelID string, remoteID string, retryMsg *syncMsg) syncTask {
+func newSyncTask(channelID string, remoteID string, retryMsg *model.SyncMsg) syncTask {
 	var retryID string
 	if retryMsg != nil {
 		retryID = retryMsg.Id
@@ -302,7 +302,7 @@ func (scs *Service) handlePostError(postId string, task syncTask, rc *model.Remo
 		return
 	}
 
-	syncMsg := newSyncMsg(task.channelID)
+	syncMsg := model.NewSyncMsg(task.channelID)
 	syncMsg.Posts = []*model.Post{post}
 
 	scs.addTask(newSyncTask(task.channelID, task.remoteID, syncMsg))
@@ -349,8 +349,10 @@ func (scs *Service) updateCursorForRemote(scrId string, rc *model.RemoteCluster,
 	scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "updated cursor for remote",
 		mlog.String("remote_id", rc.RemoteId),
 		mlog.String("remote", rc.DisplayName),
+		mlog.Int("last_post_create_at", cursor.LastPostCreateAt),
+		mlog.String("last_post_create_id", cursor.LastPostCreateID),
 		mlog.Int("last_post_update_at", cursor.LastPostUpdateAt),
-		mlog.String("last_post_id", cursor.LastPostId),
+		mlog.String("last_post_update_id", cursor.LastPostUpdateID),
 	)
 }
 

server/platform/services/sharedchannel/sync_send_remote.go

@@ -14,10 +14,11 @@ import (
 	"github.com/mattermost/mattermost/server/public/model"
 	"github.com/mattermost/mattermost/server/public/shared/mlog"
 	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
 	"github.com/mattermost/mattermost/server/v8/platform/services/remotecluster"
 )
 
-type sendSyncMsgResultFunc func(syncResp SyncResponse, err error)
+type sendSyncMsgResultFunc func(syncResp model.SyncResponse, err error)
 
 type attachment struct {
 	fi   *model.FileInfo
@@ -41,12 +42,15 @@ type syncData struct {
 
 func newSyncData(task syncTask, rc *model.RemoteCluster, scr *model.SharedChannelRemote) *syncData {
 	return &syncData{
-		task:             task,
-		rc:               rc,
-		scr:              scr,
-		users:            make(map[string]*model.User),
-		profileImages:    make(map[string]*model.User),
-		resultNextCursor: model.GetPostsSinceForSyncCursor{LastPostUpdateAt: scr.LastPostUpdateAt, LastPostId: scr.LastPostId},
+		task:          task,
+		rc:            rc,
+		scr:           scr,
+		users:         make(map[string]*model.User),
+		profileImages: make(map[string]*model.User),
+		resultNextCursor: model.GetPostsSinceForSyncCursor{
+			LastPostUpdateAt: scr.LastPostUpdateAt, LastPostUpdateID: scr.LastPostUpdateID,
+			LastPostCreateAt: scr.LastPostCreateAt, LastPostCreateID: scr.LastPostCreateID,
+		},
 	}
 }
 
@@ -55,7 +59,12 @@ func (sd *syncData) isEmpty() bool {
 }
 
 func (sd *syncData) isCursorChanged() bool {
-	return sd.scr.LastPostUpdateAt != sd.resultNextCursor.LastPostUpdateAt || sd.scr.LastPostId != sd.resultNextCursor.LastPostId
+	if sd.resultNextCursor.IsEmpty() {
+		return false
+	}
+
+	return sd.scr.LastPostCreateAt != sd.resultNextCursor.LastPostCreateAt || sd.scr.LastPostCreateID != sd.resultNextCursor.LastPostCreateID ||
+		sd.scr.LastPostUpdateAt != sd.resultNextCursor.LastPostUpdateAt || sd.scr.LastPostUpdateID != sd.resultNextCursor.LastPostUpdateID
 }
 
 // syncForRemote updates a remote cluster with any new posts/reactions for a specific
@@ -186,43 +195,67 @@ func (scs *Service) fetchUsersForSync(sd *syncData) error {
 	return nil
 }
 
-// fetchPostsForSync populates the sync data with any new posts since the last sync.
+// fetchPostsForSync populates the sync data with any new or edited posts since the last sync.
 func (scs *Service) fetchPostsForSync(sd *syncData) error {
 	options := model.GetPostsSinceForSyncOptions{
 		ChannelId:      sd.task.channelID,
 		IncludeDeleted: true,
+		SinceCreateAt:  true,
 	}
 	cursor := model.GetPostsSinceForSyncCursor{
 		LastPostUpdateAt: sd.scr.LastPostUpdateAt,
-		LastPostId:       sd.scr.LastPostId,
+		LastPostUpdateID: sd.scr.LastPostUpdateID,
+		LastPostCreateAt: sd.scr.LastPostCreateAt,
+		LastPostCreateID: sd.scr.LastPostCreateID,
 	}
 
+	// Fetch all newly created posts first. This is to ensure that post order is preserved for sync targets
+	// that cannot set the CreateAt timestamp for incoming posts (e.g. MS Teams).  If we simply used UpdateAt
+	// then posts could get out of order. For example: p1 created, p2 created, p1 updated... sync'ing on UpdateAt
+	// would order the posts p2, p1.
 	posts, nextCursor, err := scs.server.GetStore().Post().GetPostsSinceForSync(options, cursor, MaxPostsPerSync)
 	if err != nil {
 		return fmt.Errorf("could not fetch new posts for sync: %w", err)
 	}
+	count := len(posts)
+	sd.posts = appendPosts(sd.posts, posts, scs.server.GetStore().Post(), cursor.LastPostCreateAt)
 
+	// Fill remaining batch capacity with updated posts.
+	if len(posts) < MaxPostsPerSync {
+		options.SinceCreateAt = false
+		posts, nextCursor, err = scs.server.GetStore().Post().GetPostsSinceForSync(options, nextCursor, MaxPostsPerSync-len(posts))
+		if err != nil {
+			return fmt.Errorf("could not fetch modified posts for sync: %w", err)
+		}
+		count += len(posts)
+		sd.posts = appendPosts(sd.posts, posts, scs.server.GetStore().Post(), cursor.LastPostUpdateAt)
+	}
+
+	sd.resultNextCursor = nextCursor
+	sd.resultRepeat = count >= MaxPostsPerSync
+
+	return nil
+}
+
+func appendPosts(dest []*model.Post, posts []*model.Post, postStore store.PostStore, timestamp int64) []*model.Post {
 	// Append the posts individually, checking for root posts that might appear later in the list.
 	// This is due to the UpdateAt collision handling algorithm where the order of posts is not based
 	// on UpdateAt or CreateAt when the posts have the same UpdateAt value. Here we are guarding
 	// against a root post with the same UpdateAt (and probably the same CreateAt) appearing later
-	// in the list and must be sync'd before the child post. This is and edge case that likely only
+	// in the list and must be sync'd before the child post. This is an edge case that likely only
 	// happens during load testing or bulk imports.
 	for _, p := range posts {
 		if p.RootId != "" {
-			root, err := scs.server.GetStore().Post().GetSingle(p.RootId, true)
+			root, err := postStore.GetSingle(p.RootId, true)
 			if err == nil {
-				if (root.CreateAt >= cursor.LastPostUpdateAt || root.UpdateAt >= cursor.LastPostUpdateAt) && !containsPost(sd.posts, root) {
-					sd.posts = append(sd.posts, root)
+				if (root.CreateAt >= timestamp || root.UpdateAt >= timestamp) && !containsPost(dest, root) {
+					dest = append(dest, root)
 				}
 			}
 		}
-		sd.posts = append(sd.posts, p)
+		dest = append(dest, p)
 	}
-
-	sd.resultNextCursor = nextCursor
-	sd.resultRepeat = len(posts) == MaxPostsPerSync
-	return nil
+	return dest
 }
 
 func containsPost(posts []*model.Post, post *model.Post) bool {
@@ -410,10 +443,10 @@ func (scs *Service) sendSyncData(sd *syncData) error {
 
 // sendUserSyncData sends the collected user updates to the remote cluster.
 func (scs *Service) sendUserSyncData(sd *syncData) error {
-	msg := newSyncMsg(sd.task.channelID)
+	msg := model.NewSyncMsg(sd.task.channelID)
 	msg.Users = sd.users
 
-	err := scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
+	err := scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
 		for _, userID := range syncResp.UsersSyncd {
 			if err := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(userID, sd.task.channelID, sd.rc.RemoteId); err != nil {
 				scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Cannot update shared channel user LastSyncAt",
@@ -452,10 +485,10 @@ func (scs *Service) sendAttachmentSyncData(sd *syncData) {
 
 // sendPostSyncData sends the collected post updates to the remote cluster.
 func (scs *Service) sendPostSyncData(sd *syncData) error {
-	msg := newSyncMsg(sd.task.channelID)
+	msg := model.NewSyncMsg(sd.task.channelID)
 	msg.Posts = sd.posts
 
-	return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
+	return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
 		if len(syncResp.PostErrors) != 0 {
 			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Response indicates error for post(s) sync",
 				mlog.String("channel_id", sd.task.channelID),
@@ -473,10 +506,10 @@ func (scs *Service) sendPostSyncData(sd *syncData) error {
 
 // sendReactionSyncData sends the collected reaction updates to the remote cluster.
 func (scs *Service) sendReactionSyncData(sd *syncData) error {
-	msg := newSyncMsg(sd.task.channelID)
+	msg := model.NewSyncMsg(sd.task.channelID)
 	msg.Reactions = sd.reactions
 
-	return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
+	return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
 		if len(syncResp.ReactionErrors) != 0 {
 			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Response indicates error for reactions(s) sync",
 				mlog.String("channel_id", sd.task.channelID),
@@ -495,7 +528,7 @@ func (scs *Service) sendProfileImageSyncData(sd *syncData) {
 }
 
 // sendSyncMsgToRemote synchronously sends the sync message to the remote cluster.
-func (scs *Service) sendSyncMsgToRemote(msg *syncMsg, rc *model.RemoteCluster, f sendSyncMsgResultFunc) error {
+func (scs *Service) sendSyncMsgToRemote(msg *model.SyncMsg, rc *model.RemoteCluster, f sendSyncMsgResultFunc) error {
 	rcs := scs.server.GetRemoteClusterService()
 	if rcs == nil {
 		return fmt.Errorf("cannot update remote cluster %s for channel id %s; Remote Cluster Service not enabled", rc.Name, msg.ChannelId)
@@ -516,7 +549,7 @@ func (scs *Service) sendSyncMsgToRemote(msg *syncMsg, rc *model.RemoteCluster, f
 	err = rcs.SendMsg(ctx, rcMsg, rc, func(rcMsg model.RemoteClusterMsg, rc *model.RemoteCluster, rcResp *remotecluster.Response, errResp error) {
 		defer wg.Done()
 
-		var syncResp SyncResponse
+		var syncResp model.SyncResponse
 		if err2 := json.Unmarshal(rcResp.Payload, &syncResp); err2 != nil {
 			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Invalid sync msg response from remote cluster",
 				mlog.String("remote", rc.Name),