MM-27493 Shared channels (MVP) (#17301)

Remote Cluster Service
- provides ability for multiple Mattermost cluster instances to create a trusted connection with each other and exchange messages
- trusted connections are managed via slash commands (for now)
- facilitates features requiring inter-cluster communication, such as Shared Channels
Shared Channels Service
- provides ability to shared channels between one or more Mattermost cluster instances (using trusted connection)
- sharing/unsharing of channels is managed via slash commands (for now)

services/sharedchannel/attachment.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+// postToAttachments returns the file attachments for a post that need to be synchronized.
+func (scs *Service) postToAttachments(post *model.Post, rc *model.RemoteCluster) ([]*model.FileInfo, error) {
+	infos := make([]*model.FileInfo, 0)
+
+	fis, err := scs.server.GetStore().FileInfo().GetForPost(post.Id, false, true, true)
+	if err != nil {
+		return nil, fmt.Errorf("could not get file info for attachment: %w", err)
+	}
+
+	for _, fi := range fis {
+		if scs.shouldSyncAttachment(fi, rc) {
+			infos = append(infos, fi)
+		}
+	}
+	return infos, nil
+}
+
+// postsToAttachments returns the file attachments for a slice of posts that need to be synchronized.
+func (scs *Service) shouldSyncAttachment(fi *model.FileInfo, rc *model.RemoteCluster) bool {
+	sca, err := scs.server.GetStore().SharedChannel().GetAttachment(fi.Id, rc.RemoteId)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error fetching shared channel attachment",
+				mlog.String("file_id", fi.Id),
+				mlog.String("remote_id", rc.RemoteId),
+				mlog.Err(err),
+			)
+		}
+		// no record so sync is needed
+		return true
+	}
+
+	return sca.LastSyncAt < fi.UpdateAt
+}
+
+// sendAttachmentForRemote asynchronously sends a file attachment to a remote cluster.
+func (scs *Service) sendAttachmentForRemote(fi *model.FileInfo, post *model.Post, rc *model.RemoteCluster) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot update remote cluster for remote id %s; Remote Cluster Service not enabled", rc.RemoteId)
+	}
+
+	us := &model.UploadSession{
+		Id:        model.NewId(),
+		Type:      model.UploadTypeAttachment,
+		UserId:    post.UserId,
+		ChannelId: post.ChannelId,
+		Filename:  fi.Name,
+		FileSize:  fi.Size,
+		RemoteId:  rc.RemoteId,
+		ReqFileId: fi.Id,
+	}
+
+	payload, err := json.Marshal(us)
+	if err != nil {
+		return err
+	}
+
+	msg := model.NewRemoteClusterMsg(TopicUploadCreate, payload)
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	var usResp model.UploadSession
+	var respErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	// creating the upload session on the remote server needs to be done synchronously.
+	err = rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		defer wg.Done()
+		if err != nil {
+			respErr = err
+			return
+		}
+		if !resp.IsSuccess() {
+			respErr = errors.New(resp.Err)
+			return
+		}
+		respErr = json.Unmarshal(resp.Payload, &usResp)
+	})
+
+	if err != nil {
+		return fmt.Errorf("error sending create upload session to remote %s for post %s: %w", rc.RemoteId, post.Id, err)
+	}
+
+	wg.Wait()
+
+	if respErr != nil {
+		return fmt.Errorf("invalid create upload session response for remote %s and post %s: %w", rc.RemoteId, post.Id, respErr)
+	}
+
+	ctx2, cancel2 := context.WithTimeout(context.Background(), remotecluster.SendFileTimeout)
+	defer cancel2()
+
+	return rcs.SendFile(ctx2, &usResp, fi, rc, scs.app, func(us *model.UploadSession, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		if err != nil {
+			return // this means the response could not be parsed; already logged
+		}
+
+		if !resp.IsSuccess() {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "send file failed",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.String("err", resp.Err),
+			)
+			return
+		}
+
+		// response payload should be a model.FileInfo.
+		var fi model.FileInfo
+		if err2 := json.Unmarshal(resp.Payload, &fi); err2 != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "invalid file info response after send file",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.Err(err2),
+			)
+			return
+		}
+
+		// save file attachment record in SharedChannelAttachments table
+		sca := &model.SharedChannelAttachment{
+			FileId:   fi.Id,
+			RemoteId: rc.RemoteId,
+		}
+		if _, err2 := scs.server.GetStore().SharedChannel().UpsertAttachment(sca); err2 != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error saving SharedChannelAttachment",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.Err(err2),
+			)
+			return
+		}
+
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "send file successful",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("uploadId", usResp.Id),
+		)
+	})
+}
+
+// onReceiveUploadCreate is called when a message requesting to create an upload session is received.  An upload session is
+// created and the id returned in the response.
+func (scs *Service) onReceiveUploadCreate(msg model.RemoteClusterMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	var us model.UploadSession
+
+	if err := json.Unmarshal(msg.Payload, &us); err != nil {
+		return fmt.Errorf("invalid upload session request: %w", err)
+	}
+
+	// make sure channel is shared for the remote sender
+	if _, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(us.ChannelId, rc.RemoteId); err != nil {
+		return fmt.Errorf("could not validate upload session for remote: %w", err)
+	}
+
+	us.RemoteId = rc.RemoteId // don't let remotes try to impersonate each other
+
+	// create upload session.
+	usSaved, appErr := scs.app.CreateUploadSession(&us)
+	if appErr != nil {
+		return appErr
+	}
+
+	response.SetPayload(usSaved)
+	return nil
+}

services/sharedchannel/channelinvite.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+// channelInviteMsg represents an invitation for a remote cluster to start sharing a channel.
+type channelInviteMsg struct {
+	ChannelId            string   `json:"channel_id"`
+	TeamId               string   `json:"team_id"`
+	ReadOnly             bool     `json:"read_only"`
+	Name                 string   `json:"name"`
+	DisplayName          string   `json:"display_name"`
+	Header               string   `json:"header"`
+	Purpose              string   `json:"purpose"`
+	Type                 string   `json:"type"`
+	DirectParticipantIDs []string `json:"direct_participant_ids"`
+}
+
+type InviteOption func(msg *channelInviteMsg)
+
+func WithDirectParticipantID(participantID string) InviteOption {
+	return func(msg *channelInviteMsg) {
+		msg.DirectParticipantIDs = append(msg.DirectParticipantIDs, participantID)
+	}
+}
+
+// SendChannelInvite asynchronously sends a channel invite to a remote cluster. The remote cluster is
+// expected to create a new channel with the same channel id, and respond with status OK.
+// If an error occurs on the remote cluster then an ephemeral message is posted to in the channel for userId.
+func (scs *Service) SendChannelInvite(channel *model.Channel, userId string, description string, rc *model.RemoteCluster, options ...InviteOption) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot invite remote cluster for channel id %s; Remote Cluster Service not enabled", channel.Id)
+	}
+
+	sc, err := scs.server.GetStore().SharedChannel().Get(channel.Id)
+	if err != nil {
+		return err
+	}
+
+	invite := channelInviteMsg{
+		ChannelId:   channel.Id,
+		TeamId:      rc.RemoteTeamId,
+		ReadOnly:    sc.ReadOnly,
+		Name:        sc.ShareName,
+		DisplayName: sc.ShareDisplayName,
+		Header:      sc.ShareHeader,
+		Purpose:     sc.SharePurpose,
+		Type:        channel.Type,
+	}
+
+	for _, option := range options {
+		option(&invite)
+	}
+
+	json, err := json.Marshal(invite)
+	if err != nil {
+		return err
+	}
+
+	msg := model.NewRemoteClusterMsg(TopicChannelInvite, json)
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	return rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		if err != nil || !resp.IsSuccess() {
+			scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("Error sending channel invite for %s: %s", rc.DisplayName, combineErrors(err, resp.Err)))
+			return
+		}
+
+		scr := &model.SharedChannelRemote{
+			ChannelId:         sc.ChannelId,
+			Description:       description,
+			CreatorId:         userId,
+			RemoteId:          rc.RemoteId,
+			IsInviteAccepted:  true,
+			IsInviteConfirmed: true,
+		}
+		if _, err = scs.server.GetStore().SharedChannel().SaveRemote(scr); err != nil {
+			scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("Error confirming channel invite for %s: %v", rc.DisplayName, err))
+			return
+		}
+		scs.NotifyChannelChanged(sc.ChannelId)
+		scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("`%s` has been added to channel.", rc.DisplayName))
+	})
+}
+
+func combineErrors(err error, serror string) string {
+	var sb strings.Builder
+	if err != nil {
+		sb.WriteString(err.Error())
+	}
+	if serror != "" {
+		if sb.Len() > 0 {
+			sb.WriteString("; ")
+		}
+		sb.WriteString(serror)
+	}
+	return sb.String()
+}
+
+func (scs *Service) onReceiveChannelInvite(msg model.RemoteClusterMsg, rc *model.RemoteCluster, _ *remotecluster.Response) error {
+	if len(msg.Payload) == 0 {
+		return nil
+	}
+
+	var invite channelInviteMsg
+
+	if err := json.Unmarshal(msg.Payload, &invite); err != nil {
+		return fmt.Errorf("invalid channel invite: %w", err)
+	}
+
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Channel invite received",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("channel_id", invite.ChannelId),
+		mlog.String("channel_name", invite.Name),
+		mlog.String("team_id", invite.TeamId),
+	)
+
+	// create channel if it doesn't exist; the channel may already exist, such as if it was shared then unshared at some point.
+	channel, err := scs.server.GetStore().Channel().Get(invite.ChannelId, true)
+	if err != nil {
+		if channel, err = scs.handleChannelCreation(invite, rc); err != nil {
+			return err
+		}
+	}
+
+	if invite.ReadOnly {
+		if err := scs.makeChannelReadOnly(channel); err != nil {
+			return fmt.Errorf("cannot make channel readonly `%s`: %w", invite.ChannelId, err)
+		}
+	}
+
+	sharedChannel := &model.SharedChannel{
+		ChannelId:        channel.Id,
+		TeamId:           channel.TeamId,
+		Home:             false,
+		ReadOnly:         invite.ReadOnly,
+		ShareName:        channel.Name,
+		ShareDisplayName: channel.DisplayName,
+		SharePurpose:     channel.Purpose,
+		ShareHeader:      channel.Header,
+		CreatorId:        rc.CreatorId,
+		RemoteId:         rc.RemoteId,
+		Type:             channel.Type,
+	}
+
+	if _, err := scs.server.GetStore().SharedChannel().Save(sharedChannel); err != nil {
+		scs.app.PermanentDeleteChannel(channel)
+		return fmt.Errorf("cannot create shared channel (channel_id=%s): %w", invite.ChannelId, err)
+	}
+
+	sharedChannelRemote := &model.SharedChannelRemote{
+		Id:                model.NewId(),
+		ChannelId:         channel.Id,
+		Description:       invite.DisplayName,
+		CreatorId:         channel.CreatorId,
+		IsInviteAccepted:  true,
+		IsInviteConfirmed: true,
+		RemoteId:          rc.RemoteId,
+	}
+
+	if _, err := scs.server.GetStore().SharedChannel().SaveRemote(sharedChannelRemote); err != nil {
+		scs.app.PermanentDeleteChannel(channel)
+		scs.server.GetStore().SharedChannel().Delete(sharedChannel.ChannelId)
+		return fmt.Errorf("cannot create shared channel remote (channel_id=%s): %w", invite.ChannelId, err)
+	}
+	return nil
+}
+
+func (scs *Service) handleChannelCreation(invite channelInviteMsg, rc *model.RemoteCluster) (*model.Channel, error) {
+	if invite.Type == model.CHANNEL_DIRECT {
+		return scs.createDirectChannel(invite)
+	}
+
+	channelNew := &model.Channel{
+		Id:          invite.ChannelId,
+		TeamId:      invite.TeamId,
+		Type:        invite.Type,
+		DisplayName: invite.DisplayName,
+		Name:        invite.Name,
+		Header:      invite.Header,
+		Purpose:     invite.Purpose,
+		CreatorId:   rc.CreatorId,
+		Shared:      model.NewBool(true),
+	}
+
+	// check user perms?
+	channel, appErr := scs.app.CreateChannelWithUser(channelNew, rc.CreatorId)
+	if appErr != nil {
+		return nil, fmt.Errorf("cannot create channel `%s`: %w", invite.ChannelId, appErr)
+	}
+
+	return channel, nil
+}
+
+func (scs *Service) createDirectChannel(invite channelInviteMsg) (*model.Channel, error) {
+	if len(invite.DirectParticipantIDs) != 2 {
+		return nil, fmt.Errorf("cannot create direct channel `%s` insufficient participant count `%d`", invite.ChannelId, len(invite.DirectParticipantIDs))
+	}
+
+	channel, err := scs.app.GetOrCreateDirectChannel(invite.DirectParticipantIDs[0], invite.DirectParticipantIDs[1], model.WithID(invite.ChannelId))
+	if err != nil {
+		return nil, fmt.Errorf("cannot create direct channel `%s`: %w", invite.ChannelId, err)
+	}
+
+	return channel, nil
+}

services/sharedchannel/channelinvite_test.go

@@ -0,0 +1,197 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+	"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
+)
+
+type mockLogger struct {
+	mlog.LoggerIFace
+}
+
+func (ml *mockLogger) Log(level mlog.LogLevel, s string, flds ...mlog.Field) {}
+
+func TestOnReceiveChannelInvite(t *testing.T) {
+	t.Run("when msg payload is empty, it does nothing", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger := &mockLogger{}
+		mockServer.On("GetLogger").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+
+		remoteCluster := &model.RemoteCluster{}
+		msg := model.RemoteClusterMsg{}
+
+		err := scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+		mockStore.AssertNotCalled(t, "Channel")
+	})
+
+	t.Run("when invitation prescribes a readonly channel, it does create a readonly channel", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger := &mockLogger{}
+		mockServer.On("GetLogger").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{DisplayName: "test"}
+		invitation := channelInviteMsg{
+			ChannelId: model.NewId(),
+			TeamId:    model.NewId(),
+			ReadOnly:  true,
+			Type:      "0",
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		mockSharedChannelStore := mocks.SharedChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(channel, nil)
+		mockSharedChannelStore.On("Save", mock.Anything).Return(nil, nil)
+		mockSharedChannelStore.On("SaveRemote", mock.Anything).Return(nil, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+		mockStore.On("SharedChannel").Return(&mockSharedChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+		createPostPermission := model.ChannelModeratedPermissionsMap[model.PERMISSION_CREATE_POST.Id]
+		createReactionPermission := model.ChannelModeratedPermissionsMap[model.PERMISSION_ADD_REACTION.Id]
+		updateMap := model.ChannelModeratedRolesPatch{
+			Guests:  model.NewBool(false),
+			Members: model.NewBool(false),
+		}
+
+		readonlyChannelModerations := []*model.ChannelModerationPatch{
+			{
+				Name:  &createPostPermission,
+				Roles: &updateMap,
+			},
+			{
+				Name:  &createReactionPermission,
+				Roles: &updateMap,
+			},
+		}
+		mockApp.On("PatchChannelModerationsForChannel", channel, readonlyChannelModerations).Return(nil, nil)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+	})
+
+	t.Run("when invitation prescribes a readonly channel and readonly update fails, it returns an error", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger := &mockLogger{}
+		mockServer.On("GetLogger").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{DisplayName: "test"}
+		invitation := channelInviteMsg{
+			ChannelId: model.NewId(),
+			TeamId:    model.NewId(),
+			ReadOnly:  true,
+			Type:      "0",
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(channel, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+		appErr := model.NewAppError("foo", "bar", nil, "boom", http.StatusBadRequest)
+
+		mockApp.On("PatchChannelModerationsForChannel", channel, mock.Anything).Return(nil, appErr)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.Error(t, err)
+		assert.Equal(t, fmt.Sprintf("cannot make channel readonly `%s`: foo: bar, boom", invitation.ChannelId), err.Error())
+	})
+
+	t.Run("when invitation prescribes a direct channel, it does create a direct channel", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger := &mockLogger{}
+		mockServer.On("GetLogger").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{DisplayName: "test", CreatorId: model.NewId()}
+		invitation := channelInviteMsg{
+			ChannelId:            model.NewId(),
+			TeamId:               model.NewId(),
+			ReadOnly:             false,
+			Type:                 model.CHANNEL_DIRECT,
+			DirectParticipantIDs: []string{model.NewId(), model.NewId()},
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		mockSharedChannelStore := mocks.SharedChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(nil, errors.New("boom"))
+		mockSharedChannelStore.On("Save", mock.Anything).Return(nil, nil)
+		mockSharedChannelStore.On("SaveRemote", mock.Anything).Return(nil, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+		mockStore.On("SharedChannel").Return(&mockSharedChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+
+		mockApp.On("GetOrCreateDirectChannel", invitation.DirectParticipantIDs[0], invitation.DirectParticipantIDs[1], mock.AnythingOfType("model.ChannelOption")).Return(channel, nil)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+	})
+}

services/sharedchannel/getpostssince.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+type sinceResult struct {
+	posts     []*model.Post
+	hasMore   bool
+	nextSince int64
+}
+
+// getPostsSince fetches posts that need to be synchronized with a remote cluster.
+// There is a soft cap on the number of posts that will be synchronized in a single pass (MaxPostsPerSync).
+//
+// There is a special case where multiple posts have the same UpdateAt value. It is vital that this method
+// include all posts within that millisecond so that subsequent calls can use an incremented `since`. If this
+// method were to be called repeatedly with the same `since` value the same records would be returned each time
+// and the sync would never move forward.
+//
+// A boolean is also returned to indicate if there are more posts to be synchronized (true) or not (false).
+func (scs *Service) getPostsSince(channelId string, rc *model.RemoteCluster, since int64) (sinceResult, error) {
+	opts := model.GetPostsSinceForSyncOptions{
+		ChannelId:      channelId,
+		Since:          since,
+		IncludeDeleted: true,
+		Limit:          MaxPostsPerSync + 1, // ask for 1 more than needed to peek at first post in next batch
+	}
+	posts, err := scs.server.GetStore().Post().GetPostsSinceForSync(opts, true)
+	if err != nil {
+		return sinceResult{}, err
+	}
+
+	if len(posts) == 0 {
+		return sinceResult{nextSince: since}, nil
+	}
+
+	var hasMore bool
+	if len(posts) > MaxPostsPerSync {
+		hasMore = true
+		peekUpdateAt := posts[len(posts)-1].UpdateAt
+		posts = posts[:MaxPostsPerSync] // trim the peeked at record
+
+		// If the last post to be synchronized has the same Update value as the first post in the next batch
+		// then we need to grab the rest of the posts for that millisecond to ensure the next call can have an
+		// incremented `since`.
+		if peekUpdateAt == posts[len(posts)-1].UpdateAt {
+			opts.Since = peekUpdateAt
+			opts.Until = opts.Since
+			opts.Limit = 1000
+			opts.Offset = countPostsAtMillisecond(posts, peekUpdateAt)
+
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "getPostsSince handling updateAt collision",
+				mlog.String("remote", rc.DisplayName),
+				mlog.Int64("update_at", peekUpdateAt),
+				mlog.Int("offset", opts.Offset),
+			)
+
+			morePosts, err := scs.server.GetStore().Post().GetPostsSinceForSync(opts, true)
+			if err != nil {
+				return sinceResult{}, err
+			}
+			posts = append(posts, morePosts...)
+		}
+	}
+	return sinceResult{posts: posts, hasMore: hasMore, nextSince: posts[len(posts)-1].UpdateAt + 1}, nil
+}
+
+func countPostsAtMillisecond(posts []*model.Post, milli int64) int {
+	// walk backward through the slice until we find a post with UpdateAt that differs from milli.
+	var count int
+	for i := len(posts) - 1; i >= 0; i-- {
+		if posts[i].UpdateAt != milli {
+			return count
+		}
+		count++
+	}
+	return count
+}

services/sharedchannel/mock_AppIface_test.go

@@ -0,0 +1,338 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+// Regenerate this file using `make sharedchannel-mocks`.
+
+package sharedchannel
+
+import (
+	filestore "github.com/mattermost/mattermost-server/v5/shared/filestore"
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/mattermost/mattermost-server/v5/model"
+)
+
+// MockAppIface is an autogenerated mock type for the AppIface type
+type MockAppIface struct {
+	mock.Mock
+}
+
+// AddUserToChannel provides a mock function with given fields: user, channel
+func (_m *MockAppIface) AddUserToChannel(user *model.User, channel *model.Channel) (*model.ChannelMember, *model.AppError) {
+	ret := _m.Called(user, channel)
+
+	var r0 *model.ChannelMember
+	if rf, ok := ret.Get(0).(func(*model.User, *model.Channel) *model.ChannelMember); ok {
+		r0 = rf(user, channel)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.ChannelMember)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.User, *model.Channel) *model.AppError); ok {
+		r1 = rf(user, channel)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// AddUserToTeamByTeamId provides a mock function with given fields: teamId, user
+func (_m *MockAppIface) AddUserToTeamByTeamId(teamId string, user *model.User) *model.AppError {
+	ret := _m.Called(teamId, user)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(string, *model.User) *model.AppError); ok {
+		r0 = rf(teamId, user)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// CreateChannelWithUser provides a mock function with given fields: channel, userId
+func (_m *MockAppIface) CreateChannelWithUser(channel *model.Channel, userId string) (*model.Channel, *model.AppError) {
+	ret := _m.Called(channel, userId)
+
+	var r0 *model.Channel
+	if rf, ok := ret.Get(0).(func(*model.Channel, string) *model.Channel); ok {
+		r0 = rf(channel, userId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Channel)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Channel, string) *model.AppError); ok {
+		r1 = rf(channel, userId)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// CreatePost provides a mock function with given fields: post, channel, triggerWebhooks, setOnline
+func (_m *MockAppIface) CreatePost(post *model.Post, channel *model.Channel, triggerWebhooks bool, setOnline bool) (*model.Post, *model.AppError) {
+	ret := _m.Called(post, channel, triggerWebhooks, setOnline)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(*model.Post, *model.Channel, bool, bool) *model.Post); ok {
+		r0 = rf(post, channel, triggerWebhooks, setOnline)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Post, *model.Channel, bool, bool) *model.AppError); ok {
+		r1 = rf(post, channel, triggerWebhooks, setOnline)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// CreateUploadSession provides a mock function with given fields: us
+func (_m *MockAppIface) CreateUploadSession(us *model.UploadSession) (*model.UploadSession, *model.AppError) {
+	ret := _m.Called(us)
+
+	var r0 *model.UploadSession
+	if rf, ok := ret.Get(0).(func(*model.UploadSession) *model.UploadSession); ok {
+		r0 = rf(us)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.UploadSession)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.UploadSession) *model.AppError); ok {
+		r1 = rf(us)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// DeletePost provides a mock function with given fields: postID, deleteByID
+func (_m *MockAppIface) DeletePost(postID string, deleteByID string) (*model.Post, *model.AppError) {
+	ret := _m.Called(postID, deleteByID)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(string, string) *model.Post); ok {
+		r0 = rf(postID, deleteByID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, string) *model.AppError); ok {
+		r1 = rf(postID, deleteByID)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// DeleteReactionForPost provides a mock function with given fields: reaction
+func (_m *MockAppIface) DeleteReactionForPost(reaction *model.Reaction) *model.AppError {
+	ret := _m.Called(reaction)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Reaction) *model.AppError); ok {
+		r0 = rf(reaction)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// FileReader provides a mock function with given fields: path
+func (_m *MockAppIface) FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
+	ret := _m.Called(path)
+
+	var r0 filestore.ReadCloseSeeker
+	if rf, ok := ret.Get(0).(func(string) filestore.ReadCloseSeeker); ok {
+		r0 = rf(path)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(filestore.ReadCloseSeeker)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
+		r1 = rf(path)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// GetOrCreateDirectChannel provides a mock function with given fields: userId, otherUserId, channelOptions
+func (_m *MockAppIface) GetOrCreateDirectChannel(userId string, otherUserId string, channelOptions ...model.ChannelOption) (*model.Channel, *model.AppError) {
+	_va := make([]interface{}, len(channelOptions))
+	for _i := range channelOptions {
+		_va[_i] = channelOptions[_i]
+	}
+	var _ca []interface{}
+	_ca = append(_ca, userId, otherUserId)
+	_ca = append(_ca, _va...)
+	ret := _m.Called(_ca...)
+
+	var r0 *model.Channel
+	if rf, ok := ret.Get(0).(func(string, string, ...model.ChannelOption) *model.Channel); ok {
+		r0 = rf(userId, otherUserId, channelOptions...)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Channel)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, string, ...model.ChannelOption) *model.AppError); ok {
+		r1 = rf(userId, otherUserId, channelOptions...)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// PatchChannelModerationsForChannel provides a mock function with given fields: channel, channelModerationsPatch
+func (_m *MockAppIface) PatchChannelModerationsForChannel(channel *model.Channel, channelModerationsPatch []*model.ChannelModerationPatch) ([]*model.ChannelModeration, *model.AppError) {
+	ret := _m.Called(channel, channelModerationsPatch)
+
+	var r0 []*model.ChannelModeration
+	if rf, ok := ret.Get(0).(func(*model.Channel, []*model.ChannelModerationPatch) []*model.ChannelModeration); ok {
+		r0 = rf(channel, channelModerationsPatch)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.ChannelModeration)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Channel, []*model.ChannelModerationPatch) *model.AppError); ok {
+		r1 = rf(channel, channelModerationsPatch)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// PermanentDeleteChannel provides a mock function with given fields: channel
+func (_m *MockAppIface) PermanentDeleteChannel(channel *model.Channel) *model.AppError {
+	ret := _m.Called(channel)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Channel) *model.AppError); ok {
+		r0 = rf(channel)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// SaveReactionForPost provides a mock function with given fields: reaction
+func (_m *MockAppIface) SaveReactionForPost(reaction *model.Reaction) (*model.Reaction, *model.AppError) {
+	ret := _m.Called(reaction)
+
+	var r0 *model.Reaction
+	if rf, ok := ret.Get(0).(func(*model.Reaction) *model.Reaction); ok {
+		r0 = rf(reaction)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Reaction)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Reaction) *model.AppError); ok {
+		r1 = rf(reaction)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// SendEphemeralPost provides a mock function with given fields: userId, post
+func (_m *MockAppIface) SendEphemeralPost(userId string, post *model.Post) *model.Post {
+	ret := _m.Called(userId, post)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(string, *model.Post) *model.Post); ok {
+		r0 = rf(userId, post)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	return r0
+}
+
+// UpdatePost provides a mock function with given fields: post, safeUpdate
+func (_m *MockAppIface) UpdatePost(post *model.Post, safeUpdate bool) (*model.Post, *model.AppError) {
+	ret := _m.Called(post, safeUpdate)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(*model.Post, bool) *model.Post); ok {
+		r0 = rf(post, safeUpdate)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Post, bool) *model.AppError); ok {
+		r1 = rf(post, safeUpdate)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}

services/sharedchannel/mock_ServerIface_test.go

@@ -0,0 +1,118 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+// Regenerate this file using `make sharedchannel-mocks`.
+
+package sharedchannel
+
+import (
+	mlog "github.com/mattermost/mattermost-server/v5/shared/mlog"
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/mattermost/mattermost-server/v5/model"
+
+	remotecluster "github.com/mattermost/mattermost-server/v5/services/remotecluster"
+
+	store "github.com/mattermost/mattermost-server/v5/store"
+)
+
+// MockServerIface is an autogenerated mock type for the ServerIface type
+type MockServerIface struct {
+	mock.Mock
+}
+
+// AddClusterLeaderChangedListener provides a mock function with given fields: listener
+func (_m *MockServerIface) AddClusterLeaderChangedListener(listener func()) string {
+	ret := _m.Called(listener)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(func()) string); ok {
+		r0 = rf(listener)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// Config provides a mock function with given fields:
+func (_m *MockServerIface) Config() *model.Config {
+	ret := _m.Called()
+
+	var r0 *model.Config
+	if rf, ok := ret.Get(0).(func() *model.Config); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Config)
+		}
+	}
+
+	return r0
+}
+
+// GetLogger provides a mock function with given fields:
+func (_m *MockServerIface) GetLogger() mlog.LoggerIFace {
+	ret := _m.Called()
+
+	var r0 mlog.LoggerIFace
+	if rf, ok := ret.Get(0).(func() mlog.LoggerIFace); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(mlog.LoggerIFace)
+		}
+	}
+
+	return r0
+}
+
+// GetRemoteClusterService provides a mock function with given fields:
+func (_m *MockServerIface) GetRemoteClusterService() remotecluster.RemoteClusterServiceIFace {
+	ret := _m.Called()
+
+	var r0 remotecluster.RemoteClusterServiceIFace
+	if rf, ok := ret.Get(0).(func() remotecluster.RemoteClusterServiceIFace); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(remotecluster.RemoteClusterServiceIFace)
+		}
+	}
+
+	return r0
+}
+
+// GetStore provides a mock function with given fields:
+func (_m *MockServerIface) GetStore() store.Store {
+	ret := _m.Called()
+
+	var r0 store.Store
+	if rf, ok := ret.Get(0).(func() store.Store); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.Store)
+		}
+	}
+
+	return r0
+}
+
+// IsLeader provides a mock function with given fields:
+func (_m *MockServerIface) IsLeader() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// RemoveClusterLeaderChangedListener provides a mock function with given fields: id
+func (_m *MockServerIface) RemoveClusterLeaderChangedListener(id string) {
+	_m.Called(id)
+}

services/sharedchannel/msg.go

@@ -0,0 +1,216 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+// syncMsg represents a change in content (post add/edit/delete, reaction add/remove, users).
+// It is sent to remote clusters as the payload of a `RemoteClusterMsg`.
+type syncMsg struct {
+	ChannelId   string            `json:"channel_id"`
+	PostId      string            `json:"post_id"`
+	Post        *model.Post       `json:"post"`
+	Users       []*model.User     `json:"users"`
+	Reactions   []*model.Reaction `json:"reactions"`
+	Attachments []*model.FileInfo `json:"-"`
+}
+
+func (sm syncMsg) ToJSON() ([]byte, error) {
+	b, err := json.Marshal(sm)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func (sm syncMsg) String() string {
+	json, err := sm.ToJSON()
+	if err != nil {
+		return ""
+	}
+	return string(json)
+}
+
+type userCache map[string]struct{}
+
+func (u userCache) Has(id string) bool {
+	_, ok := u[id]
+	return ok
+}
+
+func (u userCache) Add(id string) {
+	u[id] = struct{}{}
+}
+
+// postsToSyncMessages takes a slice of posts and converts to a `RemoteClusterMsg` which can be
+// sent to a remote cluster.
+func (scs *Service) postsToSyncMessages(posts []*model.Post, rc *model.RemoteCluster, nextSyncAt int64) ([]syncMsg, error) {
+	syncMessages := make([]syncMsg, 0, len(posts))
+
+	uCache := make(userCache)
+
+	for _, p := range posts {
+		if p.IsSystemMessage() { // don't sync system messages
+			continue
+		}
+
+		// any reactions originating from the remote cluster are filtered out
+		reactions, err := scs.server.GetStore().Reaction().GetForPostSince(p.Id, nextSyncAt, rc.RemoteId, true)
+		if err != nil {
+			return nil, err
+		}
+
+		postSync := p
+
+		// Don't resend an existing post where only the reactions changed.
+		// Posts we must send:
+		//   - new posts (EditAt == 0)
+		//   - edited posts (EditAt >= nextSyncAt)
+		//   - deleted posts (DeleteAt > 0)
+		if p.EditAt > 0 && p.EditAt < nextSyncAt && p.DeleteAt == 0 {
+			postSync = nil
+		}
+
+		// Don't send a deleted post if it is just the original copy from an edit.
+		if p.DeleteAt > 0 && p.OriginalId != "" {
+			postSync = nil
+		}
+
+		// don't sync a post back to the remote it came from.
+		if p.RemoteId != nil && *p.RemoteId == rc.RemoteId {
+			postSync = nil
+		}
+
+		var attachments []*model.FileInfo
+		if postSync != nil {
+			// parse out all permalinks in the message.
+			postSync.Message = scs.processPermalinkToRemote(postSync)
+
+			// get any file attachments
+			attachments, err = scs.postToAttachments(postSync, rc)
+			if err != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Could not fetch attachments for post",
+					mlog.String("post_id", postSync.Id),
+					mlog.Err(err),
+				)
+			}
+		}
+
+		// any users originating from the remote cluster are filtered out
+		users := scs.usersForPost(postSync, reactions, rc, uCache)
+
+		// if everything was filtered out then don't send an empty message.
+		if postSync == nil && len(reactions) == 0 && len(users) == 0 {
+			continue
+		}
+
+		sm := syncMsg{
+			ChannelId:   p.ChannelId,
+			PostId:      p.Id,
+			Post:        postSync,
+			Users:       users,
+			Reactions:   reactions,
+			Attachments: attachments,
+		}
+		syncMessages = append(syncMessages, sm)
+	}
+	return syncMessages, nil
+}
+
+// usersForPost provides a list of Users associated with the post that need to be synchronized.
+// The user cache ensures the same user is not synchronized redundantly if they appear in multiple
+// posts for this sync batch.
+func (scs *Service) usersForPost(post *model.Post, reactions []*model.Reaction, rc *model.RemoteCluster, uCache userCache) []*model.User {
+	userIds := make([]string, 0)
+
+	if post != nil && !uCache.Has(post.UserId) {
+		userIds = append(userIds, post.UserId)
+		uCache.Add(post.UserId)
+	}
+
+	for _, r := range reactions {
+		if !uCache.Has(r.UserId) {
+			userIds = append(userIds, r.UserId)
+			uCache.Add(r.UserId)
+		}
+	}
+
+	// TODO: extract @mentions to local users and sync those as well?
+
+	users := make([]*model.User, 0)
+
+	for _, id := range userIds {
+		user, err := scs.server.GetStore().User().Get(context.Background(), id)
+		if err == nil {
+			if sync, err2 := scs.shouldUserSync(user, rc); err2 != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Could not find user for post",
+					mlog.String("user_id", id),
+					mlog.Err(err2))
+				continue
+			} else if sync {
+				users = append(users, sanitizeUserForSync(user))
+			}
+		} else {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error checking if user should sync",
+				mlog.String("user_id", id),
+				mlog.Err(err))
+		}
+	}
+	return users
+}
+
+func sanitizeUserForSync(user *model.User) *model.User {
+	user.Password = model.NewId()
+	user.AuthData = nil
+	user.AuthService = ""
+	user.Roles = "system_user"
+	user.AllowMarketing = false
+	user.Props = model.StringMap{}
+	user.NotifyProps = model.StringMap{}
+	user.LastPasswordUpdate = 0
+	user.LastPictureUpdate = 0
+	user.FailedAttempts = 0
+	user.MfaActive = false
+	user.MfaSecret = ""
+
+	return user
+}
+
+// shouldUserSync determines if a user needs to be synchronized.
+// User should be synchronized if it has no entry in the SharedChannelUsers table,
+// or there is an entry but the LastSyncAt is less than user.UpdateAt
+func (scs *Service) shouldUserSync(user *model.User, rc *model.RemoteCluster) (bool, error) {
+	// don't sync users with the remote they originated from.
+	if user.RemoteId != nil && *user.RemoteId == rc.RemoteId {
+		return false, nil
+	}
+
+	scu, err := scs.server.GetStore().SharedChannel().GetUser(user.Id, rc.RemoteId)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return false, err
+		}
+
+		// user not in the SharedChannelUsers table, so we must add them.
+		scu = &model.SharedChannelUser{
+			UserId:   user.Id,
+			RemoteId: rc.RemoteId,
+		}
+		if _, err = scs.server.GetStore().SharedChannel().SaveUser(scu); err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error adding user to shared channel users",
+				mlog.String("remote_id", rc.RemoteId),
+				mlog.String("user_id", user.Id),
+			)
+		}
+	} else if scu.LastSyncAt >= user.UpdateAt {
+		return false, nil
+	}
+	return true, nil
+}

services/sharedchannel/permalink.go

@@ -0,0 +1,81 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/i18n"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+var (
+	// Team name regex taken from model.IsValidTeamName
+	permaLinkRegex       = regexp.MustCompile(`https?://[0-9.\-A-Za-z]+/[a-z0-9]+([a-z\-0-9]+|(__)?)[a-z0-9]+/pl/([a-zA-Z0-9]+)`)
+	permaLinkSharedRegex = regexp.MustCompile(`https?://[0-9.\-A-Za-z]+/[a-z0-9]+([a-z\-0-9]+|(__)?)[a-z0-9]+/plshared/([a-zA-Z0-9]+)`)
+)
+
+const (
+	permalinkMarker = "plshared"
+)
+
+// processPermalinkToRemote processes all permalinks going towards a remote site.
+func (scs *Service) processPermalinkToRemote(p *model.Post) string {
+	var sent bool
+	return permaLinkRegex.ReplaceAllStringFunc(p.Message, func(msg string) string {
+		// Extract the postID (This is simple enough not to warrant full-blown URL parsing.)
+		lastSlash := strings.LastIndexByte(msg, '/')
+		postID := msg[lastSlash+1:]
+		postList, err := scs.server.GetStore().Post().Get(context.Background(), postID, true, false, false, "")
+		if err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceWarn, "Unable to get post during replacing permalinks", mlog.Err(err))
+			return msg
+		}
+		if len(postList.Order) == 0 {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceWarn, "No post found for permalink", mlog.String("postID", postID))
+			return msg
+		}
+
+		// If postID is for a different channel
+		if postList.Posts[postList.Order[0]].ChannelId != p.ChannelId {
+			// Send ephemeral message to OP (only once per message).
+			if !sent {
+				scs.sendEphemeralPost(p.ChannelId, p.UserId, i18n.T("sharedchannel.permalink.not_found"))
+				sent = true
+			}
+			// But don't modify msg
+			return msg
+		}
+
+		// Otherwise, modify pl to plshared as a marker to be replaced by remote sites
+		return strings.Replace(msg, "/pl/", "/"+permalinkMarker+"/", 1)
+	})
+}
+
+// processPermalinkFromRemote processes all permalinks coming from a remote site.
+func (scs *Service) processPermalinkFromRemote(p *model.Post, team *model.Team) string {
+	return permaLinkSharedRegex.ReplaceAllStringFunc(p.Message, func(remoteLink string) string {
+		// Extract host name
+		parsed, err := url.Parse(remoteLink)
+		if err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceWarn, "Unable to parse the remote link during replacing permalinks", mlog.Err(err))
+			return remoteLink
+		}
+
+		// Replace with local SiteURL
+		parsed.Scheme = scs.siteURL.Scheme
+		parsed.Host = scs.siteURL.Host
+
+		// Replace team name with local team
+		teamEnd := strings.Index(parsed.Path, "/"+permalinkMarker)
+		parsed.Path = "/" + team.Name + parsed.Path[teamEnd:]
+
+		// Replace plshared with pl
+		return strings.Replace(parsed.String(), "/"+permalinkMarker+"/", "/pl/", 1)
+	})
+}

services/sharedchannel/permalink_test.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v5/utils"
+)
+
+func TestProcessPermalinkToRemote(t *testing.T) {
+	scs := &Service{
+		server: &MockServerIface{},
+		app:    &MockAppIface{},
+	}
+
+	mockStore := &mocks.Store{}
+	mockPostStore := mocks.PostStore{}
+	utils.TranslationsPreInit()
+
+	pl := &model.PostList{}
+	mockPostStore.On("Get", context.Background(), "postID", true, false, false, "").Return(pl, nil)
+
+	mockStore.On("Post").Return(&mockPostStore)
+
+	mockServer := scs.server.(*MockServerIface)
+	mockServer.On("GetStore").Return(mockStore)
+
+	mockApp := scs.app.(*MockAppIface)
+	mockApp.On("SendEphemeralPost", "user", mock.AnythingOfType("*model.Post")).Return(&model.Post{}).Times(1)
+	defer mockApp.AssertExpectations(t)
+
+	t.Run("same channel", func(t *testing.T) {
+		post := &model.Post{
+			Message:   "hello world https://comm.matt.com/team/pl/postID link",
+			ChannelId: "sourceChan",
+			UserId:    "user",
+		}
+
+		*pl = model.PostList{
+			Order: []string{"1"},
+			Posts: map[string]*model.Post{
+				"1": {
+					ChannelId: "sourceChan",
+					UserId:    "user",
+				},
+			},
+		}
+
+		out := scs.processPermalinkToRemote(post)
+		assert.Equal(t, "hello world https://comm.matt.com/team/plshared/postID link", out)
+	})
+
+	t.Run("different channel", func(t *testing.T) {
+		post := &model.Post{
+			Message:   "hello world https://comm.matt.com/team/pl/postID link https://comm.matt.com/team/pl/postID ",
+			ChannelId: "sourceChan",
+			UserId:    "user",
+		}
+
+		*pl = model.PostList{
+			Order: []string{"1"},
+			Posts: map[string]*model.Post{
+				"1": {
+					ChannelId: "otherChan",
+				},
+			},
+		}
+		out := scs.processPermalinkToRemote(post)
+		assert.Equal(t, "hello world https://comm.matt.com/team/pl/postID link https://comm.matt.com/team/pl/postID ", out)
+	})
+}
+
+func TestProcessPermalinkFromRemote(t *testing.T) {
+	t.Run("has match", func(t *testing.T) {
+		parsed, _ := url.Parse("http://mysite.com")
+		scs := &Service{
+			server:  &MockServerIface{},
+			siteURL: parsed,
+		}
+
+		out := scs.processPermalinkFromRemote(&model.Post{Message: "hello world https://comm.matt.com/team/plshared/postID link"},
+			&model.Team{Name: "myteam"})
+		assert.Equal(t,
+			"hello world http://mysite.com/myteam/pl/postID link",
+			out)
+	})
+
+	t.Run("does not match", func(t *testing.T) {
+		parsed, _ := url.Parse("http://mysite.com")
+		scs := &Service{
+			server:  &MockServerIface{},
+			siteURL: parsed,
+		}
+
+		out := scs.processPermalinkFromRemote(&model.Post{Message: "hello world https://comm.matt.com/team/pl/postID link"},
+			&model.Team{Name: "myteam"})
+		assert.Equal(t,
+			"hello world https://comm.matt.com/team/pl/postID link",
+			out)
+	})
+}

services/sharedchannel/response.go

@@ -0,0 +1,10 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+type SyncResponse struct {
+	LastSyncAt int64    `json:"last_sync_at"`
+	PostErrors []string `json:"post_errors"`
+	UsersSyncd []string `json:"users_syncd"`
+}

services/sharedchannel/service.go

@@ -0,0 +1,239 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/filestore"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+	"github.com/mattermost/mattermost-server/v5/store"
+)
+
+const (
+	TopicSync                    = "sharedchannel_sync"
+	TopicChannelInvite           = "sharedchannel_invite"
+	TopicUploadCreate            = "sharedchannel_upload"
+	MaxRetries                   = 3
+	MaxPostsPerSync              = 12 // a bit more than one typical screenfull of posts
+	NotifyRemoteOfflineThreshold = time.Second * 10
+	NotifyMinimumDelay           = time.Second * 2
+)
+
+// Mocks can be re-generated with `make sharedchannel-mocks`.
+type ServerIface interface {
+	Config() *model.Config
+	IsLeader() bool
+	AddClusterLeaderChangedListener(listener func()) string
+	RemoveClusterLeaderChangedListener(id string)
+	GetStore() store.Store
+	GetLogger() mlog.LoggerIFace
+	GetRemoteClusterService() remotecluster.RemoteClusterServiceIFace
+}
+
+type AppIface interface {
+	SendEphemeralPost(userId string, post *model.Post) *model.Post
+	CreateChannelWithUser(channel *model.Channel, userId string) (*model.Channel, *model.AppError)
+	GetOrCreateDirectChannel(userId, otherUserId string, channelOptions ...model.ChannelOption) (*model.Channel, *model.AppError)
+	AddUserToChannel(user *model.User, channel *model.Channel) (*model.ChannelMember, *model.AppError)
+	AddUserToTeamByTeamId(teamId string, user *model.User) *model.AppError
+	PermanentDeleteChannel(channel *model.Channel) *model.AppError
+	CreatePost(post *model.Post, channel *model.Channel, triggerWebhooks bool, setOnline bool) (savedPost *model.Post, err *model.AppError)
+	UpdatePost(post *model.Post, safeUpdate bool) (*model.Post, *model.AppError)
+	DeletePost(postID, deleteByID string) (*model.Post, *model.AppError)
+	SaveReactionForPost(reaction *model.Reaction) (*model.Reaction, *model.AppError)
+	DeleteReactionForPost(reaction *model.Reaction) *model.AppError
+	PatchChannelModerationsForChannel(channel *model.Channel, channelModerationsPatch []*model.ChannelModerationPatch) ([]*model.ChannelModeration, *model.AppError)
+	CreateUploadSession(us *model.UploadSession) (*model.UploadSession, *model.AppError)
+	FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
+}
+
+// errNotFound allows checking against Store.ErrNotFound errors without making Store a dependency.
+type errNotFound interface {
+	IsErrNotFound() bool
+}
+
+// errInvalidInput allows checking against Store.ErrInvalidInput errors without making Store a dependency.
+type errInvalidInput interface {
+	InvalidInputInfo() (entity string, field string, value interface{})
+}
+
+// Service provides shared channel synchronization.
+type Service struct {
+	server       ServerIface
+	app          AppIface
+	changeSignal chan struct{}
+
+	// everything below guarded by `mux`
+	mux                       sync.RWMutex
+	active                    bool
+	leaderListenerId          string
+	connectionStateListenerId string
+	done                      chan struct{}
+	tasks                     map[string]syncTask
+	syncTopicListenerId       string
+	inviteTopicListenerId     string
+	uploadTopicListenerId     string
+	siteURL                   *url.URL
+}
+
+// NewSharedChannelService creates a RemoteClusterService instance.
+func NewSharedChannelService(server ServerIface, app AppIface) (*Service, error) {
+	service := &Service{
+		server:       server,
+		app:          app,
+		changeSignal: make(chan struct{}, 1),
+		tasks:        make(map[string]syncTask),
+	}
+	parsed, err := url.Parse(*server.Config().ServiceSettings.SiteURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse SiteURL: %w", err)
+	}
+	service.siteURL = parsed
+	return service, nil
+}
+
+// Start is called by the server on server start-up.
+func (scs *Service) Start() error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return errors.New("Shared Channel Service cannot activate: requires Remote Cluster Service")
+	}
+
+	scs.mux.Lock()
+	scs.leaderListenerId = scs.server.AddClusterLeaderChangedListener(scs.onClusterLeaderChange)
+	scs.syncTopicListenerId = rcs.AddTopicListener(TopicSync, scs.onReceiveSyncMessage)
+	scs.inviteTopicListenerId = rcs.AddTopicListener(TopicChannelInvite, scs.onReceiveChannelInvite)
+	scs.uploadTopicListenerId = rcs.AddTopicListener(TopicUploadCreate, scs.onReceiveUploadCreate)
+	scs.connectionStateListenerId = rcs.AddConnectionStateListener(scs.onConnectionStateChange)
+	scs.mux.Unlock()
+
+	scs.onClusterLeaderChange()
+
+	return nil
+}
+
+// Shutdown is called by the server on server shutdown.
+func (scs *Service) Shutdown() error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return errors.New("Shared Channel Service cannot shutdown: requires Remote Cluster Service")
+	}
+
+	scs.mux.Lock()
+	id := scs.leaderListenerId
+	rcs.RemoveTopicListener(scs.syncTopicListenerId)
+	scs.syncTopicListenerId = ""
+	rcs.RemoveTopicListener(scs.inviteTopicListenerId)
+	scs.inviteTopicListenerId = ""
+	rcs.RemoveConnectionStateListener(scs.connectionStateListenerId)
+	scs.connectionStateListenerId = ""
+	scs.mux.Unlock()
+
+	scs.server.RemoveClusterLeaderChangedListener(id)
+	scs.pause()
+	return nil
+}
+
+// Active determines whether the service is active on the node or not.
+func (scs *Service) Active() bool {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	return scs.active
+}
+
+func (scs *Service) sendEphemeralPost(channelId string, userId string, text string) {
+	ephemeral := &model.Post{
+		ChannelId: channelId,
+		Message:   text,
+		CreateAt:  model.GetMillis(),
+	}
+	scs.app.SendEphemeralPost(userId, ephemeral)
+}
+
+// onClusterLeaderChange is called whenever the cluster leader may have changed.
+func (scs *Service) onClusterLeaderChange() {
+	if scs.server.IsLeader() {
+		scs.resume()
+	} else {
+		scs.pause()
+	}
+}
+
+func (scs *Service) resume() {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	if scs.active {
+		return // already active
+	}
+
+	scs.active = true
+	scs.done = make(chan struct{})
+
+	go scs.syncLoop(scs.done)
+
+	scs.server.GetLogger().Debug("Shared Channel Service active")
+}
+
+func (scs *Service) pause() {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	if !scs.active {
+		return // already inactive
+	}
+
+	scs.active = false
+	close(scs.done)
+	scs.done = nil
+
+	scs.server.GetLogger().Debug("Shared Channel Service inactive")
+}
+
+// Makes the remote channel to be read-only(announcement mode, only admins can create posts and reactions).
+func (scs *Service) makeChannelReadOnly(channel *model.Channel) *model.AppError {
+	createPostPermission := model.ChannelModeratedPermissionsMap[model.PERMISSION_CREATE_POST.Id]
+	createReactionPermission := model.ChannelModeratedPermissionsMap[model.PERMISSION_ADD_REACTION.Id]
+	updateMap := model.ChannelModeratedRolesPatch{
+		Guests:  model.NewBool(false),
+		Members: model.NewBool(false),
+	}
+
+	readonlyChannelModerations := []*model.ChannelModerationPatch{
+		{
+			Name:  &createPostPermission,
+			Roles: &updateMap,
+		},
+		{
+			Name:  &createReactionPermission,
+			Roles: &updateMap,
+		},
+	}
+
+	_, err := scs.app.PatchChannelModerationsForChannel(channel, readonlyChannelModerations)
+	return err
+}
+
+// onConnectionStateChange is called whenever the connection state of a remote cluster changes,
+// for example when one comes back online.
+func (scs *Service) onConnectionStateChange(rc *model.RemoteCluster, online bool) {
+	if online {
+		// when a previously offline remote comes back online force a sync.
+		scs.ForceSyncForRemote(rc)
+	}
+
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Remote cluster connection status changed",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("remoteId", rc.RemoteId),
+		mlog.Bool("online", online),
+	)
+}

services/sharedchannel/sync_recv.go

@@ -0,0 +1,296 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+func (scs *Service) onReceiveSyncMessage(msg model.RemoteClusterMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	if msg.Topic != TopicSync {
+		return fmt.Errorf("wrong topic, expected `%s`, got `%s`", TopicSync, msg.Topic)
+	}
+
+	if len(msg.Payload) == 0 {
+		return errors.New("empty sync message")
+	}
+
+	if scs.server.GetLogger().IsLevelEnabled(mlog.LvlSharedChannelServiceMessagesInbound) {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceMessagesInbound, "inbound message",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("msg", string(msg.Payload)),
+		)
+	}
+
+	var syncMessages []syncMsg
+
+	if err := json.Unmarshal(msg.Payload, &syncMessages); err != nil {
+		return fmt.Errorf("invalid sync message: %w", err)
+	}
+
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Batch of sync messages received",
+		mlog.String("remote", rc.DisplayName),
+		mlog.Int("sync_msg_count", len(syncMessages)),
+	)
+
+	return scs.processSyncMessages(syncMessages, rc, response)
+}
+
+func (scs *Service) processSyncMessages(syncMessages []syncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	var channel *model.Channel
+	var team *model.Team
+
+	postErrors := make([]string, 0)
+	usersSyncd := make([]string, 0)
+	var lastSyncAt int64
+	var err error
+
+	for _, sm := range syncMessages {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Sync msg received",
+			mlog.String("post_id", sm.PostId),
+			mlog.String("channel_id", sm.ChannelId),
+			mlog.Int("reaction_count", len(sm.Reactions)),
+			mlog.Int("user_count", len(sm.Users)),
+			mlog.Bool("has_post", sm.Post != nil),
+		)
+
+		if channel == nil {
+			if channel, err = scs.server.GetStore().Channel().Get(sm.ChannelId, true); err != nil {
+				// if the channel doesn't exist then none of these sync messages are going to work.
+				return fmt.Errorf("channel not found processing sync messages: %w", err)
+			}
+		}
+
+		// add/update users before posts
+		for _, user := range sm.Users {
+			if userSaved, err := scs.upsertSyncUser(user, channel, rc); err != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync user",
+					mlog.String("post_id", sm.PostId),
+					mlog.String("channel_id", sm.ChannelId),
+					mlog.String("user_id", user.Id),
+					mlog.Err(err))
+			} else {
+				usersSyncd = append(usersSyncd, userSaved.Id)
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "User upserted via sync",
+					mlog.String("post_id", sm.PostId),
+					mlog.String("channel_id", sm.ChannelId),
+					mlog.String("user_id", user.Id),
+				)
+			}
+		}
+
+		if sm.Post != nil {
+			if sm.ChannelId != sm.Post.ChannelId {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "ChannelId mismatch",
+					mlog.String("sm.ChannelId", sm.ChannelId),
+					mlog.String("sm.Post.ChannelId", sm.Post.ChannelId),
+					mlog.String("PostId", sm.Post.Id),
+				)
+				postErrors = append(postErrors, sm.Post.Id)
+				continue
+			}
+
+			if channel.Type != model.CHANNEL_DIRECT && team == nil {
+				var err2 error
+				team, err2 = scs.server.GetStore().Channel().GetTeamForChannel(sm.ChannelId)
+				if err2 != nil {
+					scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error getting Team for Channel",
+						mlog.String("ChannelId", sm.Post.ChannelId),
+						mlog.String("PostId", sm.Post.Id),
+						mlog.Err(err2),
+					)
+					postErrors = append(postErrors, sm.Post.Id)
+					continue
+				}
+			}
+
+			// process perma-links for remote
+			if team != nil {
+				sm.Post.Message = scs.processPermalinkFromRemote(sm.Post, team)
+			}
+
+			// add/update post (may be nil if only reactions changed)
+			rpost, err := scs.upsertSyncPost(sm.Post, channel, rc)
+			if err != nil {
+				postErrors = append(postErrors, sm.Post.Id)
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync post",
+					mlog.String("post_id", sm.Post.Id),
+					mlog.String("channel_id", sm.Post.ChannelId),
+					mlog.Err(err),
+				)
+			} else if lastSyncAt < rpost.UpdateAt {
+				lastSyncAt = rpost.UpdateAt
+			}
+		}
+
+		// add/remove reactions
+		for _, reaction := range sm.Reactions {
+			if _, err := scs.upsertSyncReaction(reaction, rc); err != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync reaction",
+					mlog.String("user_id", reaction.UserId),
+					mlog.String("post_id", reaction.PostId),
+					mlog.String("emoji", reaction.EmojiName),
+					mlog.Int64("delete_at", reaction.DeleteAt),
+					mlog.Err(err),
+				)
+			} else {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Reaction upserted via sync",
+					mlog.String("user_id", reaction.UserId),
+					mlog.String("post_id", reaction.PostId),
+					mlog.String("emoji", reaction.EmojiName),
+					mlog.Int64("delete_at", reaction.DeleteAt),
+				)
+
+				if lastSyncAt < reaction.UpdateAt {
+					lastSyncAt = reaction.UpdateAt
+				}
+			}
+		}
+	}
+
+	syncResp := SyncResponse{
+		LastSyncAt: lastSyncAt, // might be zero
+		PostErrors: postErrors, // might be empty
+		UsersSyncd: usersSyncd, // might be empty
+	}
+
+	response.SetPayload(syncResp)
+
+	return nil
+}
+
+func (scs *Service) upsertSyncUser(user *model.User, channel *model.Channel, rc *model.RemoteCluster) (*model.User, error) {
+	var err error
+	var userSaved *model.User
+
+	user.RemoteId = model.NewString(rc.RemoteId)
+
+	// does the user already exist?
+	euser, err := scs.server.GetStore().User().Get(context.Background(), user.Id)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return nil, fmt.Errorf("error checking sync user: %w", err)
+		}
+	}
+
+	if euser == nil {
+		if userSaved, err = scs.server.GetStore().User().Save(user); err != nil {
+			if e, ok := err.(errInvalidInput); ok {
+				_, field, value := e.InvalidInputInfo()
+				if field == "email" || field == "username" {
+					// username or email collision
+					// TODO: handle collision by modifying username/email (MM-32133)
+					return nil, fmt.Errorf("collision inserting sync user (%s=%s): %w", field, value, err)
+				}
+			}
+			return nil, fmt.Errorf("error inserting sync user: %w", err)
+		}
+	} else {
+		patch := &model.UserPatch{
+			Nickname:  &user.Nickname,
+			FirstName: &user.FirstName,
+			LastName:  &user.LastName,
+			Position:  &user.Position,
+			Locale:    &user.Locale,
+			Timezone:  user.Timezone,
+			RemoteId:  user.RemoteId,
+		}
+		euser.Patch(patch)
+		userUpdated, err := scs.server.GetStore().User().Update(euser, false)
+		if err != nil {
+			return nil, fmt.Errorf("error updating sync user: %w", err)
+		}
+		userSaved = userUpdated.New
+	}
+
+	// add user to team. We do this here regardless of whether the user was
+	// just created or patched since there are three steps to adding a user
+	// (insert rec, add to team, add to channel) and any one could fail.
+	// Instead of undoing what succeeded on any failure we simply do all steps each
+	// time. AddUserToChannel & AddUserToTeamByTeamId do not error if user already
+	// added and exit quickly.
+	if err := scs.app.AddUserToTeamByTeamId(channel.TeamId, userSaved); err != nil {
+		return nil, fmt.Errorf("error adding sync user to Team: %w", err)
+	}
+
+	// add user to channel
+	if _, err := scs.app.AddUserToChannel(userSaved, channel); err != nil {
+		return nil, fmt.Errorf("error adding sync user to ChannelMembers: %w", err)
+	}
+	return userSaved, nil
+}
+
+func (scs *Service) upsertSyncPost(post *model.Post, channel *model.Channel, rc *model.RemoteCluster) (*model.Post, error) {
+	var appErr *model.AppError
+
+	post.RemoteId = model.NewString(rc.RemoteId)
+
+	rpost, err := scs.server.GetStore().Post().GetSingle(post.Id, true)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return nil, fmt.Errorf("error checking sync post: %w", err)
+		}
+	}
+
+	if rpost == nil {
+		// post doesn't exist; create new one
+		rpost, appErr = scs.app.CreatePost(post, channel, true, true)
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Created sync post",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	} else if post.DeleteAt > 0 {
+		// delete post
+		rpost, appErr = scs.app.DeletePost(post.Id, post.UserId)
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Deleted sync post",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	} else if post.EditAt > rpost.EditAt || post.Message != rpost.Message {
+		// update post
+		rpost, appErr = scs.app.UpdatePost(post, false)
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Updated sync post",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	} else {
+		// nothing to update
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Update to sync post ignored",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	}
+
+	var rerr error
+	if appErr != nil {
+		rerr = errors.New(appErr.Error())
+	}
+	return rpost, rerr
+}
+
+func (scs *Service) upsertSyncReaction(reaction *model.Reaction, rc *model.RemoteCluster) (*model.Reaction, error) {
+	savedReaction := reaction
+	var appErr *model.AppError
+
+	reaction.RemoteId = model.NewString(rc.RemoteId)
+
+	if reaction.DeleteAt == 0 {
+		savedReaction, appErr = scs.app.SaveReactionForPost(reaction)
+	} else {
+		appErr = scs.app.DeleteReactionForPost(reaction)
+	}
+
+	var err error
+	if appErr != nil {
+		err = errors.New(appErr.Error())
+	}
+	return savedReaction, err
+}

services/sharedchannel/sync_send.go

@@ -0,0 +1,512 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/i18n"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+type syncTask struct {
+	id         string
+	channelId  string
+	remoteId   string
+	AddedAt    time.Time
+	retryCount int
+	retryPost  *model.Post
+	schedule   time.Time
+}
+
+func newSyncTask(channelId string, remoteId string, retryPost *model.Post) syncTask {
+	var postId string
+	if retryPost != nil {
+		postId = retryPost.Id
+	}
+
+	return syncTask{
+		id:        channelId + remoteId + postId, // combination of ids to avoid duplicates
+		channelId: channelId,
+		remoteId:  remoteId, // empty means update all remote clusters
+		retryPost: retryPost,
+		schedule:  time.Now(),
+	}
+}
+
+// incRetry increments the retry counter and returns true if MaxRetries not exceeded.
+func (st *syncTask) incRetry() bool {
+	st.retryCount++
+	return st.retryCount <= MaxRetries
+}
+
+// NotifyChannelChanged is called to indicate that a shared channel has been modified,
+// thus triggering an update to all remote clusters.
+func (scs *Service) NotifyChannelChanged(channelId string) {
+	if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
+		return
+	}
+
+	task := newSyncTask(channelId, "", nil)
+	task.schedule = time.Now().Add(NotifyMinimumDelay)
+	scs.addTask(task)
+}
+
+// ForceSyncForRemote causes all channels shared with the remote to be synchronized.
+func (scs *Service) ForceSyncForRemote(rc *model.RemoteCluster) {
+	if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
+		return
+	}
+
+	// fetch all channels shared with this remote.
+	opts := model.SharedChannelRemoteFilterOpts{
+		RemoteId: rc.RemoteId,
+	}
+	scrs, err := scs.server.GetStore().SharedChannel().GetRemotes(opts)
+	if err != nil {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Failed to fetch shared channel remotes",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("remoteId", rc.RemoteId),
+			mlog.Err(err),
+		)
+		return
+	}
+
+	for _, scr := range scrs {
+		task := newSyncTask(scr.ChannelId, rc.RemoteId, nil)
+		task.schedule = time.Now().Add(NotifyMinimumDelay)
+		scs.addTask(task)
+	}
+}
+
+// addTask adds or re-adds a task to the queue.
+func (scs *Service) addTask(task syncTask) {
+	task.AddedAt = time.Now()
+	scs.mux.Lock()
+	if _, ok := scs.tasks[task.id]; !ok {
+		scs.tasks[task.id] = task
+	}
+	scs.mux.Unlock()
+
+	// wake up the sync goroutine
+	select {
+	case scs.changeSignal <- struct{}{}:
+	default:
+		// that's ok, the sync routine is already busy
+	}
+}
+
+// syncLoop is called via a dedicated goroutine to wait for notifications of channel changes and
+// updates each remote based on those changes.
+func (scs *Service) syncLoop(done chan struct{}) {
+	// create a timer to periodically check the task queue, but only if there is
+	// a delayed task in the queue.
+	delay := time.NewTimer(NotifyMinimumDelay)
+	defer stopTimer(delay)
+
+	// wait for channel changed signal and update for oldest task.
+	for {
+		select {
+		case <-scs.changeSignal:
+			if wait := scs.doSync(); wait > 0 {
+				stopTimer(delay)
+				delay.Reset(wait)
+			}
+		case <-delay.C:
+			if wait := scs.doSync(); wait > 0 {
+				delay.Reset(wait)
+			}
+		case <-done:
+			return
+		}
+	}
+}
+
+func stopTimer(timer *time.Timer) {
+	timer.Stop()
+	select {
+	case <-timer.C:
+	default:
+	}
+}
+
+// doSync checks the task queue for any tasks to be processed and processes all that are ready.
+// If any delayed tasks remain in queue then the duration until the next scheduled task is returned.
+func (scs *Service) doSync() time.Duration {
+	var task syncTask
+	var ok bool
+	var shortestWait time.Duration
+
+	for {
+		task, ok, shortestWait = scs.removeOldestTask()
+		if !ok {
+			break
+		}
+		if err := scs.processTask(task); err != nil {
+			// put task back into map so it will update again
+			if task.incRetry() {
+				scs.addTask(task)
+			} else {
+				scs.server.GetLogger().Error("Failed to synchronize shared channel",
+					mlog.String("channelId", task.channelId),
+					mlog.String("remoteId", task.remoteId),
+					mlog.Err(err),
+				)
+			}
+		}
+	}
+	return shortestWait
+}
+
+// removeOldestTask removes and returns the oldest task in the task map.
+// A task coming in via NotifyChannelChanged must stay in queue for at least
+// `NotifyMinimumDelay` to ensure we don't go nuts trying to sync during a bulk update.
+// If no tasks are available then false is returned.
+func (scs *Service) removeOldestTask() (syncTask, bool, time.Duration) {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	var oldestTask syncTask
+	var oldestKey string
+	var shortestWait time.Duration
+
+	for key, task := range scs.tasks {
+		// check if task is ready
+		if wait := time.Until(task.schedule); wait > 0 {
+			if wait < shortestWait || shortestWait == 0 {
+				shortestWait = wait
+			}
+			continue
+		}
+		// task is ready; check if it's the oldest ready task
+		if task.AddedAt.Before(oldestTask.AddedAt) || oldestTask.AddedAt.IsZero() {
+			oldestKey = key
+			oldestTask = task
+		}
+	}
+
+	if oldestKey != "" {
+		delete(scs.tasks, oldestKey)
+		return oldestTask, true, shortestWait
+	}
+	return oldestTask, false, shortestWait
+}
+
+// processTask updates one or more remote clusters with any new channel content.
+func (scs *Service) processTask(task syncTask) error {
+	var err error
+	var remotes []*model.RemoteCluster
+
+	if task.remoteId == "" {
+		filter := model.RemoteClusterQueryFilter{
+			InChannel:     task.channelId,
+			OnlyConfirmed: true,
+		}
+		remotes, err = scs.server.GetStore().RemoteCluster().GetAll(filter)
+		if err != nil {
+			return err
+		}
+	} else {
+		rc, err := scs.server.GetStore().RemoteCluster().Get(task.remoteId)
+		if err != nil {
+			return err
+		}
+		if !rc.IsOnline() {
+			return fmt.Errorf("Failed updating shared channel '%s' for offline remote cluster '%s'", task.channelId, rc.DisplayName)
+		}
+		remotes = []*model.RemoteCluster{rc}
+	}
+
+	for _, rc := range remotes {
+		rtask := task
+		rtask.remoteId = rc.RemoteId
+		if err := scs.updateForRemote(rtask, rc); err != nil {
+			// retry...
+			if rtask.incRetry() {
+				scs.addTask(rtask)
+			} else {
+				scs.server.GetLogger().Error("Failed to synchronize shared channel for remote cluster",
+					mlog.String("channelId", rtask.channelId),
+					mlog.String("remote", rc.DisplayName),
+					mlog.String("remoteId", rtask.remoteId),
+					mlog.Err(err),
+				)
+			}
+		}
+	}
+	return nil
+}
+
+// updateForRemote updates a remote cluster with any new posts/reactions for a specific
+// channel. If many changes are found, only the oldest X changes are sent and the channel
+// is re-added to the task map. This ensures no channels are starved for updates even if some
+// channels are very active.
+func (scs *Service) updateForRemote(task syncTask, rc *model.RemoteCluster) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot update remote cluster for channel id %s; Remote Cluster Service not enabled", task.channelId)
+	}
+
+	scr, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(task.channelId, rc.RemoteId)
+	if err != nil {
+		return err
+	}
+
+	var posts []*model.Post
+	var repeat bool
+	nextSince := scr.NextSyncAt
+
+	if task.retryPost != nil {
+		posts = []*model.Post{task.retryPost}
+	} else {
+		result, err2 := scs.getPostsSince(task.channelId, rc, scr.NextSyncAt)
+		if err2 != nil {
+			return err2
+		}
+		posts = result.posts
+		repeat = result.hasMore
+		nextSince = result.nextSince
+	}
+
+	if len(posts) == 0 {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task found zero posts; skipping sync",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("channel_id", task.channelId),
+			mlog.Int64("lastSyncAt", scr.NextSyncAt),
+			mlog.Int64("nextSince", nextSince),
+			mlog.Bool("repeat", repeat),
+		)
+		return nil
+	}
+
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task found posts to sync",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("channel_id", task.channelId),
+		mlog.Int64("lastSyncAt", scr.NextSyncAt),
+		mlog.Int64("nextSince", nextSince),
+		mlog.Int("count", len(posts)),
+		mlog.Bool("repeat", repeat),
+	)
+
+	if !rc.IsOnline() {
+		scs.notifyRemoteOffline(posts, rc)
+		return nil
+	}
+
+	syncMessages, err := scs.postsToSyncMessages(posts, rc, scr.NextSyncAt)
+	if err != nil {
+		return err
+	}
+
+	if len(syncMessages) == 0 {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task, all messages filtered out; skipping sync",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("channel_id", task.channelId),
+			mlog.Bool("repeat", repeat),
+		)
+
+		// All posts were filtered out, meaning no need to send them. Fast forward SharedChannelRemote's NextSyncAt.
+		scs.updateNextSyncForRemote(scr.Id, rc, nextSince)
+
+		// everything was filtered out, nothing to send.
+		if repeat {
+			scs.addTask(newSyncTask(task.channelId, task.remoteId, nil))
+		}
+		return nil
+	}
+
+	scs.sendAttachments(syncMessages, rc)
+
+	b, err := json.Marshal(syncMessages)
+	if err != nil {
+		return err
+	}
+	msg := model.NewRemoteClusterMsg(TopicSync, b)
+
+	if scs.server.GetLogger().IsLevelEnabled(mlog.LvlSharedChannelServiceMessagesOutbound) {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceMessagesOutbound, "outbound message",
+			mlog.String("remote", rc.DisplayName),
+			mlog.Int64("NextSyncAt", scr.NextSyncAt),
+			mlog.String("msg", string(b)),
+		)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	err = rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		defer wg.Done()
+		if err != nil {
+			return // this means the response could not be parsed; already logged
+		}
+
+		var syncResp SyncResponse
+		if err2 := json.Unmarshal(resp.Payload, &syncResp); err2 != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "invalid sync response after update shared channel",
+				mlog.String("remote", rc.DisplayName),
+				mlog.Err(err2),
+			)
+		}
+
+		// Any Post(s) that failed to save on remote side are included in an array of post ids in the Response payload.
+		// Handle each error by retrying the post a fixed number of times before giving up.
+		for _, p := range syncResp.PostErrors {
+			scs.handlePostError(p, task, rc)
+		}
+
+		// update NextSyncAt for all the users that were synchronized
+		scs.updateSyncUsers(syncResp.UsersSyncd, rc, nextSince)
+	})
+
+	wg.Wait()
+
+	if err == nil {
+		// Optimistically update SharedChannelRemote's NextSyncAt; if any posts failed they will be retried.
+		scs.updateNextSyncForRemote(scr.Id, rc, nextSince)
+	}
+
+	if repeat {
+		scs.addTask(newSyncTask(task.channelId, task.remoteId, nil))
+	}
+	return err
+}
+
+func (scs *Service) sendAttachments(syncMessages []syncMsg, rc *model.RemoteCluster) {
+	for _, sm := range syncMessages {
+		for _, fi := range sm.Attachments {
+			if err := scs.sendAttachmentForRemote(fi, sm.Post, rc); err != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error syncing attachment for post",
+					mlog.String("remote", rc.DisplayName),
+					mlog.String("post_id", sm.Post.Id),
+					mlog.String("file_id", fi.Id),
+					mlog.Err(err),
+				)
+			}
+		}
+	}
+}
+
+func (scs *Service) handlePostError(postId string, task syncTask, rc *model.RemoteCluster) {
+	if task.retryPost != nil && task.retryPost.Id == postId {
+		// this was a retry for specific post that failed previously. Try again if within MaxRetries.
+		if task.incRetry() {
+			scs.addTask(task)
+		} else {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error syncing post",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("post_id", postId),
+			)
+		}
+		return
+	}
+
+	// this post failed as part of a group of posts. Retry as an individual post.
+	post, err := scs.server.GetStore().Post().GetSingle(postId, true)
+	if err != nil {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error fetching post for sync retry",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("post_id", postId),
+		)
+		return
+	}
+	scs.addTask(newSyncTask(task.channelId, task.remoteId, post))
+}
+
+// notifyRemoteOffline creates an ephemeral post to the author for any posts created recently to remotes
+// that are offline.
+func (scs *Service) notifyRemoteOffline(posts []*model.Post, rc *model.RemoteCluster) {
+	// only send one ephemeral post per author.
+	notified := make(map[string]bool)
+
+	// range the slice in reverse so the newest posts are visited first; this ensures an ephemeral
+	// get added where it is mostly likely to be seen.
+	for i := len(posts) - 1; i >= 0; i-- {
+		post := posts[i]
+		if didNotify := notified[post.UserId]; didNotify {
+			continue
+		}
+
+		postCreateAt := model.GetTimeForMillis(post.CreateAt)
+
+		if post.DeleteAt == 0 && post.UserId != "" && time.Since(postCreateAt) < NotifyRemoteOfflineThreshold {
+			T := scs.getUserTranslations(post.UserId)
+			ephemeral := &model.Post{
+				ChannelId: post.ChannelId,
+				Message:   T("sharedchannel.cannot_deliver_post", map[string]interface{}{"Remote": rc.DisplayName}),
+				CreateAt:  post.CreateAt + 1,
+			}
+			scs.app.SendEphemeralPost(post.UserId, ephemeral)
+
+			notified[post.UserId] = true
+		}
+	}
+}
+
+func (scs *Service) updateNextSyncForRemote(scrId string, rc *model.RemoteCluster, nextSyncAt int64) {
+	if nextSyncAt == 0 {
+		return
+	}
+	if err := scs.server.GetStore().SharedChannel().UpdateRemoteNextSyncAt(scrId, nextSyncAt); err != nil {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error updating NextSyncAt for shared channel remote",
+			mlog.String("remote", rc.DisplayName),
+			mlog.Err(err),
+		)
+		return
+	}
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "updated NextSyncAt for remote",
+		mlog.String("remote_id", rc.RemoteId),
+		mlog.String("remote", rc.DisplayName),
+		mlog.Int64("next_update_at", nextSyncAt),
+	)
+}
+
+func (scs *Service) updateSyncUsers(userIds []string, rc *model.RemoteCluster, lastSyncAt int64) {
+	for _, uid := range userIds {
+		scu, err := scs.server.GetStore().SharedChannel().GetUser(uid, rc.RemoteId)
+		if err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error getting user for lastSyncAt update",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("user_id", uid),
+				mlog.Err(err),
+			)
+			continue
+		}
+
+		if err := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(scu.Id, lastSyncAt); err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error updating lastSyncAt for user",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("user_id", uid),
+				mlog.Err(err),
+			)
+		} else {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "updated lastSyncAt for user",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("user_id", scu.UserId),
+				mlog.Int64("last_update_at", lastSyncAt),
+			)
+		}
+	}
+}
+
+func (scs *Service) getUserTranslations(userId string) i18n.TranslateFunc {
+	var locale string
+	user, err := scs.server.GetStore().User().Get(context.Background(), userId)
+	if err == nil {
+		locale = user.Locale
+	}
+
+	if locale == "" {
+		locale = model.DEFAULT_LOCALE
+	}
+	return i18n.GetUserTranslations(locale)
+}