MM-27493 Shared channels (MVP) (#17301)

Remote Cluster Service
- provides ability for multiple Mattermost cluster instances to create a trusted connection with each other and exchange messages
- trusted connections are managed via slash commands (for now)
- facilitates features requiring inter-cluster communication, such as Shared Channels
Shared Channels Service
- provides ability to shared channels between one or more Mattermost cluster instances (using trusted connection)
- sharing/unsharing of channels is managed via slash commands (for now)

services/remotecluster/error.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import "fmt"
+
+type BufferFullError struct {
+	capacity int
+}
+
+func NewBufferFullError(capacity int) BufferFullError {
+	return BufferFullError{
+		capacity: capacity,
+	}
+}
+
+func (e BufferFullError) Capacity() int {
+	return e.capacity
+}
+
+func (e BufferFullError) Error() string {
+	return fmt.Sprintf("buffer capacity (%d) exceeded", e.capacity)
+}

services/remotecluster/invitation.go

@@ -0,0 +1,82 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+// AcceptInvitation is called when accepting an invitation to connect with a remote cluster.
+func (rcs *Service) AcceptInvitation(invite *model.RemoteClusterInvite, name string, creatorId string, teamId string, siteURL string) (*model.RemoteCluster, error) {
+	rc := &model.RemoteCluster{
+		RemoteId:     invite.RemoteId,
+		RemoteTeamId: invite.RemoteTeamId,
+		DisplayName:  name,
+		Token:        model.NewId(),
+		RemoteToken:  invite.Token,
+		SiteURL:      invite.SiteURL,
+		CreatorId:    creatorId,
+	}
+
+	rcSaved, err := rcs.server.GetStore().RemoteCluster().Save(rc)
+	if err != nil {
+		return nil, err
+	}
+
+	// confirm the invitation with the originating site
+	frame, err := makeConfirmFrame(rcSaved, teamId, siteURL)
+	if err != nil {
+		return nil, err
+	}
+
+	url := fmt.Sprintf("%s/%s", rcSaved.SiteURL, ConfirmInviteURL)
+
+	resp, err := rcs.sendFrameToRemote(PingTimeout, rc, frame, url)
+	if err != nil {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, err
+	}
+
+	var response Response
+	err = json.Unmarshal(resp, &response)
+	if err != nil {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, fmt.Errorf("invalid response from remote server: %w", err)
+	}
+
+	if !response.IsSuccess() {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, errors.New(response.Err)
+	}
+
+	// issue the first ping right away. The goroutine will exit when ping completes or PingTimeout exceeded.
+	go rcs.pingRemote(rcSaved)
+
+	return rcSaved, nil
+}
+
+func makeConfirmFrame(rc *model.RemoteCluster, teamId string, siteURL string) (*model.RemoteClusterFrame, error) {
+	confirm := model.RemoteClusterInvite{
+		RemoteId:     rc.RemoteId,
+		RemoteTeamId: teamId,
+		SiteURL:      siteURL,
+		Token:        rc.Token,
+	}
+	confirmRaw, err := json.Marshal(confirm)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := model.NewRemoteClusterMsg(InvitationTopic, confirmRaw)
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: rc.RemoteId,
+		Msg:      msg,
+	}
+	return frame, nil
+}

services/remotecluster/mocks_test.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"go.uber.org/zap/zapcore"
+
+	"github.com/mattermost/mattermost-server/v5/einterfaces"
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+	"github.com/mattermost/mattermost-server/v5/store"
+	"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
+)
+
+type mockServer struct {
+	remotes []*model.RemoteCluster
+	logger  *mockLogger
+}
+
+func newMockServer(t *testing.T, remotes []*model.RemoteCluster) *mockServer {
+	return &mockServer{
+		remotes: remotes,
+		logger:  &mockLogger{t: t},
+	}
+}
+
+func (ms *mockServer) Config() *model.Config                                  { return nil }
+func (ms *mockServer) GetMetrics() einterfaces.MetricsInterface               { return nil }
+func (ms *mockServer) IsLeader() bool                                         { return true }
+func (ms *mockServer) AddClusterLeaderChangedListener(listener func()) string { return model.NewId() }
+func (ms *mockServer) RemoveClusterLeaderChangedListener(id string)           {}
+func (ms *mockServer) GetLogger() mlog.LoggerIFace {
+	return ms.logger
+}
+func (ms *mockServer) GetStore() store.Store {
+	anyFilter := mock.MatchedBy(func(filter model.RemoteClusterQueryFilter) bool {
+		return true
+	})
+
+	remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+	remoteClusterStoreMock.On("GetByTopic", "share").Return(ms.remotes, nil)
+	remoteClusterStoreMock.On("GetAll", anyFilter).Return(ms.remotes, nil)
+
+	storeMock := &mocks.Store{}
+	storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+	return storeMock
+}
+
+type mockLogger struct {
+	t *testing.T
+}
+
+func (ml *mockLogger) IsLevelEnabled(level mlog.LogLevel) bool {
+	return true
+}
+func (ml *mockLogger) Debug(s string, flds ...mlog.Field) {
+	ml.t.Log("debug", s, fieldsToStrings(flds))
+}
+func (ml *mockLogger) Info(s string, flds ...mlog.Field) {
+	ml.t.Log("info", s, fieldsToStrings(flds))
+}
+func (ml *mockLogger) Warn(s string, flds ...mlog.Field) {
+	ml.t.Log("warn", s, fieldsToStrings(flds))
+}
+func (ml *mockLogger) Error(s string, flds ...mlog.Field) {
+	ml.t.Log("error", s, fieldsToStrings(flds))
+}
+func (ml *mockLogger) Critical(s string, flds ...mlog.Field) {
+	ml.t.Log("crit", s, fieldsToStrings(flds))
+}
+func (ml *mockLogger) Log(level mlog.LogLevel, s string, flds ...mlog.Field) {
+	ml.t.Log(level.Name, s, fieldsToStrings(flds))
+}
+func (ml *mockLogger) LogM(levels []mlog.LogLevel, s string, flds ...mlog.Field) {
+	ml.t.Log(levelsToString(levels), s, fieldsToStrings(flds))
+}
+
+func levelsToString(levels []mlog.LogLevel) string {
+	sb := strings.Builder{}
+	for _, l := range levels {
+		sb.WriteString(l.Name)
+		sb.WriteString(",")
+	}
+	return sb.String()
+}
+
+func fieldsToStrings(fields []mlog.Field) []string {
+	encoder := zapcore.NewMapObjectEncoder()
+	for _, zapField := range fields {
+		zapField.AddTo(encoder)
+	}
+
+	var result []string
+	for k, v := range encoder.Fields {
+		result = append(result, fmt.Sprintf("%s:%v", k, v))
+	}
+	return result
+}

services/remotecluster/ping.go

@@ -0,0 +1,174 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+// pingLoop periodically sends a ping to all remote clusters.
+func (rcs *Service) pingLoop(done <-chan struct{}) {
+	pingChan := make(chan *model.RemoteCluster, MaxConcurrentSends*2)
+
+	// create a thread pool to send pings concurrently to remotes.
+	for i := 0; i < MaxConcurrentSends; i++ {
+		go rcs.pingEmitter(pingChan, done)
+	}
+
+	go rcs.pingGenerator(pingChan, done)
+}
+
+func (rcs *Service) pingGenerator(pingChan chan *model.RemoteCluster, done <-chan struct{}) {
+	defer close(pingChan)
+
+	for {
+		start := time.Now()
+
+		// get all remotes, including any previously offline.
+		remotes, err := rcs.server.GetStore().RemoteCluster().GetAll(model.RemoteClusterQueryFilter{})
+		if err != nil {
+			rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceError, "Ping remote cluster failed (could not get list of remotes)", mlog.Err(err))
+			select {
+			case <-time.After(PingFreq):
+				continue
+			case <-done:
+				return
+			}
+		}
+
+		for _, rc := range remotes {
+			if rc.SiteURL != "" { // filter out unconfirmed invites
+				pingChan <- rc
+			}
+		}
+
+		// try to maintain frequency
+		elapsed := time.Since(start)
+		if elapsed < PingFreq {
+			sleep := time.Until(start.Add(PingFreq))
+			select {
+			case <-time.After(sleep):
+			case <-done:
+				return
+			}
+		}
+	}
+}
+
+// pingEmitter pulls Remotes from the ping queue (pingChan) and pings them.
+// Pinging a remote cannot take longer than PingTimeoutMillis.
+func (rcs *Service) pingEmitter(pingChan <-chan *model.RemoteCluster, done <-chan struct{}) {
+	for {
+		select {
+		case rc := <-pingChan:
+			if rc == nil {
+				return
+			}
+
+			online := rc.IsOnline()
+
+			if err := rcs.pingRemote(rc); err != nil {
+				rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceWarn, "Remote cluster ping failed",
+					mlog.String("remote", rc.DisplayName),
+					mlog.String("remoteId", rc.RemoteId),
+					mlog.Err(err),
+				)
+			}
+
+			if online != rc.IsOnline() {
+				if metrics := rcs.server.GetMetrics(); metrics != nil {
+					metrics.IncrementRemoteClusterConnStateChangeCounter(rc.RemoteId, rc.IsOnline())
+				}
+				rcs.fireConnectionStateChgEvent(rc)
+			}
+		case <-done:
+			return
+		}
+	}
+}
+
+// pingRemote make a synchronous ping to a remote cluster. Return is error if ping is
+// unsuccessful and nil on success.
+func (rcs *Service) pingRemote(rc *model.RemoteCluster) error {
+	frame, err := makePingFrame(rc)
+	if err != nil {
+		return err
+	}
+	url := fmt.Sprintf("%s/%s", rc.SiteURL, PingURL)
+
+	resp, err := rcs.sendFrameToRemote(PingTimeout, rc, frame, url)
+	if err != nil {
+		return err
+	}
+
+	ping := model.RemoteClusterPing{}
+	err = json.Unmarshal(resp, &ping)
+	if err != nil {
+		return err
+	}
+
+	if err := rcs.server.GetStore().RemoteCluster().SetLastPingAt(rc.RemoteId); err != nil {
+		rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceError, "Failed to update LastPingAt for remote cluster",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("remoteId", rc.RemoteId),
+			mlog.Err(err),
+		)
+	}
+	rc.LastPingAt = model.GetMillis()
+
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		sentAt := time.Unix(0, ping.SentAt*int64(time.Millisecond))
+		elapsed := time.Since(sentAt).Seconds()
+		metrics.ObserveRemoteClusterPingDuration(rc.RemoteId, elapsed)
+
+		// we approximate clock skew between remotes.
+		skew := elapsed/2 - float64(ping.RecvAt-ping.SentAt)/1000
+		metrics.ObserveRemoteClusterClockSkew(rc.RemoteId, skew)
+	}
+
+	rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceDebug, "Remote cluster ping",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("remoteId", rc.RemoteId),
+		mlog.Int64("SentAt", ping.SentAt),
+		mlog.Int64("RecvAt", ping.RecvAt),
+		mlog.Int64("Diff", ping.RecvAt-ping.SentAt),
+	)
+	return nil
+}
+
+func makePingFrame(rc *model.RemoteCluster) (*model.RemoteClusterFrame, error) {
+	ping := model.RemoteClusterPing{
+		SentAt: model.GetMillis(),
+	}
+	pingRaw, err := json.Marshal(ping)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := model.NewRemoteClusterMsg(PingTopic, pingRaw)
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: rc.RemoteId,
+		Msg:      msg,
+	}
+	return frame, nil
+}
+
+func (rcs *Service) fireConnectionStateChgEvent(rc *model.RemoteCluster) {
+	rcs.mux.RLock()
+	listeners := make([]ConnectionStateListener, 0, len(rcs.connectionStateListeners))
+	for _, l := range rcs.connectionStateListeners {
+		listeners = append(listeners, l)
+	}
+	rcs.mux.RUnlock()
+
+	for _, l := range listeners {
+		l(rc, rc.IsOnline())
+	}
+}

services/remotecluster/ping_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+const (
+	Recent = 60000
+)
+
+func TestPing(t *testing.T) {
+	disablePing = false
+
+	t.Run("No error", func(t *testing.T) {
+		var countWebReq int32
+		merr := merror.New()
+
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer wg.Done()
+			defer w.WriteHeader(200)
+			atomic.AddInt32(&countWebReq, 1)
+
+			frame, err := model.RemoteClusterFrameFromJSON(r.Body)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+			if len(frame.Msg.Payload) == 0 {
+				merr.Append(fmt.Errorf("Payload should not be empty; remote_id=%s", frame.RemoteId))
+				return
+			}
+
+			ping, err := model.RemoteClusterPingFromRawJSON(frame.Msg.Payload)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+			if !checkRecent(ping.SentAt, Recent) {
+				merr.Append(fmt.Errorf("timestamp out of range, got %d", ping.SentAt))
+				return
+			}
+			if ping.RecvAt != 0 {
+				merr.Append(fmt.Errorf("timestamp should be 0, got %d", ping.RecvAt))
+				return
+			}
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, ts.URL))
+		service, err := NewRemoteClusterService(mockServer)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		wg.Wait()
+
+		assert.NoError(t, merr.ErrorOrNil())
+
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countWebReq))
+		t.Log(fmt.Sprintf("%d web requests counted;  %d expected",
+			atomic.LoadInt32(&countWebReq), NumRemotes))
+	})
+
+	t.Run("HTTP errors", func(t *testing.T) {
+		var countWebReq int32
+		merr := merror.New()
+
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer wg.Done()
+			atomic.AddInt32(&countWebReq, 1)
+
+			frame, err := model.RemoteClusterFrameFromJSON(r.Body)
+			if err != nil {
+				merr.Append(err)
+			}
+			ping, err := model.RemoteClusterPingFromRawJSON(frame.Msg.Payload)
+			if err != nil {
+				merr.Append(err)
+			}
+			if !checkRecent(ping.SentAt, Recent) {
+				merr.Append(fmt.Errorf("timestamp out of range, got %d", ping.SentAt))
+			}
+			if ping.RecvAt != 0 {
+				merr.Append(fmt.Errorf("timestamp should be 0, got %d", ping.RecvAt))
+			}
+			w.WriteHeader(500)
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, ts.URL))
+		service, err := NewRemoteClusterService(mockServer)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		wg.Wait()
+
+		assert.Nil(t, merr.ErrorOrNil())
+
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countWebReq))
+		t.Log(fmt.Sprintf("%d web requests counted;  %d expected",
+			atomic.LoadInt32(&countWebReq), NumRemotes))
+	})
+}
+
+func checkRecent(millis int64, within int64) bool {
+	now := model.GetMillis()
+	return millis > now-within && millis < now+within
+}

services/remotecluster/recv.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"fmt"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+// ReceiveIncomingMsg is called by the Rest API layer, or websocket layer (future), when a Remote Cluster
+// message is received.  Here we route the message to any topic listeners.
+// `rc` and `msg` cannot be nil.
+func (rcs *Service) ReceiveIncomingMsg(rc *model.RemoteCluster, msg model.RemoteClusterMsg) Response {
+	rcs.mux.RLock()
+	defer rcs.mux.RUnlock()
+
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		metrics.IncrementRemoteClusterMsgReceivedCounter(rc.RemoteId)
+	}
+
+	rcSanitized := *rc
+	rcSanitized.Token = ""
+	rcSanitized.RemoteToken = ""
+
+	var response Response
+	response.Status = ResponseStatusOK
+
+	listeners := rcs.getTopicListeners(msg.Topic)
+
+	for _, l := range listeners {
+		if err := callback(l, msg, &rcSanitized, &response); err != nil {
+			rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceError, "Error from remote cluster message listener",
+				mlog.String("msgId", msg.Id), mlog.String("topic", msg.Topic), mlog.String("remote", rc.DisplayName), mlog.Err(err))
+
+			response.Status = ResponseStatusFail
+			response.Err = err.Error()
+		}
+	}
+	return response
+}
+
+func callback(listener TopicListener, msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("%v", r)
+		}
+	}()
+	err = listener(msg, rc, resp)
+	return
+}

services/remotecluster/response.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+)
+
+// Response represents the bytes replied from a remote server when a message is sent.
+type Response struct {
+	Status  string          `json:"status"`
+	Err     string          `json:"err"`
+	Payload json.RawMessage `json:"payload"`
+}
+
+// IsSuccess returns true if the response status indicates success.
+func (r *Response) IsSuccess() bool {
+	return r.Status == ResponseStatusOK
+}
+
+// SetPayload serializes an arbitrary struct as a RawMessage.
+func (r *Response) SetPayload(v interface{}) error {
+	raw, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	r.Payload = raw
+	return nil
+}

services/remotecluster/send.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"hash/fnv"
+)
+
+// enqueueTask adds a task to one of the send channels based on remoteId.
+//
+// There are a number  of send channels (`MaxConcurrentSends`) to allow for sending to multiple
+// remotes concurrently, while preserving message order for each remote.
+func (rcs *Service) enqueueTask(ctx context.Context, remoteId string, task interface{}) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	h := hash(remoteId)
+	idx := h % uint32(len(rcs.send))
+
+	select {
+	case rcs.send[idx] <- task:
+		return nil
+	case <-ctx.Done():
+		return NewBufferFullError(cap(rcs.send))
+	}
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+// sendLoop is called by each goroutine created for the send pool and waits for sendTask's until the
+// done channel is signalled.
+//
+// Each goroutine in the pool is assigned a specific channel, and tasks are placed in the
+// channel corresponding to the remoteId.
+func (rcs *Service) sendLoop(idx int, done chan struct{}) {
+	for {
+		select {
+		case t := <-rcs.send[idx]:
+			switch task := t.(type) {
+			case sendMsgTask:
+				rcs.sendMsg(task)
+			case sendFileTask:
+				rcs.sendFile(task)
+			}
+		case <-done:
+			return
+		}
+	}
+}

services/remotecluster/send_test.go

@@ -0,0 +1,200 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+const (
+	TestTopics  = " share incident "
+	TestTopic   = "share"
+	NumRemotes  = 50
+	NoteContent = "Woot!!"
+)
+
+type testPayload struct {
+	Note string `json:"note"`
+}
+
+func TestBroadcastMsg(t *testing.T) {
+	msgId := model.NewId()
+	disablePing = true
+
+	t.Run("No error", func(t *testing.T) {
+		var countCallbacks int32
+		var countWebReq int32
+		merr := merror.New()
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer func() {
+				w.WriteHeader(200)
+				var resp Response
+				b, errMarshall := json.Marshal(&resp)
+				if errMarshall != nil {
+					merr.Append(errMarshall)
+					return
+				}
+				w.Write(b)
+			}()
+
+			atomic.AddInt32(&countWebReq, 1)
+
+			frame, appErr := model.RemoteClusterFrameFromJSON(r.Body)
+			if appErr != nil {
+				merr.Append(appErr)
+				return
+			}
+			if len(frame.Msg.Payload) == 0 {
+				merr.Append(fmt.Errorf("webrequest missing Msg.Payload"))
+			}
+			if msgId != frame.Msg.Id {
+				merr.Append(fmt.Errorf("webrequest msgId expected %s, got %s", msgId, frame.Msg.Id))
+				return
+			}
+
+			note := testPayload{}
+			err := json.Unmarshal(frame.Msg.Payload, &note)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+			if note.Note != NoteContent {
+				merr.Append(fmt.Errorf("webrequest payload expected %s, got %s", NoteContent, note.Note))
+				return
+			}
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, ts.URL))
+		service, err := NewRemoteClusterService(mockServer)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		msg := makeRemoteClusterMsg(msgId, NoteContent)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+		defer cancel()
+
+		err = service.BroadcastMsg(ctx, msg, func(msg model.RemoteClusterMsg, remote *model.RemoteCluster, resp *Response, err error) {
+			defer wg.Done()
+			atomic.AddInt32(&countCallbacks, 1)
+
+			if err != nil {
+				merr.Append(err)
+			}
+			if msgId != msg.Id {
+				merr.Append(fmt.Errorf("result callback msgId expected %s, got %s", msgId, msg.Id))
+			}
+
+			var note testPayload
+			err2 := json.Unmarshal(msg.Payload, &note)
+			if err2 != nil {
+				merr.Append(fmt.Errorf("unmarshal payload error: %w", err2))
+				return
+			}
+			if note.Note != NoteContent {
+				merr.Append(fmt.Errorf("compare payload failed: expected '%s', got '%s'", NoteContent, note))
+			}
+		})
+		assert.NoError(t, err)
+
+		wg.Wait()
+
+		assert.NoError(t, merr.ErrorOrNil())
+
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countCallbacks))
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countWebReq))
+		t.Log(fmt.Sprintf("%d callbacks counted;  %d web requests counted;  %d expected",
+			atomic.LoadInt32(&countCallbacks), atomic.LoadInt32(&countWebReq), NumRemotes))
+	})
+
+	t.Run("HTTP error", func(t *testing.T) {
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(500)
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, ts.URL))
+		service, err := NewRemoteClusterService(mockServer)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		msg := makeRemoteClusterMsg(msgId, NoteContent)
+		var countCallbacks int32
+		var countErrors int32
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		err = service.BroadcastMsg(context.Background(), msg, func(msg model.RemoteClusterMsg, remote *model.RemoteCluster, resp *Response, err error) {
+			defer wg.Done()
+			atomic.AddInt32(&countCallbacks, 1)
+			if err != nil {
+				atomic.AddInt32(&countErrors, 1)
+			}
+		})
+		assert.NoError(t, err)
+
+		wg.Wait()
+
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countCallbacks))
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countErrors))
+	})
+}
+
+func makeRemoteClusters(num int, siteURL string) []*model.RemoteCluster {
+	var remotes []*model.RemoteCluster
+	for i := 0; i < num; i++ {
+		rc := makeRemoteCluster(fmt.Sprintf("test cluster %d", i+1), siteURL, TestTopics)
+		remotes = append(remotes, rc)
+	}
+	return remotes
+}
+
+func makeRemoteCluster(name string, siteURL string, topics string) *model.RemoteCluster {
+	return &model.RemoteCluster{
+		RemoteId:    model.NewId(),
+		DisplayName: name,
+		SiteURL:     siteURL,
+		Token:       model.NewId(),
+		Topics:      topics,
+		CreateAt:    model.GetMillis(),
+		LastPingAt:  model.GetMillis(),
+		CreatorId:   model.NewId(),
+	}
+}
+
+func makeRemoteClusterMsg(id string, note string) model.RemoteClusterMsg {
+	payload := testPayload{Note: note}
+	raw, _ := json.Marshal(payload)
+
+	return model.RemoteClusterMsg{
+		Id:       id,
+		Topic:    TestTopic,
+		CreateAt: model.GetMillis(),
+		Payload:  raw}
+}

services/remotecluster/sendfile.go

@@ -0,0 +1,147 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/filestore"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+type SendFileResultFunc func(us *model.UploadSession, rc *model.RemoteCluster, resp *Response, err error)
+
+type sendFileTask struct {
+	rc *model.RemoteCluster
+	us *model.UploadSession
+	fi *model.FileInfo
+	rp ReaderProvider
+	f  SendFileResultFunc
+}
+
+type ReaderProvider interface {
+	FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
+}
+
+// SendFile asynchronously sends a file to a remote cluster.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the file cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// Nil or error return indicates success or failure of file enqueue only.
+//
+// An optional callback can be provided that receives the response from the remote cluster. The `err` provided to the
+// callback is regarding file delivery only. The `resp` contains the decoded bytes returned from the remote.
+// If a callback is provided it should return quickly.
+func (rcs *Service) SendFile(ctx context.Context, us *model.UploadSession, fi *model.FileInfo, rc *model.RemoteCluster, rp ReaderProvider, f SendFileResultFunc) error {
+	task := sendFileTask{
+		rc: rc,
+		us: us,
+		fi: fi,
+		rp: rp,
+		f:  f,
+	}
+	return rcs.enqueueTask(ctx, rc.RemoteId, task)
+}
+
+// sendFile is called when a sendFileTask is popped from the send channel.
+func (rcs *Service) sendFile(task sendFileTask) {
+	// Ensure a panic from the callback does not exit the goroutine.
+	defer func() {
+		if r := recover(); r != nil {
+			rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceError, "Remote Cluster sendFile panic",
+				mlog.String("remote", task.rc.DisplayName),
+				mlog.String("uploadId", task.us.Id),
+				mlog.Any("panic", r),
+			)
+		}
+	}()
+
+	fi, err := rcs.sendFileToRemote(SendTimeout, task)
+	var response Response
+
+	if err != nil {
+		rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceError, "Remote Cluster send file failed",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("uploadId", task.us.Id),
+			mlog.Err(err),
+		)
+		response.Status = ResponseStatusFail
+		response.Err = err.Error()
+	} else {
+		rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceDebug, "Remote Cluster file sent successfully",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("uploadId", task.us.Id),
+		)
+		response.Status = ResponseStatusOK
+		response.SetPayload(fi)
+	}
+
+	// If callback provided then call it with the results.
+	if task.f != nil {
+		task.f(task.us, task.rc, &response, err)
+	}
+}
+
+func (rcs *Service) sendFileToRemote(timeout time.Duration, task sendFileTask) (*model.FileInfo, error) {
+	rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceDebug, "sending file to remote...",
+		mlog.String("remote", task.rc.DisplayName),
+		mlog.String("uploadId", task.us.Id),
+		mlog.String("file_path", task.us.Path),
+	)
+
+	r, appErr := task.rp.FileReader(task.fi.Path) // get Reader for the file
+	if appErr != nil {
+		return nil, fmt.Errorf("error opening file while sending file to remote %s: %w", task.rc.RemoteId, appErr)
+	}
+	defer r.Close()
+
+	u, err := url.Parse(task.rc.SiteURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid siteURL while sending file to remote %s: %w", task.rc.RemoteId, err)
+	}
+	u.Path = path.Join(u.Path, model.API_URL_SUFFIX, "remotecluster", "upload", task.us.Id)
+
+	req, err := http.NewRequest("POST", u.String(), r)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set(model.HEADER_REMOTECLUSTER_ID, task.rc.RemoteId)
+	req.Header.Set(model.HEADER_REMOTECLUSTER_TOKEN, task.rc.RemoteToken)
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	resp, err := rcs.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response: %d - %s", resp.StatusCode, resp.Status)
+	}
+
+	// body should be a FileInfo
+	var fi model.FileInfo
+	if err := json.Unmarshal(body, &fi); err != nil {
+		return nil, fmt.Errorf("unexpected response body: %w", err)
+	}
+
+	return &fi, nil
+}

services/remotecluster/sendmsg.go

@@ -0,0 +1,180 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"time"
+
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+type SendMsgResultFunc func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response, err error)
+
+type sendMsgTask struct {
+	rc  *model.RemoteCluster
+	msg model.RemoteClusterMsg
+	f   SendMsgResultFunc
+}
+
+// BroadcastMsg asynchronously sends a message to all remote clusters interested in the message's topic.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the message cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// An optional callback can be provided that receives the success or fail result of sending to each remote cluster.
+// Success or fail is regarding message delivery only.  If a callback is provided it should return quickly.
+func (rcs *Service) BroadcastMsg(ctx context.Context, msg model.RemoteClusterMsg, f SendMsgResultFunc) error {
+	// get list of interested remotes.
+	filter := model.RemoteClusterQueryFilter{
+		Topic: msg.Topic,
+	}
+	list, err := rcs.server.GetStore().RemoteCluster().GetAll(filter)
+	if err != nil {
+		return err
+	}
+
+	errs := merror.New()
+
+	for _, rc := range list {
+		if err := rcs.SendMsg(ctx, msg, rc, f); err != nil {
+			errs.Append(err)
+		}
+	}
+	return errs.ErrorOrNil()
+}
+
+// SendMsg asynchronously sends a message to a remote cluster.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the message cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// Nil or error return indicates success or failure of message enqueue only.
+//
+// An optional callback can be provided that receives the response from the remote cluster. The `err` provided to the
+// callback is regarding response decoding only. The `resp` contains the decoded bytes returned from the remote.
+// If a callback is provided it should return quickly.
+func (rcs *Service) SendMsg(ctx context.Context, msg model.RemoteClusterMsg, rc *model.RemoteCluster, f SendMsgResultFunc) error {
+	task := sendMsgTask{
+		rc:  rc,
+		msg: msg,
+		f:   f,
+	}
+	return rcs.enqueueTask(ctx, rc.RemoteId, task)
+}
+
+// sendMsg is called when a sendMsgTask is popped from the send channel.
+func (rcs *Service) sendMsg(task sendMsgTask) {
+	var errResp error
+	var response Response
+
+	// Ensure a panic from the callback does not exit the pool goroutine.
+	defer func() {
+		if r := recover(); r != nil {
+			rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceError, "Remote Cluster sendMsg panic",
+				mlog.String("remote", task.rc.DisplayName), mlog.String("msgId", task.msg.Id), mlog.Any("panic", r))
+		}
+
+		if errResp != nil {
+			response.Err = errResp.Error()
+		}
+
+		// If callback provided then call it with the results.
+		if task.f != nil {
+			task.f(task.msg, task.rc, &response, errResp)
+		}
+	}()
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: task.rc.RemoteId,
+		Msg:      task.msg,
+	}
+
+	u, err := url.Parse(task.rc.SiteURL)
+	if err != nil {
+		rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceError, "Invalid siteURL while sending message to remote",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msgId", task.msg.Id),
+			mlog.Err(err),
+		)
+		errResp = err
+		return
+	}
+	u.Path = path.Join(u.Path, SendMsgURL)
+
+	respJSON, err := rcs.sendFrameToRemote(SendTimeout, task.rc, frame, u.String())
+
+	if err != nil {
+		rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceError, "Remote Cluster send message failed",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msgId", task.msg.Id),
+			mlog.Err(err),
+		)
+		errResp = err
+	} else {
+		rcs.server.GetLogger().Log(mlog.LvlRemoteClusterServiceDebug, "Remote Cluster message sent successfully",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msgId", task.msg.Id),
+		)
+
+		if err = json.Unmarshal(respJSON, &response); err != nil {
+			rcs.server.GetLogger().Error("Invalid response sending message to remote cluster",
+				mlog.String("remote", task.rc.DisplayName),
+				mlog.Err(err),
+			)
+			errResp = err
+		}
+	}
+}
+
+func (rcs *Service) sendFrameToRemote(timeout time.Duration, rc *model.RemoteCluster, frame *model.RemoteClusterFrame, url string) ([]byte, error) {
+	body, err := json.Marshal(frame)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequest("POST", url, bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set(model.HEADER_REMOTECLUSTER_ID, rc.RemoteId)
+	req.Header.Set(model.HEADER_REMOTECLUSTER_TOKEN, rc.RemoteToken)
+
+	resp, err := rcs.httpClient.Do(req.WithContext(ctx))
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		if err != nil || resp.StatusCode != http.StatusOK {
+			metrics.IncrementRemoteClusterMsgErrorsCounter(frame.RemoteId, os.IsTimeout(err))
+		} else {
+			metrics.IncrementRemoteClusterMsgSentCounter(frame.RemoteId)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return body, fmt.Errorf("unexpected response: %d - %s", resp.StatusCode, resp.Status)
+	}
+	return body, nil
+}

services/remotecluster/service.go

@@ -0,0 +1,261 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/einterfaces"
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+	"github.com/mattermost/mattermost-server/v5/store"
+)
+
+const (
+	SendChanBuffer                = 50
+	RecvChanBuffer                = 50
+	ResultsChanBuffer             = 50
+	ResultQueueDrainTimeoutMillis = 10000
+	MaxConcurrentSends            = 10
+	SendMsgURL                    = "api/v4/remotecluster/msg"
+	SendTimeout                   = time.Minute
+	SendFileTimeout               = time.Minute * 5
+	PingURL                       = "api/v4/remotecluster/ping"
+	PingFreq                      = time.Minute
+	PingTimeout                   = time.Second * 15
+	ConfirmInviteURL              = "api/v4/remotecluster/confirm_invite"
+	InvitationTopic               = "invitation"
+	PingTopic                     = "ping"
+	ResponseStatusOK              = model.STATUS_OK
+	ResponseStatusFail            = model.STATUS_FAIL
+	InviteExpiresAfter            = time.Hour * 48
+)
+
+var (
+	disablePing bool // override for testing
+)
+
+type ServerIface interface {
+	Config() *model.Config
+	IsLeader() bool
+	AddClusterLeaderChangedListener(listener func()) string
+	RemoveClusterLeaderChangedListener(id string)
+	GetStore() store.Store
+	GetLogger() mlog.LoggerIFace
+	GetMetrics() einterfaces.MetricsInterface
+}
+
+// RemoteClusterServiceIFace is used to allow mocking where a remote cluster service is used (for testing).
+// Unfortunately it lives here because the shared channel service, app layer, and server interface all need it.
+// Putting it in app layer means shared channel service must import app package.
+type RemoteClusterServiceIFace interface {
+	Shutdown() error
+	Start() error
+	Active() bool
+	AddTopicListener(topic string, listener TopicListener) string
+	RemoveTopicListener(listenerId string)
+	AddConnectionStateListener(listener ConnectionStateListener) string
+	RemoveConnectionStateListener(listenerId string)
+	SendMsg(ctx context.Context, msg model.RemoteClusterMsg, rc *model.RemoteCluster, f SendMsgResultFunc) error
+	SendFile(ctx context.Context, us *model.UploadSession, fi *model.FileInfo, rc *model.RemoteCluster, rp ReaderProvider, f SendFileResultFunc) error
+	AcceptInvitation(invite *model.RemoteClusterInvite, name string, creatorId string, teamId string, siteURL string) (*model.RemoteCluster, error)
+	ReceiveIncomingMsg(rc *model.RemoteCluster, msg model.RemoteClusterMsg) Response
+}
+
+// TopicListener is a callback signature used to listen for incoming messages for
+// a specific topic.
+type TopicListener func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error
+
+// ConnectionStateListener is used to listen to remote cluster connection state changes.
+type ConnectionStateListener func(rc *model.RemoteCluster, online bool)
+
+// Service provides inter-cluster communication via topic based messages.
+type Service struct {
+	server     ServerIface
+	httpClient *http.Client
+	send       []chan interface{}
+
+	// everything below guarded by `mux`
+	mux                      sync.RWMutex
+	active                   bool
+	leaderListenerId         string
+	topicListeners           map[string]map[string]TopicListener // maps topic id to a map of listenerid->listener
+	connectionStateListeners map[string]ConnectionStateListener  // maps listener id to listener
+	done                     chan struct{}
+}
+
+// NewRemoteClusterService creates a RemoteClusterService instance.
+func NewRemoteClusterService(server ServerIface) (*Service, error) {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          200,
+		MaxIdleConnsPerHost:   2,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		DisableCompression:    false,
+	}
+
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   SendTimeout,
+	}
+
+	service := &Service{
+		server:                   server,
+		httpClient:               client,
+		topicListeners:           make(map[string]map[string]TopicListener),
+		connectionStateListeners: make(map[string]ConnectionStateListener),
+	}
+
+	service.send = make([]chan interface{}, MaxConcurrentSends)
+	for i := range service.send {
+		service.send[i] = make(chan interface{}, SendChanBuffer)
+	}
+
+	return service, nil
+}
+
+// Start is called by the server on server start-up.
+func (rcs *Service) Start() error {
+	rcs.mux.Lock()
+	rcs.leaderListenerId = rcs.server.AddClusterLeaderChangedListener(rcs.onClusterLeaderChange)
+	rcs.mux.Unlock()
+
+	rcs.onClusterLeaderChange()
+
+	return nil
+}
+
+// Shutdown is called by the server on server shutdown.
+func (rcs *Service) Shutdown() error {
+	rcs.server.RemoveClusterLeaderChangedListener(rcs.leaderListenerId)
+	rcs.pause()
+	return nil
+}
+
+// Active returns true if this instance of the remote cluster service is active.
+// The active instance is responsible for pinging and sending messages to remotes.
+func (rcs *Service) Active() bool {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+	return rcs.active
+}
+
+// AddTopicListener registers a callback
+func (rcs *Service) AddTopicListener(topic string, listener TopicListener) string {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	id := model.NewId()
+
+	listeners, ok := rcs.topicListeners[topic]
+	if !ok || listeners == nil {
+		rcs.topicListeners[topic] = make(map[string]TopicListener)
+	}
+	rcs.topicListeners[topic][id] = listener
+	return id
+}
+
+func (rcs *Service) RemoveTopicListener(listenerId string) {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	for topic, listeners := range rcs.topicListeners {
+		if _, ok := listeners[listenerId]; ok {
+			delete(listeners, listenerId)
+			if len(listeners) == 0 {
+				delete(rcs.topicListeners, topic)
+			}
+			break
+		}
+	}
+}
+
+func (rcs *Service) getTopicListeners(topic string) []TopicListener {
+	rcs.mux.RLock()
+	defer rcs.mux.RUnlock()
+
+	listeners, ok := rcs.topicListeners[topic]
+	if !ok {
+		return nil
+	}
+
+	listenersCopy := make([]TopicListener, 0, len(listeners))
+	for _, l := range listeners {
+		listenersCopy = append(listenersCopy, l)
+	}
+	return listenersCopy
+}
+
+func (rcs *Service) AddConnectionStateListener(listener ConnectionStateListener) string {
+	id := model.NewId()
+
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	rcs.connectionStateListeners[id] = listener
+	return id
+}
+
+func (rcs *Service) RemoveConnectionStateListener(listenerId string) {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+	delete(rcs.connectionStateListeners, listenerId)
+}
+
+// onClusterLeaderChange is called whenever the cluster leader may have changed.
+func (rcs *Service) onClusterLeaderChange() {
+	if rcs.server.IsLeader() {
+		rcs.resume()
+	} else {
+		rcs.pause()
+	}
+}
+
+func (rcs *Service) resume() {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	if rcs.active {
+		return // already active
+	}
+	rcs.active = true
+	rcs.done = make(chan struct{})
+
+	if !disablePing {
+		rcs.pingLoop(rcs.done)
+	}
+
+	// create thread pool for concurrent message sending.
+	for i := range rcs.send {
+		go rcs.sendLoop(i, rcs.done)
+	}
+
+	rcs.server.GetLogger().Debug("Remote Cluster Service active")
+}
+
+func (rcs *Service) pause() {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	if !rcs.active {
+		return // already inactive
+	}
+	rcs.active = false
+	close(rcs.done)
+	rcs.done = nil
+
+	rcs.server.GetLogger().Debug("Remote Cluster Service inactive")
+}

services/remotecluster/service_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+)
+
+func TestService_AddTopicListener(t *testing.T) {
+	var count int32
+
+	l1 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		atomic.AddInt32(&count, 1)
+		return nil
+	}
+	l2 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		atomic.AddInt32(&count, 1)
+		return nil
+	}
+	l3 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		atomic.AddInt32(&count, 1)
+		return nil
+	}
+
+	mockServer := newMockServer(t, makeRemoteClusters(NumRemotes, ""))
+	service, err := NewRemoteClusterService(mockServer)
+	require.NoError(t, err)
+
+	l1id := service.AddTopicListener("test", l1)
+	l2id := service.AddTopicListener("test", l2)
+	l3id := service.AddTopicListener("different", l3)
+
+	listeners := service.getTopicListeners("test")
+	assert.Len(t, listeners, 2)
+
+	rc := &model.RemoteCluster{}
+	msg1 := model.RemoteClusterMsg{Topic: "test"}
+	msg2 := model.RemoteClusterMsg{Topic: "different"}
+
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(2), atomic.LoadInt32(&count))
+
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(3), atomic.LoadInt32(&count))
+
+	service.RemoveTopicListener(l1id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(4), atomic.LoadInt32(&count))
+
+	service.RemoveTopicListener(l2id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(4), atomic.LoadInt32(&count))
+
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(5), atomic.LoadInt32(&count))
+
+	service.RemoveTopicListener(l3id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(5), atomic.LoadInt32(&count))
+
+	listeners = service.getTopicListeners("test")
+	assert.Empty(t, listeners)
+}

services/sharedchannel/attachment.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+// postToAttachments returns the file attachments for a post that need to be synchronized.
+func (scs *Service) postToAttachments(post *model.Post, rc *model.RemoteCluster) ([]*model.FileInfo, error) {
+	infos := make([]*model.FileInfo, 0)
+
+	fis, err := scs.server.GetStore().FileInfo().GetForPost(post.Id, false, true, true)
+	if err != nil {
+		return nil, fmt.Errorf("could not get file info for attachment: %w", err)
+	}
+
+	for _, fi := range fis {
+		if scs.shouldSyncAttachment(fi, rc) {
+			infos = append(infos, fi)
+		}
+	}
+	return infos, nil
+}
+
+// postsToAttachments returns the file attachments for a slice of posts that need to be synchronized.
+func (scs *Service) shouldSyncAttachment(fi *model.FileInfo, rc *model.RemoteCluster) bool {
+	sca, err := scs.server.GetStore().SharedChannel().GetAttachment(fi.Id, rc.RemoteId)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error fetching shared channel attachment",
+				mlog.String("file_id", fi.Id),
+				mlog.String("remote_id", rc.RemoteId),
+				mlog.Err(err),
+			)
+		}
+		// no record so sync is needed
+		return true
+	}
+
+	return sca.LastSyncAt < fi.UpdateAt
+}
+
+// sendAttachmentForRemote asynchronously sends a file attachment to a remote cluster.
+func (scs *Service) sendAttachmentForRemote(fi *model.FileInfo, post *model.Post, rc *model.RemoteCluster) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot update remote cluster for remote id %s; Remote Cluster Service not enabled", rc.RemoteId)
+	}
+
+	us := &model.UploadSession{
+		Id:        model.NewId(),
+		Type:      model.UploadTypeAttachment,
+		UserId:    post.UserId,
+		ChannelId: post.ChannelId,
+		Filename:  fi.Name,
+		FileSize:  fi.Size,
+		RemoteId:  rc.RemoteId,
+		ReqFileId: fi.Id,
+	}
+
+	payload, err := json.Marshal(us)
+	if err != nil {
+		return err
+	}
+
+	msg := model.NewRemoteClusterMsg(TopicUploadCreate, payload)
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	var usResp model.UploadSession
+	var respErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	// creating the upload session on the remote server needs to be done synchronously.
+	err = rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		defer wg.Done()
+		if err != nil {
+			respErr = err
+			return
+		}
+		if !resp.IsSuccess() {
+			respErr = errors.New(resp.Err)
+			return
+		}
+		respErr = json.Unmarshal(resp.Payload, &usResp)
+	})
+
+	if err != nil {
+		return fmt.Errorf("error sending create upload session to remote %s for post %s: %w", rc.RemoteId, post.Id, err)
+	}
+
+	wg.Wait()
+
+	if respErr != nil {
+		return fmt.Errorf("invalid create upload session response for remote %s and post %s: %w", rc.RemoteId, post.Id, respErr)
+	}
+
+	ctx2, cancel2 := context.WithTimeout(context.Background(), remotecluster.SendFileTimeout)
+	defer cancel2()
+
+	return rcs.SendFile(ctx2, &usResp, fi, rc, scs.app, func(us *model.UploadSession, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		if err != nil {
+			return // this means the response could not be parsed; already logged
+		}
+
+		if !resp.IsSuccess() {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "send file failed",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.String("err", resp.Err),
+			)
+			return
+		}
+
+		// response payload should be a model.FileInfo.
+		var fi model.FileInfo
+		if err2 := json.Unmarshal(resp.Payload, &fi); err2 != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "invalid file info response after send file",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.Err(err2),
+			)
+			return
+		}
+
+		// save file attachment record in SharedChannelAttachments table
+		sca := &model.SharedChannelAttachment{
+			FileId:   fi.Id,
+			RemoteId: rc.RemoteId,
+		}
+		if _, err2 := scs.server.GetStore().SharedChannel().UpsertAttachment(sca); err2 != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error saving SharedChannelAttachment",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.Err(err2),
+			)
+			return
+		}
+
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "send file successful",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("uploadId", usResp.Id),
+		)
+	})
+}
+
+// onReceiveUploadCreate is called when a message requesting to create an upload session is received.  An upload session is
+// created and the id returned in the response.
+func (scs *Service) onReceiveUploadCreate(msg model.RemoteClusterMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	var us model.UploadSession
+
+	if err := json.Unmarshal(msg.Payload, &us); err != nil {
+		return fmt.Errorf("invalid upload session request: %w", err)
+	}
+
+	// make sure channel is shared for the remote sender
+	if _, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(us.ChannelId, rc.RemoteId); err != nil {
+		return fmt.Errorf("could not validate upload session for remote: %w", err)
+	}
+
+	us.RemoteId = rc.RemoteId // don't let remotes try to impersonate each other
+
+	// create upload session.
+	usSaved, appErr := scs.app.CreateUploadSession(&us)
+	if appErr != nil {
+		return appErr
+	}
+
+	response.SetPayload(usSaved)
+	return nil
+}

services/sharedchannel/channelinvite.go

@@ -0,0 +1,220 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+// channelInviteMsg represents an invitation for a remote cluster to start sharing a channel.
+type channelInviteMsg struct {
+	ChannelId            string   `json:"channel_id"`
+	TeamId               string   `json:"team_id"`
+	ReadOnly             bool     `json:"read_only"`
+	Name                 string   `json:"name"`
+	DisplayName          string   `json:"display_name"`
+	Header               string   `json:"header"`
+	Purpose              string   `json:"purpose"`
+	Type                 string   `json:"type"`
+	DirectParticipantIDs []string `json:"direct_participant_ids"`
+}
+
+type InviteOption func(msg *channelInviteMsg)
+
+func WithDirectParticipantID(participantID string) InviteOption {
+	return func(msg *channelInviteMsg) {
+		msg.DirectParticipantIDs = append(msg.DirectParticipantIDs, participantID)
+	}
+}
+
+// SendChannelInvite asynchronously sends a channel invite to a remote cluster. The remote cluster is
+// expected to create a new channel with the same channel id, and respond with status OK.
+// If an error occurs on the remote cluster then an ephemeral message is posted to in the channel for userId.
+func (scs *Service) SendChannelInvite(channel *model.Channel, userId string, description string, rc *model.RemoteCluster, options ...InviteOption) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot invite remote cluster for channel id %s; Remote Cluster Service not enabled", channel.Id)
+	}
+
+	sc, err := scs.server.GetStore().SharedChannel().Get(channel.Id)
+	if err != nil {
+		return err
+	}
+
+	invite := channelInviteMsg{
+		ChannelId:   channel.Id,
+		TeamId:      rc.RemoteTeamId,
+		ReadOnly:    sc.ReadOnly,
+		Name:        sc.ShareName,
+		DisplayName: sc.ShareDisplayName,
+		Header:      sc.ShareHeader,
+		Purpose:     sc.SharePurpose,
+		Type:        channel.Type,
+	}
+
+	for _, option := range options {
+		option(&invite)
+	}
+
+	json, err := json.Marshal(invite)
+	if err != nil {
+		return err
+	}
+
+	msg := model.NewRemoteClusterMsg(TopicChannelInvite, json)
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	return rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		if err != nil || !resp.IsSuccess() {
+			scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("Error sending channel invite for %s: %s", rc.DisplayName, combineErrors(err, resp.Err)))
+			return
+		}
+
+		scr := &model.SharedChannelRemote{
+			ChannelId:         sc.ChannelId,
+			Description:       description,
+			CreatorId:         userId,
+			RemoteId:          rc.RemoteId,
+			IsInviteAccepted:  true,
+			IsInviteConfirmed: true,
+		}
+		if _, err = scs.server.GetStore().SharedChannel().SaveRemote(scr); err != nil {
+			scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("Error confirming channel invite for %s: %v", rc.DisplayName, err))
+			return
+		}
+		scs.NotifyChannelChanged(sc.ChannelId)
+		scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("`%s` has been added to channel.", rc.DisplayName))
+	})
+}
+
+func combineErrors(err error, serror string) string {
+	var sb strings.Builder
+	if err != nil {
+		sb.WriteString(err.Error())
+	}
+	if serror != "" {
+		if sb.Len() > 0 {
+			sb.WriteString("; ")
+		}
+		sb.WriteString(serror)
+	}
+	return sb.String()
+}
+
+func (scs *Service) onReceiveChannelInvite(msg model.RemoteClusterMsg, rc *model.RemoteCluster, _ *remotecluster.Response) error {
+	if len(msg.Payload) == 0 {
+		return nil
+	}
+
+	var invite channelInviteMsg
+
+	if err := json.Unmarshal(msg.Payload, &invite); err != nil {
+		return fmt.Errorf("invalid channel invite: %w", err)
+	}
+
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Channel invite received",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("channel_id", invite.ChannelId),
+		mlog.String("channel_name", invite.Name),
+		mlog.String("team_id", invite.TeamId),
+	)
+
+	// create channel if it doesn't exist; the channel may already exist, such as if it was shared then unshared at some point.
+	channel, err := scs.server.GetStore().Channel().Get(invite.ChannelId, true)
+	if err != nil {
+		if channel, err = scs.handleChannelCreation(invite, rc); err != nil {
+			return err
+		}
+	}
+
+	if invite.ReadOnly {
+		if err := scs.makeChannelReadOnly(channel); err != nil {
+			return fmt.Errorf("cannot make channel readonly `%s`: %w", invite.ChannelId, err)
+		}
+	}
+
+	sharedChannel := &model.SharedChannel{
+		ChannelId:        channel.Id,
+		TeamId:           channel.TeamId,
+		Home:             false,
+		ReadOnly:         invite.ReadOnly,
+		ShareName:        channel.Name,
+		ShareDisplayName: channel.DisplayName,
+		SharePurpose:     channel.Purpose,
+		ShareHeader:      channel.Header,
+		CreatorId:        rc.CreatorId,
+		RemoteId:         rc.RemoteId,
+		Type:             channel.Type,
+	}
+
+	if _, err := scs.server.GetStore().SharedChannel().Save(sharedChannel); err != nil {
+		scs.app.PermanentDeleteChannel(channel)
+		return fmt.Errorf("cannot create shared channel (channel_id=%s): %w", invite.ChannelId, err)
+	}
+
+	sharedChannelRemote := &model.SharedChannelRemote{
+		Id:                model.NewId(),
+		ChannelId:         channel.Id,
+		Description:       invite.DisplayName,
+		CreatorId:         channel.CreatorId,
+		IsInviteAccepted:  true,
+		IsInviteConfirmed: true,
+		RemoteId:          rc.RemoteId,
+	}
+
+	if _, err := scs.server.GetStore().SharedChannel().SaveRemote(sharedChannelRemote); err != nil {
+		scs.app.PermanentDeleteChannel(channel)
+		scs.server.GetStore().SharedChannel().Delete(sharedChannel.ChannelId)
+		return fmt.Errorf("cannot create shared channel remote (channel_id=%s): %w", invite.ChannelId, err)
+	}
+	return nil
+}
+
+func (scs *Service) handleChannelCreation(invite channelInviteMsg, rc *model.RemoteCluster) (*model.Channel, error) {
+	if invite.Type == model.CHANNEL_DIRECT {
+		return scs.createDirectChannel(invite)
+	}
+
+	channelNew := &model.Channel{
+		Id:          invite.ChannelId,
+		TeamId:      invite.TeamId,
+		Type:        invite.Type,
+		DisplayName: invite.DisplayName,
+		Name:        invite.Name,
+		Header:      invite.Header,
+		Purpose:     invite.Purpose,
+		CreatorId:   rc.CreatorId,
+		Shared:      model.NewBool(true),
+	}
+
+	// check user perms?
+	channel, appErr := scs.app.CreateChannelWithUser(channelNew, rc.CreatorId)
+	if appErr != nil {
+		return nil, fmt.Errorf("cannot create channel `%s`: %w", invite.ChannelId, appErr)
+	}
+
+	return channel, nil
+}
+
+func (scs *Service) createDirectChannel(invite channelInviteMsg) (*model.Channel, error) {
+	if len(invite.DirectParticipantIDs) != 2 {
+		return nil, fmt.Errorf("cannot create direct channel `%s` insufficient participant count `%d`", invite.ChannelId, len(invite.DirectParticipantIDs))
+	}
+
+	channel, err := scs.app.GetOrCreateDirectChannel(invite.DirectParticipantIDs[0], invite.DirectParticipantIDs[1], model.WithID(invite.ChannelId))
+	if err != nil {
+		return nil, fmt.Errorf("cannot create direct channel `%s`: %w", invite.ChannelId, err)
+	}
+
+	return channel, nil
+}

services/sharedchannel/channelinvite_test.go

@@ -0,0 +1,197 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+	"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
+)
+
+type mockLogger struct {
+	mlog.LoggerIFace
+}
+
+func (ml *mockLogger) Log(level mlog.LogLevel, s string, flds ...mlog.Field) {}
+
+func TestOnReceiveChannelInvite(t *testing.T) {
+	t.Run("when msg payload is empty, it does nothing", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger := &mockLogger{}
+		mockServer.On("GetLogger").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+
+		remoteCluster := &model.RemoteCluster{}
+		msg := model.RemoteClusterMsg{}
+
+		err := scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+		mockStore.AssertNotCalled(t, "Channel")
+	})
+
+	t.Run("when invitation prescribes a readonly channel, it does create a readonly channel", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger := &mockLogger{}
+		mockServer.On("GetLogger").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{DisplayName: "test"}
+		invitation := channelInviteMsg{
+			ChannelId: model.NewId(),
+			TeamId:    model.NewId(),
+			ReadOnly:  true,
+			Type:      "0",
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		mockSharedChannelStore := mocks.SharedChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(channel, nil)
+		mockSharedChannelStore.On("Save", mock.Anything).Return(nil, nil)
+		mockSharedChannelStore.On("SaveRemote", mock.Anything).Return(nil, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+		mockStore.On("SharedChannel").Return(&mockSharedChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+		createPostPermission := model.ChannelModeratedPermissionsMap[model.PERMISSION_CREATE_POST.Id]
+		createReactionPermission := model.ChannelModeratedPermissionsMap[model.PERMISSION_ADD_REACTION.Id]
+		updateMap := model.ChannelModeratedRolesPatch{
+			Guests:  model.NewBool(false),
+			Members: model.NewBool(false),
+		}
+
+		readonlyChannelModerations := []*model.ChannelModerationPatch{
+			{
+				Name:  &createPostPermission,
+				Roles: &updateMap,
+			},
+			{
+				Name:  &createReactionPermission,
+				Roles: &updateMap,
+			},
+		}
+		mockApp.On("PatchChannelModerationsForChannel", channel, readonlyChannelModerations).Return(nil, nil)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+	})
+
+	t.Run("when invitation prescribes a readonly channel and readonly update fails, it returns an error", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger := &mockLogger{}
+		mockServer.On("GetLogger").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{DisplayName: "test"}
+		invitation := channelInviteMsg{
+			ChannelId: model.NewId(),
+			TeamId:    model.NewId(),
+			ReadOnly:  true,
+			Type:      "0",
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(channel, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+		appErr := model.NewAppError("foo", "bar", nil, "boom", http.StatusBadRequest)
+
+		mockApp.On("PatchChannelModerationsForChannel", channel, mock.Anything).Return(nil, appErr)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.Error(t, err)
+		assert.Equal(t, fmt.Sprintf("cannot make channel readonly `%s`: foo: bar, boom", invitation.ChannelId), err.Error())
+	})
+
+	t.Run("when invitation prescribes a direct channel, it does create a direct channel", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger := &mockLogger{}
+		mockServer.On("GetLogger").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{DisplayName: "test", CreatorId: model.NewId()}
+		invitation := channelInviteMsg{
+			ChannelId:            model.NewId(),
+			TeamId:               model.NewId(),
+			ReadOnly:             false,
+			Type:                 model.CHANNEL_DIRECT,
+			DirectParticipantIDs: []string{model.NewId(), model.NewId()},
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		mockSharedChannelStore := mocks.SharedChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(nil, errors.New("boom"))
+		mockSharedChannelStore.On("Save", mock.Anything).Return(nil, nil)
+		mockSharedChannelStore.On("SaveRemote", mock.Anything).Return(nil, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+		mockStore.On("SharedChannel").Return(&mockSharedChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+
+		mockApp.On("GetOrCreateDirectChannel", invitation.DirectParticipantIDs[0], invitation.DirectParticipantIDs[1], mock.AnythingOfType("model.ChannelOption")).Return(channel, nil)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+	})
+}

services/sharedchannel/getpostssince.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+type sinceResult struct {
+	posts     []*model.Post
+	hasMore   bool
+	nextSince int64
+}
+
+// getPostsSince fetches posts that need to be synchronized with a remote cluster.
+// There is a soft cap on the number of posts that will be synchronized in a single pass (MaxPostsPerSync).
+//
+// There is a special case where multiple posts have the same UpdateAt value. It is vital that this method
+// include all posts within that millisecond so that subsequent calls can use an incremented `since`. If this
+// method were to be called repeatedly with the same `since` value the same records would be returned each time
+// and the sync would never move forward.
+//
+// A boolean is also returned to indicate if there are more posts to be synchronized (true) or not (false).
+func (scs *Service) getPostsSince(channelId string, rc *model.RemoteCluster, since int64) (sinceResult, error) {
+	opts := model.GetPostsSinceForSyncOptions{
+		ChannelId:      channelId,
+		Since:          since,
+		IncludeDeleted: true,
+		Limit:          MaxPostsPerSync + 1, // ask for 1 more than needed to peek at first post in next batch
+	}
+	posts, err := scs.server.GetStore().Post().GetPostsSinceForSync(opts, true)
+	if err != nil {
+		return sinceResult{}, err
+	}
+
+	if len(posts) == 0 {
+		return sinceResult{nextSince: since}, nil
+	}
+
+	var hasMore bool
+	if len(posts) > MaxPostsPerSync {
+		hasMore = true
+		peekUpdateAt := posts[len(posts)-1].UpdateAt
+		posts = posts[:MaxPostsPerSync] // trim the peeked at record
+
+		// If the last post to be synchronized has the same Update value as the first post in the next batch
+		// then we need to grab the rest of the posts for that millisecond to ensure the next call can have an
+		// incremented `since`.
+		if peekUpdateAt == posts[len(posts)-1].UpdateAt {
+			opts.Since = peekUpdateAt
+			opts.Until = opts.Since
+			opts.Limit = 1000
+			opts.Offset = countPostsAtMillisecond(posts, peekUpdateAt)
+
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "getPostsSince handling updateAt collision",
+				mlog.String("remote", rc.DisplayName),
+				mlog.Int64("update_at", peekUpdateAt),
+				mlog.Int("offset", opts.Offset),
+			)
+
+			morePosts, err := scs.server.GetStore().Post().GetPostsSinceForSync(opts, true)
+			if err != nil {
+				return sinceResult{}, err
+			}
+			posts = append(posts, morePosts...)
+		}
+	}
+	return sinceResult{posts: posts, hasMore: hasMore, nextSince: posts[len(posts)-1].UpdateAt + 1}, nil
+}
+
+func countPostsAtMillisecond(posts []*model.Post, milli int64) int {
+	// walk backward through the slice until we find a post with UpdateAt that differs from milli.
+	var count int
+	for i := len(posts) - 1; i >= 0; i-- {
+		if posts[i].UpdateAt != milli {
+			return count
+		}
+		count++
+	}
+	return count
+}

services/sharedchannel/mock_AppIface_test.go

@@ -0,0 +1,338 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+// Regenerate this file using `make sharedchannel-mocks`.
+
+package sharedchannel
+
+import (
+	filestore "github.com/mattermost/mattermost-server/v5/shared/filestore"
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/mattermost/mattermost-server/v5/model"
+)
+
+// MockAppIface is an autogenerated mock type for the AppIface type
+type MockAppIface struct {
+	mock.Mock
+}
+
+// AddUserToChannel provides a mock function with given fields: user, channel
+func (_m *MockAppIface) AddUserToChannel(user *model.User, channel *model.Channel) (*model.ChannelMember, *model.AppError) {
+	ret := _m.Called(user, channel)
+
+	var r0 *model.ChannelMember
+	if rf, ok := ret.Get(0).(func(*model.User, *model.Channel) *model.ChannelMember); ok {
+		r0 = rf(user, channel)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.ChannelMember)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.User, *model.Channel) *model.AppError); ok {
+		r1 = rf(user, channel)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// AddUserToTeamByTeamId provides a mock function with given fields: teamId, user
+func (_m *MockAppIface) AddUserToTeamByTeamId(teamId string, user *model.User) *model.AppError {
+	ret := _m.Called(teamId, user)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(string, *model.User) *model.AppError); ok {
+		r0 = rf(teamId, user)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// CreateChannelWithUser provides a mock function with given fields: channel, userId
+func (_m *MockAppIface) CreateChannelWithUser(channel *model.Channel, userId string) (*model.Channel, *model.AppError) {
+	ret := _m.Called(channel, userId)
+
+	var r0 *model.Channel
+	if rf, ok := ret.Get(0).(func(*model.Channel, string) *model.Channel); ok {
+		r0 = rf(channel, userId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Channel)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Channel, string) *model.AppError); ok {
+		r1 = rf(channel, userId)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// CreatePost provides a mock function with given fields: post, channel, triggerWebhooks, setOnline
+func (_m *MockAppIface) CreatePost(post *model.Post, channel *model.Channel, triggerWebhooks bool, setOnline bool) (*model.Post, *model.AppError) {
+	ret := _m.Called(post, channel, triggerWebhooks, setOnline)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(*model.Post, *model.Channel, bool, bool) *model.Post); ok {
+		r0 = rf(post, channel, triggerWebhooks, setOnline)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Post, *model.Channel, bool, bool) *model.AppError); ok {
+		r1 = rf(post, channel, triggerWebhooks, setOnline)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// CreateUploadSession provides a mock function with given fields: us
+func (_m *MockAppIface) CreateUploadSession(us *model.UploadSession) (*model.UploadSession, *model.AppError) {
+	ret := _m.Called(us)
+
+	var r0 *model.UploadSession
+	if rf, ok := ret.Get(0).(func(*model.UploadSession) *model.UploadSession); ok {
+		r0 = rf(us)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.UploadSession)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.UploadSession) *model.AppError); ok {
+		r1 = rf(us)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// DeletePost provides a mock function with given fields: postID, deleteByID
+func (_m *MockAppIface) DeletePost(postID string, deleteByID string) (*model.Post, *model.AppError) {
+	ret := _m.Called(postID, deleteByID)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(string, string) *model.Post); ok {
+		r0 = rf(postID, deleteByID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, string) *model.AppError); ok {
+		r1 = rf(postID, deleteByID)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// DeleteReactionForPost provides a mock function with given fields: reaction
+func (_m *MockAppIface) DeleteReactionForPost(reaction *model.Reaction) *model.AppError {
+	ret := _m.Called(reaction)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Reaction) *model.AppError); ok {
+		r0 = rf(reaction)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// FileReader provides a mock function with given fields: path
+func (_m *MockAppIface) FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
+	ret := _m.Called(path)
+
+	var r0 filestore.ReadCloseSeeker
+	if rf, ok := ret.Get(0).(func(string) filestore.ReadCloseSeeker); ok {
+		r0 = rf(path)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(filestore.ReadCloseSeeker)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
+		r1 = rf(path)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// GetOrCreateDirectChannel provides a mock function with given fields: userId, otherUserId, channelOptions
+func (_m *MockAppIface) GetOrCreateDirectChannel(userId string, otherUserId string, channelOptions ...model.ChannelOption) (*model.Channel, *model.AppError) {
+	_va := make([]interface{}, len(channelOptions))
+	for _i := range channelOptions {
+		_va[_i] = channelOptions[_i]
+	}
+	var _ca []interface{}
+	_ca = append(_ca, userId, otherUserId)
+	_ca = append(_ca, _va...)
+	ret := _m.Called(_ca...)
+
+	var r0 *model.Channel
+	if rf, ok := ret.Get(0).(func(string, string, ...model.ChannelOption) *model.Channel); ok {
+		r0 = rf(userId, otherUserId, channelOptions...)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Channel)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, string, ...model.ChannelOption) *model.AppError); ok {
+		r1 = rf(userId, otherUserId, channelOptions...)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// PatchChannelModerationsForChannel provides a mock function with given fields: channel, channelModerationsPatch
+func (_m *MockAppIface) PatchChannelModerationsForChannel(channel *model.Channel, channelModerationsPatch []*model.ChannelModerationPatch) ([]*model.ChannelModeration, *model.AppError) {
+	ret := _m.Called(channel, channelModerationsPatch)
+
+	var r0 []*model.ChannelModeration
+	if rf, ok := ret.Get(0).(func(*model.Channel, []*model.ChannelModerationPatch) []*model.ChannelModeration); ok {
+		r0 = rf(channel, channelModerationsPatch)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.ChannelModeration)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Channel, []*model.ChannelModerationPatch) *model.AppError); ok {
+		r1 = rf(channel, channelModerationsPatch)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// PermanentDeleteChannel provides a mock function with given fields: channel
+func (_m *MockAppIface) PermanentDeleteChannel(channel *model.Channel) *model.AppError {
+	ret := _m.Called(channel)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Channel) *model.AppError); ok {
+		r0 = rf(channel)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// SaveReactionForPost provides a mock function with given fields: reaction
+func (_m *MockAppIface) SaveReactionForPost(reaction *model.Reaction) (*model.Reaction, *model.AppError) {
+	ret := _m.Called(reaction)
+
+	var r0 *model.Reaction
+	if rf, ok := ret.Get(0).(func(*model.Reaction) *model.Reaction); ok {
+		r0 = rf(reaction)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Reaction)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Reaction) *model.AppError); ok {
+		r1 = rf(reaction)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// SendEphemeralPost provides a mock function with given fields: userId, post
+func (_m *MockAppIface) SendEphemeralPost(userId string, post *model.Post) *model.Post {
+	ret := _m.Called(userId, post)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(string, *model.Post) *model.Post); ok {
+		r0 = rf(userId, post)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	return r0
+}
+
+// UpdatePost provides a mock function with given fields: post, safeUpdate
+func (_m *MockAppIface) UpdatePost(post *model.Post, safeUpdate bool) (*model.Post, *model.AppError) {
+	ret := _m.Called(post, safeUpdate)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(*model.Post, bool) *model.Post); ok {
+		r0 = rf(post, safeUpdate)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*model.Post, bool) *model.AppError); ok {
+		r1 = rf(post, safeUpdate)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}

services/sharedchannel/mock_ServerIface_test.go

@@ -0,0 +1,118 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+// Regenerate this file using `make sharedchannel-mocks`.
+
+package sharedchannel
+
+import (
+	mlog "github.com/mattermost/mattermost-server/v5/shared/mlog"
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/mattermost/mattermost-server/v5/model"
+
+	remotecluster "github.com/mattermost/mattermost-server/v5/services/remotecluster"
+
+	store "github.com/mattermost/mattermost-server/v5/store"
+)
+
+// MockServerIface is an autogenerated mock type for the ServerIface type
+type MockServerIface struct {
+	mock.Mock
+}
+
+// AddClusterLeaderChangedListener provides a mock function with given fields: listener
+func (_m *MockServerIface) AddClusterLeaderChangedListener(listener func()) string {
+	ret := _m.Called(listener)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(func()) string); ok {
+		r0 = rf(listener)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// Config provides a mock function with given fields:
+func (_m *MockServerIface) Config() *model.Config {
+	ret := _m.Called()
+
+	var r0 *model.Config
+	if rf, ok := ret.Get(0).(func() *model.Config); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Config)
+		}
+	}
+
+	return r0
+}
+
+// GetLogger provides a mock function with given fields:
+func (_m *MockServerIface) GetLogger() mlog.LoggerIFace {
+	ret := _m.Called()
+
+	var r0 mlog.LoggerIFace
+	if rf, ok := ret.Get(0).(func() mlog.LoggerIFace); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(mlog.LoggerIFace)
+		}
+	}
+
+	return r0
+}
+
+// GetRemoteClusterService provides a mock function with given fields:
+func (_m *MockServerIface) GetRemoteClusterService() remotecluster.RemoteClusterServiceIFace {
+	ret := _m.Called()
+
+	var r0 remotecluster.RemoteClusterServiceIFace
+	if rf, ok := ret.Get(0).(func() remotecluster.RemoteClusterServiceIFace); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(remotecluster.RemoteClusterServiceIFace)
+		}
+	}
+
+	return r0
+}
+
+// GetStore provides a mock function with given fields:
+func (_m *MockServerIface) GetStore() store.Store {
+	ret := _m.Called()
+
+	var r0 store.Store
+	if rf, ok := ret.Get(0).(func() store.Store); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.Store)
+		}
+	}
+
+	return r0
+}
+
+// IsLeader provides a mock function with given fields:
+func (_m *MockServerIface) IsLeader() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// RemoveClusterLeaderChangedListener provides a mock function with given fields: id
+func (_m *MockServerIface) RemoveClusterLeaderChangedListener(id string) {
+	_m.Called(id)
+}

services/sharedchannel/msg.go

@@ -0,0 +1,216 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+// syncMsg represents a change in content (post add/edit/delete, reaction add/remove, users).
+// It is sent to remote clusters as the payload of a `RemoteClusterMsg`.
+type syncMsg struct {
+	ChannelId   string            `json:"channel_id"`
+	PostId      string            `json:"post_id"`
+	Post        *model.Post       `json:"post"`
+	Users       []*model.User     `json:"users"`
+	Reactions   []*model.Reaction `json:"reactions"`
+	Attachments []*model.FileInfo `json:"-"`
+}
+
+func (sm syncMsg) ToJSON() ([]byte, error) {
+	b, err := json.Marshal(sm)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func (sm syncMsg) String() string {
+	json, err := sm.ToJSON()
+	if err != nil {
+		return ""
+	}
+	return string(json)
+}
+
+type userCache map[string]struct{}
+
+func (u userCache) Has(id string) bool {
+	_, ok := u[id]
+	return ok
+}
+
+func (u userCache) Add(id string) {
+	u[id] = struct{}{}
+}
+
+// postsToSyncMessages takes a slice of posts and converts to a `RemoteClusterMsg` which can be
+// sent to a remote cluster.
+func (scs *Service) postsToSyncMessages(posts []*model.Post, rc *model.RemoteCluster, nextSyncAt int64) ([]syncMsg, error) {
+	syncMessages := make([]syncMsg, 0, len(posts))
+
+	uCache := make(userCache)
+
+	for _, p := range posts {
+		if p.IsSystemMessage() { // don't sync system messages
+			continue
+		}
+
+		// any reactions originating from the remote cluster are filtered out
+		reactions, err := scs.server.GetStore().Reaction().GetForPostSince(p.Id, nextSyncAt, rc.RemoteId, true)
+		if err != nil {
+			return nil, err
+		}
+
+		postSync := p
+
+		// Don't resend an existing post where only the reactions changed.
+		// Posts we must send:
+		//   - new posts (EditAt == 0)
+		//   - edited posts (EditAt >= nextSyncAt)
+		//   - deleted posts (DeleteAt > 0)
+		if p.EditAt > 0 && p.EditAt < nextSyncAt && p.DeleteAt == 0 {
+			postSync = nil
+		}
+
+		// Don't send a deleted post if it is just the original copy from an edit.
+		if p.DeleteAt > 0 && p.OriginalId != "" {
+			postSync = nil
+		}
+
+		// don't sync a post back to the remote it came from.
+		if p.RemoteId != nil && *p.RemoteId == rc.RemoteId {
+			postSync = nil
+		}
+
+		var attachments []*model.FileInfo
+		if postSync != nil {
+			// parse out all permalinks in the message.
+			postSync.Message = scs.processPermalinkToRemote(postSync)
+
+			// get any file attachments
+			attachments, err = scs.postToAttachments(postSync, rc)
+			if err != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Could not fetch attachments for post",
+					mlog.String("post_id", postSync.Id),
+					mlog.Err(err),
+				)
+			}
+		}
+
+		// any users originating from the remote cluster are filtered out
+		users := scs.usersForPost(postSync, reactions, rc, uCache)
+
+		// if everything was filtered out then don't send an empty message.
+		if postSync == nil && len(reactions) == 0 && len(users) == 0 {
+			continue
+		}
+
+		sm := syncMsg{
+			ChannelId:   p.ChannelId,
+			PostId:      p.Id,
+			Post:        postSync,
+			Users:       users,
+			Reactions:   reactions,
+			Attachments: attachments,
+		}
+		syncMessages = append(syncMessages, sm)
+	}
+	return syncMessages, nil
+}
+
+// usersForPost provides a list of Users associated with the post that need to be synchronized.
+// The user cache ensures the same user is not synchronized redundantly if they appear in multiple
+// posts for this sync batch.
+func (scs *Service) usersForPost(post *model.Post, reactions []*model.Reaction, rc *model.RemoteCluster, uCache userCache) []*model.User {
+	userIds := make([]string, 0)
+
+	if post != nil && !uCache.Has(post.UserId) {
+		userIds = append(userIds, post.UserId)
+		uCache.Add(post.UserId)
+	}
+
+	for _, r := range reactions {
+		if !uCache.Has(r.UserId) {
+			userIds = append(userIds, r.UserId)
+			uCache.Add(r.UserId)
+		}
+	}
+
+	// TODO: extract @mentions to local users and sync those as well?
+
+	users := make([]*model.User, 0)
+
+	for _, id := range userIds {
+		user, err := scs.server.GetStore().User().Get(context.Background(), id)
+		if err == nil {
+			if sync, err2 := scs.shouldUserSync(user, rc); err2 != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Could not find user for post",
+					mlog.String("user_id", id),
+					mlog.Err(err2))
+				continue
+			} else if sync {
+				users = append(users, sanitizeUserForSync(user))
+			}
+		} else {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error checking if user should sync",
+				mlog.String("user_id", id),
+				mlog.Err(err))
+		}
+	}
+	return users
+}
+
+func sanitizeUserForSync(user *model.User) *model.User {
+	user.Password = model.NewId()
+	user.AuthData = nil
+	user.AuthService = ""
+	user.Roles = "system_user"
+	user.AllowMarketing = false
+	user.Props = model.StringMap{}
+	user.NotifyProps = model.StringMap{}
+	user.LastPasswordUpdate = 0
+	user.LastPictureUpdate = 0
+	user.FailedAttempts = 0
+	user.MfaActive = false
+	user.MfaSecret = ""
+
+	return user
+}
+
+// shouldUserSync determines if a user needs to be synchronized.
+// User should be synchronized if it has no entry in the SharedChannelUsers table,
+// or there is an entry but the LastSyncAt is less than user.UpdateAt
+func (scs *Service) shouldUserSync(user *model.User, rc *model.RemoteCluster) (bool, error) {
+	// don't sync users with the remote they originated from.
+	if user.RemoteId != nil && *user.RemoteId == rc.RemoteId {
+		return false, nil
+	}
+
+	scu, err := scs.server.GetStore().SharedChannel().GetUser(user.Id, rc.RemoteId)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return false, err
+		}
+
+		// user not in the SharedChannelUsers table, so we must add them.
+		scu = &model.SharedChannelUser{
+			UserId:   user.Id,
+			RemoteId: rc.RemoteId,
+		}
+		if _, err = scs.server.GetStore().SharedChannel().SaveUser(scu); err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error adding user to shared channel users",
+				mlog.String("remote_id", rc.RemoteId),
+				mlog.String("user_id", user.Id),
+			)
+		}
+	} else if scu.LastSyncAt >= user.UpdateAt {
+		return false, nil
+	}
+	return true, nil
+}

services/sharedchannel/permalink.go

@@ -0,0 +1,81 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/shared/i18n"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+var (
+	// Team name regex taken from model.IsValidTeamName
+	permaLinkRegex       = regexp.MustCompile(`https?://[0-9.\-A-Za-z]+/[a-z0-9]+([a-z\-0-9]+|(__)?)[a-z0-9]+/pl/([a-zA-Z0-9]+)`)
+	permaLinkSharedRegex = regexp.MustCompile(`https?://[0-9.\-A-Za-z]+/[a-z0-9]+([a-z\-0-9]+|(__)?)[a-z0-9]+/plshared/([a-zA-Z0-9]+)`)
+)
+
+const (
+	permalinkMarker = "plshared"
+)
+
+// processPermalinkToRemote processes all permalinks going towards a remote site.
+func (scs *Service) processPermalinkToRemote(p *model.Post) string {
+	var sent bool
+	return permaLinkRegex.ReplaceAllStringFunc(p.Message, func(msg string) string {
+		// Extract the postID (This is simple enough not to warrant full-blown URL parsing.)
+		lastSlash := strings.LastIndexByte(msg, '/')
+		postID := msg[lastSlash+1:]
+		postList, err := scs.server.GetStore().Post().Get(context.Background(), postID, true, false, false, "")
+		if err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceWarn, "Unable to get post during replacing permalinks", mlog.Err(err))
+			return msg
+		}
+		if len(postList.Order) == 0 {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceWarn, "No post found for permalink", mlog.String("postID", postID))
+			return msg
+		}
+
+		// If postID is for a different channel
+		if postList.Posts[postList.Order[0]].ChannelId != p.ChannelId {
+			// Send ephemeral message to OP (only once per message).
+			if !sent {
+				scs.sendEphemeralPost(p.ChannelId, p.UserId, i18n.T("sharedchannel.permalink.not_found"))
+				sent = true
+			}
+			// But don't modify msg
+			return msg
+		}
+
+		// Otherwise, modify pl to plshared as a marker to be replaced by remote sites
+		return strings.Replace(msg, "/pl/", "/"+permalinkMarker+"/", 1)
+	})
+}
+
+// processPermalinkFromRemote processes all permalinks coming from a remote site.
+func (scs *Service) processPermalinkFromRemote(p *model.Post, team *model.Team) string {
+	return permaLinkSharedRegex.ReplaceAllStringFunc(p.Message, func(remoteLink string) string {
+		// Extract host name
+		parsed, err := url.Parse(remoteLink)
+		if err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceWarn, "Unable to parse the remote link during replacing permalinks", mlog.Err(err))
+			return remoteLink
+		}
+
+		// Replace with local SiteURL
+		parsed.Scheme = scs.siteURL.Scheme
+		parsed.Host = scs.siteURL.Host
+
+		// Replace team name with local team
+		teamEnd := strings.Index(parsed.Path, "/"+permalinkMarker)
+		parsed.Path = "/" + team.Name + parsed.Path[teamEnd:]
+
+		// Replace plshared with pl
+		return strings.Replace(parsed.String(), "/"+permalinkMarker+"/", "/pl/", 1)
+	})
+}

services/sharedchannel/permalink_test.go

@@ -0,0 +1,110 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v5/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v5/utils"
+)
+
+func TestProcessPermalinkToRemote(t *testing.T) {
+	scs := &Service{
+		server: &MockServerIface{},
+		app:    &MockAppIface{},
+	}
+
+	mockStore := &mocks.Store{}
+	mockPostStore := mocks.PostStore{}
+	utils.TranslationsPreInit()
+
+	pl := &model.PostList{}
+	mockPostStore.On("Get", context.Background(), "postID", true, false, false, "").Return(pl, nil)
+
+	mockStore.On("Post").Return(&mockPostStore)
+
+	mockServer := scs.server.(*MockServerIface)
+	mockServer.On("GetStore").Return(mockStore)
+
+	mockApp := scs.app.(*MockAppIface)
+	mockApp.On("SendEphemeralPost", "user", mock.AnythingOfType("*model.Post")).Return(&model.Post{}).Times(1)
+	defer mockApp.AssertExpectations(t)
+
+	t.Run("same channel", func(t *testing.T) {
+		post := &model.Post{
+			Message:   "hello world https://comm.matt.com/team/pl/postID link",
+			ChannelId: "sourceChan",
+			UserId:    "user",
+		}
+
+		*pl = model.PostList{
+			Order: []string{"1"},
+			Posts: map[string]*model.Post{
+				"1": {
+					ChannelId: "sourceChan",
+					UserId:    "user",
+				},
+			},
+		}
+
+		out := scs.processPermalinkToRemote(post)
+		assert.Equal(t, "hello world https://comm.matt.com/team/plshared/postID link", out)
+	})
+
+	t.Run("different channel", func(t *testing.T) {
+		post := &model.Post{
+			Message:   "hello world https://comm.matt.com/team/pl/postID link https://comm.matt.com/team/pl/postID ",
+			ChannelId: "sourceChan",
+			UserId:    "user",
+		}
+
+		*pl = model.PostList{
+			Order: []string{"1"},
+			Posts: map[string]*model.Post{
+				"1": {
+					ChannelId: "otherChan",
+				},
+			},
+		}
+		out := scs.processPermalinkToRemote(post)
+		assert.Equal(t, "hello world https://comm.matt.com/team/pl/postID link https://comm.matt.com/team/pl/postID ", out)
+	})
+}
+
+func TestProcessPermalinkFromRemote(t *testing.T) {
+	t.Run("has match", func(t *testing.T) {
+		parsed, _ := url.Parse("http://mysite.com")
+		scs := &Service{
+			server:  &MockServerIface{},
+			siteURL: parsed,
+		}
+
+		out := scs.processPermalinkFromRemote(&model.Post{Message: "hello world https://comm.matt.com/team/plshared/postID link"},
+			&model.Team{Name: "myteam"})
+		assert.Equal(t,
+			"hello world http://mysite.com/myteam/pl/postID link",
+			out)
+	})
+
+	t.Run("does not match", func(t *testing.T) {
+		parsed, _ := url.Parse("http://mysite.com")
+		scs := &Service{
+			server:  &MockServerIface{},
+			siteURL: parsed,
+		}
+
+		out := scs.processPermalinkFromRemote(&model.Post{Message: "hello world https://comm.matt.com/team/pl/postID link"},
+			&model.Team{Name: "myteam"})
+		assert.Equal(t,
+			"hello world https://comm.matt.com/team/pl/postID link",
+			out)
+	})
+}

services/sharedchannel/response.go

@@ -0,0 +1,10 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+type SyncResponse struct {
+	LastSyncAt int64    `json:"last_sync_at"`
+	PostErrors []string `json:"post_errors"`
+	UsersSyncd []string `json:"users_syncd"`
+}

services/sharedchannel/service.go

@@ -0,0 +1,239 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/filestore"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+	"github.com/mattermost/mattermost-server/v5/store"
+)
+
+const (
+	TopicSync                    = "sharedchannel_sync"
+	TopicChannelInvite           = "sharedchannel_invite"
+	TopicUploadCreate            = "sharedchannel_upload"
+	MaxRetries                   = 3
+	MaxPostsPerSync              = 12 // a bit more than one typical screenfull of posts
+	NotifyRemoteOfflineThreshold = time.Second * 10
+	NotifyMinimumDelay           = time.Second * 2
+)
+
+// Mocks can be re-generated with `make sharedchannel-mocks`.
+type ServerIface interface {
+	Config() *model.Config
+	IsLeader() bool
+	AddClusterLeaderChangedListener(listener func()) string
+	RemoveClusterLeaderChangedListener(id string)
+	GetStore() store.Store
+	GetLogger() mlog.LoggerIFace
+	GetRemoteClusterService() remotecluster.RemoteClusterServiceIFace
+}
+
+type AppIface interface {
+	SendEphemeralPost(userId string, post *model.Post) *model.Post
+	CreateChannelWithUser(channel *model.Channel, userId string) (*model.Channel, *model.AppError)
+	GetOrCreateDirectChannel(userId, otherUserId string, channelOptions ...model.ChannelOption) (*model.Channel, *model.AppError)
+	AddUserToChannel(user *model.User, channel *model.Channel) (*model.ChannelMember, *model.AppError)
+	AddUserToTeamByTeamId(teamId string, user *model.User) *model.AppError
+	PermanentDeleteChannel(channel *model.Channel) *model.AppError
+	CreatePost(post *model.Post, channel *model.Channel, triggerWebhooks bool, setOnline bool) (savedPost *model.Post, err *model.AppError)
+	UpdatePost(post *model.Post, safeUpdate bool) (*model.Post, *model.AppError)
+	DeletePost(postID, deleteByID string) (*model.Post, *model.AppError)
+	SaveReactionForPost(reaction *model.Reaction) (*model.Reaction, *model.AppError)
+	DeleteReactionForPost(reaction *model.Reaction) *model.AppError
+	PatchChannelModerationsForChannel(channel *model.Channel, channelModerationsPatch []*model.ChannelModerationPatch) ([]*model.ChannelModeration, *model.AppError)
+	CreateUploadSession(us *model.UploadSession) (*model.UploadSession, *model.AppError)
+	FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
+}
+
+// errNotFound allows checking against Store.ErrNotFound errors without making Store a dependency.
+type errNotFound interface {
+	IsErrNotFound() bool
+}
+
+// errInvalidInput allows checking against Store.ErrInvalidInput errors without making Store a dependency.
+type errInvalidInput interface {
+	InvalidInputInfo() (entity string, field string, value interface{})
+}
+
+// Service provides shared channel synchronization.
+type Service struct {
+	server       ServerIface
+	app          AppIface
+	changeSignal chan struct{}
+
+	// everything below guarded by `mux`
+	mux                       sync.RWMutex
+	active                    bool
+	leaderListenerId          string
+	connectionStateListenerId string
+	done                      chan struct{}
+	tasks                     map[string]syncTask
+	syncTopicListenerId       string
+	inviteTopicListenerId     string
+	uploadTopicListenerId     string
+	siteURL                   *url.URL
+}
+
+// NewSharedChannelService creates a RemoteClusterService instance.
+func NewSharedChannelService(server ServerIface, app AppIface) (*Service, error) {
+	service := &Service{
+		server:       server,
+		app:          app,
+		changeSignal: make(chan struct{}, 1),
+		tasks:        make(map[string]syncTask),
+	}
+	parsed, err := url.Parse(*server.Config().ServiceSettings.SiteURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse SiteURL: %w", err)
+	}
+	service.siteURL = parsed
+	return service, nil
+}
+
+// Start is called by the server on server start-up.
+func (scs *Service) Start() error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return errors.New("Shared Channel Service cannot activate: requires Remote Cluster Service")
+	}
+
+	scs.mux.Lock()
+	scs.leaderListenerId = scs.server.AddClusterLeaderChangedListener(scs.onClusterLeaderChange)
+	scs.syncTopicListenerId = rcs.AddTopicListener(TopicSync, scs.onReceiveSyncMessage)
+	scs.inviteTopicListenerId = rcs.AddTopicListener(TopicChannelInvite, scs.onReceiveChannelInvite)
+	scs.uploadTopicListenerId = rcs.AddTopicListener(TopicUploadCreate, scs.onReceiveUploadCreate)
+	scs.connectionStateListenerId = rcs.AddConnectionStateListener(scs.onConnectionStateChange)
+	scs.mux.Unlock()
+
+	scs.onClusterLeaderChange()
+
+	return nil
+}
+
+// Shutdown is called by the server on server shutdown.
+func (scs *Service) Shutdown() error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return errors.New("Shared Channel Service cannot shutdown: requires Remote Cluster Service")
+	}
+
+	scs.mux.Lock()
+	id := scs.leaderListenerId
+	rcs.RemoveTopicListener(scs.syncTopicListenerId)
+	scs.syncTopicListenerId = ""
+	rcs.RemoveTopicListener(scs.inviteTopicListenerId)
+	scs.inviteTopicListenerId = ""
+	rcs.RemoveConnectionStateListener(scs.connectionStateListenerId)
+	scs.connectionStateListenerId = ""
+	scs.mux.Unlock()
+
+	scs.server.RemoveClusterLeaderChangedListener(id)
+	scs.pause()
+	return nil
+}
+
+// Active determines whether the service is active on the node or not.
+func (scs *Service) Active() bool {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	return scs.active
+}
+
+func (scs *Service) sendEphemeralPost(channelId string, userId string, text string) {
+	ephemeral := &model.Post{
+		ChannelId: channelId,
+		Message:   text,
+		CreateAt:  model.GetMillis(),
+	}
+	scs.app.SendEphemeralPost(userId, ephemeral)
+}
+
+// onClusterLeaderChange is called whenever the cluster leader may have changed.
+func (scs *Service) onClusterLeaderChange() {
+	if scs.server.IsLeader() {
+		scs.resume()
+	} else {
+		scs.pause()
+	}
+}
+
+func (scs *Service) resume() {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	if scs.active {
+		return // already active
+	}
+
+	scs.active = true
+	scs.done = make(chan struct{})
+
+	go scs.syncLoop(scs.done)
+
+	scs.server.GetLogger().Debug("Shared Channel Service active")
+}
+
+func (scs *Service) pause() {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	if !scs.active {
+		return // already inactive
+	}
+
+	scs.active = false
+	close(scs.done)
+	scs.done = nil
+
+	scs.server.GetLogger().Debug("Shared Channel Service inactive")
+}
+
+// Makes the remote channel to be read-only(announcement mode, only admins can create posts and reactions).
+func (scs *Service) makeChannelReadOnly(channel *model.Channel) *model.AppError {
+	createPostPermission := model.ChannelModeratedPermissionsMap[model.PERMISSION_CREATE_POST.Id]
+	createReactionPermission := model.ChannelModeratedPermissionsMap[model.PERMISSION_ADD_REACTION.Id]
+	updateMap := model.ChannelModeratedRolesPatch{
+		Guests:  model.NewBool(false),
+		Members: model.NewBool(false),
+	}
+
+	readonlyChannelModerations := []*model.ChannelModerationPatch{
+		{
+			Name:  &createPostPermission,
+			Roles: &updateMap,
+		},
+		{
+			Name:  &createReactionPermission,
+			Roles: &updateMap,
+		},
+	}
+
+	_, err := scs.app.PatchChannelModerationsForChannel(channel, readonlyChannelModerations)
+	return err
+}
+
+// onConnectionStateChange is called whenever the connection state of a remote cluster changes,
+// for example when one comes back online.
+func (scs *Service) onConnectionStateChange(rc *model.RemoteCluster, online bool) {
+	if online {
+		// when a previously offline remote comes back online force a sync.
+		scs.ForceSyncForRemote(rc)
+	}
+
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Remote cluster connection status changed",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("remoteId", rc.RemoteId),
+		mlog.Bool("online", online),
+	)
+}

services/sharedchannel/sync_recv.go

@@ -0,0 +1,296 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+func (scs *Service) onReceiveSyncMessage(msg model.RemoteClusterMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	if msg.Topic != TopicSync {
+		return fmt.Errorf("wrong topic, expected `%s`, got `%s`", TopicSync, msg.Topic)
+	}
+
+	if len(msg.Payload) == 0 {
+		return errors.New("empty sync message")
+	}
+
+	if scs.server.GetLogger().IsLevelEnabled(mlog.LvlSharedChannelServiceMessagesInbound) {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceMessagesInbound, "inbound message",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("msg", string(msg.Payload)),
+		)
+	}
+
+	var syncMessages []syncMsg
+
+	if err := json.Unmarshal(msg.Payload, &syncMessages); err != nil {
+		return fmt.Errorf("invalid sync message: %w", err)
+	}
+
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Batch of sync messages received",
+		mlog.String("remote", rc.DisplayName),
+		mlog.Int("sync_msg_count", len(syncMessages)),
+	)
+
+	return scs.processSyncMessages(syncMessages, rc, response)
+}
+
+func (scs *Service) processSyncMessages(syncMessages []syncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	var channel *model.Channel
+	var team *model.Team
+
+	postErrors := make([]string, 0)
+	usersSyncd := make([]string, 0)
+	var lastSyncAt int64
+	var err error
+
+	for _, sm := range syncMessages {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Sync msg received",
+			mlog.String("post_id", sm.PostId),
+			mlog.String("channel_id", sm.ChannelId),
+			mlog.Int("reaction_count", len(sm.Reactions)),
+			mlog.Int("user_count", len(sm.Users)),
+			mlog.Bool("has_post", sm.Post != nil),
+		)
+
+		if channel == nil {
+			if channel, err = scs.server.GetStore().Channel().Get(sm.ChannelId, true); err != nil {
+				// if the channel doesn't exist then none of these sync messages are going to work.
+				return fmt.Errorf("channel not found processing sync messages: %w", err)
+			}
+		}
+
+		// add/update users before posts
+		for _, user := range sm.Users {
+			if userSaved, err := scs.upsertSyncUser(user, channel, rc); err != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync user",
+					mlog.String("post_id", sm.PostId),
+					mlog.String("channel_id", sm.ChannelId),
+					mlog.String("user_id", user.Id),
+					mlog.Err(err))
+			} else {
+				usersSyncd = append(usersSyncd, userSaved.Id)
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "User upserted via sync",
+					mlog.String("post_id", sm.PostId),
+					mlog.String("channel_id", sm.ChannelId),
+					mlog.String("user_id", user.Id),
+				)
+			}
+		}
+
+		if sm.Post != nil {
+			if sm.ChannelId != sm.Post.ChannelId {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "ChannelId mismatch",
+					mlog.String("sm.ChannelId", sm.ChannelId),
+					mlog.String("sm.Post.ChannelId", sm.Post.ChannelId),
+					mlog.String("PostId", sm.Post.Id),
+				)
+				postErrors = append(postErrors, sm.Post.Id)
+				continue
+			}
+
+			if channel.Type != model.CHANNEL_DIRECT && team == nil {
+				var err2 error
+				team, err2 = scs.server.GetStore().Channel().GetTeamForChannel(sm.ChannelId)
+				if err2 != nil {
+					scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error getting Team for Channel",
+						mlog.String("ChannelId", sm.Post.ChannelId),
+						mlog.String("PostId", sm.Post.Id),
+						mlog.Err(err2),
+					)
+					postErrors = append(postErrors, sm.Post.Id)
+					continue
+				}
+			}
+
+			// process perma-links for remote
+			if team != nil {
+				sm.Post.Message = scs.processPermalinkFromRemote(sm.Post, team)
+			}
+
+			// add/update post (may be nil if only reactions changed)
+			rpost, err := scs.upsertSyncPost(sm.Post, channel, rc)
+			if err != nil {
+				postErrors = append(postErrors, sm.Post.Id)
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync post",
+					mlog.String("post_id", sm.Post.Id),
+					mlog.String("channel_id", sm.Post.ChannelId),
+					mlog.Err(err),
+				)
+			} else if lastSyncAt < rpost.UpdateAt {
+				lastSyncAt = rpost.UpdateAt
+			}
+		}
+
+		// add/remove reactions
+		for _, reaction := range sm.Reactions {
+			if _, err := scs.upsertSyncReaction(reaction, rc); err != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync reaction",
+					mlog.String("user_id", reaction.UserId),
+					mlog.String("post_id", reaction.PostId),
+					mlog.String("emoji", reaction.EmojiName),
+					mlog.Int64("delete_at", reaction.DeleteAt),
+					mlog.Err(err),
+				)
+			} else {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Reaction upserted via sync",
+					mlog.String("user_id", reaction.UserId),
+					mlog.String("post_id", reaction.PostId),
+					mlog.String("emoji", reaction.EmojiName),
+					mlog.Int64("delete_at", reaction.DeleteAt),
+				)
+
+				if lastSyncAt < reaction.UpdateAt {
+					lastSyncAt = reaction.UpdateAt
+				}
+			}
+		}
+	}
+
+	syncResp := SyncResponse{
+		LastSyncAt: lastSyncAt, // might be zero
+		PostErrors: postErrors, // might be empty
+		UsersSyncd: usersSyncd, // might be empty
+	}
+
+	response.SetPayload(syncResp)
+
+	return nil
+}
+
+func (scs *Service) upsertSyncUser(user *model.User, channel *model.Channel, rc *model.RemoteCluster) (*model.User, error) {
+	var err error
+	var userSaved *model.User
+
+	user.RemoteId = model.NewString(rc.RemoteId)
+
+	// does the user already exist?
+	euser, err := scs.server.GetStore().User().Get(context.Background(), user.Id)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return nil, fmt.Errorf("error checking sync user: %w", err)
+		}
+	}
+
+	if euser == nil {
+		if userSaved, err = scs.server.GetStore().User().Save(user); err != nil {
+			if e, ok := err.(errInvalidInput); ok {
+				_, field, value := e.InvalidInputInfo()
+				if field == "email" || field == "username" {
+					// username or email collision
+					// TODO: handle collision by modifying username/email (MM-32133)
+					return nil, fmt.Errorf("collision inserting sync user (%s=%s): %w", field, value, err)
+				}
+			}
+			return nil, fmt.Errorf("error inserting sync user: %w", err)
+		}
+	} else {
+		patch := &model.UserPatch{
+			Nickname:  &user.Nickname,
+			FirstName: &user.FirstName,
+			LastName:  &user.LastName,
+			Position:  &user.Position,
+			Locale:    &user.Locale,
+			Timezone:  user.Timezone,
+			RemoteId:  user.RemoteId,
+		}
+		euser.Patch(patch)
+		userUpdated, err := scs.server.GetStore().User().Update(euser, false)
+		if err != nil {
+			return nil, fmt.Errorf("error updating sync user: %w", err)
+		}
+		userSaved = userUpdated.New
+	}
+
+	// add user to team. We do this here regardless of whether the user was
+	// just created or patched since there are three steps to adding a user
+	// (insert rec, add to team, add to channel) and any one could fail.
+	// Instead of undoing what succeeded on any failure we simply do all steps each
+	// time. AddUserToChannel & AddUserToTeamByTeamId do not error if user already
+	// added and exit quickly.
+	if err := scs.app.AddUserToTeamByTeamId(channel.TeamId, userSaved); err != nil {
+		return nil, fmt.Errorf("error adding sync user to Team: %w", err)
+	}
+
+	// add user to channel
+	if _, err := scs.app.AddUserToChannel(userSaved, channel); err != nil {
+		return nil, fmt.Errorf("error adding sync user to ChannelMembers: %w", err)
+	}
+	return userSaved, nil
+}
+
+func (scs *Service) upsertSyncPost(post *model.Post, channel *model.Channel, rc *model.RemoteCluster) (*model.Post, error) {
+	var appErr *model.AppError
+
+	post.RemoteId = model.NewString(rc.RemoteId)
+
+	rpost, err := scs.server.GetStore().Post().GetSingle(post.Id, true)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return nil, fmt.Errorf("error checking sync post: %w", err)
+		}
+	}
+
+	if rpost == nil {
+		// post doesn't exist; create new one
+		rpost, appErr = scs.app.CreatePost(post, channel, true, true)
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Created sync post",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	} else if post.DeleteAt > 0 {
+		// delete post
+		rpost, appErr = scs.app.DeletePost(post.Id, post.UserId)
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Deleted sync post",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	} else if post.EditAt > rpost.EditAt || post.Message != rpost.Message {
+		// update post
+		rpost, appErr = scs.app.UpdatePost(post, false)
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Updated sync post",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	} else {
+		// nothing to update
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "Update to sync post ignored",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	}
+
+	var rerr error
+	if appErr != nil {
+		rerr = errors.New(appErr.Error())
+	}
+	return rpost, rerr
+}
+
+func (scs *Service) upsertSyncReaction(reaction *model.Reaction, rc *model.RemoteCluster) (*model.Reaction, error) {
+	savedReaction := reaction
+	var appErr *model.AppError
+
+	reaction.RemoteId = model.NewString(rc.RemoteId)
+
+	if reaction.DeleteAt == 0 {
+		savedReaction, appErr = scs.app.SaveReactionForPost(reaction)
+	} else {
+		appErr = scs.app.DeleteReactionForPost(reaction)
+	}
+
+	var err error
+	if appErr != nil {
+		err = errors.New(appErr.Error())
+	}
+	return savedReaction, err
+}

services/sharedchannel/sync_send.go

@@ -0,0 +1,512 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v5/shared/i18n"
+	"github.com/mattermost/mattermost-server/v5/shared/mlog"
+)
+
+type syncTask struct {
+	id         string
+	channelId  string
+	remoteId   string
+	AddedAt    time.Time
+	retryCount int
+	retryPost  *model.Post
+	schedule   time.Time
+}
+
+func newSyncTask(channelId string, remoteId string, retryPost *model.Post) syncTask {
+	var postId string
+	if retryPost != nil {
+		postId = retryPost.Id
+	}
+
+	return syncTask{
+		id:        channelId + remoteId + postId, // combination of ids to avoid duplicates
+		channelId: channelId,
+		remoteId:  remoteId, // empty means update all remote clusters
+		retryPost: retryPost,
+		schedule:  time.Now(),
+	}
+}
+
+// incRetry increments the retry counter and returns true if MaxRetries not exceeded.
+func (st *syncTask) incRetry() bool {
+	st.retryCount++
+	return st.retryCount <= MaxRetries
+}
+
+// NotifyChannelChanged is called to indicate that a shared channel has been modified,
+// thus triggering an update to all remote clusters.
+func (scs *Service) NotifyChannelChanged(channelId string) {
+	if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
+		return
+	}
+
+	task := newSyncTask(channelId, "", nil)
+	task.schedule = time.Now().Add(NotifyMinimumDelay)
+	scs.addTask(task)
+}
+
+// ForceSyncForRemote causes all channels shared with the remote to be synchronized.
+func (scs *Service) ForceSyncForRemote(rc *model.RemoteCluster) {
+	if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
+		return
+	}
+
+	// fetch all channels shared with this remote.
+	opts := model.SharedChannelRemoteFilterOpts{
+		RemoteId: rc.RemoteId,
+	}
+	scrs, err := scs.server.GetStore().SharedChannel().GetRemotes(opts)
+	if err != nil {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "Failed to fetch shared channel remotes",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("remoteId", rc.RemoteId),
+			mlog.Err(err),
+		)
+		return
+	}
+
+	for _, scr := range scrs {
+		task := newSyncTask(scr.ChannelId, rc.RemoteId, nil)
+		task.schedule = time.Now().Add(NotifyMinimumDelay)
+		scs.addTask(task)
+	}
+}
+
+// addTask adds or re-adds a task to the queue.
+func (scs *Service) addTask(task syncTask) {
+	task.AddedAt = time.Now()
+	scs.mux.Lock()
+	if _, ok := scs.tasks[task.id]; !ok {
+		scs.tasks[task.id] = task
+	}
+	scs.mux.Unlock()
+
+	// wake up the sync goroutine
+	select {
+	case scs.changeSignal <- struct{}{}:
+	default:
+		// that's ok, the sync routine is already busy
+	}
+}
+
+// syncLoop is called via a dedicated goroutine to wait for notifications of channel changes and
+// updates each remote based on those changes.
+func (scs *Service) syncLoop(done chan struct{}) {
+	// create a timer to periodically check the task queue, but only if there is
+	// a delayed task in the queue.
+	delay := time.NewTimer(NotifyMinimumDelay)
+	defer stopTimer(delay)
+
+	// wait for channel changed signal and update for oldest task.
+	for {
+		select {
+		case <-scs.changeSignal:
+			if wait := scs.doSync(); wait > 0 {
+				stopTimer(delay)
+				delay.Reset(wait)
+			}
+		case <-delay.C:
+			if wait := scs.doSync(); wait > 0 {
+				delay.Reset(wait)
+			}
+		case <-done:
+			return
+		}
+	}
+}
+
+func stopTimer(timer *time.Timer) {
+	timer.Stop()
+	select {
+	case <-timer.C:
+	default:
+	}
+}
+
+// doSync checks the task queue for any tasks to be processed and processes all that are ready.
+// If any delayed tasks remain in queue then the duration until the next scheduled task is returned.
+func (scs *Service) doSync() time.Duration {
+	var task syncTask
+	var ok bool
+	var shortestWait time.Duration
+
+	for {
+		task, ok, shortestWait = scs.removeOldestTask()
+		if !ok {
+			break
+		}
+		if err := scs.processTask(task); err != nil {
+			// put task back into map so it will update again
+			if task.incRetry() {
+				scs.addTask(task)
+			} else {
+				scs.server.GetLogger().Error("Failed to synchronize shared channel",
+					mlog.String("channelId", task.channelId),
+					mlog.String("remoteId", task.remoteId),
+					mlog.Err(err),
+				)
+			}
+		}
+	}
+	return shortestWait
+}
+
+// removeOldestTask removes and returns the oldest task in the task map.
+// A task coming in via NotifyChannelChanged must stay in queue for at least
+// `NotifyMinimumDelay` to ensure we don't go nuts trying to sync during a bulk update.
+// If no tasks are available then false is returned.
+func (scs *Service) removeOldestTask() (syncTask, bool, time.Duration) {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	var oldestTask syncTask
+	var oldestKey string
+	var shortestWait time.Duration
+
+	for key, task := range scs.tasks {
+		// check if task is ready
+		if wait := time.Until(task.schedule); wait > 0 {
+			if wait < shortestWait || shortestWait == 0 {
+				shortestWait = wait
+			}
+			continue
+		}
+		// task is ready; check if it's the oldest ready task
+		if task.AddedAt.Before(oldestTask.AddedAt) || oldestTask.AddedAt.IsZero() {
+			oldestKey = key
+			oldestTask = task
+		}
+	}
+
+	if oldestKey != "" {
+		delete(scs.tasks, oldestKey)
+		return oldestTask, true, shortestWait
+	}
+	return oldestTask, false, shortestWait
+}
+
+// processTask updates one or more remote clusters with any new channel content.
+func (scs *Service) processTask(task syncTask) error {
+	var err error
+	var remotes []*model.RemoteCluster
+
+	if task.remoteId == "" {
+		filter := model.RemoteClusterQueryFilter{
+			InChannel:     task.channelId,
+			OnlyConfirmed: true,
+		}
+		remotes, err = scs.server.GetStore().RemoteCluster().GetAll(filter)
+		if err != nil {
+			return err
+		}
+	} else {
+		rc, err := scs.server.GetStore().RemoteCluster().Get(task.remoteId)
+		if err != nil {
+			return err
+		}
+		if !rc.IsOnline() {
+			return fmt.Errorf("Failed updating shared channel '%s' for offline remote cluster '%s'", task.channelId, rc.DisplayName)
+		}
+		remotes = []*model.RemoteCluster{rc}
+	}
+
+	for _, rc := range remotes {
+		rtask := task
+		rtask.remoteId = rc.RemoteId
+		if err := scs.updateForRemote(rtask, rc); err != nil {
+			// retry...
+			if rtask.incRetry() {
+				scs.addTask(rtask)
+			} else {
+				scs.server.GetLogger().Error("Failed to synchronize shared channel for remote cluster",
+					mlog.String("channelId", rtask.channelId),
+					mlog.String("remote", rc.DisplayName),
+					mlog.String("remoteId", rtask.remoteId),
+					mlog.Err(err),
+				)
+			}
+		}
+	}
+	return nil
+}
+
+// updateForRemote updates a remote cluster with any new posts/reactions for a specific
+// channel. If many changes are found, only the oldest X changes are sent and the channel
+// is re-added to the task map. This ensures no channels are starved for updates even if some
+// channels are very active.
+func (scs *Service) updateForRemote(task syncTask, rc *model.RemoteCluster) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot update remote cluster for channel id %s; Remote Cluster Service not enabled", task.channelId)
+	}
+
+	scr, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(task.channelId, rc.RemoteId)
+	if err != nil {
+		return err
+	}
+
+	var posts []*model.Post
+	var repeat bool
+	nextSince := scr.NextSyncAt
+
+	if task.retryPost != nil {
+		posts = []*model.Post{task.retryPost}
+	} else {
+		result, err2 := scs.getPostsSince(task.channelId, rc, scr.NextSyncAt)
+		if err2 != nil {
+			return err2
+		}
+		posts = result.posts
+		repeat = result.hasMore
+		nextSince = result.nextSince
+	}
+
+	if len(posts) == 0 {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task found zero posts; skipping sync",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("channel_id", task.channelId),
+			mlog.Int64("lastSyncAt", scr.NextSyncAt),
+			mlog.Int64("nextSince", nextSince),
+			mlog.Bool("repeat", repeat),
+		)
+		return nil
+	}
+
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task found posts to sync",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("channel_id", task.channelId),
+		mlog.Int64("lastSyncAt", scr.NextSyncAt),
+		mlog.Int64("nextSince", nextSince),
+		mlog.Int("count", len(posts)),
+		mlog.Bool("repeat", repeat),
+	)
+
+	if !rc.IsOnline() {
+		scs.notifyRemoteOffline(posts, rc)
+		return nil
+	}
+
+	syncMessages, err := scs.postsToSyncMessages(posts, rc, scr.NextSyncAt)
+	if err != nil {
+		return err
+	}
+
+	if len(syncMessages) == 0 {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "sync task, all messages filtered out; skipping sync",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("channel_id", task.channelId),
+			mlog.Bool("repeat", repeat),
+		)
+
+		// All posts were filtered out, meaning no need to send them. Fast forward SharedChannelRemote's NextSyncAt.
+		scs.updateNextSyncForRemote(scr.Id, rc, nextSince)
+
+		// everything was filtered out, nothing to send.
+		if repeat {
+			scs.addTask(newSyncTask(task.channelId, task.remoteId, nil))
+		}
+		return nil
+	}
+
+	scs.sendAttachments(syncMessages, rc)
+
+	b, err := json.Marshal(syncMessages)
+	if err != nil {
+		return err
+	}
+	msg := model.NewRemoteClusterMsg(TopicSync, b)
+
+	if scs.server.GetLogger().IsLevelEnabled(mlog.LvlSharedChannelServiceMessagesOutbound) {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceMessagesOutbound, "outbound message",
+			mlog.String("remote", rc.DisplayName),
+			mlog.Int64("NextSyncAt", scr.NextSyncAt),
+			mlog.String("msg", string(b)),
+		)
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	err = rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		defer wg.Done()
+		if err != nil {
+			return // this means the response could not be parsed; already logged
+		}
+
+		var syncResp SyncResponse
+		if err2 := json.Unmarshal(resp.Payload, &syncResp); err2 != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "invalid sync response after update shared channel",
+				mlog.String("remote", rc.DisplayName),
+				mlog.Err(err2),
+			)
+		}
+
+		// Any Post(s) that failed to save on remote side are included in an array of post ids in the Response payload.
+		// Handle each error by retrying the post a fixed number of times before giving up.
+		for _, p := range syncResp.PostErrors {
+			scs.handlePostError(p, task, rc)
+		}
+
+		// update NextSyncAt for all the users that were synchronized
+		scs.updateSyncUsers(syncResp.UsersSyncd, rc, nextSince)
+	})
+
+	wg.Wait()
+
+	if err == nil {
+		// Optimistically update SharedChannelRemote's NextSyncAt; if any posts failed they will be retried.
+		scs.updateNextSyncForRemote(scr.Id, rc, nextSince)
+	}
+
+	if repeat {
+		scs.addTask(newSyncTask(task.channelId, task.remoteId, nil))
+	}
+	return err
+}
+
+func (scs *Service) sendAttachments(syncMessages []syncMsg, rc *model.RemoteCluster) {
+	for _, sm := range syncMessages {
+		for _, fi := range sm.Attachments {
+			if err := scs.sendAttachmentForRemote(fi, sm.Post, rc); err != nil {
+				scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error syncing attachment for post",
+					mlog.String("remote", rc.DisplayName),
+					mlog.String("post_id", sm.Post.Id),
+					mlog.String("file_id", fi.Id),
+					mlog.Err(err),
+				)
+			}
+		}
+	}
+}
+
+func (scs *Service) handlePostError(postId string, task syncTask, rc *model.RemoteCluster) {
+	if task.retryPost != nil && task.retryPost.Id == postId {
+		// this was a retry for specific post that failed previously. Try again if within MaxRetries.
+		if task.incRetry() {
+			scs.addTask(task)
+		} else {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error syncing post",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("post_id", postId),
+			)
+		}
+		return
+	}
+
+	// this post failed as part of a group of posts. Retry as an individual post.
+	post, err := scs.server.GetStore().Post().GetSingle(postId, true)
+	if err != nil {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error fetching post for sync retry",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("post_id", postId),
+		)
+		return
+	}
+	scs.addTask(newSyncTask(task.channelId, task.remoteId, post))
+}
+
+// notifyRemoteOffline creates an ephemeral post to the author for any posts created recently to remotes
+// that are offline.
+func (scs *Service) notifyRemoteOffline(posts []*model.Post, rc *model.RemoteCluster) {
+	// only send one ephemeral post per author.
+	notified := make(map[string]bool)
+
+	// range the slice in reverse so the newest posts are visited first; this ensures an ephemeral
+	// get added where it is mostly likely to be seen.
+	for i := len(posts) - 1; i >= 0; i-- {
+		post := posts[i]
+		if didNotify := notified[post.UserId]; didNotify {
+			continue
+		}
+
+		postCreateAt := model.GetTimeForMillis(post.CreateAt)
+
+		if post.DeleteAt == 0 && post.UserId != "" && time.Since(postCreateAt) < NotifyRemoteOfflineThreshold {
+			T := scs.getUserTranslations(post.UserId)
+			ephemeral := &model.Post{
+				ChannelId: post.ChannelId,
+				Message:   T("sharedchannel.cannot_deliver_post", map[string]interface{}{"Remote": rc.DisplayName}),
+				CreateAt:  post.CreateAt + 1,
+			}
+			scs.app.SendEphemeralPost(post.UserId, ephemeral)
+
+			notified[post.UserId] = true
+		}
+	}
+}
+
+func (scs *Service) updateNextSyncForRemote(scrId string, rc *model.RemoteCluster, nextSyncAt int64) {
+	if nextSyncAt == 0 {
+		return
+	}
+	if err := scs.server.GetStore().SharedChannel().UpdateRemoteNextSyncAt(scrId, nextSyncAt); err != nil {
+		scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error updating NextSyncAt for shared channel remote",
+			mlog.String("remote", rc.DisplayName),
+			mlog.Err(err),
+		)
+		return
+	}
+	scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "updated NextSyncAt for remote",
+		mlog.String("remote_id", rc.RemoteId),
+		mlog.String("remote", rc.DisplayName),
+		mlog.Int64("next_update_at", nextSyncAt),
+	)
+}
+
+func (scs *Service) updateSyncUsers(userIds []string, rc *model.RemoteCluster, lastSyncAt int64) {
+	for _, uid := range userIds {
+		scu, err := scs.server.GetStore().SharedChannel().GetUser(uid, rc.RemoteId)
+		if err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error getting user for lastSyncAt update",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("user_id", uid),
+				mlog.Err(err),
+			)
+			continue
+		}
+
+		if err := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(scu.Id, lastSyncAt); err != nil {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceError, "error updating lastSyncAt for user",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("user_id", uid),
+				mlog.Err(err),
+			)
+		} else {
+			scs.server.GetLogger().Log(mlog.LvlSharedChannelServiceDebug, "updated lastSyncAt for user",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("user_id", scu.UserId),
+				mlog.Int64("last_update_at", lastSyncAt),
+			)
+		}
+	}
+}
+
+func (scs *Service) getUserTranslations(userId string) i18n.TranslateFunc {
+	var locale string
+	user, err := scs.server.GetStore().User().Get(context.Background(), userId)
+	if err == nil {
+		locale = user.Locale
+	}
+
+	if locale == "" {
+		locale = model.DEFAULT_LOCALE
+	}
+	return i18n.GetUserTranslations(locale)
+}

services/slackimport/slackimport.go

@@ -85,7 +85,7 @@ type Actions struct {
 	UpdateActive           func(*model.User, bool) (*model.User, *model.AppError)
 	AddUserToChannel       func(*model.User, *model.Channel) (*model.ChannelMember, *model.AppError)
 	JoinUserToTeam         func(*model.Team, *model.User, string) *model.AppError
-	CreateDirectChannel    func(string, string) (*model.Channel, *model.AppError)
+	CreateDirectChannel    func(string, string, ...model.ChannelOption) (*model.Channel, *model.AppError)
 	CreateGroupChannel     func([]string) (*model.Channel, *model.AppError)
 	CreateChannel          func(*model.Channel, bool) (*model.Channel, *model.AppError)
 	DoUploadFile           func(time.Time, string, string, string, string, []byte) (*model.FileInfo, *model.AppError)

services/telemetry/telemetry.go

@@ -734,6 +734,7 @@ func (ts *TelemetryService) trackConfig() {
 		"cloud_billing":                      *cfg.ExperimentalSettings.CloudBilling,
 		"cloud_user_limit":                   *cfg.ExperimentalSettings.CloudUserLimit,
 		"enable_shared_channels":             *cfg.ExperimentalSettings.EnableSharedChannels,
+		"enable_remote_cluster_service":      *cfg.ExperimentalSettings.EnableRemoteClusterService && cfg.FeatureFlags.EnableRemoteClusterService,
 	})
 
 	ts.sendTelemetry(TrackConfigAnalytics, map[string]interface{}{