Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

server/platform/services/awsmeter/awsmeter.go

@@ -0,0 +1,154 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package awsmeter
+
+import (
+	"encoding/json"
+	"os"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/marketplacemetering"
+	"github.com/aws/aws-sdk-go/service/marketplacemetering/marketplacemeteringiface"
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type AwsMeter struct {
+	store   store.Store
+	service *AWSMeterService
+	config  *model.Config
+}
+
+type AWSMeterService struct {
+	AwsDryRun      bool
+	AwsProductCode string
+	AwsMeteringSvc marketplacemeteringiface.MarketplaceMeteringAPI
+}
+
+type AWSMeterReport struct {
+	Dimension string    `json:"dimension"`
+	Value     int64     `json:"value"`
+	Timestamp time.Time `json:"timestamp"`
+}
+
+func (o *AWSMeterReport) ToJSON() string {
+	b, _ := json.Marshal(o)
+	return string(b)
+}
+
+func New(store store.Store, config *model.Config) *AwsMeter {
+	svc := &AWSMeterService{
+		AwsDryRun:      false,
+		AwsProductCode: "12345", //TODO
+	}
+
+	service, err := newAWSMarketplaceMeteringService()
+	if err != nil {
+		mlog.Debug("Could not create AWS metering service", mlog.String("error", err.Error()))
+		return nil
+	}
+
+	svc.AwsMeteringSvc = service
+	return &AwsMeter{
+		store:   store,
+		service: svc,
+		config:  config,
+	}
+}
+
+func newAWSMarketplaceMeteringService() (*marketplacemetering.MarketplaceMetering, error) {
+	region := os.Getenv("AWS_REGION")
+	s, err := session.NewSession(&aws.Config{Region: &region})
+	if err != nil {
+		return nil, err
+	}
+
+	creds := credentials.NewChainCredentials(
+		[]credentials.Provider{
+			&ec2rolecreds.EC2RoleProvider{
+				Client: ec2metadata.New(s),
+			},
+		})
+
+	_, err = creds.Get()
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot obtain credentials")
+	}
+
+	return marketplacemetering.New(session.Must(session.NewSession(&aws.Config{
+		Credentials: creds,
+	}))), nil
+}
+
+// a report entry is for all metrics
+func (awsm *AwsMeter) GetUserCategoryUsage(dimensions []string, startTime time.Time, endTime time.Time) []*AWSMeterReport {
+	reports := make([]*AWSMeterReport, 0)
+
+	for _, dimension := range dimensions {
+		var userCount int64
+		var err error
+
+		switch dimension {
+		case model.AwsMeteringDimensionUsageHrs:
+			userCount, err = awsm.store.User().AnalyticsActiveCountForPeriod(model.GetMillisForTime(startTime), model.GetMillisForTime(endTime), model.UserCountOptions{})
+			if err != nil {
+				mlog.Warn("Failed to obtain usage data", mlog.String("dimension", dimension), mlog.String("start", startTime.String()), mlog.Int64("count", userCount), mlog.Err(err))
+				continue
+			}
+		default:
+			mlog.Debug("Dimension does not exist!", mlog.String("dimension", dimension))
+			continue
+		}
+
+		report := &AWSMeterReport{
+			Dimension: dimension,
+			Value:     userCount,
+			Timestamp: startTime,
+		}
+
+		reports = append(reports, report)
+	}
+
+	return reports
+}
+
+func (awsm *AwsMeter) ReportUserCategoryUsage(reports []*AWSMeterReport) error {
+	for _, report := range reports {
+		err := sendReportToMeteringService(awsm.service, report)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func sendReportToMeteringService(ams *AWSMeterService, report *AWSMeterReport) error {
+	params := &marketplacemetering.MeterUsageInput{
+		DryRun:         aws.Bool(ams.AwsDryRun),
+		ProductCode:    aws.String(ams.AwsProductCode),
+		UsageDimension: aws.String(report.Dimension),
+		UsageQuantity:  aws.Int64(report.Value),
+		Timestamp:      aws.Time(report.Timestamp),
+	}
+
+	resp, err := ams.AwsMeteringSvc.MeterUsage(params)
+	if err != nil {
+		return errors.Wrap(err, "Invalid metering service id.")
+	}
+	if resp.MeteringRecordId == nil {
+		return errors.Wrap(err, "Invalid metering service id.")
+	}
+
+	mlog.Debug("Sent record to AWS metering service", mlog.String("dimension", report.Dimension), mlog.Int64("value", report.Value), mlog.String("timestamp", report.Timestamp.String()))
+
+	return nil
+}

server/platform/services/awsmeter/awsmeter_test.go

@@ -0,0 +1,149 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package awsmeter
+
+import (
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/marketplacemetering"
+	"github.com/aws/aws-sdk-go/service/marketplacemetering/marketplacemeteringiface"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest/mocks"
+)
+
+type mockMarketplaceMeteringClient struct {
+	marketplacemeteringiface.MarketplaceMeteringAPI
+}
+
+func (m *mockMarketplaceMeteringClient) MeterUsage(input *marketplacemetering.MeterUsageInput) (*marketplacemetering.MeterUsageOutput, error) {
+	return &marketplacemetering.MeterUsageOutput{
+		MeteringRecordId: String("1"),
+	}, nil
+}
+
+type mockMarketplaceMeteringClientWithError struct {
+	marketplacemeteringiface.MarketplaceMeteringAPI
+}
+
+func (m *mockMarketplaceMeteringClientWithError) MeterUsage(input *marketplacemetering.MeterUsageInput) (*marketplacemetering.MeterUsageOutput, error) {
+	return nil, errors.New("error")
+}
+
+func String(i string) *string {
+	return &i
+}
+func TestAwsMeterUsage(t *testing.T) {
+	startTime := time.Now()
+	endTime := time.Now()
+	dimensions := []string{model.AwsMeteringDimensionUsageHrs}
+
+	userStoreMock := mocks.UserStore{}
+	userStoreMock.On("AnalyticsActiveCountForPeriod", model.GetMillisForTime(startTime), model.GetMillisForTime(endTime), mock.AnythingOfType("model.UserCountOptions")).Return(int64(2), nil)
+
+	storeMock := mocks.Store{}
+	storeMock.On("User").Return(&userStoreMock)
+
+	reports := make([]*AWSMeterReport, 1)
+	reports[0] = &AWSMeterReport{
+		Dimension: model.AwsMeteringDimensionUsageHrs,
+		Value:     2,
+		Timestamp: startTime,
+	}
+
+	// Define a mock struct to be used in your unit tests of myFunc.
+	svc := &AWSMeterService{
+		AwsDryRun:      false,
+		AwsProductCode: "12345",
+		AwsMeteringSvc: &mockMarketplaceMeteringClient{},
+	}
+
+	config := &model.Config{}
+	config.SetDefaults()
+
+	awsmeter := &AwsMeter{
+		store:   &storeMock,
+		service: svc,
+		config:  config,
+	}
+
+	t.Run("Send report for one usage category", func(t *testing.T) {
+		resultReports := awsmeter.GetUserCategoryUsage(dimensions, startTime, endTime)
+		require.NotNil(t, resultReports)
+		assert.Equal(t, 1, len(resultReports))
+		assert.Equal(t, reports[0].Dimension, resultReports[0].Dimension)
+		assert.Equal(t, reports[0].Value, resultReports[0].Value)
+		assert.Equal(t, reports[0].Timestamp, resultReports[0].Timestamp)
+
+		err := awsmeter.ReportUserCategoryUsage(resultReports)
+		require.NoError(t, err)
+	})
+
+	t.Run("Error in AWS service call", func(t *testing.T) {
+		awsmeter.service.AwsMeteringSvc = &mockMarketplaceMeteringClientWithError{}
+		resultReports := awsmeter.GetUserCategoryUsage(dimensions, startTime, endTime)
+		require.NotNil(t, resultReports)
+		assert.Equal(t, 1, len(resultReports))
+		err := awsmeter.ReportUserCategoryUsage(resultReports)
+		require.Error(t, err)
+	})
+
+	t.Run("Invalid dimension", func(t *testing.T) {
+		awsmeter.service.AwsMeteringSvc = &mockMarketplaceMeteringClient{}
+		dimensions = []string{"invalid dimension"}
+		resultReports := awsmeter.GetUserCategoryUsage(dimensions, startTime, endTime)
+		require.NotNil(t, resultReports)
+		assert.Equal(t, 0, len(resultReports))
+		err := awsmeter.ReportUserCategoryUsage(resultReports)
+		require.NoError(t, err)
+	})
+}
+
+func TestAwsMeterUsageWithDBError(t *testing.T) {
+	startTime := time.Now()
+	endTime := time.Now()
+	dimensions := []string{model.AwsMeteringDimensionUsageHrs}
+
+	userStoreMock := mocks.UserStore{}
+	userStoreMock.On("AnalyticsActiveCountForPeriod", model.GetMillisForTime(startTime), model.GetMillisForTime(endTime), mock.AnythingOfType("model.UserCountOptions")).Return(int64(0), errors.New("error"))
+
+	storeMock := mocks.Store{}
+	storeMock.On("User").Return(&userStoreMock)
+
+	reports := make([]*AWSMeterReport, 1)
+	reports[0] = &AWSMeterReport{
+		Dimension: model.AwsMeteringDimensionUsageHrs,
+		Value:     2,
+		Timestamp: startTime,
+	}
+
+	// Define a mock struct to be used in your unit tests of myFunc.
+	svc := &AWSMeterService{
+		AwsDryRun:      false,
+		AwsProductCode: "12345",
+		AwsMeteringSvc: &mockMarketplaceMeteringClient{},
+	}
+
+	config := &model.Config{}
+	config.SetDefaults()
+
+	awsmeter := &AwsMeter{
+		store:   &storeMock,
+		service: svc,
+		config:  config,
+	}
+
+	t.Run("Error in DB query", func(t *testing.T) {
+		resultReports := awsmeter.GetUserCategoryUsage(dimensions, startTime, endTime)
+		require.NotNil(t, resultReports)
+		assert.Equal(t, 0, len(resultReports))
+		err := awsmeter.ReportUserCategoryUsage(resultReports)
+		require.NoError(t, err)
+	})
+}

server/platform/services/cache/cache.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cache
+
+import (
+	"errors"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// ErrKeyNotFound is the error when the given key is not found
+var ErrKeyNotFound = errors.New("key not found")
+
+// Cache is a representation of a cache store that aims to replace cache.Cache
+type Cache interface {
+	// Purge is used to completely clear the cache.
+	Purge() error
+
+	// Set adds the given key and value to the store without an expiry. If the key already exists,
+	// it will overwrite the previous value.
+	Set(key string, value any) error
+
+	// SetWithDefaultExpiry adds the given key and value to the store with the default expiry. If
+	// the key already exists, it will overwrite the previous value
+	SetWithDefaultExpiry(key string, value any) error
+
+	// SetWithExpiry adds the given key and value to the cache with the given expiry. If the key
+	// already exists, it will overwrite the previous value
+	SetWithExpiry(key string, value any, ttl time.Duration) error
+
+	// Get the content stored in the cache for the given key, and decode it into the value interface.
+	// Return ErrKeyNotFound if the key is missing from the cache
+	Get(key string, value any) error
+
+	// Remove deletes the value for a given key.
+	Remove(key string) error
+
+	// Keys returns a slice of the keys in the cache.
+	Keys() ([]string, error)
+
+	// Len returns the number of items in the cache.
+	Len() (int, error)
+
+	// GetInvalidateClusterEvent returns the cluster event configured when this cache was created.
+	GetInvalidateClusterEvent() model.ClusterEvent
+
+	// Name returns the name of the cache
+	Name() string
+}

server/platform/services/cache/lru.go

@@ -0,0 +1,248 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cache
+
+import (
+	"container/list"
+	"sync"
+	"time"
+
+	"github.com/tinylib/msgp/msgp"
+	"github.com/vmihailenco/msgpack/v5"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// LRU is a thread-safe fixed size LRU cache.
+type LRU struct {
+	lock                   sync.RWMutex
+	size                   int
+	len                    int
+	currentGeneration      int64
+	evictList              *list.List
+	items                  map[string]*list.Element
+	defaultExpiry          time.Duration
+	name                   string
+	invalidateClusterEvent model.ClusterEvent
+}
+
+// LRUOptions contains options for initializing LRU cache
+type LRUOptions struct {
+	Name                   string
+	Size                   int
+	DefaultExpiry          time.Duration
+	InvalidateClusterEvent model.ClusterEvent
+	// StripedBuckets is used only by LRUStriped and shouldn't be greater than the number
+	// of CPUs available on the machine running this cache.
+	StripedBuckets int
+}
+
+// entry is used to hold a value in the evictList.
+type entry struct {
+	key        string
+	value      []byte
+	expires    time.Time
+	generation int64
+}
+
+// NewLRU creates an LRU of the given size.
+func NewLRU(opts LRUOptions) Cache {
+	return &LRU{
+		name:                   opts.Name,
+		size:                   opts.Size,
+		evictList:              list.New(),
+		items:                  make(map[string]*list.Element, opts.Size),
+		defaultExpiry:          opts.DefaultExpiry,
+		invalidateClusterEvent: opts.InvalidateClusterEvent,
+	}
+}
+
+// Purge is used to completely clear the cache.
+func (l *LRU) Purge() error {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	l.len = 0
+	l.currentGeneration++
+	return nil
+}
+
+// Set adds the given key and value to the store without an expiry. If the key already exists,
+// it will overwrite the previous value.
+func (l *LRU) Set(key string, value any) error {
+	return l.SetWithExpiry(key, value, 0)
+}
+
+// SetWithDefaultExpiry adds the given key and value to the store with the default expiry. If
+// the key already exists, it will overwrite the previous value
+func (l *LRU) SetWithDefaultExpiry(key string, value any) error {
+	return l.SetWithExpiry(key, value, l.defaultExpiry)
+}
+
+// SetWithExpiry adds the given key and value to the cache with the given expiry. If the key
+// already exists, it will overwrite the previous value
+func (l *LRU) SetWithExpiry(key string, value any, ttl time.Duration) error {
+	return l.set(key, value, ttl)
+}
+
+// Get the content stored in the cache for the given key, and decode it into the value interface.
+// return ErrKeyNotFound if the key is missing from the cache
+func (l *LRU) Get(key string, value any) error {
+	return l.get(key, value)
+}
+
+// Remove deletes the value for a key.
+func (l *LRU) Remove(key string) error {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	if ent, ok := l.items[key]; ok {
+		l.removeElement(ent)
+	}
+	return nil
+}
+
+// Keys returns a slice of the keys in the cache.
+func (l *LRU) Keys() ([]string, error) {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+
+	keys := make([]string, l.len)
+	i := 0
+	for ent := l.evictList.Back(); ent != nil; ent = ent.Prev() {
+		e := ent.Value.(*entry)
+		if e.generation == l.currentGeneration {
+			keys[i] = e.key
+			i++
+		}
+	}
+	return keys, nil
+}
+
+// Len returns the number of items in the cache.
+func (l *LRU) Len() (int, error) {
+	l.lock.RLock()
+	defer l.lock.RUnlock()
+	return l.len, nil
+}
+
+// GetInvalidateClusterEvent returns the cluster event configured when this cache was created.
+func (l *LRU) GetInvalidateClusterEvent() model.ClusterEvent {
+	return l.invalidateClusterEvent
+}
+
+// Name returns the name of the cache
+func (l *LRU) Name() string {
+	return l.name
+}
+
+func (l *LRU) set(key string, value any, ttl time.Duration) error {
+	var expires time.Time
+	if ttl > 0 {
+		expires = time.Now().Add(ttl)
+	}
+
+	var buf []byte
+	var err error
+	// We use a fast path for hot structs.
+	if msgpVal, ok := value.(msgp.Marshaler); ok {
+		buf, err = msgpVal.MarshalMsg(nil)
+	} else {
+		// Slow path for other structs.
+		buf, err = msgpack.Marshal(value)
+	}
+	if err != nil {
+		return err
+	}
+
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	// Check for existing item, ignoring expiry since we'd update anyway.
+	if ent, ok := l.items[key]; ok {
+		l.evictList.MoveToFront(ent)
+		e := ent.Value.(*entry)
+		e.value = buf
+		e.expires = expires
+		if e.generation != l.currentGeneration {
+			e.generation = l.currentGeneration
+			l.len++
+		}
+		return nil
+	}
+
+	// Add new item
+	ent := &entry{key, buf, expires, l.currentGeneration}
+	entry := l.evictList.PushFront(ent)
+	l.items[key] = entry
+	l.len++
+
+	if l.evictList.Len() > l.size {
+		l.removeElement(l.evictList.Back())
+	}
+	return nil
+}
+
+func (l *LRU) get(key string, value any) error {
+	val, err := l.getItem(key)
+	if err != nil {
+		return err
+	}
+
+	// We use a fast path for hot structs.
+	if msgpVal, ok := value.(msgp.Unmarshaler); ok {
+		_, err := msgpVal.UnmarshalMsg(val)
+		return err
+	}
+
+	// This is ugly and makes the cache package aware of the model package.
+	// But this is due to 2 things.
+	// 1. The msgp package works on methods on structs rather than functions.
+	// 2. Our cache interface passes pointers to empty pointers, and not pointers
+	// to values. This is mainly how all our model structs are passed around.
+	// It might be technically possible to use values _just_ for hot structs
+	// like these and then return a pointer while returning from the cache function,
+	// but it will make the codebase inconsistent, and has some edge-cases to take care of.
+	switch v := value.(type) {
+	case **model.User:
+		var u model.User
+		_, err := u.UnmarshalMsg(val)
+		*v = &u
+		return err
+	case *map[string]*model.User:
+		var u model.UserMap
+		_, err := u.UnmarshalMsg(val)
+		*v = u
+		return err
+	}
+
+	// Slow path for other structs.
+	return msgpack.Unmarshal(val, value)
+}
+
+func (l *LRU) getItem(key string) ([]byte, error) {
+	l.lock.Lock()
+	defer l.lock.Unlock()
+
+	ent, ok := l.items[key]
+	if !ok {
+		return nil, ErrKeyNotFound
+	}
+	e := ent.Value.(*entry)
+	if e.generation != l.currentGeneration || (!e.expires.IsZero() && time.Now().After(e.expires)) {
+		l.removeElement(ent)
+		return nil, ErrKeyNotFound
+	}
+	l.evictList.MoveToFront(ent)
+	return e.value, nil
+}
+
+func (l *LRU) removeElement(e *list.Element) {
+	l.evictList.Remove(e)
+	kv := e.Value.(*entry)
+	if kv.generation == l.currentGeneration {
+		l.len--
+	}
+	delete(l.items, kv.key)
+}

server/platform/services/cache/lru_striped.go

@@ -0,0 +1,151 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cache
+
+import (
+	"fmt"
+	"math"
+	"time"
+
+	"github.com/cespare/xxhash/v2"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// LRUStriped keeps LRU caches in buckets in order to lower mutex contention.
+// This is achieved by hashing the input key to map it to a dedicated bucket.
+// Each bucket (an LRU cache) has its own lock that helps distributing the lock
+// contention on multiple threads/cores, leading to less wait times.
+//
+// LRUStriped implements the Cache interface with the same behavior as LRU.
+//
+// Note that, because of it's distributed nature, the fixed size cannot be strictly respected
+// and you may have a tiny bit more space for keys than you defined through LRUOptions.
+// Bucket size is computed as follows: (size / nbuckets) + (size % nbuckets)
+//
+// Because of this size limit per bucket, and because of the nature of the data, you
+// may have buckets filled unevenly, and because of this, keys will be evicted from the entire
+// cache where a simple LRU wouldn't have. Example:
+//
+// Two buckets B1 and B2, of max size 2 each, meaning, theoretically, a max size of 4:
+//   - Say you have a set of 3 keys, they could fill an entire LRU cache.
+//   - But if all those keys are assigned to a single bucket B1, the first key will be evicted from B1
+//   - B2 will remain empty, even though there was enough memory allocated
+//
+// With 4 buckets and random UUIDs as keys, the amount of false evictions is around 5%.
+//
+// By default, the number of buckets equals the number of cpus returned from runtime.NumCPU.
+//
+// This struct is lock-free and intended to be used without lock.
+type LRUStriped struct {
+	buckets                []*LRU
+	name                   string
+	invalidateClusterEvent model.ClusterEvent
+}
+
+func (L LRUStriped) hashkeyMapHash(key string) uint64 {
+	return xxhash.Sum64String(key)
+}
+
+func (L LRUStriped) keyBucket(key string) *LRU {
+	return L.buckets[L.hashkeyMapHash(key)%uint64(len(L.buckets))]
+}
+
+// Purge loops through each LRU cache for purging. Since LRUStriped doesn't use any lock,
+// each LRU bucket is purged after another one, which means that keys could still
+// be present after a call to Purge.
+func (L LRUStriped) Purge() error {
+	for _, lru := range L.buckets {
+		lru.Purge() // errors from purging LRU can be ignored as they always return nil
+	}
+	return nil
+}
+
+// Set does the same as LRU.Set
+func (L LRUStriped) Set(key string, value any) error {
+	return L.keyBucket(key).Set(key, value)
+}
+
+// SetWithDefaultExpiry does the same as LRU.SetWithDefaultExpiry
+func (L LRUStriped) SetWithDefaultExpiry(key string, value any) error {
+	return L.keyBucket(key).SetWithDefaultExpiry(key, value)
+}
+
+// SetWithExpiry does the same as LRU.SetWithExpiry
+func (L LRUStriped) SetWithExpiry(key string, value any, ttl time.Duration) error {
+	return L.keyBucket(key).SetWithExpiry(key, value, ttl)
+}
+
+// Get does the same as LRU.Get
+func (L LRUStriped) Get(key string, value any) error {
+	return L.keyBucket(key).Get(key, value)
+}
+
+// Remove does the same as LRU.Remove
+func (L LRUStriped) Remove(key string) error {
+	return L.keyBucket(key).Remove(key)
+}
+
+// Keys does the same as LRU.Keys. However, because this is lock-free, keys might be
+// inserted or removed from a previously scanned LRU cache.
+// This is not as precise as using a single LRU instance.
+func (L LRUStriped) Keys() ([]string, error) {
+	var keys []string
+	for _, lru := range L.buckets {
+		k, _ := lru.Keys() // Keys never returns any error
+		keys = append(keys, k...)
+	}
+	return keys, nil
+}
+
+// Len does the same as LRU.Len. As for LRUStriped.Keys, this call cannot be precise.
+func (L LRUStriped) Len() (int, error) {
+	var size int
+	for _, lru := range L.buckets {
+		s, _ := lru.Len() // Len never returns any error
+		size += s
+	}
+	return size, nil
+}
+
+// GetInvalidateClusterEvent does the same as LRU.GetInvalidateClusterEvent
+func (L LRUStriped) GetInvalidateClusterEvent() model.ClusterEvent {
+	return L.invalidateClusterEvent
+}
+
+// Name does the same as LRU.Name
+func (L LRUStriped) Name() string {
+	return L.name
+}
+
+// NewLRUStriped creates a striped LRU cache using the special LRUOptions.StripedBuckets value.
+// See LRUStriped and LRUOptions for more details.
+//
+// Not that in order to prevent false eviction, this LRU cache adds 10% (computation is rounded up) of the
+// requested size to the total cache size.
+func NewLRUStriped(opts LRUOptions) (Cache, error) {
+	if opts.StripedBuckets == 0 {
+		return nil, fmt.Errorf("number of buckets is mandatory")
+	}
+
+	if opts.Size < opts.StripedBuckets {
+		return nil, fmt.Errorf("cache size must at least be equal to the number of buckets")
+	}
+
+	// add 10% to the total size, before splitting
+	opts.Size += int(math.Ceil(float64(opts.Size) * 10.0 / 100.0))
+	// now this is the size for each bucket
+	opts.Size = (opts.Size / opts.StripedBuckets) + (opts.Size % opts.StripedBuckets)
+
+	buckets := make([]*LRU, opts.StripedBuckets)
+	for i := 0; i < opts.StripedBuckets; i++ {
+		buckets[i] = NewLRU(opts).(*LRU)
+	}
+
+	return LRUStriped{
+		buckets:                buckets,
+		invalidateClusterEvent: opts.InvalidateClusterEvent,
+		name:                   opts.Name,
+	}, nil
+}

server/platform/services/cache/lru_striped_bench_test.go

@@ -0,0 +1,91 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cache_test
+
+import (
+	"fmt"
+	"runtime"
+	"sync"
+	"testing"
+
+	"github.com/cespare/xxhash/v2"
+
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/cache"
+)
+
+const (
+	m = 500_000
+)
+
+func BenchmarkLRUStriped(b *testing.B) {
+	opts := cache.LRUOptions{
+		Name:                   "",
+		Size:                   128,
+		DefaultExpiry:          0,
+		InvalidateClusterEvent: "",
+		StripedBuckets:         runtime.NumCPU() - 1,
+	}
+
+	cache, err := cache.NewLRUStriped(opts)
+	if err != nil {
+		panic(err)
+	}
+	// prepare keys and initial cache values and set routine
+	keys := make([]string, 0, m)
+	// bucketKeys is to demonstrate that splitted locks is working correctly
+	// by assigning one sequence of key for each bucket.
+	bucketKeys := make([][]string, opts.StripedBuckets)
+	for i := 0; i < m; i++ {
+		key := fmt.Sprintf("%d-key-%d", i, i)
+		keys = append(keys, key)
+		bucketKey := xxhash.Sum64String(key) % uint64(opts.StripedBuckets)
+		bucketKeys[bucketKey] = append(bucketKeys[bucketKey], key)
+	}
+	for i := 0; i < opts.Size; i++ {
+		cache.Set(keys[i], "preflight")
+	}
+
+	wgGet := &sync.WaitGroup{}
+	wgSet := &sync.WaitGroup{}
+	// need buffered chan because if the set routine finished before we write into the chan,
+	// we're left without any consumer, making any write to the chan waiting forever.
+	stopSet := make(chan bool, 1)
+	set := func() {
+		defer wgSet.Done()
+		for i := 0; i < m; i++ {
+			select {
+			case <-stopSet:
+				return
+			default:
+				_ = cache.Set(keys[i], "ignored")
+			}
+		}
+	}
+
+	get := func(bucket int) {
+		defer wgGet.Done()
+		var out string
+		for i := 0; i < m; i++ {
+			_ = cache.Get(bucketKeys[bucket][i%opts.Size], &out)
+		}
+	}
+
+	b.StopTimer()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		wgSet.Add(1)
+		go set()
+		for j := 0; j < opts.StripedBuckets; j++ {
+			wgGet.Add(1)
+			go get(j)
+		}
+
+		b.StartTimer()
+		wgGet.Wait()
+		b.StopTimer()
+
+		stopSet <- true
+		wgSet.Wait()
+	}
+}

server/platform/services/cache/lru_striped_test.go

@@ -0,0 +1,129 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cache
+
+import (
+	"fmt"
+	"hash/maphash"
+	"testing"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func makeLRUPredictableTestData(num int) [][2]string {
+	kv := make([][2]string, num)
+	for i := 0; i < len(kv); i++ {
+		kv[i] = [2]string{
+			fmt.Sprintf("%d-key-%d", i, i),
+			fmt.Sprintf("%d-val-%d", i, i),
+		}
+	}
+	return kv
+}
+
+func TestNewLRUStriped(t *testing.T) {
+	scache, err := NewLRUStriped(LRUOptions{StripedBuckets: 3, Size: 20})
+	require.NoError(t, err)
+
+	cache := scache.(LRUStriped)
+
+	require.Len(t, cache.buckets, 3)
+	assert.Equal(t, 8, cache.buckets[0].size)
+	assert.Equal(t, 8, cache.buckets[1].size)
+	assert.Equal(t, 8, cache.buckets[2].size)
+}
+
+func TestLRUStripedKeyDistribution(t *testing.T) {
+	dataset := makeLRUPredictableTestData(100)
+
+	scache, err := NewLRUStriped(LRUOptions{StripedBuckets: 4, Size: len(dataset)})
+	require.NoError(t, err)
+	cache := scache.(LRUStriped)
+	for _, kv := range dataset {
+		require.NoError(t, cache.Set(kv[0], kv[1]))
+		var out string
+		require.NoError(t, cache.Get(kv[0], &out))
+		require.Equal(t, kv[1], out)
+	}
+
+	require.Len(t, cache.buckets, 4)
+	acc := 0
+	for i := 0; i < 4; i++ {
+		clen, err := cache.buckets[i].Len()
+		acc += clen
+		assert.NoError(t, err)
+		assert.GreaterOrEqual(t, clen, len(dataset)/2/4, "at least 50%/nbuckets of all keys in each bucket")
+	}
+	// because of the limited size of each bucket and the nature of our data,
+	// we may have around 10% of our keys evicted in this scenario. removing 1% because we cannot predict
+	// accurately what is happening with random data.
+	assert.GreaterOrEqual(t, acc, len(dataset)-(len(dataset)*1.0/100.0))
+}
+
+func TestLRUStriped_Size(t *testing.T) {
+	scache, err := NewLRUStriped(LRUOptions{StripedBuckets: 2, Size: 128})
+	require.NoError(t, err)
+	cache := scache.(LRUStriped)
+	acc := 0
+	for _, bucket := range cache.buckets {
+		acc += bucket.size
+	}
+	assert.Equal(t, 128+13+1, acc) // +10% +modulo padding
+}
+
+func TestLRUStriped_HashKey(t *testing.T) {
+	scache, err := NewLRUStriped(LRUOptions{StripedBuckets: 2, Size: 128})
+	require.NoError(t, err)
+	cache := scache.(LRUStriped)
+	first := cache.hashkeyMapHash("key")
+	cache.hashkeyMapHash("other_key_to_ensure_that_result_it’s_not_dependent_on_previous_input")
+	second := cache.hashkeyMapHash("key")
+	require.Equal(t, first, second)
+}
+
+func TestLRUStriped_Get(t *testing.T) {
+	cache, err := NewLRUStriped(LRUOptions{StripedBuckets: 4, Size: 128})
+	require.NoError(t, err)
+	var out string
+	require.Equal(t, ErrKeyNotFound, cache.Get("key", &out))
+	require.Zero(t, out)
+
+	require.NoError(t, cache.Set("key", "value"))
+	require.NoError(t, cache.Get("key", &out))
+	require.Equal(t, "value", out)
+}
+
+var hashSink uint64
+
+func BenchmarkSum64(b *testing.B) {
+	cases := []string{
+		"1",
+		"22",
+		"333",
+		model.NewId(),
+		model.NewId() + model.NewId(),
+	}
+
+	for _, case_ := range cases {
+		b.Run(fmt.Sprintf("maphash_string_len_%d", len(case_)), func(b *testing.B) {
+			seed := maphash.MakeSeed()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				var h maphash.Hash
+				h.SetSeed(seed)
+				h.WriteString(case_) // documentation and code says it never fails
+				hashSink = h.Sum64()
+			}
+		})
+		b.Run(fmt.Sprintf("xxhash_string_len_%d", len(case_)), func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				hashSink = xxhash.Sum64String(case_)
+			}
+		})
+	}
+}

server/platform/services/cache/lru_test.go

@@ -0,0 +1,646 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cache
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestLRU(t *testing.T) {
+	l := NewLRU(LRUOptions{
+		Size:                   128,
+		DefaultExpiry:          0,
+		InvalidateClusterEvent: "",
+	})
+
+	for i := 0; i < 256; i++ {
+		err := l.Set(fmt.Sprintf("%d", i), i)
+		require.NoError(t, err)
+	}
+	size, err := l.Len()
+	require.NoError(t, err)
+	require.Equalf(t, size, 128, "bad len: %v", size)
+
+	keys, err := l.Keys()
+	require.NoError(t, err)
+	for i, k := range keys {
+		var v int
+		err = l.Get(k, &v)
+		require.NoError(t, err, "bad key: %v", k)
+		require.Equalf(t, fmt.Sprintf("%d", v), k, "bad key: %v", k)
+		require.Equalf(t, i+128, v, "bad value: %v", k)
+	}
+	for i := 0; i < 128; i++ {
+		var v int
+		err = l.Get(fmt.Sprintf("%d", i), &v)
+		require.Equal(t, ErrKeyNotFound, err, "should be evicted %v: %v", i, err)
+	}
+	for i := 128; i < 256; i++ {
+		var v int
+		err = l.Get(fmt.Sprintf("%d", i), &v)
+		require.NoError(t, err, "should not be evicted %v: %v", i, err)
+	}
+	for i := 128; i < 192; i++ {
+		l.Remove(fmt.Sprintf("%d", i))
+		var v int
+		err = l.Get(fmt.Sprintf("%d", i), &v)
+		require.Equal(t, ErrKeyNotFound, err, "should be deleted %v: %v", i, err)
+	}
+
+	var v int
+	err = l.Get("192", &v) // expect 192 to be last key in l.Keys()
+	require.NoError(t, err, "should exist")
+	require.Equalf(t, 192, v, "bad value: %v", v)
+
+	keys, err = l.Keys()
+	require.NoError(t, err)
+	for i, k := range keys {
+		require.Falsef(t, i < 63 && k != fmt.Sprintf("%d", i+193), "out of order key: %v", k)
+		require.Falsef(t, i == 63 && k != "192", "out of order key: %v", k)
+	}
+
+	l.Purge()
+	size, err = l.Len()
+	require.NoError(t, err)
+	require.Equalf(t, size, 0, "bad len: %v", size)
+	err = l.Get("200", &v)
+	require.Equal(t, err, ErrKeyNotFound, "should contain nothing")
+
+	err = l.Set("201", 301)
+	require.NoError(t, err)
+	err = l.Get("201", &v)
+	require.NoError(t, err)
+	require.Equal(t, 301, v)
+
+}
+
+func TestLRUExpire(t *testing.T) {
+	l := NewLRU(LRUOptions{
+		Size:                   128,
+		DefaultExpiry:          1 * time.Second,
+		InvalidateClusterEvent: "",
+	})
+
+	l.SetWithDefaultExpiry("1", 1)
+	l.SetWithExpiry("3", 3, 0*time.Second)
+
+	time.Sleep(time.Second * 2)
+
+	var r1 int
+	err := l.Get("1", &r1)
+	require.Equal(t, err, ErrKeyNotFound, "should not exist")
+
+	var r2 int
+	err2 := l.Get("3", &r2)
+	require.NoError(t, err2, "should exist")
+	require.Equal(t, 3, r2)
+}
+
+func TestLRUMarshalUnMarshal(t *testing.T) {
+	l := NewLRU(LRUOptions{
+		Size:                   1,
+		DefaultExpiry:          0,
+		InvalidateClusterEvent: "",
+	})
+
+	value1 := map[string]any{
+		"key1": 1,
+		"key2": "value2",
+	}
+	err := l.Set("test", value1)
+
+	require.NoError(t, err)
+
+	var value2 map[string]any
+	err = l.Get("test", &value2)
+	require.NoError(t, err)
+	assert.EqualValues(t, 1, value2["key1"])
+
+	v2, ok := value2["key2"].(string)
+	require.True(t, ok, "unable to cast value")
+	assert.Equal(t, "value2", v2)
+
+	post := model.Post{
+		Id:            "id",
+		CreateAt:      11111,
+		UpdateAt:      11111,
+		DeleteAt:      11111,
+		EditAt:        111111,
+		IsPinned:      true,
+		UserId:        "UserId",
+		ChannelId:     "ChannelId",
+		RootId:        "RootId",
+		OriginalId:    "OriginalId",
+		Message:       "OriginalId",
+		MessageSource: "MessageSource",
+		Type:          "Type",
+		Props: map[string]any{
+			"key": "val",
+		},
+		Hashtags:      "Hashtags",
+		Filenames:     []string{"item1", "item2"},
+		FileIds:       []string{"item1", "item2"},
+		PendingPostId: "PendingPostId",
+		HasReactions:  true,
+		ReplyCount:    11111,
+		Metadata: &model.PostMetadata{
+			Embeds: []*model.PostEmbed{
+				{
+					Type: "Type",
+					URL:  "URL",
+					Data: "some data",
+				},
+				{
+					Type: "Type 2",
+					URL:  "URL 2",
+					Data: "some data 2",
+				},
+			},
+			Emojis: []*model.Emoji{
+				{
+					Id:   "id",
+					Name: "name",
+				},
+			},
+			Files: nil,
+			Images: map[string]*model.PostImage{
+				"key": {
+					Width:      1,
+					Height:     1,
+					Format:     "format",
+					FrameCount: 1,
+				},
+				"key2": {
+					Width:      999,
+					Height:     888,
+					Format:     "format 2",
+					FrameCount: 1000,
+				},
+			},
+			Reactions: []*model.Reaction{
+				{
+					UserId:    "user_id",
+					PostId:    "post_id",
+					EmojiName: "emoji_name",
+					CreateAt:  111,
+				},
+			},
+		},
+	}
+	err = l.Set("post", post.Clone())
+	require.NoError(t, err)
+
+	var p model.Post
+	err = l.Get("post", &p)
+	require.NoError(t, err)
+	require.Equal(t, post.Clone(), p.Clone())
+
+	session := &model.Session{
+		Id:             "ty7ia14yuty5bmpt8wmz6da1fw",
+		Token:          "79c3iq6nzpycmkkawudanqhg5c",
+		CreateAt:       1595445296960,
+		ExpiresAt:      1598296496960,
+		LastActivityAt: 1595445296960,
+		UserId:         "rpgh1q5ra38y9xjn9z8fjctezr",
+		Roles:          "system_admin system_user",
+		IsOAuth:        false,
+		ExpiredNotify:  false,
+		Props: map[string]string{
+			"csrf":     "33zb7h7rk3rfffztojn5pxbkxe",
+			"isMobile": "false",
+			"isSaml":   "false",
+			"is_guest": "false",
+			"os":       "",
+			"platform": "Windows",
+		},
+	}
+
+	err = l.Set("session", session)
+	require.NoError(t, err)
+	var s = &model.Session{}
+	err = l.Get("session", s)
+
+	require.NoError(t, err)
+	require.Equal(t, session, s)
+
+	user := &model.User{
+		Id:             "id",
+		CreateAt:       11111,
+		UpdateAt:       11111,
+		DeleteAt:       11111,
+		Username:       "username",
+		Password:       "password",
+		AuthService:    "AuthService",
+		AuthData:       nil,
+		Email:          "Email",
+		EmailVerified:  true,
+		Nickname:       "Nickname",
+		FirstName:      "FirstName",
+		LastName:       "LastName",
+		Position:       "Position",
+		Roles:          "Roles",
+		AllowMarketing: true,
+		Props: map[string]string{
+			"key0": "value0",
+		},
+		NotifyProps: map[string]string{
+			"key0": "value0",
+		},
+		LastPasswordUpdate:     111111,
+		LastPictureUpdate:      111111,
+		FailedAttempts:         111111,
+		Locale:                 "Locale",
+		MfaActive:              true,
+		MfaSecret:              "MfaSecret",
+		LastActivityAt:         111111,
+		IsBot:                  true,
+		TermsOfServiceId:       "TermsOfServiceId",
+		TermsOfServiceCreateAt: 111111,
+	}
+
+	err = l.Set("user", user)
+	require.NoError(t, err)
+
+	var u *model.User
+	err = l.Get("user", &u)
+	require.NoError(t, err)
+	// msgp returns an empty map instead of a nil map.
+	// This does not make an actual difference in terms of functionality.
+	u.Timezone = nil
+	require.Equal(t, user, u)
+
+	tt := make(map[string]*model.User)
+	tt["1"] = u
+	err = l.Set("mm", model.UserMap(tt))
+	require.NoError(t, err)
+
+	var out map[string]*model.User
+	err = l.Get("mm", &out)
+	require.NoError(t, err)
+	out["1"].Timezone = nil
+	require.Equal(t, tt, out)
+}
+
+func BenchmarkLRU(b *testing.B) {
+
+	value1 := "simplestring"
+
+	b.Run("simple=new", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			l2 := NewLRU(LRUOptions{
+				Size:                   1,
+				DefaultExpiry:          0,
+				InvalidateClusterEvent: "",
+			})
+			err := l2.Set("test", value1)
+			require.NoError(b, err)
+
+			var val string
+			err = l2.Get("test", &val)
+			require.NoError(b, err)
+		}
+	})
+
+	type obj struct {
+		Field1 int
+		Field2 string
+		Field3 struct {
+			Field4 int
+			Field5 string
+		}
+		Field6 map[string]string
+	}
+
+	value2 := obj{
+		1,
+		"field2",
+		struct {
+			Field4 int
+			Field5 string
+		}{
+			6,
+			"field5 is a looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong string",
+		},
+		map[string]string{
+			"key0": "value0",
+			"key1": "value value1",
+			"key2": "value value value2",
+			"key3": "value value value value3",
+			"key4": "value value value value value4",
+			"key5": "value value value value value value5",
+			"key6": "value value value value value value value6",
+			"key7": "value value value value value value value value7",
+			"key8": "value value value value value value value value value8",
+			"key9": "value value value value value value value value value value9",
+		},
+	}
+
+	b.Run("complex=new", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			l2 := NewLRU(LRUOptions{
+				Size:                   1,
+				DefaultExpiry:          0,
+				InvalidateClusterEvent: "",
+			})
+			err := l2.Set("test", value2)
+			require.NoError(b, err)
+
+			var val obj
+			err = l2.Get("test", &val)
+			require.NoError(b, err)
+		}
+	})
+
+	user := &model.User{
+		Id:             "id",
+		CreateAt:       11111,
+		UpdateAt:       11111,
+		DeleteAt:       11111,
+		Username:       "username",
+		Password:       "password",
+		AuthService:    "AuthService",
+		AuthData:       nil,
+		Email:          "Email",
+		EmailVerified:  true,
+		Nickname:       "Nickname",
+		FirstName:      "FirstName",
+		LastName:       "LastName",
+		Position:       "Position",
+		Roles:          "Roles",
+		AllowMarketing: true,
+		Props: map[string]string{
+			"key0": "value0",
+			"key1": "value value1",
+			"key2": "value value value2",
+			"key3": "value value value value3",
+			"key4": "value value value value value4",
+			"key5": "value value value value value value5",
+			"key6": "value value value value value value value6",
+			"key7": "value value value value value value value value7",
+			"key8": "value value value value value value value value value8",
+			"key9": "value value value value value value value value value value9",
+		},
+		NotifyProps: map[string]string{
+			"key0": "value0",
+			"key1": "value value1",
+			"key2": "value value value2",
+			"key3": "value value value value3",
+			"key4": "value value value value value4",
+			"key5": "value value value value value value5",
+			"key6": "value value value value value value value6",
+			"key7": "value value value value value value value value7",
+			"key8": "value value value value value value value value value8",
+			"key9": "value value value value value value value value value value9",
+		},
+		LastPasswordUpdate: 111111,
+		LastPictureUpdate:  111111,
+		FailedAttempts:     111111,
+		Locale:             "Locale",
+		Timezone: map[string]string{
+			"key0": "value0",
+			"key1": "value value1",
+			"key2": "value value value2",
+			"key3": "value value value value3",
+			"key4": "value value value value value4",
+			"key5": "value value value value value value5",
+			"key6": "value value value value value value value6",
+			"key7": "value value value value value value value value7",
+			"key8": "value value value value value value value value value8",
+			"key9": "value value value value value value value value value value9",
+		},
+		MfaActive:              true,
+		MfaSecret:              "MfaSecret",
+		LastActivityAt:         111111,
+		IsBot:                  true,
+		BotDescription:         "field5 is a looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong string",
+		BotLastIconUpdate:      111111,
+		TermsOfServiceId:       "TermsOfServiceId",
+		TermsOfServiceCreateAt: 111111,
+	}
+
+	b.Run("User=new", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			l2 := NewLRU(LRUOptions{
+				Size:                   1,
+				DefaultExpiry:          0,
+				InvalidateClusterEvent: "",
+			})
+			err := l2.Set("test", user)
+			require.NoError(b, err)
+
+			var val model.User
+			err = l2.Get("test", &val)
+			require.NoError(b, err)
+		}
+	})
+
+	uMap := map[string]*model.User{
+		"id1": {
+			Id:       "id1",
+			CreateAt: 1111,
+			UpdateAt: 1112,
+			Username: "user1",
+			Password: "pass",
+		},
+		"id2": {
+			Id:       "id2",
+			CreateAt: 1113,
+			UpdateAt: 1114,
+			Username: "user2",
+			Password: "pass2",
+		},
+	}
+
+	b.Run("UserMap=new", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			l2 := NewLRU(LRUOptions{
+				Size:                   1,
+				DefaultExpiry:          0,
+				InvalidateClusterEvent: "",
+			})
+			err := l2.Set("test", model.UserMap(uMap))
+			require.NoError(b, err)
+
+			var val map[string]*model.User
+			err = l2.Get("test", &val)
+			require.NoError(b, err)
+		}
+	})
+
+	post := &model.Post{
+		Id:            "id",
+		CreateAt:      11111,
+		UpdateAt:      11111,
+		DeleteAt:      11111,
+		EditAt:        111111,
+		IsPinned:      true,
+		UserId:        "UserId",
+		ChannelId:     "ChannelId",
+		RootId:        "RootId",
+		OriginalId:    "OriginalId",
+		Message:       "OriginalId",
+		MessageSource: "MessageSource",
+		Type:          "Type",
+		Props: map[string]any{
+			"key": "val",
+		},
+		Hashtags:      "Hashtags",
+		Filenames:     []string{"item1", "item2"},
+		FileIds:       []string{"item1", "item2"},
+		PendingPostId: "PendingPostId",
+		HasReactions:  true,
+
+		// Transient data populated before sending a post to the client
+		ReplyCount: 11111,
+		Metadata: &model.PostMetadata{
+			Embeds: []*model.PostEmbed{
+				{
+					Type: "Type",
+					URL:  "URL",
+					Data: "some data",
+				},
+				{
+					Type: "Type 2",
+					URL:  "URL 2",
+					Data: "some data 2",
+				},
+			},
+			Emojis: []*model.Emoji{
+				{
+					Id:   "id",
+					Name: "name",
+				},
+			},
+			Files: nil,
+			Images: map[string]*model.PostImage{
+				"key": {
+					Width:      1,
+					Height:     1,
+					Format:     "format",
+					FrameCount: 1,
+				},
+				"key2": {
+					Width:      999,
+					Height:     888,
+					Format:     "format 2",
+					FrameCount: 1000,
+				},
+			},
+			Reactions: []*model.Reaction{},
+		},
+	}
+
+	b.Run("Post=new", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			l2 := NewLRU(LRUOptions{
+				Size:                   1,
+				DefaultExpiry:          0,
+				InvalidateClusterEvent: "",
+			})
+			err := l2.Set("test", post)
+			require.NoError(b, err)
+
+			var val model.Post
+			err = l2.Get("test", &val)
+			require.NoError(b, err)
+		}
+	})
+
+	status := model.Status{
+		UserId:         "UserId",
+		Status:         "Status",
+		Manual:         true,
+		LastActivityAt: 111111,
+		ActiveChannel:  "ActiveChannel",
+	}
+
+	b.Run("Status=new", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			l2 := NewLRU(LRUOptions{
+				Size:                   1,
+				DefaultExpiry:          0,
+				InvalidateClusterEvent: "",
+			})
+			err := l2.Set("test", status)
+			require.NoError(b, err)
+
+			var val *model.Status
+			err = l2.Get("test", &val)
+			require.NoError(b, err)
+		}
+	})
+
+	session := model.Session{
+		Id:             "ty7ia14yuty5bmpt8wmz6da1fw",
+		Token:          "79c3iq6nzpycmkkawudanqhg5c",
+		CreateAt:       1595445296960,
+		ExpiresAt:      1598296496960,
+		LastActivityAt: 1595445296960,
+		UserId:         "rpgh1q5ra38y9xjn9z8fjctezr",
+		Roles:          "system_admin system_user",
+		IsOAuth:        false,
+		ExpiredNotify:  false,
+		Props: map[string]string{
+			"csrf":     "33zb7h7rk3rfffztojn5pxbkxe",
+			"isMobile": "false",
+			"isSaml":   "false",
+			"is_guest": "false",
+			"os":       "",
+			"platform": "Windows",
+		},
+	}
+
+	b.Run("Session=new", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			l2 := NewLRU(LRUOptions{
+				Size:                   1,
+				DefaultExpiry:          0,
+				InvalidateClusterEvent: "",
+			})
+			err := l2.Set("test", &session)
+			require.NoError(b, err)
+
+			var val *model.Session
+			err = l2.Get("test", &val)
+			require.NoError(b, err)
+		}
+	})
+}
+
+func TestLRURace(t *testing.T) {
+	l2 := NewLRU(LRUOptions{
+		Size:                   1,
+		DefaultExpiry:          0,
+		InvalidateClusterEvent: "",
+	})
+	var wg sync.WaitGroup
+	l2.Set("test", "value1")
+
+	wg.Add(2)
+
+	go func() {
+		defer wg.Done()
+		value1 := "simplestring"
+		err := l2.Set("test", value1)
+		require.NoError(t, err)
+	}()
+
+	go func() {
+		defer wg.Done()
+
+		var val string
+		err := l2.Get("test", &val)
+		require.NoError(t, err)
+	}()
+
+	wg.Wait()
+}

server/platform/services/cache/mocks/Cache.go

@@ -0,0 +1,170 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	time "time"
+
+	mock "github.com/stretchr/testify/mock"
+)
+
+// Cache is an autogenerated mock type for the Cache type
+type Cache struct {
+	mock.Mock
+}
+
+// Get provides a mock function with given fields: key, value
+func (_m *Cache) Get(key string, value any) error {
+	ret := _m.Called(key, value)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, any) error); ok {
+		r0 = rf(key, value)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// GetInvalidateClusterEvent provides a mock function with given fields:
+func (_m *Cache) GetInvalidateClusterEvent() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// Keys provides a mock function with given fields:
+func (_m *Cache) Keys() ([]string, error) {
+	ret := _m.Called()
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func() []string); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Len provides a mock function with given fields:
+func (_m *Cache) Len() (int, error) {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func() error); ok {
+		r1 = rf()
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
+// Name provides a mock function with given fields:
+func (_m *Cache) Name() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// Purge provides a mock function with given fields:
+func (_m *Cache) Purge() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Remove provides a mock function with given fields: key
+func (_m *Cache) Remove(key string) error {
+	ret := _m.Called(key)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string) error); ok {
+		r0 = rf(key)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Set provides a mock function with given fields: key, value
+func (_m *Cache) Set(key string, value any) error {
+	ret := _m.Called(key, value)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, any) error); ok {
+		r0 = rf(key, value)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// SetWithDefaultExpiry provides a mock function with given fields: key, value
+func (_m *Cache) SetWithDefaultExpiry(key string, value any) error {
+	ret := _m.Called(key, value)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, any) error); ok {
+		r0 = rf(key, value)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// SetWithExpiry provides a mock function with given fields: key, value, ttl
+func (_m *Cache) SetWithExpiry(key string, value any, ttl time.Duration) error {
+	ret := _m.Called(key, value, ttl)
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, any, time.Duration) error); ok {
+		r0 = rf(key, value, ttl)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}

server/platform/services/cache/mocks/Provider.go

@@ -0,0 +1,65 @@
+// Code generated by mockery v1.0.0. DO NOT EDIT.
+
+package mocks
+
+import (
+	mock "github.com/stretchr/testify/mock"
+
+	cache "github.com/mattermost/mattermost-server/v6/server/platform/services/cache"
+)
+
+// Provider is an autogenerated mock type for the Provider type
+type Provider struct {
+	mock.Mock
+}
+
+// Close provides a mock function with given fields:
+func (_m *Provider) Close() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// Connect provides a mock function with given fields:
+func (_m *Provider) Connect() error {
+	ret := _m.Called()
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func() error); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
+// NewCache provides a mock function with given fields: opts
+func (_m *Provider) NewCache(opts *cache.CacheOptions) (cache.Cache, error) {
+	ret := _m.Called(opts)
+
+	var r0 cache.Cache
+	if rf, ok := ret.Get(0).(func(*cache.CacheOptions) cache.Cache); ok {
+		r0 = rf(opts)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(cache.Cache)
+		}
+	}
+
+	var r1 error
+	if rf, ok := ret.Get(1).(func(*cache.CacheOptions) error); ok {
+		r1 = rf(opts)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}

server/platform/services/cache/provider.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cache
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// CacheOptions contains options for initializing a cache
+type CacheOptions struct {
+	Size                   int
+	DefaultExpiry          time.Duration
+	Name                   string
+	InvalidateClusterEvent model.ClusterEvent
+	Striped                bool
+	StripedBuckets         int
+}
+
+// Provider is a provider for Cache
+type Provider interface {
+	// NewCache creates a new cache with given options.
+	NewCache(opts *CacheOptions) (Cache, error)
+	// Connect opens a new connection to the cache using specific provider parameters.
+	Connect() error
+	// Close releases any resources used by the cache provider.
+	Close() error
+}
+
+type cacheProvider struct {
+}
+
+// NewProvider creates a new CacheProvider
+func NewProvider() Provider {
+	return &cacheProvider{}
+}
+
+// NewCache creates a new cache with given opts
+func (c *cacheProvider) NewCache(opts *CacheOptions) (Cache, error) {
+	if opts.Striped {
+		return NewLRUStriped(LRUOptions{
+			Name:                   opts.Name,
+			Size:                   opts.Size,
+			DefaultExpiry:          opts.DefaultExpiry,
+			InvalidateClusterEvent: opts.InvalidateClusterEvent,
+			StripedBuckets:         opts.StripedBuckets,
+		})
+	}
+	return NewLRU(LRUOptions{
+		Name:                   opts.Name,
+		Size:                   opts.Size,
+		DefaultExpiry:          opts.DefaultExpiry,
+		InvalidateClusterEvent: opts.InvalidateClusterEvent,
+	}), nil
+}
+
+// Connect opens a new connection to the cache using specific provider parameters.
+func (c *cacheProvider) Connect() error {
+	return nil
+}
+
+// Close releases any resources used by the cache provider.
+func (c *cacheProvider) Close() error {
+	return nil
+}

server/platform/services/cache/provider_test.go

@@ -0,0 +1,187 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package cache
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestNewCache(t *testing.T) {
+	t.Run("with only size option given", func(t *testing.T) {
+		p := NewProvider()
+
+		size := 1
+		c, err := p.NewCache(&CacheOptions{
+			Size: size,
+		})
+		require.NoError(t, err)
+
+		err = c.Set("key1", "val1")
+		require.NoError(t, err)
+		err = c.Set("key2", "val2")
+		require.NoError(t, err)
+		err = c.Set("key3", "val3")
+		require.NoError(t, err)
+		l, err := c.Len()
+		require.NoError(t, err)
+		require.Equal(t, size, l)
+	})
+
+	t.Run("with only size option given", func(t *testing.T) {
+		p := NewProvider()
+
+		size := 1
+		c, err := p.NewCache(&CacheOptions{
+			Size: size,
+		})
+		require.NoError(t, err)
+
+		err = c.Set("key1", "val1")
+		require.NoError(t, err)
+		err = c.Set("key2", "val2")
+		require.NoError(t, err)
+		err = c.Set("key3", "val3")
+		require.NoError(t, err)
+		l, err := c.Len()
+		require.NoError(t, err)
+		require.Equal(t, size, l)
+	})
+
+	t.Run("with all options specified", func(t *testing.T) {
+		p := NewProvider()
+
+		size := 1
+		expiry := 1 * time.Second
+		event := model.ClusterEvent("clusterEvent")
+		c, err := p.NewCache(&CacheOptions{
+			Size:                   size,
+			Name:                   "name",
+			DefaultExpiry:          expiry,
+			InvalidateClusterEvent: event,
+		})
+		require.NoError(t, err)
+
+		require.Equal(t, event, c.GetInvalidateClusterEvent())
+
+		err = c.SetWithDefaultExpiry("key1", "val1")
+		require.NoError(t, err)
+		err = c.SetWithDefaultExpiry("key2", "val2")
+		require.NoError(t, err)
+		err = c.SetWithDefaultExpiry("key3", "val3")
+		require.NoError(t, err)
+		l, err := c.Len()
+		require.NoError(t, err)
+		require.Equal(t, size, l)
+
+		time.Sleep(expiry + 1*time.Second)
+
+		var v string
+		err = c.Get("key1", &v)
+		require.Equal(t, ErrKeyNotFound, err)
+		err = c.Get("key2", &v)
+		require.Equal(t, ErrKeyNotFound, err)
+		err = c.Get("key3", &v)
+		require.Equal(t, ErrKeyNotFound, err)
+	})
+}
+
+func TestNewCache_Striped(t *testing.T) {
+	t.Run("with only size option given", func(t *testing.T) {
+		p := NewProvider()
+
+		size := 1
+		c, err := p.NewCache(&CacheOptions{
+			Size:           size,
+			Striped:        true,
+			StripedBuckets: 1,
+		})
+		require.NoError(t, err)
+
+		err = c.Set("key1", "val1")
+		require.NoError(t, err)
+		err = c.Set("key2", "val2")
+		require.NoError(t, err)
+		err = c.Set("key3", "val3")
+		require.NoError(t, err)
+		l, err := c.Len()
+		require.NoError(t, err)
+		require.Equal(t, size+1, l) // +10% from striping
+	})
+
+	t.Run("with only size option given", func(t *testing.T) {
+		p := NewProvider()
+
+		size := 1
+		c, err := p.NewCache(&CacheOptions{
+			Size:           size,
+			Striped:        true,
+			StripedBuckets: 1,
+		})
+		require.NoError(t, err)
+
+		err = c.Set("key1", "val1")
+		require.NoError(t, err)
+		err = c.Set("key2", "val2")
+		require.NoError(t, err)
+		err = c.Set("key3", "val3")
+		require.NoError(t, err)
+		l, err := c.Len()
+		require.NoError(t, err)
+		require.Equal(t, size+1, l) // +10% rounded up from striped lru
+	})
+
+	t.Run("with all options specified", func(t *testing.T) {
+		p := NewProvider()
+
+		size := 1
+		expiry := 1 * time.Second
+		event := model.ClusterEvent("clusterEvent")
+		c, err := p.NewCache(&CacheOptions{
+			Size:                   size,
+			Name:                   "name",
+			DefaultExpiry:          expiry,
+			InvalidateClusterEvent: event,
+			Striped:                true,
+			StripedBuckets:         1,
+		})
+		require.NoError(t, err)
+
+		require.Equal(t, event, c.GetInvalidateClusterEvent())
+
+		err = c.SetWithDefaultExpiry("key1", "val1")
+		require.NoError(t, err)
+		err = c.SetWithDefaultExpiry("key2", "val2")
+		require.NoError(t, err)
+		err = c.SetWithDefaultExpiry("key3", "val3")
+		require.NoError(t, err)
+		l, err := c.Len()
+		require.NoError(t, err)
+		require.Equal(t, size+1, l) // +10% from striping
+
+		time.Sleep(expiry + 1*time.Second)
+
+		var v string
+		err = c.Get("key1", &v)
+		require.Equal(t, ErrKeyNotFound, err)
+		err = c.Get("key2", &v)
+		require.Equal(t, ErrKeyNotFound, err)
+		err = c.Get("key3", &v)
+		require.Equal(t, ErrKeyNotFound, err)
+	})
+}
+
+func TestConnectClose(t *testing.T) {
+	p := NewProvider()
+
+	err := p.Connect()
+	require.NoError(t, err)
+
+	err = p.Close()
+	require.NoError(t, err)
+}

server/platform/services/configservice/configservice.go

@@ -0,0 +1,15 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package configservice
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// An interface representing something that contains a Config, such as the app.App struct
+type ConfigService interface {
+	Config() *model.Config
+	AddConfigListener(func(old, current *model.Config)) string
+	RemoveConfigListener(string)
+}

server/platform/services/docextractor/archive.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mholt/archiver/v3"
+)
+
+type archiveExtractor struct {
+	SubExtractor Extractor
+}
+
+func (ae *archiveExtractor) Match(filename string) bool {
+	_, err := archiver.ByExtension(filename)
+	return err == nil
+}
+
+func (ae *archiveExtractor) Extract(name string, r io.ReadSeeker) (string, error) {
+	dir, err := os.MkdirTemp(os.TempDir(), "archiver")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	f, err := os.Create(filepath.Join(dir, name))
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+	_, err = io.Copy(f, r)
+	f.Close()
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	var text strings.Builder
+	err = archiver.Walk(f.Name(), func(file archiver.File) error {
+		text.WriteString(file.Name() + " ")
+		if ae.SubExtractor != nil {
+			filename := filepath.Base(file.Name())
+			filename = strings.ReplaceAll(filename, "-", " ")
+			filename = strings.ReplaceAll(filename, ".", " ")
+			filename = strings.ReplaceAll(filename, ",", " ")
+			data, err2 := io.ReadAll(file)
+			if err2 != nil {
+				return err2
+			}
+			subtext, extractErr := ae.SubExtractor.Extract(filename, bytes.NewReader(data))
+			if extractErr == nil {
+				text.WriteString(subtext + " ")
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return text.String(), nil
+}

server/platform/services/docextractor/combine.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type combineExtractor struct {
+	SubExtractors []Extractor
+}
+
+func (ce *combineExtractor) Add(extractor Extractor) {
+	ce.SubExtractors = append(ce.SubExtractors, extractor)
+}
+
+func (ce *combineExtractor) Match(filename string) bool {
+	for _, extractor := range ce.SubExtractors {
+		if extractor.Match(filename) {
+			return true
+		}
+	}
+	return false
+}
+
+func (ce *combineExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
+	for _, extractor := range ce.SubExtractors {
+		if extractor.Match(filename) {
+			r.Seek(0, io.SeekStart)
+			text, err := extractor.Extract(filename, r)
+			if err != nil {
+				mlog.Warn("unable to extract file content", mlog.Err(err))
+				continue
+			}
+			return text, nil
+		}
+	}
+	return "", nil
+}

server/platform/services/docextractor/docextractor.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+)
+
+// ExtractSettings defines the features enabled/disable during the document text extraction.
+type ExtractSettings struct {
+	ArchiveRecursion bool
+	MMPreviewURL     string
+	MMPreviewSecret  string
+}
+
+// Extract extract the text from a document using the system default extractors
+func Extract(filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) {
+	return ExtractWithExtraExtractors(filename, r, settings, []Extractor{})
+}
+
+// ExtractWithExtraExtractors extract the text from a document using the provided extractors beside the system default extractors.
+func ExtractWithExtraExtractors(filename string, r io.ReadSeeker, settings ExtractSettings, extraExtractors []Extractor) (string, error) {
+	enabledExtractors := &combineExtractor{}
+	for _, extraExtractor := range extraExtractors {
+		enabledExtractors.Add(extraExtractor)
+	}
+	enabledExtractors.Add(&documentExtractor{})
+	enabledExtractors.Add(&pdfExtractor{})
+
+	if settings.ArchiveRecursion {
+		enabledExtractors.Add(&archiveExtractor{SubExtractor: enabledExtractors})
+	} else {
+		enabledExtractors.Add(&archiveExtractor{})
+	}
+
+	if settings.MMPreviewURL != "" {
+		enabledExtractors.Add(newMMPreviewExtractor(settings.MMPreviewURL, settings.MMPreviewSecret, pdfExtractor{}))
+	}
+	enabledExtractors.Add(&plainExtractor{})
+
+	if enabledExtractors.Match(filename) {
+		return enabledExtractors.Extract(filename, r)
+	}
+	return "", nil
+}

server/platform/services/docextractor/docextractor_test.go

@@ -0,0 +1,202 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+)
+
+func TestExtract(t *testing.T) {
+	testCases := []struct {
+		Name         string
+		TestFileName string
+		Settings     ExtractSettings
+		Contains     []string
+		NotContains  []string
+		ExpectError  bool
+	}{
+		{
+			"Plain text file",
+			"test-markdown-basics.md",
+			ExtractSettings{},
+			[]string{"followed", "separated", "Basic"},
+			[]string{},
+			false,
+		},
+		{
+			"Plain small text file",
+			"test-hashtags.md",
+			ExtractSettings{},
+			[]string{"should", "render", "strings"},
+			[]string{},
+			false,
+		},
+		{
+			"Zip file without recursion",
+			"Fake_Team_Import.zip",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Zip file with recursion",
+			"Fake_Team_Import.zip",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Rar file without recursion",
+			"Fake_Team_Import.rar",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Rar file with recursion",
+			"Fake_Team_Import.rar",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Tar.gz file without recursion",
+			"Fake_Team_Import.tar.gz",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Tar.gz file with recursion",
+			"Fake_Team_Import.tar.gz",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Pdf file",
+			"sample-doc.pdf",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Docx file",
+			"sample-doc.docx",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Odt file",
+			"sample-doc.odt",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Pptx file",
+			"sample-doc.pptx",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			data, err := testutils.ReadTestFile(tc.TestFileName)
+			require.NoError(t, err)
+			text, err := Extract(tc.TestFileName, bytes.NewReader(data), tc.Settings)
+			if tc.ExpectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				for _, expectedString := range tc.Contains {
+					assert.Contains(t, text, expectedString)
+				}
+				for _, notExpectedString := range tc.NotContains {
+					assert.NotContains(t, text, notExpectedString)
+				}
+			}
+		})
+	}
+
+	t.Run("Unsupported binary file", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("testjpg.jpg")
+		require.NoError(t, err)
+		text, err := Extract("testjpg.jpg", bytes.NewReader(data), ExtractSettings{})
+		require.NoError(t, err)
+		require.Equal(t, "", text)
+	})
+
+	t.Run("Wrong docx extension", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.NoError(t, err)
+		text, err := Extract("sample-doc.docx", bytes.NewReader(data), ExtractSettings{})
+		require.NoError(t, err)
+		require.Equal(t, "", text)
+	})
+}
+
+type customTestPdfExtractor struct{}
+
+func (te *customTestPdfExtractor) Match(filename string) bool {
+	return strings.HasSuffix(filename, ".pdf")
+}
+
+func (te *customTestPdfExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
+	return "this is a text generated content", nil
+}
+
+type failingExtractor struct{}
+
+func (te *failingExtractor) Match(filename string) bool {
+	return true
+}
+
+func (te *failingExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
+	return "", errors.New("this always fail")
+}
+
+func TestExtractWithExtraExtractors(t *testing.T) {
+	t.Run("override existing extractor", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.NoError(t, err)
+
+		text, err := ExtractWithExtraExtractors("sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&customTestPdfExtractor{}})
+		require.NoError(t, err)
+		require.Equal(t, text, "this is a text generated content")
+	})
+
+	t.Run("failing extractor", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.NoError(t, err)
+
+		text, err := ExtractWithExtraExtractors("sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&failingExtractor{}})
+		require.NoError(t, err)
+		assert.Contains(t, text, "simple")
+		assert.Contains(t, text, "document")
+		assert.Contains(t, text, "contains")
+	})
+}

server/platform/services/docextractor/documents.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"errors"
+	"io"
+	"path"
+	"strings"
+
+	"code.sajari.com/docconv"
+)
+
+type documentExtractor struct{}
+
+var doconvConverterByExtensions = map[string]func(io.Reader) (string, map[string]string, error){
+	"doc":  docconv.ConvertDoc,
+	"docx": docconv.ConvertDocx,
+	"pptx": docconv.ConvertPptx,
+	"odt":  docconv.ConvertODT,
+	"html": func(r io.Reader) (string, map[string]string, error) { return docconv.ConvertHTML(r, true) },
+	// Temporarily disabled to avoid crashes on malicious .pages files
+	// "pages": docconv.ConvertPages,
+	"rtf": docconv.ConvertRTF,
+	"pdf": docconv.ConvertPDF,
+}
+
+func (de *documentExtractor) Match(filename string) bool {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	_, ok := doconvConverterByExtensions[extension]
+	return ok
+}
+
+func (de *documentExtractor) Extract(filename string, r io.ReadSeeker) (out string, outErr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			out = ""
+			outErr = errors.New("error extracting document text")
+		}
+	}()
+
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	converter, ok := doconvConverterByExtensions[extension]
+	if !ok {
+		return "", errors.New("unknown converter")
+	}
+
+	text, _, err := converter(r)
+	if err != nil {
+		return "", err
+	}
+
+	return text, nil
+}

server/platform/services/docextractor/interface.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+)
+
+// Extractors define the interface needed to extract file content
+type Extractor interface {
+	Match(filename string) bool
+	Extract(filename string, file io.ReadSeeker) (string, error)
+}

server/platform/services/docextractor/mmpreview.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+// MMPreview is a micro-service to convert from any libreoffice supported
+// format into a PDF file, and then we use the regular pdf extractor to convert
+// it into plain text.
+
+import (
+	"bytes"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type mmPreviewExtractor struct {
+	url          string
+	secret       string
+	pdfExtractor pdfExtractor
+}
+
+var mmpreviewSupportedExtensions = map[string]bool{
+	"ppt":  true,
+	"odp":  true,
+	"xls":  true,
+	"xlsx": true,
+	"ods":  true,
+}
+
+func newMMPreviewExtractor(url string, secret string, pdfExtractor pdfExtractor) *mmPreviewExtractor {
+	return &mmPreviewExtractor{url: url, secret: secret, pdfExtractor: pdfExtractor}
+}
+
+func (mpe *mmPreviewExtractor) Match(filename string) bool {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	return mmpreviewSupportedExtensions[extension]
+}
+
+func (mpe *mmPreviewExtractor) Extract(filename string, file io.ReadSeeker) (string, error) {
+	b, w, err := createMultipartFormData("file", filename, file)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	req, err := http.NewRequest("POST", mpe.url+"/toPDF", &b)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	if mpe.secret != "" {
+		req.Header.Add("Authentication", mpe.secret)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)")
+	}
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "unable to read the response from mmpreview")
+	}
+	return mpe.pdfExtractor.Extract(filename, bytes.NewReader(data))
+}
+
+func createMultipartFormData(fieldName, fileName string, fileData io.ReadSeeker) (bytes.Buffer, *multipart.Writer, error) {
+	var b bytes.Buffer
+	var err error
+	w := multipart.NewWriter(&b)
+	var fw io.Writer
+	if fw, err = w.CreateFormFile(fieldName, fileName); err != nil {
+		return b, nil, err
+	}
+	if _, err = io.Copy(fw, fileData); err != nil {
+		return b, nil, err
+	}
+	w.Close()
+	return b, w, nil
+}

server/platform/services/docextractor/pdf.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/ledongthuc/pdf"
+)
+
+type pdfExtractor struct{}
+
+func (pe *pdfExtractor) Match(filename string) bool {
+	supportedExtensions := map[string]bool{
+		"pdf": true,
+	}
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	return supportedExtensions[extension]
+}
+
+func (pe *pdfExtractor) Extract(filename string, r io.ReadSeeker) (out string, outErr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			out = ""
+			outErr = errors.New("error extracting pdf text")
+		}
+	}()
+	f, err := os.CreateTemp(os.TempDir(), "pdflib")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+	size, err := io.Copy(f, r)
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	reader, err := pdf.NewReader(f, size)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	b, err := reader.GetPlainText()
+	if err != nil {
+		return "", err
+	}
+	buf.ReadFrom(b)
+	return buf.String(), nil
+}

server/platform/services/docextractor/pdf_test.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+)
+
+func TestPdfEmptyFile(t *testing.T) {
+	extractor := pdfExtractor{}
+	_, err := extractor.Extract("test.pdf", bytes.NewReader([]byte{}))
+	require.Error(t, err)
+}
+
+func TestPdfFile(t *testing.T) {
+	extractor := pdfExtractor{}
+	contentText := "This is a simple document that contains some text."
+	content, err := testutils.ReadTestFile("sample-doc.pdf")
+	require.NoError(t, err)
+	extractedText, err := extractor.Extract("sample-doc.pdf", bytes.NewReader(content))
+	require.NoError(t, err)
+	require.Equal(t, contentText, extractedText)
+}
+
+func TestWrongPdfFile(t *testing.T) {
+	extractor := pdfExtractor{}
+	content, err := testutils.ReadTestFile("sample-doc.docx")
+	require.NoError(t, err)
+	_, err = extractor.Extract("sample-doc.pdf", bytes.NewReader(content))
+	require.Error(t, err)
+}

server/platform/services/docextractor/plain.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+	"unicode"
+	"unicode/utf8"
+)
+
+type plainExtractor struct{}
+
+func (pe *plainExtractor) Match(filename string) bool {
+	return true
+}
+
+func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
+	// This detects any visible character plus any whitespace
+	validRanges := append(unicode.GraphicRanges, unicode.White_Space)
+
+	runes := make([]byte, 1024)
+	total, err := r.Read(runes)
+	if err != nil && err != io.EOF {
+		return "", err
+	}
+
+	if total == 0 {
+		return "", nil
+	}
+
+	count := 0
+	for {
+		c, size := utf8.DecodeRune(runes[count:])
+		if !unicode.In(c, validRanges...) {
+			return "", nil
+		}
+		if size == 0 {
+			break
+		}
+		count += size
+
+		// subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes
+		if count > total-utf8.UTFMax {
+			break
+		}
+	}
+
+	text, _ := io.ReadAll(r)
+	return string(runes[0:total]) + string(text), nil
+}

server/platform/services/docextractor/plain_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPlainEmptyFile(t *testing.T) {
+	extractor := plainExtractor{}
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte{}))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}
+
+func TestPlainTextSmallFile(t *testing.T) {
+	extractor := plainExtractor{}
+	content := strings.Repeat("test \n", 5)
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte(content)))
+	require.NoError(t, err)
+	require.Equal(t, content, extractedText)
+}
+
+func TestPlainBigFile(t *testing.T) {
+	extractor := plainExtractor{}
+	content := strings.Repeat("test \n", 1000)
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte(content)))
+	require.NoError(t, err)
+	require.Equal(t, content, extractedText)
+}
+
+func TestSmallBinaryFile(t *testing.T) {
+	extractor := plainExtractor{}
+	notUTF8Char := byte(0x7)
+	content := bytes.Repeat([]byte{notUTF8Char}, 1000)
+	extractedText, err := extractor.Extract("test.bin", bytes.NewReader(content))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}
+
+func TestBigBinaryFile(t *testing.T) {
+	extractor := plainExtractor{}
+	notUTF8Char := byte(0x7)
+	content := bytes.Repeat([]byte{notUTF8Char}, 10000)
+	extractedText, err := extractor.Extract("test.bin", bytes.NewReader(content))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}

server/platform/services/httpservice/client.go

@@ -0,0 +1,160 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package httpservice
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net"
+	"net/http"
+	"time"
+)
+
+const (
+	ConnectTimeout = 3 * time.Second
+	RequestTimeout = 30 * time.Second
+)
+
+var reservedIPRanges []*net.IPNet
+
+// IsReservedIP checks whether the target IP belongs to reserved IP address ranges to avoid SSRF attacks to the internal
+// network of the Mattermost server
+func IsReservedIP(ip net.IP) bool {
+	for _, ipRange := range reservedIPRanges {
+		if ipRange.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// IsOwnIP handles the special case that a request might be made to the public IP of the host which on Linux is routed
+// directly via the loopback IP to any listening sockets, effectively bypassing host-based firewalls such as firewalld
+func IsOwnIP(ip net.IP) (bool, error) {
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return false, err
+	}
+
+	for _, interf := range interfaces {
+		addresses, err := interf.Addrs()
+		if err != nil {
+			return false, err
+		}
+
+		for _, addr := range addresses {
+			var selfIP net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				selfIP = v.IP
+			case *net.IPAddr:
+				selfIP = v.IP
+			}
+
+			if ip.Equal(selfIP) {
+				return true, nil
+			}
+		}
+	}
+
+	return false, nil
+}
+
+var defaultUserAgent string
+
+func init() {
+	for _, cidr := range []string{
+		// See https://tools.ietf.org/html/rfc6890
+		"0.0.0.0/8",      // This host on this network
+		"10.0.0.0/8",     // Private-Use
+		"127.0.0.0/8",    // Loopback
+		"169.254.0.0/16", // Link Local
+		"172.16.0.0/12",  // Private-Use Networks
+		"192.168.0.0/16", // Private-Use Networks
+		"::/128",         // Unspecified Address
+		"::1/128",        // Loopback Address
+		"fc00::/7",       // Unique-Local
+		"fe80::/10",      // Linked-Scoped Unicast
+	} {
+		_, parsed, err := net.ParseCIDR(cidr)
+		if err != nil {
+			panic(err)
+		}
+		reservedIPRanges = append(reservedIPRanges, parsed)
+	}
+	defaultUserAgent = "Mattermost-Bot/1.1"
+}
+
+type DialContextFunction func(ctx context.Context, network, addr string) (net.Conn, error)
+
+var AddressForbidden error = errors.New("address forbidden, you may need to set AllowedUntrustedInternalConnections to allow an integration access to your internal network")
+
+func dialContextFilter(dial DialContextFunction, allowHost func(host string) bool, allowIP func(ip net.IP) bool) DialContextFunction {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+
+		if allowHost != nil && allowHost(host) {
+			return dial(ctx, network, addr)
+		}
+
+		ips, err := net.LookupIP(host)
+		if err != nil {
+			return nil, err
+		}
+
+		var firstErr error
+		for _, ip := range ips {
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			default:
+			}
+
+			if allowIP == nil || !allowIP(ip) {
+				continue
+			}
+
+			conn, err := dial(ctx, network, net.JoinHostPort(ip.String(), port))
+			if err == nil {
+				return conn, nil
+			}
+			if firstErr == nil {
+				firstErr = err
+			}
+		}
+		if firstErr == nil {
+			return nil, AddressForbidden
+		}
+		return nil, firstErr
+	}
+}
+
+func NewTransport(enableInsecureConnections bool, allowHost func(host string) bool, allowIP func(ip net.IP) bool) *MattermostTransport {
+	dialContext := (&net.Dialer{
+		Timeout:   ConnectTimeout,
+		KeepAlive: 30 * time.Second,
+	}).DialContext
+
+	if allowHost != nil || allowIP != nil {
+		dialContext = dialContextFilter(dialContext, allowHost, allowIP)
+	}
+
+	return &MattermostTransport{
+		&http.Transport{
+			Proxy:                 http.ProxyFromEnvironment,
+			DialContext:           dialContext,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   ConnectTimeout,
+			ExpectContinueTimeout: 1 * time.Second,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: enableInsecureConnections,
+			},
+		},
+	}
+}

server/platform/services/httpservice/client_test.go

@@ -0,0 +1,263 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package httpservice
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPClient(t *testing.T) {
+	mockHTTP := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer mockHTTP.Close()
+
+	mockSelfSignedHTTPS := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer mockSelfSignedHTTPS.Close()
+
+	t.Run("insecure connections", func(t *testing.T) {
+		disableInsecureConnections := false
+		enableInsecureConnections := true
+
+		testCases := []struct {
+			description               string
+			enableInsecureConnections bool
+			url                       string
+			expectedAllowed           bool
+		}{
+			{"allow HTTP even when insecure disabled", disableInsecureConnections, mockHTTP.URL, true},
+			{"allow HTTP when insecure enabled", enableInsecureConnections, mockHTTP.URL, true},
+			{"reject self-signed HTTPS even when insecure disabled", disableInsecureConnections, mockSelfSignedHTTPS.URL, false},
+			{"allow self-signed HTTPS when insecure enabled", enableInsecureConnections, mockSelfSignedHTTPS.URL, true},
+		}
+
+		for _, testCase := range testCases {
+			t.Run(testCase.description, func(t *testing.T) {
+				c := NewHTTPClient(NewTransport(testCase.enableInsecureConnections, nil, nil))
+				if _, err := c.Get(testCase.url); testCase.expectedAllowed {
+					require.NoError(t, err)
+				} else {
+					require.Error(t, err)
+				}
+
+			})
+		}
+	})
+
+	t.Run("checks", func(t *testing.T) {
+		allowHost := func(_ string) bool { return true }
+		rejectHost := func(_ string) bool { return false }
+		allowIP := func(_ net.IP) bool { return true }
+		rejectIP := func(_ net.IP) bool { return false }
+
+		testCases := []struct {
+			description     string
+			allowHost       func(string) bool
+			allowIP         func(net.IP) bool
+			expectedAllowed bool
+		}{
+			{"allow with no checks", nil, nil, true},
+			{"reject without host check when ip rejected", nil, rejectIP, false},
+			{"allow without host check when ip allowed", nil, allowIP, true},
+
+			{"reject when host rejected since no ip check", rejectHost, nil, false},
+			{"reject when host and ip rejected", rejectHost, rejectIP, false},
+			{"allow when host rejected since ip allowed", rejectHost, allowIP, true},
+
+			{"allow when host allowed even without ip check", allowHost, nil, true},
+			{"allow when host allowed even if ip rejected", allowHost, rejectIP, true},
+			{"allow when host and ip allowed", allowHost, allowIP, true},
+		}
+		for _, testCase := range testCases {
+			t.Run(testCase.description, func(t *testing.T) {
+				c := NewHTTPClient(NewTransport(false, testCase.allowHost, testCase.allowIP))
+				if _, err := c.Get(mockHTTP.URL); testCase.expectedAllowed {
+					require.NoError(t, err)
+				} else {
+					require.IsType(t, &url.Error{}, err)
+					require.Equal(t, AddressForbidden, err.(*url.Error).Err)
+				}
+			})
+		}
+	})
+}
+
+func TestHTTPClientWithProxy(t *testing.T) {
+	proxy := createProxyServer()
+	defer proxy.Close()
+
+	c := NewHTTPClient(NewTransport(true, nil, nil))
+	purl, _ := url.Parse(proxy.URL)
+	c.Transport.(*MattermostTransport).Transport.(*http.Transport).Proxy = http.ProxyURL(purl)
+
+	resp, err := c.Get("http://acme.com")
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, "proxy", string(body))
+}
+
+func createProxyServer() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		w.Header().Set("Content-Type", "text/plain; charset=us-ascii")
+		fmt.Fprint(w, "proxy")
+	}))
+}
+
+func TestDialContextFilter(t *testing.T) {
+	for _, tc := range []struct {
+		Addr    string
+		IsValid bool
+	}{
+		{
+			Addr:    "google.com:80",
+			IsValid: true,
+		},
+		{
+			Addr:    "8.8.8.8:53",
+			IsValid: true,
+		},
+		{
+			Addr: "127.0.0.1:80",
+		},
+		{
+			Addr:    "10.0.0.1:80",
+			IsValid: true,
+		},
+	} {
+		didDial := false
+		filter := dialContextFilter(func(ctx context.Context, network, addr string) (net.Conn, error) {
+			didDial = true
+			return nil, nil
+		}, func(host string) bool { return host == "10.0.0.1" }, func(ip net.IP) bool { return !IsReservedIP(ip) })
+		_, err := filter(context.Background(), "", tc.Addr)
+
+		if tc.IsValid {
+			require.NoError(t, err)
+			require.True(t, didDial)
+		} else {
+			require.Error(t, err)
+			require.Equal(t, err, AddressForbidden)
+			require.False(t, didDial)
+		}
+	}
+}
+
+func TestUserAgentIsSet(t *testing.T) {
+	testUserAgent := "test-user-agent"
+	defaultUserAgent = testUserAgent
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		ua := req.UserAgent()
+		assert.NotEqual(t, "", ua, "expected user-agent to be non-empty")
+		assert.Equalf(t, testUserAgent, ua, "expected user-agent to be %q but was %q", testUserAgent, ua)
+	}))
+	defer ts.Close()
+	client := NewHTTPClient(NewTransport(true, nil, nil))
+	req, err := http.NewRequest("GET", ts.URL, nil)
+
+	require.NoError(t, err, "NewRequest failed", err)
+
+	client.Do(req)
+}
+
+func NewHTTPClient(transport http.RoundTripper) *http.Client {
+	return &http.Client{
+		Transport: transport,
+	}
+}
+
+func TestIsReservedIP(t *testing.T) {
+	tests := []struct {
+		name string
+		ip   net.IP
+		want bool
+	}{
+		{"127.8.3.5", net.IPv4(127, 8, 3, 5), true},
+		{"192.168.0.1", net.IPv4(192, 168, 0, 1), true},
+		{"169.254.0.6", net.IPv4(169, 254, 0, 6), true},
+		{"127.120.6.3", net.IPv4(127, 120, 6, 3), true},
+		{"8.8.8.8", net.IPv4(8, 8, 8, 8), false},
+		{"9.9.9.9", net.IPv4(9, 9, 9, 8), false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := IsReservedIP(tt.ip)
+			assert.Equalf(t, tt.want, got, "IsReservedIP() = %v, want %v", got, tt.want)
+		})
+	}
+}
+
+func TestIsOwnIP(t *testing.T) {
+	tests := []struct {
+		name string
+		ip   net.IP
+		want bool
+	}{
+		{"127.0.0.1", net.IPv4(127, 0, 0, 1), true},
+		{"8.8.8.8", net.IPv4(8, 0, 0, 8), false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, _ := IsOwnIP(tt.ip)
+			assert.Equalf(t, tt.want, got, "IsOwnIP() = %v, want %v for IP %s", got, tt.want, tt.ip.String())
+		})
+	}
+}
+
+func TestSplitHostnames(t *testing.T) {
+	var config string
+	var hostnames []string
+
+	config = ""
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{}, hostnames)
+
+	config = "127.0.0.1 localhost"
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{"127.0.0.1", "localhost"}, hostnames)
+
+	config = "127.0.0.1,localhost"
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{"127.0.0.1", "localhost"}, hostnames)
+
+	config = "127.0.0.1,,localhost"
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{"127.0.0.1", "localhost"}, hostnames)
+
+	config = "127.0.0.1  localhost"
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{"127.0.0.1", "localhost"}, hostnames)
+
+	config = "127.0.0.1 , localhost"
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{"127.0.0.1", "localhost"}, hostnames)
+
+	config = "127.0.0.1  localhost  "
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{"127.0.0.1", "localhost"}, hostnames)
+
+	config = " 127.0.0.1  ,,localhost  , , ,,"
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{"127.0.0.1", "localhost"}, hostnames)
+
+	config = "127.0.0.1 localhost, 192.168.1.0"
+	hostnames = strings.FieldsFunc(config, splitFields)
+	require.Equal(t, []string{"127.0.0.1", "localhost", "192.168.1.0"}, hostnames)
+}

server/platform/services/httpservice/httpservice.go

@@ -0,0 +1,102 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package httpservice
+
+import (
+	"net"
+	"net/http"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+)
+
+// HTTPService wraps the functionality for making http requests to provide some improvements to the default client
+// behaviour.
+type HTTPService interface {
+	// MakeClient returns an http client constructed with a RoundTripper as returned by MakeTransport.
+	MakeClient(trustURLs bool) *http.Client
+
+	// MakeTransport returns a RoundTripper that is suitable for making requests to external resources. The default
+	// implementation provides:
+	// - A shorter timeout for dial and TLS handshake (defined as constant "ConnectTimeout")
+	// - A timeout for end-to-end requests
+	// - A Mattermost-specific user agent header
+	// - Additional security for untrusted and insecure connections
+	MakeTransport(trustURLs bool) *MattermostTransport
+}
+
+type HTTPServiceImpl struct {
+	configService configservice.ConfigService
+
+	RequestTimeout time.Duration
+}
+
+func splitFields(c rune) bool {
+	return unicode.IsSpace(c) || c == ','
+}
+
+func MakeHTTPService(configService configservice.ConfigService) HTTPService {
+	return &HTTPServiceImpl{
+		configService,
+		RequestTimeout,
+	}
+}
+
+func (h *HTTPServiceImpl) MakeClient(trustURLs bool) *http.Client {
+	return &http.Client{
+		Transport: h.MakeTransport(trustURLs),
+		Timeout:   h.RequestTimeout,
+	}
+}
+
+func (h *HTTPServiceImpl) MakeTransport(trustURLs bool) *MattermostTransport {
+	insecure := h.configService.Config().ServiceSettings.EnableInsecureOutgoingConnections != nil && *h.configService.Config().ServiceSettings.EnableInsecureOutgoingConnections
+
+	if trustURLs {
+		return NewTransport(insecure, nil, nil)
+	}
+
+	allowHost := func(host string) bool {
+		if h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections == nil {
+			return false
+		}
+		for _, allowed := range strings.FieldsFunc(*h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections, splitFields) {
+			if host == allowed {
+				return true
+			}
+		}
+		return false
+	}
+
+	allowIP := func(ip net.IP) bool {
+		reservedIP := IsReservedIP(ip)
+		ownIP, err := IsOwnIP(ip)
+
+		// If there is an error getting the self-assigned IPs, default to the secure option
+		if err != nil {
+			return false
+		}
+
+		// If it's not a reserved IP and it's not self-assigned IP, accept the IP
+		if !reservedIP && !ownIP {
+			return true
+		}
+
+		if h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections == nil {
+			return false
+		}
+
+		// In the case it's the self-assigned IP, enforce that it needs to be explicitly added to the AllowedUntrustedInternalConnections
+		for _, allowed := range strings.FieldsFunc(*h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections, splitFields) {
+			if _, ipRange, err := net.ParseCIDR(allowed); err == nil && ipRange.Contains(ip) {
+				return true
+			}
+		}
+		return false
+	}
+
+	return NewTransport(insecure, allowHost, allowIP)
+}

server/platform/services/httpservice/transport.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package httpservice
+
+import (
+	"net/http"
+)
+
+// MattermostTransport is an implementation of http.RoundTripper that ensures each request contains a custom user agent
+// string to indicate that the request is coming from a Mattermost instance.
+type MattermostTransport struct {
+	// Transport is the underlying http.RoundTripper that is actually used to make the request
+	Transport http.RoundTripper
+}
+
+func (t *MattermostTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.Header.Set("User-Agent", defaultUserAgent)
+
+	return t.Transport.RoundTrip(req)
+}

server/platform/services/imageproxy/atmos_camo.go

@@ -0,0 +1,92 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package imageproxy
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+type AtmosCamoBackend struct {
+	proxy     *ImageProxy
+	siteURL   *url.URL
+	remoteURL *url.URL
+	client    *http.Client
+}
+
+func makeAtmosCamoBackend(proxy *ImageProxy) *AtmosCamoBackend {
+	// We deliberately ignore the error because it's from config.json.
+	// The function returns a nil pointer in case of error, and we handle it when it's used.
+	siteURL, _ := url.Parse(*proxy.ConfigService.Config().ServiceSettings.SiteURL)
+	remoteURL, _ := url.Parse(*proxy.ConfigService.Config().ImageProxySettings.RemoteImageProxyURL)
+
+	return &AtmosCamoBackend{
+		proxy:     proxy,
+		siteURL:   siteURL,
+		remoteURL: remoteURL,
+		client:    proxy.HTTPService.MakeClient(false),
+	}
+}
+
+func (backend *AtmosCamoBackend) GetImage(w http.ResponseWriter, r *http.Request, imageURL string) {
+	http.Redirect(w, r, backend.getAtmosCamoImageURL(imageURL), http.StatusFound)
+}
+
+func (backend *AtmosCamoBackend) GetImageDirect(imageURL string) (io.ReadCloser, string, error) {
+	req, err := http.NewRequest("GET", backend.getAtmosCamoImageURL(imageURL), nil)
+	if err != nil {
+		return nil, "", Error{err}
+	}
+
+	resp, err := backend.client.Do(req)
+	if err != nil {
+		return nil, "", Error{err}
+	}
+
+	// Note that we don't do any additional validation of the received data since we expect the image proxy to do that
+	return resp.Body, resp.Header.Get("Content-Type"), nil
+}
+
+func (backend *AtmosCamoBackend) getAtmosCamoImageURL(imageURL string) string {
+	cfg := *backend.proxy.ConfigService.Config()
+	options := *cfg.ImageProxySettings.RemoteImageProxyOptions
+
+	if imageURL == "" || backend.siteURL == nil {
+		return imageURL
+	}
+
+	// Parse url, return siteURL in case of failure.
+	// Also if the URL is opaque.
+	parsedURL, err := url.Parse(imageURL)
+	if err != nil || parsedURL.Opaque != "" {
+		return backend.siteURL.String()
+	}
+
+	// If host is same as siteURL host/ remoteURL host, return.
+	if parsedURL.Host == backend.siteURL.Host || parsedURL.Host == backend.remoteURL.Host {
+		return parsedURL.String()
+	}
+
+	// Handle protocol-relative URLs.
+	if parsedURL.Scheme == "" {
+		parsedURL.Scheme = backend.siteURL.Scheme
+	}
+
+	// If it's a relative URL, fill up the hostname and scheme and return.
+	if parsedURL.Host == "" {
+		parsedURL.Host = backend.siteURL.Host
+		return parsedURL.String()
+	}
+
+	urlBytes := []byte(parsedURL.String())
+	mac := hmac.New(sha1.New, []byte(options))
+	mac.Write(urlBytes)
+	digest := hex.EncodeToString(mac.Sum(nil))
+
+	return backend.remoteURL.String() + "/" + digest + "/" + hex.EncodeToString(urlBytes)
+}

server/platform/services/imageproxy/atmos_camo_test.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package imageproxy
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/httpservice"
+)
+
+func makeTestAtmosCamoProxy() *ImageProxy {
+	configService := &testutils.StaticConfigService{
+		Cfg: &model.Config{
+			ServiceSettings: model.ServiceSettings{
+				SiteURL:                             model.NewString("https://mattermost.example.com"),
+				AllowedUntrustedInternalConnections: model.NewString("127.0.0.1"),
+			},
+			ImageProxySettings: model.ImageProxySettings{
+				Enable:                  model.NewBool(true),
+				ImageProxyType:          model.NewString(model.ImageProxyTypeAtmosCamo),
+				RemoteImageProxyURL:     model.NewString("http://images.example.com"),
+				RemoteImageProxyOptions: model.NewString("7e5f3fab20b94782b43cdb022a66985ef28ba355df2c5d5da3c9a05e4b697bac"),
+			},
+		},
+	}
+
+	return MakeImageProxy(configService, httpservice.MakeHTTPService(configService), nil)
+}
+
+func TestAtmosCamoBackend_GetImage(t *testing.T) {
+	imageURL := "https://www.mattermost.com/wp-content/uploads/2022/02/logoHorizontalWhite.png"
+	proxiedURL := "http://images.example.com/b569ce17f1be4550cffa8d8dd3a9e80e6d209584/68747470733a2f2f7777772e6d61747465726d6f73742e636f6d2f77702d636f6e74656e742f75706c6f6164732f323032322f30322f6c6f676f486f72697a6f6e74616c57686974652e706e67"
+
+	proxy := makeTestAtmosCamoProxy()
+
+	recorder := httptest.NewRecorder()
+	request, _ := http.NewRequest(http.MethodGet, "", nil)
+	proxy.GetImage(recorder, request, imageURL)
+	resp := recorder.Result()
+
+	assert.Equal(t, http.StatusFound, resp.StatusCode)
+	assert.Equal(t, proxiedURL, resp.Header.Get("Location"))
+}
+
+func TestAtmosCamoBackend_GetImageDirect(t *testing.T) {
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Cache-Control", "max-age=2592000, private")
+		w.Header().Set("Content-Type", "image/png")
+		w.Header().Set("Content-Length", "10")
+
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte("1111111111"))
+	})
+
+	mock := httptest.NewServer(handler)
+	defer mock.Close()
+
+	proxy := makeTestAtmosCamoProxy()
+	parsedURL, err := url.Parse(*proxy.ConfigService.Config().ServiceSettings.SiteURL)
+	require.NoError(t, err)
+
+	remoteURL, err := url.Parse(mock.URL)
+	require.NoError(t, err)
+
+	backend := &AtmosCamoBackend{
+		proxy:     proxy,
+		siteURL:   parsedURL,
+		remoteURL: remoteURL,
+		client:    proxy.HTTPService.MakeClient(false),
+	}
+
+	body, contentType, err := backend.GetImageDirect("https://example.com/image.png")
+
+	assert.NoError(t, err)
+	assert.Equal(t, "image/png", contentType)
+
+	require.NotNil(t, body)
+	respBody, _ := io.ReadAll(body)
+	assert.Equal(t, []byte("1111111111"), respBody)
+}
+
+func TestGetAtmosCamoImageURL(t *testing.T) {
+	imageURL := "https://mattermost.com/wp-content/uploads/2022/02/logoHorizontal.png"
+	proxiedURL := "http://images.example.com/03b122734ae088d10cb46ea05512ec7dc852299e/68747470733a2f2f6d61747465726d6f73742e636f6d2f77702d636f6e74656e742f75706c6f6164732f323032322f30322f6c6f676f486f72697a6f6e74616c2e706e67"
+
+	defaultSiteURL := "https://mattermost.example.com"
+	proxyURL := "http://images.example.com"
+
+	for _, test := range []struct {
+		Name     string
+		Input    string
+		SiteURL  string
+		Expected string
+	}{
+		{
+			Name:     "should proxy image",
+			Input:    imageURL,
+			SiteURL:  defaultSiteURL,
+			Expected: proxiedURL,
+		},
+		{
+			Name:     "should proxy image when no site URL is set",
+			Input:    imageURL,
+			SiteURL:  "",
+			Expected: proxiedURL,
+		},
+		{
+			Name:     "should proxy image when a site URL with a subpath is set",
+			Input:    imageURL,
+			SiteURL:  proxyURL + "/subpath",
+			Expected: proxiedURL,
+		},
+		{
+			Name:     "should not proxy a relative image",
+			Input:    "/static/logo.png",
+			SiteURL:  defaultSiteURL,
+			Expected: "https://mattermost.example.com/static/logo.png",
+		},
+		{
+			Name:     "should bypass opaque URLs",
+			Input:    "http:xyz123?query",
+			SiteURL:  defaultSiteURL,
+			Expected: defaultSiteURL,
+		},
+		{
+			Name:     "should not proxy an image on the Mattermost server",
+			Input:    "https://mattermost.example.com/static/logo.png",
+			SiteURL:  defaultSiteURL,
+			Expected: "https://mattermost.example.com/static/logo.png",
+		},
+		{
+			Name:     "should not proxy an image on the Mattermost server when a subpath is set",
+			Input:    "https://mattermost.example.com/static/logo.png",
+			SiteURL:  defaultSiteURL + "/static",
+			Expected: "https://mattermost.example.com/static/logo.png",
+		},
+		{
+			Name:     "should not proxy an image that has already been proxied",
+			Input:    proxiedURL,
+			SiteURL:  defaultSiteURL,
+			Expected: proxiedURL,
+		},
+		{
+			Name:     "should not bypass protocol relative URLs",
+			Input:    "https://mattermost.com/wp-content/uploads/2022/02/logoHorizontal.png",
+			SiteURL:  "http://mattermost.example.com",
+			Expected: proxiedURL,
+		},
+		{
+			Name:     "should not bypass if the host prefix is same",
+			Input:    "https://mattermost.com/wp-content/uploads/2022/02/logoHorizontal.png",
+			SiteURL:  defaultSiteURL,
+			Expected: "http://images.example.com/03b122734ae088d10cb46ea05512ec7dc852299e/68747470733a2f2f6d61747465726d6f73742e636f6d2f77702d636f6e74656e742f75706c6f6164732f323032322f30322f6c6f676f486f72697a6f6e74616c2e706e67",
+		},
+		{
+			Name:     "should not bypass for user auth URLs",
+			Input:    "https://mattermost.com/wp-content/uploads/2022/02/logoHorizontal.png",
+			SiteURL:  defaultSiteURL,
+			Expected: "http://images.example.com/03b122734ae088d10cb46ea05512ec7dc852299e/68747470733a2f2f6d61747465726d6f73742e636f6d2f77702d636f6e74656e742f75706c6f6164732f323032322f30322f6c6f676f486f72697a6f6e74616c2e706e67",
+		},
+	} {
+		t.Run(test.Name, func(t *testing.T) {
+			parsedURL, err := url.Parse(test.SiteURL)
+			require.NoError(t, err)
+
+			remoteURL, err := url.Parse(proxyURL)
+			require.NoError(t, err)
+
+			backend := &AtmosCamoBackend{
+				proxy:     makeTestAtmosCamoProxy(),
+				siteURL:   parsedURL,
+				remoteURL: remoteURL,
+			}
+
+			assert.Equal(t, test.Expected, backend.getAtmosCamoImageURL(test.Input))
+		})
+	}
+
+}

server/platform/services/imageproxy/error.go

@@ -0,0 +1,8 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package imageproxy
+
+type Error struct {
+	error
+}

server/platform/services/imageproxy/imageproxy.go

@@ -0,0 +1,176 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package imageproxy
+
+import (
+	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/configservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/httpservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+var ErrNotEnabled = Error{errors.New("imageproxy.ImageProxy: image proxy not enabled")}
+
+// An ImageProxy is the public interface for Mattermost's image proxy. An instance of ImageProxy should be created
+// using MakeImageProxy which requires a configService and an HTTPService provided by the server.
+type ImageProxy struct {
+	ConfigService    configservice.ConfigService
+	configListenerID string
+
+	HTTPService httpservice.HTTPService
+
+	Logger *mlog.Logger
+
+	siteURL *url.URL
+	lock    sync.RWMutex
+	backend ImageProxyBackend
+}
+
+// An ImageProxyBackend provides the functionality for different types of image proxies. An ImageProxy will construct
+// the required backend depending on the ImageProxySettings provided by the ConfigService.
+type ImageProxyBackend interface {
+	// GetImage provides a proxied image in response to an HTTP request.
+	GetImage(w http.ResponseWriter, r *http.Request, imageURL string)
+
+	// GetImageDirect returns a proxied image along with its content type.
+	GetImageDirect(imageURL string) (io.ReadCloser, string, error)
+}
+
+func MakeImageProxy(configService configservice.ConfigService, httpService httpservice.HTTPService, logger *mlog.Logger) *ImageProxy {
+	proxy := &ImageProxy{
+		ConfigService: configService,
+		HTTPService:   httpService,
+		Logger:        logger,
+	}
+
+	// We deliberately ignore the error because it's from config.json.
+	// The function returns a nil pointer in case of error, and we handle it when it's used.
+	siteURL, _ := url.Parse(*configService.Config().ServiceSettings.SiteURL)
+	proxy.siteURL = siteURL
+
+	proxy.configListenerID = proxy.ConfigService.AddConfigListener(proxy.OnConfigChange)
+
+	config := proxy.ConfigService.Config()
+	proxy.backend = proxy.makeBackend(*config.ImageProxySettings.Enable, *config.ImageProxySettings.ImageProxyType)
+
+	return proxy
+}
+
+func (proxy *ImageProxy) makeBackend(enable bool, proxyType string) ImageProxyBackend {
+	if !enable {
+		return nil
+	}
+
+	switch proxyType {
+	case model.ImageProxyTypeLocal:
+		return makeLocalBackend(proxy)
+	case model.ImageProxyTypeAtmosCamo:
+		return makeAtmosCamoBackend(proxy)
+	default:
+		return nil
+	}
+}
+
+func (proxy *ImageProxy) Close() {
+	proxy.lock.Lock()
+	defer proxy.lock.Unlock()
+
+	proxy.ConfigService.RemoveConfigListener(proxy.configListenerID)
+}
+
+func (proxy *ImageProxy) OnConfigChange(oldConfig, newConfig *model.Config) {
+	if *oldConfig.ImageProxySettings.Enable != *newConfig.ImageProxySettings.Enable ||
+		*oldConfig.ImageProxySettings.ImageProxyType != *newConfig.ImageProxySettings.ImageProxyType {
+		proxy.lock.Lock()
+		defer proxy.lock.Unlock()
+
+		proxy.backend = proxy.makeBackend(*newConfig.ImageProxySettings.Enable, *newConfig.ImageProxySettings.ImageProxyType)
+	}
+}
+
+// GetImage takes an HTTP request for an image and requests that image using the image proxy.
+func (proxy *ImageProxy) GetImage(w http.ResponseWriter, r *http.Request, imageURL string) {
+	proxy.lock.RLock()
+	defer proxy.lock.RUnlock()
+
+	if proxy.backend == nil {
+		w.WriteHeader(http.StatusNotImplemented)
+		return
+	}
+
+	proxy.backend.GetImage(w, r, imageURL)
+}
+
+// GetImageDirect takes the URL of an image and returns the image along with its content type.
+func (proxy *ImageProxy) GetImageDirect(imageURL string) (io.ReadCloser, string, error) {
+	proxy.lock.RLock()
+	defer proxy.lock.RUnlock()
+
+	if proxy.backend == nil {
+		return nil, "", ErrNotEnabled
+	}
+
+	return proxy.backend.GetImageDirect(imageURL)
+}
+
+// GetProxiedImageURL takes the URL of an image and returns a URL that can be used to view that image through the
+// image proxy.
+func (proxy *ImageProxy) GetProxiedImageURL(imageURL string) string {
+	if imageURL == "" || proxy.siteURL == nil {
+		return imageURL
+	}
+	// Parse url, return siteURL in case of failure.
+	// Also if the URL is opaque.
+	parsedURL, err := url.Parse(imageURL)
+	if err != nil || parsedURL.Opaque != "" {
+		return proxy.siteURL.String()
+	}
+	// If host is same as siteURL host, return.
+	if parsedURL.Host == proxy.siteURL.Host {
+		return parsedURL.String()
+	}
+
+	// Handle protocol-relative URLs.
+	if parsedURL.Scheme == "" {
+		parsedURL.Scheme = proxy.siteURL.Scheme
+	}
+
+	// If it's a relative URL, fill up the hostname and return.
+	if parsedURL.Host == "" {
+		parsedURL.Host = proxy.siteURL.Host
+		return parsedURL.String()
+	}
+
+	return proxy.siteURL.String() + "/api/v4/image?url=" + url.QueryEscape(parsedURL.String())
+}
+
+// GetUnproxiedImageURL takes the URL of an image on the image proxy and returns the original URL of the image.
+func (proxy *ImageProxy) GetUnproxiedImageURL(proxiedURL string) string {
+	return getUnproxiedImageURL(proxiedURL, *proxy.ConfigService.Config().ServiceSettings.SiteURL)
+}
+
+func getUnproxiedImageURL(proxiedURL, siteURL string) string {
+	if !strings.HasPrefix(proxiedURL, siteURL+"/api/v4/image?url=") {
+		return proxiedURL
+	}
+
+	parsed, err := url.Parse(proxiedURL)
+	if err != nil {
+		return proxiedURL
+	}
+
+	u := parsed.Query()["url"]
+	if len(u) == 0 {
+		return proxiedURL
+	}
+
+	return u[0]
+}

server/platform/services/imageproxy/imageproxy_test.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package imageproxy
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetProxiedImageURL(t *testing.T) {
+	siteURL := "https://mattermost.example.com"
+	parsedURL, err := url.Parse(siteURL)
+	require.NoError(t, err)
+
+	imageURL := "https://mattermost.com/wp-content/uploads/2022/02/logoHorizontal.png"
+	proxiedURL := "https://mattermost.example.com/api/v4/image?url=https%3A%2F%2Fmattermost.com%2Fwp-content%2Fuploads%2F2022%2F02%2FlogoHorizontal.png"
+
+	proxy := ImageProxy{siteURL: parsedURL}
+
+	for _, test := range []struct {
+		Name     string
+		Input    string
+		Expected string
+	}{
+		{
+			Name:     "should proxy an image",
+			Input:    imageURL,
+			Expected: proxiedURL,
+		},
+		{
+			Name:     "should not proxy a relative image",
+			Input:    "/static/logo.png",
+			Expected: "https://mattermost.example.com/static/logo.png",
+		},
+		{
+			Name:     "should bypass opaque URLs",
+			Input:    "http:xyz123?query",
+			Expected: siteURL,
+		},
+		{
+			Name:     "should not proxy an image on the Mattermost server",
+			Input:    "https://mattermost.example.com/static/logo.png",
+			Expected: "https://mattermost.example.com/static/logo.png",
+		},
+		{
+			Name:     "should not proxy an image that has already been proxied",
+			Input:    proxiedURL,
+			Expected: proxiedURL,
+		},
+		{
+			Name:     "should not bypass protocol relative URLs",
+			Input:    "//mattermost.com/static/logo.png",
+			Expected: "https://mattermost.example.com/api/v4/image?url=https%3A%2F%2Fmattermost.com%2Fstatic%2Flogo.png",
+		},
+		{
+			Name:     "should not bypass if the host prefix is same",
+			Input:    "https://mattermost.example.com.anothersite.com/static/logo.png",
+			Expected: "https://mattermost.example.com/api/v4/image?url=https%3A%2F%2Fmattermost.example.com.anothersite.com%2Fstatic%2Flogo.png",
+		},
+		{
+			Name:     "should not bypass for user auth URLs",
+			Input:    "https://mattermost.example.com@anothersite.com/static/logo.png",
+			Expected: "https://mattermost.example.com/api/v4/image?url=https%3A%2F%2Fmattermost.example.com%40anothersite.com%2Fstatic%2Flogo.png",
+		},
+	} {
+		t.Run(test.Name, func(t *testing.T) {
+			assert.Equal(t, test.Expected, proxy.GetProxiedImageURL(test.Input))
+		})
+	}
+}
+
+func TestGetUnproxiedImageURL(t *testing.T) {
+	siteURL := "https://mattermost.example.com"
+
+	imageURL := "https://mattermost.com/wp-content/uploads/2022/02/logoHorizontal.png"
+	proxiedURL := "https://mattermost.example.com/api/v4/image?url=https%3A%2F%2Fmattermost.com%2Fwp-content%2Fuploads%2F2022%2F02%2FlogoHorizontal.png"
+
+	for _, test := range []struct {
+		Name     string
+		Input    string
+		Expected string
+	}{
+		{
+			Name:     "should remove proxy",
+			Input:    proxiedURL,
+			Expected: imageURL,
+		},
+		{
+			Name:     "should not remove proxy from a relative image",
+			Input:    "/static/logo.png",
+			Expected: "/static/logo.png",
+		},
+		{
+			Name:     "should not remove proxy from an image on the Mattermost server",
+			Input:    "https://mattermost.example.com/static/logo.png",
+			Expected: "https://mattermost.example.com/static/logo.png",
+		},
+		{
+			Name:     "should not remove proxy from a non-proxied image",
+			Input:    imageURL,
+			Expected: imageURL,
+		},
+	} {
+		t.Run(test.Name, func(t *testing.T) {
+			assert.Equal(t, test.Expected, getUnproxiedImageURL(test.Input, siteURL))
+		})
+	}
+}

server/platform/services/imageproxy/local.go

@@ -0,0 +1,328 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package imageproxy
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"io"
+	"mime"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+var imageContentTypes = []string{
+	"image/bmp", "image/cgm", "image/g3fax", "image/gif", "image/ief", "image/jp2",
+	"image/jpeg", "image/jpg", "image/pict", "image/png", "image/prs.btif", "image/svg+xml",
+	"image/tiff", "image/vnd.adobe.photoshop", "image/vnd.djvu", "image/vnd.dwg",
+	"image/vnd.dxf", "image/vnd.fastbidsheet", "image/vnd.fpx", "image/vnd.fst",
+	"image/vnd.fujixerox.edmics-mmr", "image/vnd.fujixerox.edmics-rlc",
+	"image/vnd.microsoft.icon", "image/vnd.ms-modi", "image/vnd.net-fpx", "image/vnd.wap.wbmp",
+	"image/vnd.xiff", "image/webp", "image/x-cmu-raster", "image/x-cmx", "image/x-icon",
+	"image/x-macpaint", "image/x-pcx", "image/x-pict", "image/x-portable-anymap",
+	"image/x-portable-bitmap", "image/x-portable-graymap", "image/x-portable-pixmap",
+	"image/x-quicktime", "image/x-rgb", "image/x-xbitmap", "image/x-xpixmap", "image/x-xwindowdump",
+}
+
+var msgNotAllowed = "requested URL is not allowed"
+
+var ErrLocalRequestFailed = Error{errors.New("imageproxy.LocalBackend: failed to request proxied image")}
+
+type LocalBackend struct {
+	proxy *ImageProxy
+
+	client  *http.Client
+	baseURL *url.URL
+}
+
+// URLError reports a malformed URL error.
+type URLError struct {
+	Message string
+	URL     *url.URL
+}
+
+func (e URLError) Error() string {
+	return fmt.Sprintf("malformed URL %q: %s", e.URL, e.Message)
+}
+
+func makeLocalBackend(proxy *ImageProxy) *LocalBackend {
+	baseURL, err := url.Parse(*proxy.ConfigService.Config().ServiceSettings.SiteURL)
+	if err != nil {
+		mlog.Warn("Failed to set base URL for image proxy. Relative image links may not work.", mlog.Err(err))
+	}
+
+	client := proxy.HTTPService.MakeClient(false)
+
+	return &LocalBackend{
+		proxy:   proxy,
+		client:  client,
+		baseURL: baseURL,
+	}
+}
+
+type contentTypeRecorder struct {
+	http.ResponseWriter
+	filename string
+}
+
+func (rec *contentTypeRecorder) WriteHeader(code int) {
+	hdr := rec.ResponseWriter.Header()
+	contentType := hdr.Get("Content-Type")
+	mediaType, _, err := mime.ParseMediaType(contentType)
+	// The error is caused by a malformed input and there's not much use logging it.
+	// Therefore, even in the error case we set it to attachment mode to be safe.
+	if err != nil || mediaType == "image/svg+xml" {
+		hdr.Set("Content-Disposition", fmt.Sprintf("attachment;filename=%q", rec.filename))
+	}
+
+	rec.ResponseWriter.WriteHeader(code)
+}
+
+func (backend *LocalBackend) GetImage(w http.ResponseWriter, r *http.Request, imageURL string) {
+	// The interface to the proxy only exposes a ServeHTTP method, so fake a request to it
+	req, err := http.NewRequest(http.MethodGet, "/"+imageURL, nil)
+	if err != nil {
+		// http.NewRequest should only return an error on an invalid URL
+		mlog.Debug("Failed to create request for proxied image", mlog.String("url", imageURL), mlog.Err(err))
+
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte{})
+		return
+	}
+
+	u, err := url.Parse(imageURL)
+	if err != nil {
+		mlog.Debug("Failed to parse URL for proxied image", mlog.String("url", imageURL), mlog.Err(err))
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte{})
+		return
+	}
+
+	w.Header().Set("X-Frame-Options", "deny")
+	w.Header().Set("X-XSS-Protection", "1; mode=block")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.Header().Set("Content-Security-Policy", "default-src 'none'; img-src data:; style-src 'unsafe-inline'")
+
+	rec := contentTypeRecorder{w, filepath.Base(u.Path)}
+	backend.ServeImage(&rec, req)
+}
+
+func (backend *LocalBackend) GetImageDirect(imageURL string) (io.ReadCloser, string, error) {
+	// The interface to the proxy only exposes a ServeHTTP method, so fake a request to it
+	req, err := http.NewRequest(http.MethodGet, "/"+imageURL, nil)
+	if err != nil {
+		return nil, "", Error{err}
+	}
+
+	recorder := httptest.NewRecorder()
+
+	backend.ServeImage(recorder, req)
+
+	if recorder.Code != http.StatusOK {
+		return nil, "", ErrLocalRequestFailed
+	}
+
+	return io.NopCloser(recorder.Body), recorder.Header().Get("Content-Type"), nil
+}
+
+func (backend *LocalBackend) ServeImage(w http.ResponseWriter, req *http.Request) {
+	proxyReq, err := newProxyRequest(req, backend.baseURL)
+	if err != nil {
+		http.Error(w, fmt.Sprintf("invalid request URL: %v", err), http.StatusBadRequest)
+		return
+	}
+
+	actualReq, err := http.NewRequest("GET", proxyReq.String(), nil)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	actualReq.Header.Set("Accept", strings.Join(imageContentTypes, ", "))
+
+	resp, err := backend.client.Do(actualReq)
+	if err != nil {
+		msg := fmt.Sprintf("error fetching remote image: %v", err)
+		mlog.Warn(msg)
+		statusCode := http.StatusInternalServerError
+		if e, ok := err.(net.Error); ok && e.Timeout() {
+			statusCode = http.StatusGatewayTimeout
+		}
+		http.Error(w, msg, statusCode)
+		return
+	}
+	// close the original resp.Body, even if we wrap it in a NopCloser below
+	defer resp.Body.Close()
+
+	copyHeader(w.Header(), resp.Header, "Cache-Control", "Last-Modified", "Expires", "Etag", "Link")
+
+	if should304(req, resp) {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+
+	contentType, _, _ := mime.ParseMediaType(resp.Header.Get("Content-Type"))
+	if contentType == "" || contentType == "application/octet-stream" || contentType == "binary/octet-stream" {
+		// try to detect content type
+		b := bufio.NewReader(resp.Body)
+		resp.Body = io.NopCloser(b)
+		contentType = peekContentType(b)
+	}
+	if resp.ContentLength != 0 && !contentTypeMatches(imageContentTypes, contentType) {
+		http.Error(w, msgNotAllowed, http.StatusForbidden)
+		return
+	}
+	w.Header().Set("Content-Type", contentType)
+
+	copyHeader(w.Header(), resp.Header, "Content-Length")
+
+	// Enable CORS for 3rd party applications
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+
+	// Add a Content-Security-Policy to prevent stored-XSS attacks via SVG files
+	w.Header().Set("Content-Security-Policy", "script-src 'none'")
+
+	// Disable Content-Type sniffing
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+
+	// Block potential XSS attacks especially in legacy browsers which do not support CSP
+	w.Header().Set("X-XSS-Protection", "1; mode=block")
+
+	w.WriteHeader(resp.StatusCode)
+	if _, err := io.Copy(w, resp.Body); err != nil {
+		mlog.Warn("error copying response", mlog.Err(err))
+	}
+}
+
+// copyHeader copies header values from src to dst, adding to any existing
+// values with the same header name.  If keys is not empty, only those header
+// keys will be copied.
+func copyHeader(dst, src http.Header, keys ...string) {
+	if len(keys) == 0 {
+		for k := range src {
+			keys = append(keys, k)
+		}
+	}
+	for _, key := range keys {
+		k := http.CanonicalHeaderKey(key)
+		for _, v := range src[k] {
+			dst.Add(k, v)
+		}
+	}
+}
+
+func should304(req *http.Request, resp *http.Response) bool {
+	etag := resp.Header.Get("Etag")
+	if etag != "" && etag == req.Header.Get("If-None-Match") {
+		return true
+	}
+
+	lastModified, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
+	if err != nil {
+		return false
+	}
+	ifModSince, err := time.Parse(time.RFC1123, req.Header.Get("If-Modified-Since"))
+	if err != nil {
+		return false
+	}
+	if lastModified.Before(ifModSince) || lastModified.Equal(ifModSince) {
+		return true
+	}
+
+	return false
+}
+
+// peekContentType peeks at the first 512 bytes of p, and attempts to detect
+// the content type.  Returns empty string if error occurs.
+func peekContentType(p *bufio.Reader) string {
+	byt, err := p.Peek(512)
+	if err != nil && err != bufio.ErrBufferFull && err != io.EOF {
+		return ""
+	}
+	return http.DetectContentType(byt)
+}
+
+// contentTypeMatches returns whether contentType matches one of the allowed patterns.
+func contentTypeMatches(patterns []string, contentType string) bool {
+	if len(patterns) == 0 {
+		return true
+	}
+
+	for _, pattern := range patterns {
+		if ok, err := path.Match(pattern, contentType); ok && err == nil {
+			return true
+		}
+	}
+
+	return false
+}
+
+// proxyRequest is an imageproxy request which includes a remote URL of an image to
+// proxy.
+type proxyRequest struct {
+	URL      *url.URL      // URL of the image to proxy
+	Original *http.Request // The original HTTP request
+}
+
+// String returns the request URL as a string, with r.Options encoded in the
+// URL fragment.
+func (r proxyRequest) String() string {
+	return r.URL.String()
+}
+
+func newProxyRequest(r *http.Request, baseURL *url.URL) (*proxyRequest, error) {
+	var err error
+	req := &proxyRequest{Original: r}
+
+	path := r.URL.EscapedPath()[1:] // strip leading slash
+	req.URL, err = parseURL(path)
+	if err != nil || !req.URL.IsAbs() {
+		// first segment should be options
+		parts := strings.SplitN(path, "/", 2)
+		if len(parts) != 2 {
+			return nil, URLError{"too few path segments", r.URL}
+		}
+
+		var err error
+		req.URL, err = parseURL(parts[1])
+		if err != nil {
+			return nil, URLError{fmt.Sprintf("unable to parse remote URL: %v", err), r.URL}
+		}
+
+	}
+
+	if baseURL != nil {
+		req.URL = baseURL.ResolveReference(req.URL)
+	}
+
+	if !req.URL.IsAbs() {
+		return nil, URLError{"must provide absolute remote URL", r.URL}
+	}
+
+	if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
+		return nil, URLError{"remote URL must have http or https scheme", r.URL}
+	}
+
+	// query string is always part of the remote URL
+	req.URL.RawQuery = r.URL.RawQuery
+	return req, nil
+}
+
+var reCleanedURL = regexp.MustCompile(`^(https?):/+([^/])`)
+
+// parseURL parses s as a URL, handling URLs that have been munged by
+// path.Clean or a webserver that collapses multiple slashes.
+func parseURL(s string) (*url.URL, error) {
+	s = reCleanedURL.ReplaceAllString(s, "$1://$2")
+	return url.Parse(s)
+}

server/platform/services/imageproxy/local_test.go

@@ -0,0 +1,355 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package imageproxy
+
+import (
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/httpservice"
+)
+
+func makeTestLocalProxy() *ImageProxy {
+	configService := &testutils.StaticConfigService{
+		Cfg: &model.Config{
+			ServiceSettings: model.ServiceSettings{
+				SiteURL:                             model.NewString("https://mattermost.example.com"),
+				AllowedUntrustedInternalConnections: model.NewString("127.0.0.1"),
+			},
+			ImageProxySettings: model.ImageProxySettings{
+				Enable:         model.NewBool(true),
+				ImageProxyType: model.NewString(model.ImageProxyTypeLocal),
+			},
+		},
+	}
+
+	return MakeImageProxy(configService, httpservice.MakeHTTPService(configService), nil)
+}
+
+func TestLocalBackend_GetImage(t *testing.T) {
+	t.Run("image", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Cache-Control", "max-age=2592000, private")
+			w.Header().Set("Content-Type", "image/png")
+			w.Header().Set("Content-Length", "10")
+
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("1111111111"))
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		recorder := httptest.NewRecorder()
+		request, _ := http.NewRequest(http.MethodGet, "", nil)
+		proxy.GetImage(recorder, request, mock.URL+"/image.png")
+		resp := recorder.Result()
+
+		require.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, "max-age=2592000, private", resp.Header.Get("Cache-Control"))
+		assert.Equal(t, "10", resp.Header.Get("Content-Length"))
+
+		respBody, _ := io.ReadAll(resp.Body)
+		assert.Equal(t, []byte("1111111111"), respBody)
+	})
+
+	t.Run("not an image", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNotAcceptable)
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		recorder := httptest.NewRecorder()
+		request, _ := http.NewRequest(http.MethodGet, "", nil)
+		proxy.GetImage(recorder, request, mock.URL+"/file.pdf")
+		resp := recorder.Result()
+
+		require.Equal(t, http.StatusNotAcceptable, resp.StatusCode)
+	})
+
+	t.Run("not an image, but remote server ignores accept header", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Cache-Control", "max-age=2592000, private")
+			w.Header().Set("Content-Type", "application/pdf")
+			w.Header().Set("Content-Length", "10")
+
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("1111111111"))
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		recorder := httptest.NewRecorder()
+		request, _ := http.NewRequest(http.MethodGet, "", nil)
+		proxy.GetImage(recorder, request, mock.URL+"/file.pdf")
+		resp := recorder.Result()
+
+		require.Equal(t, http.StatusForbidden, resp.StatusCode)
+	})
+
+	t.Run("not found", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNotFound)
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		recorder := httptest.NewRecorder()
+		request, _ := http.NewRequest(http.MethodGet, "", nil)
+		proxy.GetImage(recorder, request, mock.URL+"/image.png")
+		resp := recorder.Result()
+
+		require.Equal(t, http.StatusNotFound, resp.StatusCode)
+	})
+
+	t.Run("other server error", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusInternalServerError)
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		recorder := httptest.NewRecorder()
+		request, _ := http.NewRequest(http.MethodGet, "", nil)
+		proxy.GetImage(recorder, request, mock.URL+"/image.png")
+		resp := recorder.Result()
+
+		require.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+	})
+
+	t.Run("timeout", func(t *testing.T) {
+		wait := make(chan bool, 1)
+
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			<-wait
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		// Modify the timeout to be much shorter than the default 30 seconds
+		proxy.backend.(*LocalBackend).client.Timeout = time.Millisecond
+
+		recorder := httptest.NewRecorder()
+		request, _ := http.NewRequest(http.MethodGet, "", nil)
+		proxy.GetImage(recorder, request, mock.URL+"/image.png")
+		resp := recorder.Result()
+
+		require.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)
+
+		wait <- true
+	})
+
+	t.Run("SVG attachment", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Cache-Control", "max-age=2592000, private")
+			w.Header().Set("Content-Type", "image/svg+xml")
+			w.Header().Set("Content-Length", "10")
+
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("1111111111"))
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		recorder := httptest.NewRecorder()
+		request, err := http.NewRequest(http.MethodGet, "", nil)
+		require.NoError(t, err)
+		proxy.GetImage(recorder, request, mock.URL+"/test.svg")
+		resp := recorder.Result()
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, "attachment;filename=\"test.svg\"", resp.Header.Get("Content-Disposition"))
+
+		_, err = io.ReadAll(resp.Body)
+		require.NoError(t, err)
+	})
+
+	t.Run("Redirect", func(t *testing.T) {
+		var mock *httptest.Server
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			switch r.URL.Path {
+			case "/image.png":
+				w.Header().Set("Location", mock.URL+"/image2.png")
+				w.WriteHeader(http.StatusMovedPermanently)
+			case "/image2.png":
+				w.Header().Set("Cache-Control", "max-age=2592000, private")
+				w.Header().Set("Content-Type", "image/png")
+				w.Header().Set("Content-Length", "10")
+
+				w.WriteHeader(http.StatusOK)
+				w.Write([]byte("1111111111"))
+			}
+		})
+
+		mock = httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		recorder := httptest.NewRecorder()
+		request, _ := http.NewRequest(http.MethodGet, "", nil)
+		proxy.GetImage(recorder, request, mock.URL+"/image.png")
+		resp := recorder.Result()
+
+		require.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, "10", resp.Header.Get("Content-Length"))
+	})
+}
+
+func TestLocalBackend_GetImageDirect(t *testing.T) {
+	t.Run("image", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Cache-Control", "max-age=2592000, private")
+			w.Header().Set("Content-Type", "image/png")
+			w.Header().Set("Content-Length", "10")
+
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("1111111111"))
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		body, contentType, err := proxy.GetImageDirect(mock.URL + "/image.png")
+
+		assert.NoError(t, err)
+		assert.Equal(t, "image/png", contentType)
+
+		respBody, _ := io.ReadAll(body)
+		assert.Equal(t, []byte("1111111111"), respBody)
+	})
+
+	t.Run("not an image", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNotAcceptable)
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		body, contentType, err := proxy.GetImageDirect(mock.URL + "/file.pdf")
+
+		assert.Error(t, err)
+		assert.Equal(t, "", contentType)
+		assert.Equal(t, ErrLocalRequestFailed, err)
+		assert.Nil(t, body)
+	})
+
+	t.Run("not an image, but remote server ignores accept header", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Cache-Control", "max-age=2592000, private")
+			w.Header().Set("Content-Type", "application/pdf")
+			w.Header().Set("Content-Length", "10")
+
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("1111111111"))
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		body, contentType, err := proxy.GetImageDirect(mock.URL + "/file.pdf")
+
+		assert.Error(t, err)
+		assert.Equal(t, "", contentType)
+		assert.Equal(t, ErrLocalRequestFailed, err)
+		assert.Nil(t, body)
+	})
+
+	t.Run("not found", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusNotFound)
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		body, contentType, err := proxy.GetImageDirect(mock.URL + "/image.png")
+
+		assert.Error(t, err)
+		assert.Equal(t, "", contentType)
+		assert.Equal(t, ErrLocalRequestFailed, err)
+		assert.Nil(t, body)
+	})
+
+	t.Run("other server error", func(t *testing.T) {
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(http.StatusInternalServerError)
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		body, contentType, err := proxy.GetImageDirect(mock.URL + "/image.png")
+
+		assert.Error(t, err)
+		assert.Equal(t, "", contentType)
+		assert.Equal(t, ErrLocalRequestFailed, err)
+		assert.Nil(t, body)
+	})
+
+	t.Run("timeout", func(t *testing.T) {
+		wait := make(chan bool, 1)
+
+		handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			<-wait
+		})
+
+		mock := httptest.NewServer(handler)
+		defer mock.Close()
+
+		proxy := makeTestLocalProxy()
+
+		// Modify the timeout to be much shorter than the default 30 seconds
+		proxy.backend.(*LocalBackend).client.Timeout = time.Millisecond
+
+		body, contentType, err := proxy.GetImageDirect(mock.URL + "/image.png")
+
+		assert.Error(t, err)
+		assert.Equal(t, "", contentType)
+		assert.Equal(t, ErrLocalRequestFailed, err)
+		assert.Nil(t, body)
+
+		wait <- true
+	})
+}

server/platform/services/marketplace/client.go

@@ -0,0 +1,127 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package marketplace
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/httpservice"
+)
+
+// Client is the programmatic interface to the marketplace server API.
+type Client struct {
+	address    string
+	httpClient *http.Client
+}
+
+// NewClient creates a client to the marketplace server at the given address.
+func NewClient(address string, httpService httpservice.HTTPService) (*Client, error) {
+	var httpClient *http.Client
+	addressURL, err := url.Parse(address)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to parse marketplace address")
+	}
+	if addressURL.Hostname() == "localhost" || addressURL.Hostname() == "127.0.0.1" {
+		httpClient = httpService.MakeClient(true)
+	} else {
+		httpClient = httpService.MakeClient(false)
+	}
+
+	return &Client{
+		address:    address,
+		httpClient: httpClient,
+	}, nil
+}
+
+// GetPlugins fetches the list of plugins from the configured server.
+func (c *Client) GetPlugins(request *model.MarketplacePluginFilter) ([]*model.BaseMarketplacePlugin, error) {
+	u, err := url.Parse(c.buildURL("/api/v1/plugins"))
+	if err != nil {
+		return nil, err
+	}
+
+	request.ApplyToURL(u)
+
+	resp, err := c.doGet(u.String())
+	if err != nil {
+		return nil, err
+	}
+	defer closeBody(resp)
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return model.BaseMarketplacePluginsFromReader(resp.Body)
+	default:
+		return nil, errors.Errorf("failed with status code %d", resp.StatusCode)
+	}
+}
+
+func (c *Client) GetPlugin(filter *model.MarketplacePluginFilter, pluginVersion string) (*model.BaseMarketplacePlugin, error) {
+	filter.ReturnAllVersions = true
+
+	if filter.PluginId == "" {
+		return nil, errors.New("missing pluginID")
+	}
+
+	if pluginVersion == "" {
+		return nil, errors.New("missing pluginVersion")
+	}
+
+	plugins, err := c.GetPlugins(filter)
+	if err != nil {
+		return nil, err
+	}
+	for _, plugin := range plugins {
+		if plugin.Manifest.Version == pluginVersion {
+			return plugin, nil
+		}
+	}
+	return nil, errors.New("plugin not found")
+}
+
+func (c *Client) GetLatestPlugin(filter *model.MarketplacePluginFilter) (*model.BaseMarketplacePlugin, error) {
+	filter.ReturnAllVersions = false
+
+	if filter.PluginId == "" {
+		return nil, errors.New("no pluginID provided")
+	}
+
+	plugins, err := c.GetPlugins(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(plugins) == 0 {
+		return nil, errors.New("plugin not found")
+	}
+
+	if len(plugins) > 1 {
+		return nil, errors.Errorf("unexpectedly more then one plugin was returned from the marketplace")
+	}
+
+	return plugins[0], nil
+}
+
+// closeBody ensures the Body of an http.Response is properly closed.
+func closeBody(r *http.Response) {
+	if r.Body != nil {
+		_, _ = io.Copy(io.Discard, r.Body)
+		_ = r.Body.Close()
+	}
+}
+
+func (c *Client) buildURL(urlPath string, args ...any) string {
+	return fmt.Sprintf("%s/%s", strings.TrimRight(c.address, "/"), strings.TrimLeft(fmt.Sprintf(urlPath, args...), "/"))
+}
+
+func (c *Client) doGet(u string) (*http.Response, error) {
+	return c.httpClient.Get(u)
+}

server/platform/services/marketplace/client_test.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package marketplace
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildURL(t *testing.T) {
+	config := &Client{}
+
+	testCases := map[string]struct {
+		base     string
+		path     string
+		expected string
+	}{
+		"Base url with trailing slash and path with leading slash": {
+			base:     "https://api.integrations.mattermost.com/",
+			path:     "/api/v1/plugins",
+			expected: "https://api.integrations.mattermost.com/api/v1/plugins",
+		},
+		"Base url without trailing slash and path with leading slash": {
+			base:     "https://api.integrations.mattermost.com",
+			path:     "/api/v1/plugins",
+			expected: "https://api.integrations.mattermost.com/api/v1/plugins",
+		},
+		"Base url without trailing slash and path without leading slash": {
+			base:     "https://api.integrations.mattermost.com",
+			path:     "api/v1/plugins",
+			expected: "https://api.integrations.mattermost.com/api/v1/plugins",
+		},
+	}
+
+	for name, tt := range testCases {
+		t.Run(name, func(t *testing.T) {
+			config.address = tt.base
+			actual := config.buildURL(tt.path)
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}

server/platform/services/remotecluster/error.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import "fmt"
+
+type BufferFullError struct {
+	capacity int
+}
+
+func NewBufferFullError(capacity int) BufferFullError {
+	return BufferFullError{
+		capacity: capacity,
+	}
+}
+
+func (e BufferFullError) Capacity() int {
+	return e.capacity
+}
+
+func (e BufferFullError) Error() string {
+	return fmt.Sprintf("buffer capacity (%d) exceeded", e.capacity)
+}

server/platform/services/remotecluster/invitation.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// AcceptInvitation is called when accepting an invitation to connect with a remote cluster.
+func (rcs *Service) AcceptInvitation(invite *model.RemoteClusterInvite, name string, displayName, creatorId string, teamId string, siteURL string) (*model.RemoteCluster, error) {
+	rc := &model.RemoteCluster{
+		RemoteId:     invite.RemoteId,
+		RemoteTeamId: invite.RemoteTeamId,
+		Name:         name,
+		DisplayName:  displayName,
+		Token:        model.NewId(),
+		RemoteToken:  invite.Token,
+		SiteURL:      invite.SiteURL,
+		CreatorId:    creatorId,
+	}
+
+	rcSaved, err := rcs.server.GetStore().RemoteCluster().Save(rc)
+	if err != nil {
+		return nil, err
+	}
+
+	// confirm the invitation with the originating site
+	frame, err := makeConfirmFrame(rcSaved, teamId, siteURL)
+	if err != nil {
+		return nil, err
+	}
+
+	url := fmt.Sprintf("%s/%s", rcSaved.SiteURL, ConfirmInviteURL)
+
+	resp, err := rcs.sendFrameToRemote(PingTimeout, rc, frame, url)
+	if err != nil {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, err
+	}
+
+	var response Response
+	err = json.Unmarshal(resp, &response)
+	if err != nil {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, fmt.Errorf("invalid response from remote server: %w", err)
+	}
+
+	if !response.IsSuccess() {
+		rcs.server.GetStore().RemoteCluster().Delete(rcSaved.RemoteId)
+		return nil, errors.New(response.Err)
+	}
+
+	// issue the first ping right away. The goroutine will exit when ping completes or PingTimeout exceeded.
+	go rcs.pingRemote(rcSaved)
+
+	return rcSaved, nil
+}
+
+func makeConfirmFrame(rc *model.RemoteCluster, teamId string, siteURL string) (*model.RemoteClusterFrame, error) {
+	confirm := model.RemoteClusterInvite{
+		RemoteId:     rc.RemoteId,
+		RemoteTeamId: teamId,
+		SiteURL:      siteURL,
+		Token:        rc.Token,
+	}
+	confirmRaw, err := json.Marshal(confirm)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := model.NewRemoteClusterMsg(InvitationTopic, confirmRaw)
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: rc.RemoteId,
+		Msg:      msg,
+	}
+	return frame, nil
+}

server/platform/services/remotecluster/mocks_test.go

@@ -0,0 +1,62 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v6/server/channels/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type mockServer struct {
+	remotes []*model.RemoteCluster
+	logger  *mlog.Logger
+	user    *model.User
+}
+
+func newMockServer(remotes []*model.RemoteCluster) *mockServer {
+	testLogger := mlog.CreateConsoleTestLogger(true, mlog.LvlDebug)
+
+	return &mockServer{
+		remotes: remotes,
+		logger:  testLogger,
+	}
+}
+
+func (ms *mockServer) SetUser(user *model.User) {
+	ms.user = user
+}
+
+func (ms *mockServer) Config() *model.Config                                  { return nil }
+func (ms *mockServer) GetMetrics() einterfaces.MetricsInterface               { return nil }
+func (ms *mockServer) IsLeader() bool                                         { return true }
+func (ms *mockServer) AddClusterLeaderChangedListener(listener func()) string { return model.NewId() }
+func (ms *mockServer) RemoveClusterLeaderChangedListener(id string)           {}
+func (ms *mockServer) Log() *mlog.Logger {
+	return ms.logger
+}
+func (ms *mockServer) GetStore() store.Store {
+	anyQueryFilter := mock.MatchedBy(func(filter model.RemoteClusterQueryFilter) bool {
+		return true
+	})
+	anyUserId := mock.AnythingOfType("string")
+
+	remoteClusterStoreMock := &mocks.RemoteClusterStore{}
+	remoteClusterStoreMock.On("GetByTopic", "share").Return(ms.remotes, nil)
+	remoteClusterStoreMock.On("GetAll", anyQueryFilter).Return(ms.remotes, nil)
+
+	userStoreMock := &mocks.UserStore{}
+	userStoreMock.On("Get", context.Background(), anyUserId).Return(ms.user, nil)
+
+	storeMock := &mocks.Store{}
+	storeMock.On("RemoteCluster").Return(remoteClusterStoreMock)
+	storeMock.On("User").Return(userStoreMock)
+	return storeMock
+}
+func (ms *mockServer) Shutdown() { ms.logger.Shutdown() }

server/platform/services/remotecluster/ping.go

@@ -0,0 +1,174 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+// pingLoop periodically sends a ping to all remote clusters.
+func (rcs *Service) pingLoop(done <-chan struct{}) {
+	pingChan := make(chan *model.RemoteCluster, MaxConcurrentSends*2)
+
+	// create a thread pool to send pings concurrently to remotes.
+	for i := 0; i < MaxConcurrentSends; i++ {
+		go rcs.pingEmitter(pingChan, done)
+	}
+
+	go rcs.pingGenerator(pingChan, done)
+}
+
+func (rcs *Service) pingGenerator(pingChan chan *model.RemoteCluster, done <-chan struct{}) {
+	defer close(pingChan)
+
+	for {
+		start := time.Now()
+
+		// get all remotes, including any previously offline.
+		remotes, err := rcs.server.GetStore().RemoteCluster().GetAll(model.RemoteClusterQueryFilter{})
+		if err != nil {
+			rcs.server.Log().Log(mlog.LvlRemoteClusterServiceError, "Ping remote cluster failed (could not get list of remotes)", mlog.Err(err))
+			select {
+			case <-time.After(PingFreq):
+				continue
+			case <-done:
+				return
+			}
+		}
+
+		for _, rc := range remotes {
+			if rc.SiteURL != "" { // filter out unconfirmed invites
+				pingChan <- rc
+			}
+		}
+
+		// try to maintain frequency
+		elapsed := time.Since(start)
+		if elapsed < PingFreq {
+			sleep := time.Until(start.Add(PingFreq))
+			select {
+			case <-time.After(sleep):
+			case <-done:
+				return
+			}
+		}
+	}
+}
+
+// pingEmitter pulls Remotes from the ping queue (pingChan) and pings them.
+// Pinging a remote cannot take longer than PingTimeoutMillis.
+func (rcs *Service) pingEmitter(pingChan <-chan *model.RemoteCluster, done <-chan struct{}) {
+	for {
+		select {
+		case rc := <-pingChan:
+			if rc == nil {
+				return
+			}
+
+			online := rc.IsOnline()
+
+			if err := rcs.pingRemote(rc); err != nil {
+				rcs.server.Log().Log(mlog.LvlRemoteClusterServiceWarn, "Remote cluster ping failed",
+					mlog.String("remote", rc.DisplayName),
+					mlog.String("remoteId", rc.RemoteId),
+					mlog.Err(err),
+				)
+			}
+
+			if online != rc.IsOnline() {
+				if metrics := rcs.server.GetMetrics(); metrics != nil {
+					metrics.IncrementRemoteClusterConnStateChangeCounter(rc.RemoteId, rc.IsOnline())
+				}
+				rcs.fireConnectionStateChgEvent(rc)
+			}
+		case <-done:
+			return
+		}
+	}
+}
+
+// pingRemote make a synchronous ping to a remote cluster. Return is error if ping is
+// unsuccessful and nil on success.
+func (rcs *Service) pingRemote(rc *model.RemoteCluster) error {
+	frame, err := makePingFrame(rc)
+	if err != nil {
+		return err
+	}
+	url := fmt.Sprintf("%s/%s", rc.SiteURL, PingURL)
+
+	resp, err := rcs.sendFrameToRemote(PingTimeout, rc, frame, url)
+	if err != nil {
+		return err
+	}
+
+	ping := model.RemoteClusterPing{}
+	err = json.Unmarshal(resp, &ping)
+	if err != nil {
+		return err
+	}
+
+	if err := rcs.server.GetStore().RemoteCluster().SetLastPingAt(rc.RemoteId); err != nil {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceError, "Failed to update LastPingAt for remote cluster",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("remoteId", rc.RemoteId),
+			mlog.Err(err),
+		)
+	}
+	rc.LastPingAt = model.GetMillis()
+
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		sentAt := time.Unix(0, ping.SentAt*int64(time.Millisecond))
+		elapsed := time.Since(sentAt).Seconds()
+		metrics.ObserveRemoteClusterPingDuration(rc.RemoteId, elapsed)
+
+		// we approximate clock skew between remotes.
+		skew := elapsed/2 - float64(ping.RecvAt-ping.SentAt)/1000
+		metrics.ObserveRemoteClusterClockSkew(rc.RemoteId, skew)
+	}
+
+	rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "Remote cluster ping",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("remoteId", rc.RemoteId),
+		mlog.Int64("SentAt", ping.SentAt),
+		mlog.Int64("RecvAt", ping.RecvAt),
+		mlog.Int64("Diff", ping.RecvAt-ping.SentAt),
+	)
+	return nil
+}
+
+func makePingFrame(rc *model.RemoteCluster) (*model.RemoteClusterFrame, error) {
+	ping := model.RemoteClusterPing{
+		SentAt: model.GetMillis(),
+	}
+	pingRaw, err := json.Marshal(ping)
+	if err != nil {
+		return nil, err
+	}
+
+	msg := model.NewRemoteClusterMsg(PingTopic, pingRaw)
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: rc.RemoteId,
+		Msg:      msg,
+	}
+	return frame, nil
+}
+
+func (rcs *Service) fireConnectionStateChgEvent(rc *model.RemoteCluster) {
+	rcs.mux.RLock()
+	listeners := make([]ConnectionStateListener, 0, len(rcs.connectionStateListeners))
+	for _, l := range rcs.connectionStateListeners {
+		listeners = append(listeners, l)
+	}
+	rcs.mux.RUnlock()
+
+	for _, l := range listeners {
+		l(rc, rc.IsOnline())
+	}
+}

server/platform/services/remotecluster/ping_test.go

@@ -0,0 +1,142 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+const (
+	Recent = 60000
+)
+
+func TestPing(t *testing.T) {
+	disablePing = false
+
+	t.Run("No error", func(t *testing.T) {
+		var countWebReq int32
+		merr := merror.New()
+
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer wg.Done()
+			defer w.WriteHeader(200)
+			atomic.AddInt32(&countWebReq, 1)
+
+			var frame model.RemoteClusterFrame
+			err := json.NewDecoder(r.Body).Decode(&frame)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+			if len(frame.Msg.Payload) == 0 {
+				merr.Append(fmt.Errorf("Payload should not be empty; remote_id=%s", frame.RemoteId))
+				return
+			}
+
+			var ping model.RemoteClusterPing
+			err = json.Unmarshal(frame.Msg.Payload, &ping)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+			if !checkRecent(ping.SentAt, Recent) {
+				merr.Append(fmt.Errorf("timestamp out of range, got %d", ping.SentAt))
+				return
+			}
+			if ping.RecvAt != 0 {
+				merr.Append(fmt.Errorf("timestamp should be 0, got %d", ping.RecvAt))
+				return
+			}
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(makeRemoteClusters(NumRemotes, ts.URL))
+		defer mockServer.Shutdown()
+
+		service, err := NewRemoteClusterService(mockServer)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		wg.Wait()
+
+		assert.NoError(t, merr.ErrorOrNil())
+
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countWebReq))
+		t.Logf("%d web requests counted;  %d expected",
+			atomic.LoadInt32(&countWebReq), NumRemotes)
+	})
+
+	t.Run("HTTP errors", func(t *testing.T) {
+		var countWebReq int32
+		merr := merror.New()
+
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer wg.Done()
+			atomic.AddInt32(&countWebReq, 1)
+
+			var frame model.RemoteClusterFrame
+			err := json.NewDecoder(r.Body).Decode(&frame)
+			if err != nil {
+				merr.Append(err)
+			}
+			var ping model.RemoteClusterPing
+			err = json.Unmarshal(frame.Msg.Payload, &ping)
+			if err != nil {
+				merr.Append(err)
+			}
+			if !checkRecent(ping.SentAt, Recent) {
+				merr.Append(fmt.Errorf("timestamp out of range, got %d", ping.SentAt))
+			}
+			if ping.RecvAt != 0 {
+				merr.Append(fmt.Errorf("timestamp should be 0, got %d", ping.RecvAt))
+			}
+			w.WriteHeader(500)
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(makeRemoteClusters(NumRemotes, ts.URL))
+		defer mockServer.Shutdown()
+
+		service, err := NewRemoteClusterService(mockServer)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		wg.Wait()
+
+		assert.NoError(t, merr.ErrorOrNil())
+
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countWebReq))
+		t.Logf("%d web requests counted;  %d expected",
+			atomic.LoadInt32(&countWebReq), NumRemotes)
+	})
+}
+
+func checkRecent(millis int64, within int64) bool {
+	now := model.GetMillis()
+	return millis > now-within && millis < now+within
+}

server/platform/services/remotecluster/recv.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"fmt"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+// ReceiveIncomingMsg is called by the Rest API layer, or websocket layer (future), when a Remote Cluster
+// message is received.  Here we route the message to any topic listeners.
+// `rc` and `msg` cannot be nil.
+func (rcs *Service) ReceiveIncomingMsg(rc *model.RemoteCluster, msg model.RemoteClusterMsg) Response {
+	rcs.mux.RLock()
+	defer rcs.mux.RUnlock()
+
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		metrics.IncrementRemoteClusterMsgReceivedCounter(rc.RemoteId)
+	}
+
+	rcSanitized := *rc
+	rcSanitized.Token = ""
+	rcSanitized.RemoteToken = ""
+
+	var response Response
+	response.Status = ResponseStatusOK
+
+	listeners := rcs.getTopicListeners(msg.Topic)
+
+	for _, l := range listeners {
+		if err := callback(l, msg, &rcSanitized, &response); err != nil {
+			rcs.server.Log().Log(mlog.LvlRemoteClusterServiceError, "Error from remote cluster message listener",
+				mlog.String("msgId", msg.Id), mlog.String("topic", msg.Topic), mlog.String("remote", rc.DisplayName), mlog.Err(err))
+
+			response.Status = ResponseStatusFail
+			response.Err = err.Error()
+		}
+	}
+	return response
+}
+
+func callback(listener TopicListener, msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("%v", r)
+		}
+	}()
+	err = listener(msg, rc, resp)
+	return
+}

server/platform/services/remotecluster/response.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"encoding/json"
+)
+
+// Response represents the bytes replied from a remote server when a message is sent.
+type Response struct {
+	Status  string          `json:"status"`
+	Err     string          `json:"err"`
+	Payload json.RawMessage `json:"payload"`
+}
+
+// IsSuccess returns true if the response status indicates success.
+func (r *Response) IsSuccess() bool {
+	return r.Status == ResponseStatusOK
+}
+
+// SetPayload serializes an arbitrary struct as a RawMessage.
+func (r *Response) SetPayload(v any) error {
+	raw, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	r.Payload = raw
+	return nil
+}

server/platform/services/remotecluster/send.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"hash/fnv"
+)
+
+// enqueueTask adds a task to one of the send channels based on remoteId.
+//
+// There are a number  of send channels (`MaxConcurrentSends`) to allow for sending to multiple
+// remotes concurrently, while preserving message order for each remote.
+func (rcs *Service) enqueueTask(ctx context.Context, remoteId string, task any) error {
+	if ctx == nil {
+		ctx = context.Background()
+	}
+
+	h := hash(remoteId)
+	idx := h % uint32(len(rcs.send))
+
+	select {
+	case rcs.send[idx] <- task:
+		return nil
+	case <-ctx.Done():
+		return NewBufferFullError(cap(rcs.send))
+	}
+}
+
+func hash(s string) uint32 {
+	h := fnv.New32a()
+	h.Write([]byte(s))
+	return h.Sum32()
+}
+
+// sendLoop is called by each goroutine created for the send pool and waits for sendTask's until the
+// done channel is signalled.
+//
+// Each goroutine in the pool is assigned a specific channel, and tasks are placed in the
+// channel corresponding to the remoteId.
+func (rcs *Service) sendLoop(idx int, done chan struct{}) {
+	for {
+		select {
+		case t := <-rcs.send[idx]:
+			switch task := t.(type) {
+			case sendMsgTask:
+				rcs.sendMsg(task)
+			case sendFileTask:
+				rcs.sendFile(task)
+			case sendProfileImageTask:
+				rcs.sendProfileImage(task)
+			}
+		case <-done:
+			return
+		}
+	}
+}

server/platform/services/remotecluster/send_test.go

@@ -0,0 +1,205 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+const (
+	TestTopics  = " share incident "
+	TestTopic   = "share"
+	NumRemotes  = 50
+	NoteContent = "Woot!!"
+)
+
+type testPayload struct {
+	Note string `json:"note"`
+}
+
+func TestBroadcastMsg(t *testing.T) {
+	msgId := model.NewId()
+	disablePing = true
+
+	t.Run("No error", func(t *testing.T) {
+		var countCallbacks int32
+		var countWebReq int32
+		merr := merror.New()
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer func() {
+				w.WriteHeader(200)
+				var resp Response
+				b, errMarshall := json.Marshal(&resp)
+				if errMarshall != nil {
+					merr.Append(errMarshall)
+					return
+				}
+				w.Write(b)
+			}()
+
+			atomic.AddInt32(&countWebReq, 1)
+
+			var frame model.RemoteClusterFrame
+			jsonErr := json.NewDecoder(r.Body).Decode(&frame)
+			if jsonErr != nil {
+				merr.Append(jsonErr)
+				return
+			}
+			if len(frame.Msg.Payload) == 0 {
+				merr.Append(fmt.Errorf("webrequest missing Msg.Payload"))
+			}
+			if msgId != frame.Msg.Id {
+				merr.Append(fmt.Errorf("webrequest msgId expected %s, got %s", msgId, frame.Msg.Id))
+				return
+			}
+
+			note := testPayload{}
+			err := json.Unmarshal(frame.Msg.Payload, &note)
+			if err != nil {
+				merr.Append(err)
+				return
+			}
+			if note.Note != NoteContent {
+				merr.Append(fmt.Errorf("webrequest payload expected %s, got %s", NoteContent, note.Note))
+				return
+			}
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(makeRemoteClusters(NumRemotes, ts.URL))
+		defer mockServer.Shutdown()
+
+		service, err := NewRemoteClusterService(mockServer)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		msg := makeRemoteClusterMsg(msgId, NoteContent)
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*15)
+		defer cancel()
+
+		err = service.BroadcastMsg(ctx, msg, func(msg model.RemoteClusterMsg, remote *model.RemoteCluster, resp *Response, err error) {
+			defer wg.Done()
+			atomic.AddInt32(&countCallbacks, 1)
+
+			if err != nil {
+				merr.Append(err)
+			}
+			if msgId != msg.Id {
+				merr.Append(fmt.Errorf("result callback msgId expected %s, got %s", msgId, msg.Id))
+			}
+
+			var note testPayload
+			err2 := json.Unmarshal(msg.Payload, &note)
+			if err2 != nil {
+				merr.Append(fmt.Errorf("unmarshal payload error: %w", err2))
+				return
+			}
+			if note.Note != NoteContent {
+				merr.Append(fmt.Errorf("compare payload failed: expected '%s', got '%s'", NoteContent, note))
+			}
+		})
+		assert.NoError(t, err)
+
+		wg.Wait()
+
+		assert.NoError(t, merr.ErrorOrNil())
+
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countCallbacks))
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countWebReq))
+		t.Logf("%d callbacks counted;  %d web requests counted;  %d expected",
+			atomic.LoadInt32(&countCallbacks), atomic.LoadInt32(&countWebReq), NumRemotes)
+	})
+
+	t.Run("HTTP error", func(t *testing.T) {
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(500)
+		}))
+		defer ts.Close()
+
+		mockServer := newMockServer(makeRemoteClusters(NumRemotes, ts.URL))
+		defer mockServer.Shutdown()
+
+		service, err := NewRemoteClusterService(mockServer)
+		require.NoError(t, err)
+
+		err = service.Start()
+		require.NoError(t, err)
+		defer service.Shutdown()
+
+		msg := makeRemoteClusterMsg(msgId, NoteContent)
+		var countCallbacks int32
+		var countErrors int32
+		wg := &sync.WaitGroup{}
+		wg.Add(NumRemotes)
+
+		err = service.BroadcastMsg(context.Background(), msg, func(msg model.RemoteClusterMsg, remote *model.RemoteCluster, resp *Response, err error) {
+			defer wg.Done()
+			atomic.AddInt32(&countCallbacks, 1)
+			if err != nil {
+				atomic.AddInt32(&countErrors, 1)
+			}
+		})
+		assert.NoError(t, err)
+
+		wg.Wait()
+
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countCallbacks))
+		assert.Equal(t, int32(NumRemotes), atomic.LoadInt32(&countErrors))
+	})
+}
+
+func makeRemoteClusters(num int, siteURL string) []*model.RemoteCluster {
+	var remotes []*model.RemoteCluster
+	for i := 0; i < num; i++ {
+		rc := makeRemoteCluster(fmt.Sprintf("test cluster %d", i+1), siteURL, TestTopics)
+		remotes = append(remotes, rc)
+	}
+	return remotes
+}
+
+func makeRemoteCluster(name string, siteURL string, topics string) *model.RemoteCluster {
+	return &model.RemoteCluster{
+		RemoteId:   model.NewId(),
+		Name:       name,
+		SiteURL:    siteURL,
+		Token:      model.NewId(),
+		Topics:     topics,
+		CreateAt:   model.GetMillis(),
+		LastPingAt: model.GetMillis(),
+		CreatorId:  model.NewId(),
+	}
+}
+
+func makeRemoteClusterMsg(id string, note string) model.RemoteClusterMsg {
+	payload := testPayload{Note: note}
+	raw, _ := json.Marshal(payload)
+
+	return model.RemoteClusterMsg{
+		Id:       id,
+		Topic:    TestTopic,
+		CreateAt: model.GetMillis(),
+		Payload:  raw}
+}

server/platform/services/remotecluster/sendfile.go

@@ -0,0 +1,136 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/filestore"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type SendFileResultFunc func(us *model.UploadSession, rc *model.RemoteCluster, resp *Response, err error)
+
+type sendFileTask struct {
+	rc *model.RemoteCluster
+	us *model.UploadSession
+	fi *model.FileInfo
+	rp ReaderProvider
+	f  SendFileResultFunc
+}
+
+type ReaderProvider interface {
+	FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
+}
+
+// SendFile asynchronously sends a file to a remote cluster.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the task cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// Nil or error return indicates success or failure of task enqueue only.
+//
+// An optional callback can be provided that receives the response from the remote cluster. The `err` provided to the
+// callback is regarding file delivery only. The `resp` contains the decoded bytes returned from the remote.
+// If a callback is provided it should return quickly.
+func (rcs *Service) SendFile(ctx context.Context, us *model.UploadSession, fi *model.FileInfo, rc *model.RemoteCluster, rp ReaderProvider, f SendFileResultFunc) error {
+	task := sendFileTask{
+		rc: rc,
+		us: us,
+		fi: fi,
+		rp: rp,
+		f:  f,
+	}
+	return rcs.enqueueTask(ctx, rc.RemoteId, task)
+}
+
+// sendFile is called when a sendFileTask is popped from the send channel.
+func (rcs *Service) sendFile(task sendFileTask) {
+	fi, err := rcs.sendFileToRemote(SendTimeout, task)
+	var response Response
+
+	if err != nil {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceError, "Remote Cluster send file failed",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("uploadId", task.us.Id),
+			mlog.Err(err),
+		)
+		response.Status = ResponseStatusFail
+		response.Err = err.Error()
+	} else {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "Remote Cluster file sent successfully",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("uploadId", task.us.Id),
+		)
+		response.Status = ResponseStatusOK
+		response.SetPayload(fi)
+	}
+
+	// If callback provided then call it with the results.
+	if task.f != nil {
+		task.f(task.us, task.rc, &response, err)
+	}
+}
+
+func (rcs *Service) sendFileToRemote(timeout time.Duration, task sendFileTask) (*model.FileInfo, error) {
+	rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "sending file to remote...",
+		mlog.String("remote", task.rc.DisplayName),
+		mlog.String("uploadId", task.us.Id),
+		mlog.String("file_path", task.us.Path),
+	)
+
+	r, appErr := task.rp.FileReader(task.fi.Path) // get Reader for the file
+	if appErr != nil {
+		return nil, fmt.Errorf("error opening file while sending file to remote %s: %w", task.rc.RemoteId, appErr)
+	}
+	defer r.Close()
+
+	u, err := url.Parse(task.rc.SiteURL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid siteURL while sending file to remote %s: %w", task.rc.RemoteId, err)
+	}
+	u.Path = path.Join(u.Path, model.APIURLSuffix, "remotecluster", "upload", task.us.Id)
+
+	req, err := http.NewRequest("POST", u.String(), r)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set(model.HeaderRemoteclusterId, task.rc.RemoteId)
+	req.Header.Set(model.HeaderRemoteclusterToken, task.rc.RemoteToken)
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	resp, err := rcs.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected response: %d - %s", resp.StatusCode, resp.Status)
+	}
+
+	// body should be a FileInfo
+	var fi model.FileInfo
+	if err := json.Unmarshal(body, &fi); err != nil {
+		return nil, fmt.Errorf("unexpected response body: %w", err)
+	}
+
+	return &fi, nil
+}

server/platform/services/remotecluster/sendmsg.go

@@ -0,0 +1,175 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"time"
+
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type SendMsgResultFunc func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response, err error)
+
+type sendMsgTask struct {
+	rc  *model.RemoteCluster
+	msg model.RemoteClusterMsg
+	f   SendMsgResultFunc
+}
+
+// BroadcastMsg asynchronously sends a message to all remote clusters interested in the message's topic.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the message cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// An optional callback can be provided that receives the success or fail result of sending to each remote cluster.
+// Success or fail is regarding message delivery only.  If a callback is provided it should return quickly.
+func (rcs *Service) BroadcastMsg(ctx context.Context, msg model.RemoteClusterMsg, f SendMsgResultFunc) error {
+	// get list of interested remotes.
+	filter := model.RemoteClusterQueryFilter{
+		Topic: msg.Topic,
+	}
+	list, err := rcs.server.GetStore().RemoteCluster().GetAll(filter)
+	if err != nil {
+		return err
+	}
+
+	errs := merror.New()
+
+	for _, rc := range list {
+		if err := rcs.SendMsg(ctx, msg, rc, f); err != nil {
+			errs.Append(err)
+		}
+	}
+	return errs.ErrorOrNil()
+}
+
+// SendMsg asynchronously sends a message to a remote cluster.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the message cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// Nil or error return indicates success or failure of message enqueue only.
+//
+// An optional callback can be provided that receives the response from the remote cluster. The `err` provided to the
+// callback is regarding response decoding only. The `resp` contains the decoded bytes returned from the remote.
+// If a callback is provided it should return quickly.
+func (rcs *Service) SendMsg(ctx context.Context, msg model.RemoteClusterMsg, rc *model.RemoteCluster, f SendMsgResultFunc) error {
+	task := sendMsgTask{
+		rc:  rc,
+		msg: msg,
+		f:   f,
+	}
+	return rcs.enqueueTask(ctx, rc.RemoteId, task)
+}
+
+// sendMsg is called when a sendMsgTask is popped from the send channel.
+func (rcs *Service) sendMsg(task sendMsgTask) {
+	var errResp error
+	var response Response
+
+	// Ensure a panic from the callback does not exit the pool goroutine.
+	defer func() {
+		if errResp != nil {
+			response.Err = errResp.Error()
+		}
+
+		// If callback provided then call it with the results.
+		if task.f != nil {
+			task.f(task.msg, task.rc, &response, errResp)
+		}
+	}()
+
+	frame := &model.RemoteClusterFrame{
+		RemoteId: task.rc.RemoteId,
+		Msg:      task.msg,
+	}
+
+	u, err := url.Parse(task.rc.SiteURL)
+	if err != nil {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceError, "Invalid siteURL while sending message to remote",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msgId", task.msg.Id),
+			mlog.Err(err),
+		)
+		errResp = err
+		return
+	}
+	u.Path = path.Join(u.Path, SendMsgURL)
+
+	respJSON, err := rcs.sendFrameToRemote(SendTimeout, task.rc, frame, u.String())
+
+	if err != nil {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceError, "Remote Cluster send message failed",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msgId", task.msg.Id),
+			mlog.Err(err),
+		)
+		errResp = err
+	} else {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "Remote Cluster message sent successfully",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("msgId", task.msg.Id),
+		)
+
+		if err = json.Unmarshal(respJSON, &response); err != nil {
+			rcs.server.Log().Error("Invalid response sending message to remote cluster",
+				mlog.String("remote", task.rc.DisplayName),
+				mlog.Err(err),
+			)
+			errResp = err
+		}
+	}
+}
+
+func (rcs *Service) sendFrameToRemote(timeout time.Duration, rc *model.RemoteCluster, frame *model.RemoteClusterFrame, url string) ([]byte, error) {
+	body, err := json.Marshal(frame)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	req, err := http.NewRequest("POST", url, bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set(model.HeaderRemoteclusterId, rc.RemoteId)
+	req.Header.Set(model.HeaderRemoteclusterToken, rc.RemoteToken)
+
+	resp, err := rcs.httpClient.Do(req.WithContext(ctx))
+	if metrics := rcs.server.GetMetrics(); metrics != nil {
+		if err != nil || resp.StatusCode != http.StatusOK {
+			metrics.IncrementRemoteClusterMsgErrorsCounter(frame.RemoteId, os.IsTimeout(err))
+		} else {
+			metrics.IncrementRemoteClusterMsgSentCounter(frame.RemoteId)
+		}
+	}
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return body, fmt.Errorf("unexpected response: %d - %s", resp.StatusCode, resp.Status)
+	}
+	return body, nil
+}

server/platform/services/remotecluster/sendprofileImage.go

@@ -0,0 +1,145 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type SendProfileImageResultFunc func(userId string, rc *model.RemoteCluster, resp *Response, err error)
+
+type sendProfileImageTask struct {
+	rc       *model.RemoteCluster
+	userID   string
+	provider ProfileImageProvider
+	f        SendProfileImageResultFunc
+}
+
+type ProfileImageProvider interface {
+	GetProfileImage(user *model.User) ([]byte, bool, *model.AppError)
+}
+
+// SendProfileImage asynchronously sends a user's profile image to a remote cluster.
+//
+// `ctx` determines behaviour when the outbound queue is full. A timeout or deadline context will return a
+// BufferFullError if the task cannot be enqueued before the timeout. A background context will block indefinitely.
+//
+// Nil or error return indicates success or failure of task enqueue only.
+//
+// An optional callback can be provided that receives the response from the remote cluster. The `err` provided to the
+// callback is regarding image delivery only. The `resp` contains the decoded bytes returned from the remote.
+// If a callback is provided it should return quickly.
+func (rcs *Service) SendProfileImage(ctx context.Context, userID string, rc *model.RemoteCluster, provider ProfileImageProvider, f SendProfileImageResultFunc) error {
+	task := sendProfileImageTask{
+		rc:       rc,
+		userID:   userID,
+		provider: provider,
+		f:        f,
+	}
+	return rcs.enqueueTask(ctx, rc.RemoteId, task)
+}
+
+// sendProfileImage is called when a sendProfileImageTask is popped from the send channel.
+func (rcs *Service) sendProfileImage(task sendProfileImageTask) {
+	err := rcs.sendProfileImageToRemote(SendTimeout, task)
+	var response Response
+
+	if err != nil {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceError, "Remote Cluster send profile image failed",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("UserId", task.userID),
+			mlog.Err(err),
+		)
+		response.Status = ResponseStatusFail
+		response.Err = err.Error()
+	} else {
+		rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "Remote Cluster profile image sent successfully",
+			mlog.String("remote", task.rc.DisplayName),
+			mlog.String("UserId", task.userID),
+		)
+		response.Status = ResponseStatusOK
+	}
+
+	// If callback provided then call it with the results.
+	if task.f != nil {
+		task.f(task.userID, task.rc, &response, err)
+	}
+}
+
+func (rcs *Service) sendProfileImageToRemote(timeout time.Duration, task sendProfileImageTask) error {
+	rcs.server.Log().Log(mlog.LvlRemoteClusterServiceDebug, "sending profile image to remote...",
+		mlog.String("remote", task.rc.DisplayName),
+		mlog.String("UserId", task.userID),
+	)
+
+	user, err := rcs.server.GetStore().User().Get(context.Background(), task.userID)
+	if err != nil {
+		return fmt.Errorf("error fetching user while sending profile image to remote %s: %w", task.rc.RemoteId, err)
+	}
+
+	img, _, appErr := task.provider.GetProfileImage(user) // get Reader for the file
+	if appErr != nil {
+		return fmt.Errorf("error fetching profile image for user (%s) while sending to remote %s: %w", task.userID, task.rc.RemoteId, appErr)
+	}
+
+	u, err := url.Parse(task.rc.SiteURL)
+	if err != nil {
+		return fmt.Errorf("invalid siteURL while sending file to remote %s: %w", task.rc.RemoteId, err)
+	}
+	u.Path = path.Join(u.Path, model.APIURLSuffix, "remotecluster", task.userID, "image")
+
+	body := &bytes.Buffer{}
+	writer := multipart.NewWriter(body)
+
+	part, err := writer.CreateFormFile("image", "profile.png")
+	if err != nil {
+		return err
+	}
+
+	if _, err = io.Copy(part, bytes.NewBuffer(img)); err != nil {
+		return err
+	}
+
+	if err = writer.Close(); err != nil {
+		return err
+	}
+
+	req, err := http.NewRequest("POST", u.String(), body)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", writer.FormDataContentType())
+	req.Header.Set(model.HeaderRemoteclusterId, task.rc.RemoteId)
+	req.Header.Set(model.HeaderRemoteclusterToken, task.rc.RemoteToken)
+
+	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	defer cancel()
+
+	resp, err := rcs.httpClient.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	_, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("unexpected response: %d - %s", resp.StatusCode, resp.Status)
+	}
+	return nil
+}

server/platform/services/remotecluster/sendprofileImage_test.go

@@ -0,0 +1,187 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"bytes"
+	"image"
+	"image/color"
+	"image/png"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+const (
+	imageWidth  = 128
+	imageHeight = 128
+)
+
+func TestService_sendProfileImageToRemote(t *testing.T) {
+	hadPing := disablePing
+	disablePing = true
+	defer func() { disablePing = hadPing }()
+
+	shouldError := &flag{}
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		defer io.Copy(io.Discard, r.Body)
+
+		if shouldError.get() {
+			w.WriteHeader(http.StatusInternalServerError)
+			resp := make(map[string]string)
+			resp[model.STATUS] = model.StatusFail
+			w.Write([]byte(model.MapToJSON(resp)))
+			return
+		}
+
+		status := model.StatusOk
+		defer func(s *string) {
+			if *s != model.StatusOk {
+				w.WriteHeader(http.StatusInternalServerError)
+			}
+			resp := make(map[string]string)
+			resp[model.STATUS] = *s
+			w.Write([]byte(model.MapToJSON(resp)))
+		}(&status)
+
+		if err := r.ParseMultipartForm(1024 * 1024); err != nil {
+			status = model.StatusFail
+			assert.Fail(t, "connect parse multipart form", err)
+			return
+		}
+		m := r.MultipartForm
+		if m == nil {
+			status = model.StatusFail
+			assert.Fail(t, "multipart form missing")
+			return
+		}
+
+		imageArray, ok := m.File["image"]
+		if !ok || len(imageArray) != 1 {
+			status = model.StatusFail
+			assert.Fail(t, "image missing")
+			return
+		}
+
+		imageData := imageArray[0]
+		file, err := imageData.Open()
+		if err != nil {
+			status = model.StatusFail
+			assert.Fail(t, "cannot open multipart form file")
+			return
+		}
+		defer file.Close()
+
+		img, err := png.Decode(file)
+		if err != nil || imageWidth != img.Bounds().Max.X || imageHeight != img.Bounds().Max.Y {
+			status = model.StatusFail
+			assert.Fail(t, "cannot decode png", err)
+			return
+		}
+	}))
+	defer ts.Close()
+
+	rc := makeRemoteCluster("remote_test_profile_image", ts.URL, TestTopics)
+
+	user := &model.User{
+		Id:       model.NewId(),
+		RemoteId: model.NewString(rc.RemoteId),
+	}
+
+	provider := testImageProvider{}
+
+	mockServer := newMockServer(makeRemoteClusters(NumRemotes, ts.URL))
+	defer mockServer.Shutdown()
+	mockServer.SetUser(user)
+	service, err := NewRemoteClusterService(mockServer)
+	require.NoError(t, err)
+
+	err = service.Start()
+	require.NoError(t, err)
+	defer service.Shutdown()
+
+	t.Run("Server response 200", func(t *testing.T) {
+		shouldError.set(false)
+
+		resultFunc := func(userId string, rc *model.RemoteCluster, resp *Response, err error) {
+			assert.Equal(t, user.Id, userId, "user ids should match")
+			assert.NoError(t, err)
+			assert.True(t, resp.IsSuccess())
+		}
+
+		task := sendProfileImageTask{
+			rc:       rc,
+			userID:   user.Id,
+			provider: provider,
+			f:        resultFunc,
+		}
+
+		err := service.sendProfileImageToRemote(time.Second*15, task)
+		assert.NoError(t, err, "request should not error")
+	})
+
+	t.Run("Server response 500", func(t *testing.T) {
+		shouldError.set(true)
+
+		resultFunc := func(userId string, rc *model.RemoteCluster, resp *Response, err error) {
+			assert.Equal(t, user.Id, userId, "user ids should match")
+			assert.False(t, resp.IsSuccess())
+		}
+
+		task := sendProfileImageTask{
+			rc:       rc,
+			userID:   user.Id,
+			provider: provider,
+			f:        resultFunc,
+		}
+
+		err := service.sendProfileImageToRemote(time.Second*15, task)
+		assert.Error(t, err, "request should error")
+	})
+}
+
+type testImageProvider struct {
+}
+
+func (tip testImageProvider) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) {
+	img := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{imageWidth, imageHeight}})
+	red := color.RGBA{255, 50, 50, 0xff}
+
+	for x := 0; x < imageWidth; x++ {
+		for y := 0; y < imageHeight; y++ {
+			img.Set(x, y, red)
+		}
+	}
+
+	buf := &bytes.Buffer{}
+	png.Encode(buf, img)
+
+	return buf.Bytes(), true, nil
+}
+
+type flag struct {
+	mux sync.RWMutex
+	b   bool
+}
+
+func (f *flag) get() bool {
+	f.mux.RLock()
+	defer f.mux.RUnlock()
+	return f.b
+}
+
+func (f *flag) set(b bool) {
+	f.mux.Lock()
+	defer f.mux.Unlock()
+	f.b = b
+}

server/platform/services/remotecluster/service.go

@@ -0,0 +1,262 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	SendChanBuffer                = 50
+	RecvChanBuffer                = 50
+	ResultsChanBuffer             = 50
+	ResultQueueDrainTimeoutMillis = 10000
+	MaxConcurrentSends            = 10
+	SendMsgURL                    = "api/v4/remotecluster/msg"
+	SendTimeout                   = time.Minute
+	SendFileTimeout               = time.Minute * 5
+	PingURL                       = "api/v4/remotecluster/ping"
+	PingFreq                      = time.Minute
+	PingTimeout                   = time.Second * 15
+	ConfirmInviteURL              = "api/v4/remotecluster/confirm_invite"
+	InvitationTopic               = "invitation"
+	PingTopic                     = "ping"
+	ResponseStatusOK              = model.StatusOk
+	ResponseStatusFail            = model.StatusFail
+	InviteExpiresAfter            = time.Hour * 48
+)
+
+var (
+	disablePing bool // override for testing
+)
+
+type ServerIface interface {
+	Config() *model.Config
+	IsLeader() bool
+	AddClusterLeaderChangedListener(listener func()) string
+	RemoveClusterLeaderChangedListener(id string)
+	GetStore() store.Store
+	Log() *mlog.Logger
+	GetMetrics() einterfaces.MetricsInterface
+}
+
+// RemoteClusterServiceIFace is used to allow mocking where a remote cluster service is used (for testing).
+// Unfortunately it lives here because the shared channel service, app layer, and server interface all need it.
+// Putting it in app layer means shared channel service must import app package.
+type RemoteClusterServiceIFace interface {
+	Shutdown() error
+	Start() error
+	Active() bool
+	AddTopicListener(topic string, listener TopicListener) string
+	RemoveTopicListener(listenerId string)
+	AddConnectionStateListener(listener ConnectionStateListener) string
+	RemoveConnectionStateListener(listenerId string)
+	SendMsg(ctx context.Context, msg model.RemoteClusterMsg, rc *model.RemoteCluster, f SendMsgResultFunc) error
+	SendFile(ctx context.Context, us *model.UploadSession, fi *model.FileInfo, rc *model.RemoteCluster, rp ReaderProvider, f SendFileResultFunc) error
+	SendProfileImage(ctx context.Context, userID string, rc *model.RemoteCluster, provider ProfileImageProvider, f SendProfileImageResultFunc) error
+	AcceptInvitation(invite *model.RemoteClusterInvite, name string, displayName string, creatorId string, teamId string, siteURL string) (*model.RemoteCluster, error)
+	ReceiveIncomingMsg(rc *model.RemoteCluster, msg model.RemoteClusterMsg) Response
+}
+
+// TopicListener is a callback signature used to listen for incoming messages for
+// a specific topic.
+type TopicListener func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error
+
+// ConnectionStateListener is used to listen to remote cluster connection state changes.
+type ConnectionStateListener func(rc *model.RemoteCluster, online bool)
+
+// Service provides inter-cluster communication via topic based messages. In product these are called "Secured Connections".
+type Service struct {
+	server     ServerIface
+	httpClient *http.Client
+	send       []chan any
+
+	// everything below guarded by `mux`
+	mux                      sync.RWMutex
+	active                   bool
+	leaderListenerId         string
+	topicListeners           map[string]map[string]TopicListener // maps topic id to a map of listenerid->listener
+	connectionStateListeners map[string]ConnectionStateListener  // maps listener id to listener
+	done                     chan struct{}
+}
+
+// NewRemoteClusterService creates a RemoteClusterService instance. In product this is called a "Secured Connection".
+func NewRemoteClusterService(server ServerIface) (*Service, error) {
+	transport := &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          200,
+		MaxIdleConnsPerHost:   2,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		DisableCompression:    false,
+	}
+
+	client := &http.Client{
+		Transport: transport,
+		Timeout:   SendTimeout,
+	}
+
+	service := &Service{
+		server:                   server,
+		httpClient:               client,
+		topicListeners:           make(map[string]map[string]TopicListener),
+		connectionStateListeners: make(map[string]ConnectionStateListener),
+	}
+
+	service.send = make([]chan any, MaxConcurrentSends)
+	for i := range service.send {
+		service.send[i] = make(chan any, SendChanBuffer)
+	}
+
+	return service, nil
+}
+
+// Start is called by the server on server start-up.
+func (rcs *Service) Start() error {
+	rcs.mux.Lock()
+	rcs.leaderListenerId = rcs.server.AddClusterLeaderChangedListener(rcs.onClusterLeaderChange)
+	rcs.mux.Unlock()
+
+	rcs.onClusterLeaderChange()
+
+	return nil
+}
+
+// Shutdown is called by the server on server shutdown.
+func (rcs *Service) Shutdown() error {
+	rcs.server.RemoveClusterLeaderChangedListener(rcs.leaderListenerId)
+	rcs.pause()
+	return nil
+}
+
+// Active returns true if this instance of the remote cluster service is active.
+// The active instance is responsible for pinging and sending messages to remotes.
+func (rcs *Service) Active() bool {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+	return rcs.active
+}
+
+// AddTopicListener registers a callback
+func (rcs *Service) AddTopicListener(topic string, listener TopicListener) string {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	id := model.NewId()
+
+	listeners, ok := rcs.topicListeners[topic]
+	if !ok || listeners == nil {
+		rcs.topicListeners[topic] = make(map[string]TopicListener)
+	}
+	rcs.topicListeners[topic][id] = listener
+	return id
+}
+
+func (rcs *Service) RemoveTopicListener(listenerId string) {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	for topic, listeners := range rcs.topicListeners {
+		if _, ok := listeners[listenerId]; ok {
+			delete(listeners, listenerId)
+			if len(listeners) == 0 {
+				delete(rcs.topicListeners, topic)
+			}
+			break
+		}
+	}
+}
+
+func (rcs *Service) getTopicListeners(topic string) []TopicListener {
+	rcs.mux.RLock()
+	defer rcs.mux.RUnlock()
+
+	listeners, ok := rcs.topicListeners[topic]
+	if !ok {
+		return nil
+	}
+
+	listenersCopy := make([]TopicListener, 0, len(listeners))
+	for _, l := range listeners {
+		listenersCopy = append(listenersCopy, l)
+	}
+	return listenersCopy
+}
+
+func (rcs *Service) AddConnectionStateListener(listener ConnectionStateListener) string {
+	id := model.NewId()
+
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	rcs.connectionStateListeners[id] = listener
+	return id
+}
+
+func (rcs *Service) RemoveConnectionStateListener(listenerId string) {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+	delete(rcs.connectionStateListeners, listenerId)
+}
+
+// onClusterLeaderChange is called whenever the cluster leader may have changed.
+func (rcs *Service) onClusterLeaderChange() {
+	if rcs.server.IsLeader() {
+		rcs.resume()
+	} else {
+		rcs.pause()
+	}
+}
+
+func (rcs *Service) resume() {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	if rcs.active {
+		return // already active
+	}
+	rcs.active = true
+	rcs.done = make(chan struct{})
+
+	if !disablePing {
+		rcs.pingLoop(rcs.done)
+	}
+
+	// create thread pool for concurrent message sending.
+	for i := range rcs.send {
+		go rcs.sendLoop(i, rcs.done)
+	}
+
+	rcs.server.Log().Debug("Remote Cluster Service active")
+}
+
+func (rcs *Service) pause() {
+	rcs.mux.Lock()
+	defer rcs.mux.Unlock()
+
+	if !rcs.active {
+		return // already inactive
+	}
+	rcs.active = false
+	close(rcs.done)
+	rcs.done = nil
+
+	rcs.server.Log().Debug("Remote Cluster Service inactive")
+}

server/platform/services/remotecluster/service_test.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package remotecluster
+
+import (
+	"sync/atomic"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestService_AddTopicListener(t *testing.T) {
+	var count int32
+
+	l1 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		atomic.AddInt32(&count, 1)
+		return nil
+	}
+	l2 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		atomic.AddInt32(&count, 1)
+		return nil
+	}
+	l3 := func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *Response) error {
+		atomic.AddInt32(&count, 1)
+		return nil
+	}
+
+	mockServer := newMockServer(makeRemoteClusters(NumRemotes, ""))
+	defer mockServer.Shutdown()
+
+	service, err := NewRemoteClusterService(mockServer)
+	require.NoError(t, err)
+
+	l1id := service.AddTopicListener("test", l1)
+	l2id := service.AddTopicListener("test", l2)
+	l3id := service.AddTopicListener("different", l3)
+
+	listeners := service.getTopicListeners("test")
+	assert.Len(t, listeners, 2)
+
+	rc := &model.RemoteCluster{}
+	msg1 := model.RemoteClusterMsg{Topic: "test"}
+	msg2 := model.RemoteClusterMsg{Topic: "different"}
+
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(2), atomic.LoadInt32(&count))
+
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(3), atomic.LoadInt32(&count))
+
+	service.RemoveTopicListener(l1id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(4), atomic.LoadInt32(&count))
+
+	service.RemoveTopicListener(l2id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	assert.Equal(t, int32(4), atomic.LoadInt32(&count))
+
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(5), atomic.LoadInt32(&count))
+
+	service.RemoveTopicListener(l3id)
+	service.ReceiveIncomingMsg(rc, msg1)
+	service.ReceiveIncomingMsg(rc, msg2)
+	assert.Equal(t, int32(5), atomic.LoadInt32(&count))
+
+	listeners = service.getTopicListeners("test")
+	assert.Empty(t, listeners)
+}

server/platform/services/searchengine/bleveengine/bleve.go

@@ -0,0 +1,328 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
+	"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
+	"github.com/blevesearch/bleve/v2/mapping"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	EngineName   = "bleve"
+	PostIndex    = "posts"
+	FileIndex    = "files"
+	UserIndex    = "users"
+	ChannelIndex = "channels"
+)
+
+type BleveEngine struct {
+	PostIndex    bleve.Index
+	FileIndex    bleve.Index
+	UserIndex    bleve.Index
+	ChannelIndex bleve.Index
+	Mutex        sync.RWMutex
+	ready        int32
+	cfg          *model.Config
+	indexSync    bool
+}
+
+var keywordMapping *mapping.FieldMapping
+var standardMapping *mapping.FieldMapping
+var dateMapping *mapping.FieldMapping
+
+func init() {
+	keywordMapping = bleve.NewTextFieldMapping()
+	keywordMapping.Analyzer = keyword.Name
+
+	standardMapping = bleve.NewTextFieldMapping()
+	standardMapping.Analyzer = standard.Name
+
+	dateMapping = bleve.NewNumericFieldMapping()
+}
+
+func getChannelIndexMapping() *mapping.IndexMappingImpl {
+	channelMapping := bleve.NewDocumentMapping()
+	channelMapping.AddFieldMappingsAt("Id", keywordMapping)
+	channelMapping.AddFieldMappingsAt("Type", keywordMapping)
+	channelMapping.AddFieldMappingsAt("TeamId", keywordMapping)
+	channelMapping.AddFieldMappingsAt("NameSuggest", keywordMapping)
+	channelMapping.AddFieldMappingsAt("UserIDs", keywordMapping)
+	channelMapping.AddFieldMappingsAt("TeamMemberIDs", keywordMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("_default", channelMapping)
+
+	return indexMapping
+}
+
+func getPostIndexMapping() *mapping.IndexMappingImpl {
+	postMapping := bleve.NewDocumentMapping()
+	postMapping.AddFieldMappingsAt("Id", keywordMapping)
+	postMapping.AddFieldMappingsAt("TeamId", keywordMapping)
+	postMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
+	postMapping.AddFieldMappingsAt("UserId", keywordMapping)
+	postMapping.AddFieldMappingsAt("CreateAt", dateMapping)
+	postMapping.AddFieldMappingsAt("Message", standardMapping)
+	postMapping.AddFieldMappingsAt("Type", keywordMapping)
+	postMapping.AddFieldMappingsAt("Hashtags", standardMapping)
+	postMapping.AddFieldMappingsAt("Attachments", standardMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("_default", postMapping)
+
+	return indexMapping
+}
+
+func getFileIndexMapping() *mapping.IndexMappingImpl {
+	fileMapping := bleve.NewDocumentMapping()
+	fileMapping.AddFieldMappingsAt("Id", keywordMapping)
+	fileMapping.AddFieldMappingsAt("CreatorId", keywordMapping)
+	fileMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
+	fileMapping.AddFieldMappingsAt("CreateAt", dateMapping)
+	fileMapping.AddFieldMappingsAt("Name", standardMapping)
+	fileMapping.AddFieldMappingsAt("Content", standardMapping)
+	fileMapping.AddFieldMappingsAt("Extension", keywordMapping)
+	fileMapping.AddFieldMappingsAt("Content", standardMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("_default", fileMapping)
+
+	return indexMapping
+}
+
+func getUserIndexMapping() *mapping.IndexMappingImpl {
+	userMapping := bleve.NewDocumentMapping()
+	userMapping.AddFieldMappingsAt("Id", keywordMapping)
+	userMapping.AddFieldMappingsAt("SuggestionsWithFullname", keywordMapping)
+	userMapping.AddFieldMappingsAt("SuggestionsWithoutFullname", keywordMapping)
+	userMapping.AddFieldMappingsAt("TeamsIds", keywordMapping)
+	userMapping.AddFieldMappingsAt("ChannelsIds", keywordMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("_default", userMapping)
+
+	return indexMapping
+}
+
+func NewBleveEngine(cfg *model.Config) *BleveEngine {
+	return &BleveEngine{
+		cfg: cfg,
+	}
+}
+
+func (b *BleveEngine) getIndexDir(indexName string) string {
+	return filepath.Join(*b.cfg.BleveSettings.IndexDir, indexName+".bleve")
+}
+
+func (b *BleveEngine) createOrOpenIndex(indexName string, mapping *mapping.IndexMappingImpl) (bleve.Index, error) {
+	indexPath := b.getIndexDir(indexName)
+	if index, err := bleve.Open(indexPath); err == nil {
+		return index, nil
+	}
+
+	index, err := bleve.NewUsing(indexPath, mapping, "scorch", "scorch", map[string]any{
+		"forceSegmentType":    "zap",
+		"forceSegmentVersion": 15,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return index, nil
+}
+
+func (b *BleveEngine) openIndexes() *model.AppError {
+	if atomic.LoadInt32(&b.ready) != 0 {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.already_started.error", nil, "", http.StatusInternalServerError)
+	}
+
+	var err error
+	b.PostIndex, err = b.createOrOpenIndex(PostIndex, getPostIndexMapping())
+	if err != nil {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	b.FileIndex, err = b.createOrOpenIndex(FileIndex, getFileIndexMapping())
+	if err != nil {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.create_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	b.UserIndex, err = b.createOrOpenIndex(UserIndex, getUserIndexMapping())
+	if err != nil {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	b.ChannelIndex, err = b.createOrOpenIndex(ChannelIndex, getChannelIndexMapping())
+	if err != nil {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.create_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	atomic.StoreInt32(&b.ready, 1)
+	return nil
+}
+
+func (b *BleveEngine) Start() *model.AppError {
+	if !*b.cfg.BleveSettings.EnableIndexing || *b.cfg.BleveSettings.IndexDir == "" {
+		return nil
+	}
+
+	b.Mutex.Lock()
+	defer b.Mutex.Unlock()
+
+	mlog.Info("EXPERIMENTAL: Starting Bleve")
+
+	return b.openIndexes()
+}
+
+func (b *BleveEngine) closeIndexes() *model.AppError {
+	if b.IsActive() {
+		if err := b.PostIndex.Close(); err != nil {
+			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		if err := b.FileIndex.Close(); err != nil {
+			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		if err := b.UserIndex.Close(); err != nil {
+			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		if err := b.ChannelIndex.Close(); err != nil {
+			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+	}
+
+	atomic.StoreInt32(&b.ready, 0)
+	return nil
+}
+
+func (b *BleveEngine) Stop() *model.AppError {
+	b.Mutex.Lock()
+	defer b.Mutex.Unlock()
+
+	mlog.Info("Stopping Bleve")
+
+	return b.closeIndexes()
+}
+
+func (b *BleveEngine) IsActive() bool {
+	return atomic.LoadInt32(&b.ready) == 1
+}
+
+func (b *BleveEngine) IsIndexingSync() bool {
+	return b.indexSync
+}
+
+func (b *BleveEngine) RefreshIndexes() *model.AppError {
+	return nil
+}
+
+func (b *BleveEngine) GetVersion() int {
+	return 0
+}
+
+func (b *BleveEngine) GetFullVersion() string {
+	return "0"
+}
+
+func (b *BleveEngine) GetPlugins() []string {
+	return []string{}
+}
+
+func (b *BleveEngine) GetName() string {
+	return EngineName
+}
+
+func (b *BleveEngine) TestConfig(cfg *model.Config) *model.AppError {
+	return nil
+}
+
+func (b *BleveEngine) deleteIndexes() *model.AppError {
+	if err := os.RemoveAll(b.getIndexDir(PostIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if err := os.RemoveAll(b.getIndexDir(UserIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if err := os.RemoveAll(b.getIndexDir(ChannelIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if err := os.RemoveAll(b.getIndexDir(FileIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) PurgeIndexes() *model.AppError {
+	if *b.cfg.BleveSettings.IndexDir == "" {
+		return nil
+	}
+
+	b.Mutex.Lock()
+	defer b.Mutex.Unlock()
+
+	mlog.Info("PurgeIndexes Bleve")
+	if err := b.closeIndexes(); err != nil {
+		return err
+	}
+
+	if err := b.deleteIndexes(); err != nil {
+		return err
+	}
+
+	return b.openIndexes()
+}
+
+func (b *BleveEngine) DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError {
+	return nil
+}
+
+func (b *BleveEngine) IsAutocompletionEnabled() bool {
+	return *b.cfg.BleveSettings.EnableAutocomplete
+}
+
+func (b *BleveEngine) IsIndexingEnabled() bool {
+	return *b.cfg.BleveSettings.EnableIndexing
+}
+
+func (b *BleveEngine) IsSearchEnabled() bool {
+	return *b.cfg.BleveSettings.EnableSearching
+}
+
+func (b *BleveEngine) UpdateConfig(cfg *model.Config) {
+	b.Mutex.Lock()
+	defer b.Mutex.Unlock()
+
+	if reflect.DeepEqual(cfg.BleveSettings, b.cfg.BleveSettings) {
+		return
+	}
+
+	mlog.Info("UpdateConf Bleve")
+
+	if *cfg.BleveSettings.EnableIndexing != *b.cfg.BleveSettings.EnableIndexing || *cfg.BleveSettings.IndexDir != *b.cfg.BleveSettings.IndexDir {
+		if err := b.closeIndexes(); err != nil {
+			mlog.Error("Error closing Bleve indexes to update the config", mlog.Err(err))
+			return
+		}
+		b.cfg = cfg
+		if err := b.openIndexes(); err != nil {
+			mlog.Error("Error opening Bleve indexes after updating the config", mlog.Err(err))
+		}
+		return
+	}
+	b.cfg = cfg
+}

server/platform/services/searchengine/bleveengine/bleve_test.go

@@ -0,0 +1,226 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"os"
+	"testing"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/searchlayer"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/searchtest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/sqlstore"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/testlib"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine"
+)
+
+type BleveEngineTestSuite struct {
+	suite.Suite
+
+	SQLSettings  *model.SqlSettings
+	SQLStore     *sqlstore.SqlStore
+	SearchEngine *searchengine.Broker
+	Store        *searchlayer.SearchStore
+	BleveEngine  *BleveEngine
+	IndexDir     string
+}
+
+func TestBleveEngineTestSuite(t *testing.T) {
+	suite.Run(t, new(BleveEngineTestSuite))
+}
+
+func (s *BleveEngineTestSuite) setupIndexes() {
+	indexDir, err := os.MkdirTemp("", "mmbleve")
+	if err != nil {
+		s.Require().FailNow("Cannot setup bleveengine tests: %s", err.Error())
+	}
+	s.IndexDir = indexDir
+}
+
+func (s *BleveEngineTestSuite) setupStore() {
+	driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
+	if driverName == "" {
+		driverName = model.DatabaseDriverPostgres
+	}
+	s.SQLSettings = storetest.MakeSqlSettings(driverName, false)
+	s.SQLStore = sqlstore.New(*s.SQLSettings, nil)
+
+	cfg := &model.Config{}
+	cfg.SetDefaults()
+	cfg.BleveSettings.EnableIndexing = model.NewBool(true)
+	cfg.BleveSettings.EnableSearching = model.NewBool(true)
+	cfg.BleveSettings.EnableAutocomplete = model.NewBool(true)
+	cfg.BleveSettings.IndexDir = model.NewString(s.IndexDir)
+	cfg.SqlSettings.DisableDatabaseSearch = model.NewBool(true)
+
+	s.SearchEngine = searchengine.NewBroker(cfg)
+	s.Store = searchlayer.NewSearchLayer(&testlib.TestStore{Store: s.SQLStore}, s.SearchEngine, cfg)
+
+	s.BleveEngine = NewBleveEngine(cfg)
+	s.BleveEngine.indexSync = true
+	s.SearchEngine.RegisterBleveEngine(s.BleveEngine)
+	if err := s.BleveEngine.Start(); err != nil {
+		s.Require().FailNow("Cannot start bleveengine: %s", err.Error())
+	}
+}
+
+func (s *BleveEngineTestSuite) SetupSuite() {
+	s.setupIndexes()
+	s.setupStore()
+}
+
+func (s *BleveEngineTestSuite) TearDownSuite() {
+	os.RemoveAll(s.IndexDir)
+	s.SQLStore.Close()
+	storetest.CleanupSqlSettings(s.SQLSettings)
+}
+
+func (s *BleveEngineTestSuite) TestBleveSearchStoreTests() {
+	searchTestEngine := &searchtest.SearchTestEngine{
+		Driver: searchtest.EngineBleve,
+	}
+
+	s.Run("TestSearchChannelStore", func() {
+		searchtest.TestSearchChannelStore(s.T(), s.Store, searchTestEngine)
+	})
+
+	s.Run("TestSearchUserStore", func() {
+		searchtest.TestSearchUserStore(s.T(), s.Store, searchTestEngine)
+	})
+
+	s.Run("TestSearchPostStore", func() {
+		searchtest.TestSearchPostStore(s.T(), s.Store, searchTestEngine)
+	})
+
+	s.Run("TestSearchFileInfoStore", func() {
+		searchtest.TestSearchFileInfoStore(s.T(), s.Store, searchTestEngine)
+	})
+}
+
+func (s *BleveEngineTestSuite) TestDeleteChannelPosts() {
+	s.Run("Should remove all the posts that belongs to a channel", func() {
+		s.BleveEngine.PurgeIndexes()
+		teamID := model.NewId()
+		userID := model.NewId()
+		channelID := model.NewId()
+		channelToAvoidID := model.NewId()
+		for i := 0; i < 10; i++ {
+			post := createPost(userID, channelID)
+			appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+			require.Nil(s.T(), appErr)
+		}
+		postToAvoid := createPost(userID, channelToAvoidID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
+		require.Nil(s.T(), appErr)
+
+		s.SearchEngine.BleveEngine.DeleteChannelPosts(channelID)
+
+		doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), postToAvoid.Id, doc.ID())
+		numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), 1, int(numberDocs))
+	})
+
+	s.Run("Shouldn't do anything if there is not posts for the selected channel", func() {
+		s.BleveEngine.PurgeIndexes()
+		teamID := model.NewId()
+		userID := model.NewId()
+		channelID := model.NewId()
+		channelToDeleteID := model.NewId()
+		post := createPost(userID, channelID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+		require.Nil(s.T(), appErr)
+
+		s.SearchEngine.BleveEngine.DeleteChannelPosts(channelToDeleteID)
+
+		_, err := s.BleveEngine.PostIndex.Document(post.Id)
+		require.NoError(s.T(), err)
+		numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), 1, int(numberDocs))
+	})
+}
+
+func (s *BleveEngineTestSuite) TestDeleteUserPosts() {
+	s.Run("Should remove all the posts that belongs to a user", func() {
+		s.BleveEngine.PurgeIndexes()
+		teamID := model.NewId()
+		userID := model.NewId()
+		userToAvoidID := model.NewId()
+		channelID := model.NewId()
+		for i := 0; i < 10; i++ {
+			post := createPost(userID, channelID)
+			appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+			require.Nil(s.T(), appErr)
+		}
+		postToAvoid := createPost(userToAvoidID, channelID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
+		require.Nil(s.T(), appErr)
+
+		s.SearchEngine.BleveEngine.DeleteUserPosts(userID)
+
+		doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), postToAvoid.Id, doc.ID())
+		numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), 1, int(numberDocs))
+	})
+
+	s.Run("Shouldn't do anything if there is not posts for the selected user", func() {
+		s.BleveEngine.PurgeIndexes()
+		teamID := model.NewId()
+		userID := model.NewId()
+		userToDeleteID := model.NewId()
+		channelID := model.NewId()
+		post := createPost(userID, channelID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+		require.Nil(s.T(), appErr)
+
+		s.SearchEngine.BleveEngine.DeleteUserPosts(userToDeleteID)
+
+		_, err := s.BleveEngine.PostIndex.Document(post.Id)
+		require.NoError(s.T(), err)
+		numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), 1, int(numberDocs))
+	})
+}
+
+func (s *BleveEngineTestSuite) TestDeletePosts() {
+	s.BleveEngine.PurgeIndexes()
+	teamID := model.NewId()
+	userID := model.NewId()
+	userToAvoidID := model.NewId()
+	channelID := model.NewId()
+	for i := 0; i < 10; i++ {
+		post := createPost(userID, channelID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+		require.Nil(s.T(), appErr)
+	}
+	postToAvoid := createPost(userToAvoidID, channelID)
+	appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
+	require.Nil(s.T(), appErr)
+
+	query := bleve.NewTermQuery(userID)
+	query.SetField("UserId")
+	search := bleve.NewSearchRequest(query)
+	count, err := s.BleveEngine.deletePosts(search, 1)
+	require.NoError(s.T(), err)
+	require.Equal(s.T(), 10, int(count))
+
+	doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
+	require.NoError(s.T(), err)
+	require.Equal(s.T(), postToAvoid.Id, doc.ID())
+	numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+	require.NoError(s.T(), err)
+	require.Equal(s.T(), 1, int(numberDocs))
+}

server/platform/services/searchengine/bleveengine/common.go

@@ -0,0 +1,163 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine"
+)
+
+type BLVChannel struct {
+	Id            string
+	Type          model.ChannelType
+	UserIDs       []string
+	TeamId        []string
+	TeamMemberIDs []string
+	NameSuggest   []string
+}
+
+type BLVUser struct {
+	Id                         string
+	SuggestionsWithFullname    []string
+	SuggestionsWithoutFullname []string
+	TeamsIds                   []string
+	ChannelsIds                []string
+}
+
+type BLVPost struct {
+	Id          string
+	TeamId      string
+	ChannelId   string
+	UserId      string
+	CreateAt    int64
+	Message     string
+	Type        string
+	Hashtags    []string
+	Attachments string
+}
+
+type BLVFile struct {
+	Id        string
+	CreatorId string
+	ChannelId string
+	CreateAt  int64
+	Name      string
+	Content   string
+	Extension string
+}
+
+func BLVChannelFromChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *BLVChannel {
+	displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ")
+	nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"})
+
+	return &BLVChannel{
+		Id:            channel.Id,
+		Type:          channel.Type,
+		TeamId:        []string{channel.TeamId},
+		NameSuggest:   append(displayNameInputs, nameInputs...),
+		UserIDs:       userIDs,
+		TeamMemberIDs: teamMemberIDs,
+	}
+}
+
+func BLVUserFromUserAndTeams(user *model.User, teamsIds, channelsIds []string) *BLVUser {
+	usernameSuggestions := searchengine.GetSuggestionInputsSplitByMultiple(user.Username, []string{".", "-", "_"})
+
+	fullnameStrings := []string{}
+	if user.FirstName != "" {
+		fullnameStrings = append(fullnameStrings, user.FirstName)
+	}
+	if user.LastName != "" {
+		fullnameStrings = append(fullnameStrings, user.LastName)
+	}
+
+	fullnameSuggestions := []string{}
+	if len(fullnameStrings) > 0 {
+		fullname := strings.Join(fullnameStrings, " ")
+		fullnameSuggestions = searchengine.GetSuggestionInputsSplitBy(fullname, " ")
+	}
+
+	nicknameSuggestions := []string{}
+	if user.Nickname != "" {
+		nicknameSuggestions = searchengine.GetSuggestionInputsSplitBy(user.Nickname, " ")
+	}
+
+	usernameAndNicknameSuggestions := append(usernameSuggestions, nicknameSuggestions...)
+
+	return &BLVUser{
+		Id:                         user.Id,
+		SuggestionsWithFullname:    append(usernameAndNicknameSuggestions, fullnameSuggestions...),
+		SuggestionsWithoutFullname: usernameAndNicknameSuggestions,
+		TeamsIds:                   teamsIds,
+		ChannelsIds:                channelsIds,
+	}
+}
+
+func BLVUserFromUserForIndexing(userForIndexing *model.UserForIndexing) *BLVUser {
+	user := &model.User{
+		Id:        userForIndexing.Id,
+		Username:  userForIndexing.Username,
+		Nickname:  userForIndexing.Nickname,
+		FirstName: userForIndexing.FirstName,
+		LastName:  userForIndexing.LastName,
+		CreateAt:  userForIndexing.CreateAt,
+		DeleteAt:  userForIndexing.DeleteAt,
+	}
+
+	return BLVUserFromUserAndTeams(user, userForIndexing.TeamsIds, userForIndexing.ChannelsIds)
+}
+
+func BLVPostFromPost(post *model.Post, teamId string) *BLVPost {
+	p := &model.PostForIndexing{
+		TeamId: teamId,
+	}
+	post.ShallowCopy(&p.Post)
+	return BLVPostFromPostForIndexing(p)
+}
+
+func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost {
+	return &BLVPost{
+		Id:        post.Id,
+		TeamId:    post.TeamId,
+		ChannelId: post.ChannelId,
+		UserId:    post.UserId,
+		CreateAt:  post.CreateAt,
+		Message:   post.Message,
+		Type:      post.Type,
+		Hashtags:  strings.Fields(post.Hashtags),
+	}
+}
+
+func splitFilenameWords(name string) string {
+	result := name
+	result = strings.ReplaceAll(result, "-", " ")
+	result = strings.ReplaceAll(result, ".", " ")
+	return result
+}
+
+func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile {
+	return &BLVFile{
+		Id:        fileInfo.Id,
+		ChannelId: channelId,
+		CreatorId: fileInfo.CreatorId,
+		CreateAt:  fileInfo.CreateAt,
+		Content:   fileInfo.Content,
+		Extension: fileInfo.Extension,
+		Name:      fileInfo.Name + " " + splitFilenameWords(fileInfo.Name),
+	}
+}
+
+func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile {
+	return &BLVFile{
+		Id:        file.Id,
+		ChannelId: file.ChannelId,
+		CreatorId: file.CreatorId,
+		CreateAt:  file.CreateAt,
+		Content:   file.Content,
+		Extension: file.Extension,
+		Name:      file.Name + " " + splitFilenameWords(file.Name),
+	}
+}

server/platform/services/searchengine/bleveengine/indexer/indexing_job.go

@@ -0,0 +1,614 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package indexer
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine/bleveengine"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	timeBetweenBatches = 100 * time.Millisecond
+
+	estimatedPostCount    = 10000000
+	estimatedFilesCount   = 100000
+	estimatedChannelCount = 100000
+	estimatedUserCount    = 10000
+)
+
+type BleveIndexerWorker struct {
+	name      string
+	stop      chan struct{}
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	engine    *bleveengine.BleveEngine
+	closed    int32
+}
+
+func MakeWorker(jobServer *jobs.JobServer, engine *bleveengine.BleveEngine) model.Worker {
+	if engine == nil {
+		return nil
+	}
+	return &BleveIndexerWorker{
+		name:      "BleveIndexer",
+		stop:      make(chan struct{}),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		engine:    engine,
+	}
+}
+
+type IndexingProgress struct {
+	Now            time.Time
+	StartAtTime    int64
+	EndAtTime      int64
+	LastEntityTime int64
+
+	TotalPostsCount int64
+	DonePostsCount  int64
+	DonePosts       bool
+	LastPostID      string
+
+	TotalFilesCount int64
+	DoneFilesCount  int64
+	DoneFiles       bool
+	LastFileID      string
+
+	TotalChannelsCount int64
+	DoneChannelsCount  int64
+	DoneChannels       bool
+	LastChannelID      string
+
+	TotalUsersCount int64
+	DoneUsersCount  int64
+	DoneUsers       bool
+	LastUserID      string
+}
+
+func (ip *IndexingProgress) CurrentProgress() int64 {
+	return (ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount) * 100 / (ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalUsersCount + ip.TotalFilesCount)
+}
+
+func (ip *IndexingProgress) IsDone() bool {
+	return ip.DonePosts && ip.DoneChannels && ip.DoneUsers && ip.DoneFiles
+}
+
+func (worker *BleveIndexerWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *BleveIndexerWorker) IsEnabled(cfg *model.Config) bool {
+	return true
+}
+
+func (worker *BleveIndexerWorker) Run() {
+	// Set to open if closed before. We are not bothered about multiple opens.
+	if atomic.CompareAndSwapInt32(&worker.closed, 1, 0) {
+		worker.stop = make(chan struct{})
+	}
+	mlog.Debug("Worker Started", mlog.String("workername", worker.name))
+
+	defer func() {
+		mlog.Debug("Worker: Finished", mlog.String("workername", worker.name))
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			mlog.Debug("Worker: Received stop signal", mlog.String("workername", worker.name))
+			return
+		case job := <-worker.jobs:
+			mlog.Debug("Worker: Received a new candidate job.", mlog.String("workername", worker.name))
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *BleveIndexerWorker) Stop() {
+	// Set to close, and if already closed before, then return.
+	if !atomic.CompareAndSwapInt32(&worker.closed, 0, 1) {
+		return
+	}
+	mlog.Debug("Worker Stopping", mlog.String("workername", worker.name))
+	close(worker.stop)
+	<-worker.stopped
+}
+
+func (worker *BleveIndexerWorker) DoJob(job *model.Job) {
+	claimed, err := worker.jobServer.ClaimJob(job)
+	if err != nil {
+		mlog.Warn("Worker: Error occurred while trying to claim job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		return
+	}
+	if !claimed {
+		return
+	}
+
+	mlog.Info("Worker: Indexing job claimed by worker", mlog.String("workername", worker.name), mlog.String("job_id", job.Id))
+
+	if !worker.engine.IsActive() {
+		appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.engine_inactive", nil, "", http.StatusInternalServerError)
+		if err := worker.jobServer.SetJobError(job, appError); err != nil {
+			mlog.Error("Worker: Failed to run job as ")
+		}
+		return
+	}
+
+	progress := IndexingProgress{
+		Now:          time.Now(),
+		DonePosts:    false,
+		DoneChannels: false,
+		DoneUsers:    false,
+		DoneFiles:    false,
+		StartAtTime:  0,
+		EndAtTime:    model.GetMillis(),
+	}
+
+	// Extract the start and end times, if they are set.
+	if startString, ok := job.Data["start_time"]; ok {
+		startInt, err := strconv.ParseInt(startString, 10, 64)
+		if err != nil {
+			mlog.Error("Worker: Failed to parse start_time for job", mlog.String("workername", worker.name), mlog.String("start_time", startString), mlog.String("job_id", job.Id), mlog.Err(err))
+			appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.parse_start_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := worker.jobServer.SetJobError(job, appError); err != nil {
+				mlog.Error("Worker: Failed to set job error", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return
+		}
+		progress.StartAtTime = startInt
+	} else {
+		// Set start time to oldest entity in the database.
+		// A user or a channel may be created before any post.
+		oldestEntityCreationTime, err := worker.jobServer.Store.Post().GetOldestEntityCreationTime()
+		if err != nil {
+			mlog.Error("Worker: Failed to fetch oldest entity for job.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.String("start_time", startString), mlog.Err(err))
+			appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.get_oldest_entity.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := worker.jobServer.SetJobError(job, appError); err != nil {
+				mlog.Error("Worker: Failed to set job error", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return
+		}
+		progress.StartAtTime = oldestEntityCreationTime
+	}
+	progress.LastEntityTime = progress.StartAtTime
+
+	if endString, ok := job.Data["end_time"]; ok {
+		endInt, err := strconv.ParseInt(endString, 10, 64)
+		if err != nil {
+			mlog.Error("Worker: Failed to parse end_time for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.String("end_time", endString), mlog.Err(err))
+			appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.parse_end_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := worker.jobServer.SetJobError(job, appError); err != nil {
+				mlog.Error("Worker: Failed to set job errorv", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return
+		}
+		progress.EndAtTime = endInt
+	}
+
+	if id, ok := job.Data["start_post_id"]; ok {
+		progress.LastPostID = id
+	}
+	if id, ok := job.Data["start_channel_id"]; ok {
+		progress.LastChannelID = id
+	}
+	if id, ok := job.Data["start_user_id"]; ok {
+		progress.LastUserID = id
+	}
+	if id, ok := job.Data["start_file_id"]; ok {
+		progress.LastFileID = id
+	}
+
+	// Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry
+	// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
+	if count, err := worker.jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}); err != nil {
+		mlog.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalPostsCount = estimatedPostCount
+	} else {
+		progress.TotalPostsCount = count
+	}
+
+	// Same possible fail as above can happen when counting channels
+	if count, err := worker.jobServer.Store.Channel().AnalyticsTypeCount("", ""); err != nil {
+		mlog.Warn("Worker: Failed to fetch total channel count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalChannelsCount = estimatedChannelCount
+	} else {
+		progress.TotalChannelsCount = count
+	}
+
+	// Same possible fail as above can happen when counting users
+	if count, err := worker.jobServer.Store.User().Count(model.UserCountOptions{
+		IncludeBotAccounts: true, // This actually doesn't join with the bots table
+		// since ExcludeRegularUsers is set to false
+	}); err != nil {
+		mlog.Warn("Worker: Failed to fetch total user count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalUsersCount = estimatedUserCount
+	} else {
+		progress.TotalUsersCount = count
+	}
+
+	// Counting all files may fail or timeout when the file_info table is large. If this happens, log a warning, but carry
+	// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
+	if count, err := worker.jobServer.Store.FileInfo().CountAll(); err != nil {
+		mlog.Warn("Worker: Failed to fetch total file info count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalFilesCount = estimatedFilesCount
+	} else {
+		progress.TotalFilesCount = count
+	}
+
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan)
+
+	defer cancelCancelWatcher()
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			mlog.Info("Worker: Indexing job has been canceled via CancellationWatcher", mlog.String("workername", worker.name), mlog.String("job_id", job.Id))
+			if err := worker.jobServer.SetJobCanceled(job); err != nil {
+				mlog.Error("Worker: Failed to mark job as cancelled", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+			}
+			return
+
+		case <-worker.stop:
+			mlog.Info("Worker: Indexing has been canceled via Worker Stop", mlog.String("workername", worker.name), mlog.String("job_id", job.Id))
+			if err := worker.jobServer.SetJobCanceled(job); err != nil {
+				mlog.Error("Worker: Failed to mark job as canceled", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+			}
+			return
+
+		case <-time.After(timeBetweenBatches):
+			var err *model.AppError
+			if progress, err = worker.IndexBatch(progress); err != nil {
+				mlog.Error("Worker: Failed to index batch for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+				if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+					mlog.Error("Worker: Failed to set job error", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err2), mlog.NamedErr("set_error", err))
+				}
+				return
+			}
+
+			// Storing the batch progress in metadata.
+			if job.Data == nil {
+				job.Data = make(model.StringMap)
+			}
+
+			job.Data["start_time"] = strconv.FormatInt(progress.LastEntityTime, 10)
+			job.Data["start_post_id"] = progress.LastPostID
+			job.Data["start_channel_id"] = progress.LastChannelID
+			job.Data["start_user_id"] = progress.LastUserID
+			job.Data["start_file_id"] = progress.LastFileID
+			job.Data["original_start_time"] = strconv.FormatInt(progress.StartAtTime, 10)
+			job.Data["end_time"] = strconv.FormatInt(progress.EndAtTime, 10)
+
+			if err := worker.jobServer.SetJobProgress(job, progress.CurrentProgress()); err != nil {
+				mlog.Error("Worker: Failed to set progress for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+				if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+					mlog.Error("Worker: Failed to set error for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err2), mlog.NamedErr("set_error", err))
+				}
+				return
+			}
+
+			if progress.IsDone() {
+				if err := worker.jobServer.SetJobSuccess(job); err != nil {
+					mlog.Error("Worker: Failed to set success for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+					if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+						mlog.Error("Worker: Failed to set error for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err2), mlog.NamedErr("set_error", err))
+					}
+				}
+				mlog.Info("Worker: Indexing job finished successfully", mlog.String("workername", worker.name), mlog.String("job_id", job.Id))
+				return
+			}
+		}
+	}
+}
+
+func (worker *BleveIndexerWorker) IndexBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	if !progress.DonePosts {
+		return worker.IndexPostsBatch(progress)
+	}
+	if !progress.DoneChannels {
+		return worker.IndexChannelsBatch(progress)
+	}
+	if !progress.DoneUsers {
+		return worker.IndexUsersBatch(progress)
+	}
+	if !progress.DoneFiles {
+		return worker.IndexFilesBatch(progress)
+	}
+	return progress, model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError)
+}
+
+func (worker *BleveIndexerWorker) IndexPostsBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var posts []*model.PostForIndexing
+
+	tries := 0
+	for posts == nil {
+		var err error
+		posts, err = worker.jobServer.Store.Post().GetPostsBatchForIndexing(progress.LastEntityTime, progress.LastPostID, *worker.jobServer.Config().BleveSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexPostsBatch", "app.post.get_posts_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			mlog.Warn("Failed to get posts batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	// Handle zero messages.
+	if len(posts) == 0 {
+		progress.DonePosts = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastPost, err := worker.BulkIndexPosts(posts, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last post's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastPost.CreateAt {
+		progress.DonePosts = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastPost.CreateAt
+	}
+
+	progress.LastPostID = lastPost.Id
+	progress.DonePostsCount += int64(len(posts))
+
+	return progress, nil
+}
+
+func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, progress IndexingProgress) (*model.Post, *model.AppError) {
+	batch := worker.engine.PostIndex.NewBatch()
+
+	for _, post := range posts {
+		if post.DeleteAt == 0 {
+			searchPost := bleveengine.BLVPostFromPostForIndexing(post)
+			batch.Index(searchPost.Id, searchPost)
+		} else {
+			batch.Delete(post.Id)
+		}
+	}
+
+	worker.engine.Mutex.RLock()
+	defer worker.engine.Mutex.RUnlock()
+
+	if err := worker.engine.PostIndex.Batch(batch); err != nil {
+		return nil, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_posts.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return &posts[len(posts)-1].Post, nil
+}
+
+func (worker *BleveIndexerWorker) IndexFilesBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var files []*model.FileForIndexing
+
+	tries := 0
+	for files == nil {
+		var err error
+		files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, progress.LastFileID, *worker.jobServer.Config().BleveSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexFilesBatch", "app.post.get_files_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			mlog.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	if len(files) == 0 {
+		progress.DoneFiles = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastFile, err := worker.BulkIndexFiles(files, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last file's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastFile.CreateAt {
+		progress.DoneFiles = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastFile.CreateAt
+	}
+
+	progress.LastFileID = lastFile.Id
+	progress.DoneFilesCount += int64(len(files))
+
+	return progress, nil
+}
+
+func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (*model.FileInfo, *model.AppError) {
+	batch := worker.engine.FileIndex.NewBatch()
+
+	for _, file := range files {
+		if file.DeleteAt == 0 {
+			searchFile := bleveengine.BLVFileFromFileForIndexing(file)
+			batch.Index(searchFile.Id, searchFile)
+		} else {
+			batch.Delete(file.Id)
+		}
+	}
+
+	worker.engine.Mutex.RLock()
+	defer worker.engine.Mutex.RUnlock()
+
+	if err := worker.engine.FileIndex.Batch(batch); err != nil {
+		return nil, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_files.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return &files[len(files)-1].FileInfo, nil
+}
+
+func (worker *BleveIndexerWorker) IndexChannelsBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var channels []*model.Channel
+
+	tries := 0
+	for channels == nil {
+		var nErr error
+		channels, nErr = worker.jobServer.Store.Channel().GetChannelsBatchForIndexing(progress.LastEntityTime, progress.LastChannelID, *worker.jobServer.Config().BleveSettings.BatchSize)
+		if nErr != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("BleveIndexerWorker.IndexChannelsBatch", "app.channel.get_channels_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+			}
+
+			mlog.Warn("Failed to get channels batch for indexing. Retrying.", mlog.Err(nErr))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+		tries++
+	}
+
+	if len(channels) == 0 {
+		progress.DoneChannels = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastChannel, err := worker.BulkIndexChannels(channels, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last channel's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastChannel.CreateAt {
+		progress.DoneChannels = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastChannel.CreateAt
+	}
+
+	progress.LastChannelID = lastChannel.Id
+	progress.DoneChannelsCount += int64(len(channels))
+
+	return progress, nil
+}
+
+func (worker *BleveIndexerWorker) BulkIndexChannels(channels []*model.Channel, progress IndexingProgress) (*model.Channel, *model.AppError) {
+	batch := worker.engine.ChannelIndex.NewBatch()
+
+	for _, channel := range channels {
+		if channel.DeleteAt == 0 {
+			var userIDs []string
+			var err error
+			if channel.Type == model.ChannelTypePrivate {
+				userIDs, err = worker.jobServer.Store.Channel().GetAllChannelMembersById(channel.Id)
+				if err != nil {
+					return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+				}
+			}
+
+			// Get teamMember ids from channelid
+			teamMemberIDs, err := worker.jobServer.Store.Channel().GetTeamMembersForChannel(channel.Id)
+			if err != nil {
+				return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			searchChannel := bleveengine.BLVChannelFromChannel(channel, userIDs, teamMemberIDs)
+			batch.Index(searchChannel.Id, searchChannel)
+		} else {
+			batch.Delete(channel.Id)
+		}
+	}
+
+	worker.engine.Mutex.RLock()
+	defer worker.engine.Mutex.RUnlock()
+
+	if err := worker.engine.ChannelIndex.Batch(batch); err != nil {
+		return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return channels[len(channels)-1], nil
+}
+
+func (worker *BleveIndexerWorker) IndexUsersBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var users []*model.UserForIndexing
+
+	tries := 0
+	for users == nil {
+		if usersBatch, err := worker.jobServer.Store.User().GetUsersBatchForIndexing(progress.LastEntityTime, progress.LastUserID, *worker.jobServer.Config().BleveSettings.BatchSize); err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexUsersBatch", "app.user.get_users_batch_for_indexing.get_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			mlog.Warn("Failed to get users batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		} else {
+			users = usersBatch
+		}
+
+		tries++
+	}
+
+	if len(users) == 0 {
+		progress.DoneUsers = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastUser, err := worker.BulkIndexUsers(users, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last user's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastUser.CreateAt {
+		progress.DoneUsers = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastUser.CreateAt
+	}
+	progress.LastUserID = lastUser.Id
+	progress.DoneUsersCount += int64(len(users))
+
+	return progress, nil
+}
+
+func (worker *BleveIndexerWorker) BulkIndexUsers(users []*model.UserForIndexing, progress IndexingProgress) (*model.UserForIndexing, *model.AppError) {
+	batch := worker.engine.UserIndex.NewBatch()
+
+	for _, user := range users {
+		if user.DeleteAt == 0 {
+			searchUser := bleveengine.BLVUserFromUserForIndexing(user)
+			batch.Index(searchUser.Id, searchUser)
+		} else {
+			batch.Delete(user.Id)
+		}
+	}
+
+	worker.engine.Mutex.RLock()
+	defer worker.engine.Mutex.RUnlock()
+
+	if err := worker.engine.UserIndex.Batch(batch); err != nil {
+		return nil, model.NewAppError("BleveIndexerWorker.BulkIndexUsers", "bleveengine.indexer.do_job.bulk_index_users.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return users[len(users)-1], nil
+}

server/platform/services/searchengine/bleveengine/indexer/indexing_job_test.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package indexer
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine/bleveengine"
+)
+
+func TestBleveIndexer(t *testing.T) {
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	t.Run("Call GetOldestEntityCreationTime for the first indexing call", func(t *testing.T) {
+		job := &model.Job{
+			Id:       model.NewId(),
+			CreateAt: model.GetMillis(),
+			Status:   model.JobStatusPending,
+			Type:     model.JobTypeBlevePostIndexing,
+		}
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).Return(true, nil)
+		mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+		mockStore.PostStore.On("GetOldestEntityCreationTime").Return(int64(1), errors.New("")) // intentionally return error to return from function
+
+		tempDir, err := os.MkdirTemp("", "setupConfigFile")
+		require.NoError(t, err)
+
+		t.Cleanup(func() {
+			os.RemoveAll(tempDir)
+		})
+
+		cfg := &model.Config{
+			BleveSettings: model.BleveSettings{
+				EnableIndexing: model.NewBool(true),
+				IndexDir:       model.NewString(tempDir),
+			},
+		}
+
+		jobServer := &jobs.JobServer{
+			Store: mockStore,
+			ConfigService: &testutils.StaticConfigService{
+				Cfg: cfg,
+			},
+		}
+
+		bleveEngine := bleveengine.NewBleveEngine(cfg)
+		aErr := bleveEngine.Start()
+		require.Nil(t, aErr)
+
+		worker := &BleveIndexerWorker{
+			jobServer: jobServer,
+			engine:    bleveEngine,
+		}
+
+		worker.DoJob(job)
+	})
+}

server/platform/services/searchengine/bleveengine/search.go

@@ -0,0 +1,879 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/search/query"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const DeletePostsBatchSize = 500
+const DeleteFilesBatchSize = 500
+
+func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	blvPost := BLVPostFromPost(post, teamId)
+	if err := b.PostIndex.Index(blvPost.Id, blvPost); err != nil {
+		return model.NewAppError("Bleveengine.IndexPost", "bleveengine.index_post.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
+	channelQueries := []query.Query{}
+	for _, channel := range channels {
+		channelIdQ := bleve.NewTermQuery(channel.Id)
+		channelIdQ.SetField("ChannelId")
+		channelQueries = append(channelQueries, channelIdQ)
+	}
+	channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
+
+	var termQueries []query.Query
+	var notTermQueries []query.Query
+	var filters []query.Query
+	var notFilters []query.Query
+
+	typeQ := bleve.NewTermQuery("")
+	typeQ.SetField("Type")
+	filters = append(filters, typeQ)
+
+	for i, params := range searchParams {
+		var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
+		if searchParams[0].OrTerms {
+			termOperator = query.MatchQueryOperatorOr
+		}
+
+		// Date, channels and FromUsers filters come in all
+		// searchParams iteration, and as they are global to the
+		// query, we only need to process them once
+		if i == 0 {
+			if len(params.InChannels) > 0 {
+				inChannels := []query.Query{}
+				for _, channelId := range params.InChannels {
+					channelQ := bleve.NewTermQuery(channelId)
+					channelQ.SetField("ChannelId")
+					inChannels = append(inChannels, channelQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
+			}
+
+			if len(params.ExcludedChannels) > 0 {
+				excludedChannels := []query.Query{}
+				for _, channelId := range params.ExcludedChannels {
+					channelQ := bleve.NewTermQuery(channelId)
+					channelQ.SetField("ChannelId")
+					excludedChannels = append(excludedChannels, channelQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
+			}
+
+			if len(params.FromUsers) > 0 {
+				fromUsers := []query.Query{}
+				for _, userId := range params.FromUsers {
+					userQ := bleve.NewTermQuery(userId)
+					userQ.SetField("UserId")
+					fromUsers = append(fromUsers, userQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
+			}
+
+			if len(params.ExcludedUsers) > 0 {
+				excludedUsers := []query.Query{}
+				for _, userId := range params.ExcludedUsers {
+					userQ := bleve.NewTermQuery(userId)
+					userQ.SetField("UserId")
+					excludedUsers = append(excludedUsers, userQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
+			}
+
+			if params.OnDate != "" {
+				before, after := params.GetOnDateMillis()
+				beforeFloat64 := float64(before)
+				afterFloat64 := float64(after)
+				onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
+				onDateQ.SetField("CreateAt")
+				filters = append(filters, onDateQ)
+			} else {
+				if params.AfterDate != "" || params.BeforeDate != "" {
+					var min, max *float64
+					if params.AfterDate != "" {
+						minf := float64(params.GetAfterDateMillis())
+						min = &minf
+					}
+
+					if params.BeforeDate != "" {
+						maxf := float64(params.GetBeforeDateMillis())
+						max = &maxf
+					}
+
+					dateQ := bleve.NewNumericRangeQuery(min, max)
+					dateQ.SetField("CreateAt")
+					filters = append(filters, dateQ)
+				}
+
+				if params.ExcludedAfterDate != "" {
+					minf := float64(params.GetExcludedAfterDateMillis())
+					dateQ := bleve.NewNumericRangeQuery(&minf, nil)
+					dateQ.SetField("CreateAt")
+					notFilters = append(notFilters, dateQ)
+				}
+
+				if params.ExcludedBeforeDate != "" {
+					maxf := float64(params.GetExcludedBeforeDateMillis())
+					dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
+					dateQ.SetField("CreateAt")
+					notFilters = append(notFilters, dateQ)
+				}
+
+				if params.ExcludedDate != "" {
+					before, after := params.GetExcludedDateMillis()
+					beforef := float64(before)
+					afterf := float64(after)
+					onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
+					onDateQ.SetField("CreateAt")
+					notFilters = append(notFilters, onDateQ)
+				}
+			}
+		}
+
+		if params.IsHashtag {
+			if params.Terms != "" {
+				hashtagQ := bleve.NewMatchQuery(params.Terms)
+				hashtagQ.SetField("Hashtags")
+				hashtagQ.SetOperator(termOperator)
+				termQueries = append(termQueries, hashtagQ)
+			} else if params.ExcludedTerms != "" {
+				hashtagQ := bleve.NewMatchQuery(params.ExcludedTerms)
+				hashtagQ.SetField("Hashtags")
+				hashtagQ.SetOperator(termOperator)
+				notTermQueries = append(notTermQueries, hashtagQ)
+			}
+		} else {
+			if params.Terms != "" {
+				terms := []string{}
+				for _, term := range strings.Split(params.Terms, " ") {
+					if strings.HasSuffix(term, "*") {
+						messageQ := bleve.NewWildcardQuery(term)
+						messageQ.SetField("Message")
+						termQueries = append(termQueries, messageQ)
+					} else {
+						terms = append(terms, term)
+					}
+				}
+
+				if len(terms) > 0 {
+					messageQ := bleve.NewMatchQuery(strings.Join(terms, " "))
+					messageQ.SetField("Message")
+					messageQ.SetOperator(termOperator)
+					termQueries = append(termQueries, messageQ)
+				}
+			}
+
+			if params.ExcludedTerms != "" {
+				messageQ := bleve.NewMatchQuery(params.ExcludedTerms)
+				messageQ.SetField("Message")
+				messageQ.SetOperator(termOperator)
+				notTermQueries = append(notTermQueries, messageQ)
+			}
+		}
+	}
+
+	allTermsQ := bleve.NewBooleanQuery()
+	allTermsQ.AddMustNot(notTermQueries...)
+	if searchParams[0].OrTerms {
+		allTermsQ.AddShould(termQueries...)
+	} else {
+		allTermsQ.AddMust(termQueries...)
+	}
+
+	query := bleve.NewBooleanQuery()
+	query.AddMust(channelDisjunctionQ)
+
+	if len(termQueries) > 0 || len(notTermQueries) > 0 {
+		query.AddMust(allTermsQ)
+	}
+
+	if len(filters) > 0 {
+		query.AddMust(bleve.NewConjunctionQuery(filters...))
+	}
+	if len(notFilters) > 0 {
+		query.AddMustNot(notFilters...)
+	}
+
+	search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
+	search.SortBy([]string{"-CreateAt"})
+	results, err := b.PostIndex.Search(search)
+	if err != nil {
+		return nil, nil, model.NewAppError("Bleveengine.SearchPosts", "bleveengine.search_posts.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	postIds := []string{}
+	matches := model.PostSearchMatches{}
+
+	for _, r := range results.Hits {
+		postIds = append(postIds, r.ID)
+	}
+
+	return postIds, matches, nil
+}
+
+func (b *BleveEngine) deletePosts(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
+	resultsCount := int64(0)
+
+	for {
+		// As we are deleting the posts after fetching them, we need to keep
+		// From fixed always to 0
+		searchRequest.From = 0
+		searchRequest.Size = batchSize
+		results, err := b.PostIndex.Search(searchRequest)
+		if err != nil {
+			return -1, err
+		}
+		batch := b.PostIndex.NewBatch()
+		for _, post := range results.Hits {
+			batch.Delete(post.ID)
+		}
+		if err := b.PostIndex.Batch(batch); err != nil {
+			return -1, err
+		}
+		resultsCount += int64(results.Hits.Len())
+		if results.Hits.Len() < batchSize {
+			break
+		}
+	}
+
+	return resultsCount, nil
+}
+
+func (b *BleveEngine) DeleteChannelPosts(channelID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	query := bleve.NewTermQuery(channelID)
+	query.SetField("ChannelId")
+	search := bleve.NewSearchRequest(query)
+	deleted, err := b.deletePosts(search, DeletePostsBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeleteChannelPosts",
+			"bleveengine.delete_channel_posts.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Posts for channel deleted", mlog.String("channel_id", channelID), mlog.Int64("deleted", deleted))
+
+	return nil
+}
+
+func (b *BleveEngine) DeleteUserPosts(userID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	query := bleve.NewTermQuery(userID)
+	query.SetField("UserId")
+	search := bleve.NewSearchRequest(query)
+	deleted, err := b.deletePosts(search, DeletePostsBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeleteUserPosts",
+			"bleveengine.delete_user_posts.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Posts for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted))
+
+	return nil
+}
+
+func (b *BleveEngine) DeletePost(post *model.Post) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	if err := b.PostIndex.Delete(post.Id); err != nil {
+		return model.NewAppError("Bleveengine.DeletePost", "bleveengine.delete_post.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) IndexChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	blvChannel := BLVChannelFromChannel(channel, userIDs, teamMemberIDs)
+	if err := b.ChannelIndex.Index(blvChannel.Id, blvChannel); err != nil {
+		return model.NewAppError("Bleveengine.IndexChannel", "bleveengine.index_channel.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) SearchChannels(teamId, userID, term string, isGuest bool) ([]string, *model.AppError) {
+	// This query essentially boils down to (if teamID is passed):
+	// match teamID == <>
+	// AND
+	// match term == <>
+	// AND
+	// match (channelType != 'P' || (<> in userIDs && channelType == 'P'))
+
+	// (or if teamID is not passed)
+	// <> in teamMemberIds
+	// AND
+	// match term == <>
+	// AND
+	// match (channelType != 'P' || (<> in userIDs && channelType == 'P'))
+
+	// (or if isGuest is true)
+	// <> in teamMemberIds
+	// AND
+	// match term == <>
+	// AND
+	// match (<> in userIDs)
+
+	queries := []query.Query{}
+	if teamId != "" {
+		teamIdQ := bleve.NewTermQuery(teamId)
+		teamIdQ.SetField("TeamId")
+		queries = append(queries, teamIdQ)
+	} else {
+		teamMemberQ := bleve.NewTermQuery(userID)
+		teamMemberQ.SetField("TeamMemberIDs")
+		queries = append(queries, teamMemberQ)
+	}
+
+	if isGuest {
+		userQ := bleve.NewBooleanQuery()
+		userIDQ := bleve.NewTermQuery(userID)
+		userIDQ.SetField("UserIDs")
+		userQ.AddMust(userIDQ)
+		queries = append(queries, userIDQ)
+	} else {
+		boolNotPrivate := bleve.NewBooleanQuery()
+		privateQ := bleve.NewTermQuery(string(model.ChannelTypePrivate))
+		privateQ.SetField("Type")
+		boolNotPrivate.AddMustNot(privateQ)
+
+		userQ := bleve.NewBooleanQuery()
+		userIDQ := bleve.NewTermQuery(userID)
+		userIDQ.SetField("UserIDs")
+		userQ.AddMust(userIDQ)
+		userQ.AddMust(privateQ)
+
+		channelTypeQ := bleve.NewDisjunctionQuery()
+		channelTypeQ.AddQuery(boolNotPrivate)
+		channelTypeQ.AddQuery(userQ) // userID && 'p'
+		queries = append(queries, channelTypeQ)
+	}
+
+	if term != "" {
+		nameSuggestQ := bleve.NewPrefixQuery(strings.ToLower(term))
+		nameSuggestQ.SetField("NameSuggest")
+		queries = append(queries, nameSuggestQ)
+	}
+
+	query := bleve.NewSearchRequest(bleve.NewConjunctionQuery(queries...))
+	query.Size = model.ChannelSearchDefaultLimit
+	results, err := b.ChannelIndex.Search(query)
+	if err != nil {
+		return nil, model.NewAppError("Bleveengine.SearchChannels", "bleveengine.search_channels.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	channelIds := []string{}
+	for _, result := range results.Hits {
+		channelIds = append(channelIds, result.ID)
+	}
+
+	return channelIds, nil
+}
+
+func (b *BleveEngine) DeleteChannel(channel *model.Channel) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	if err := b.ChannelIndex.Delete(channel.Id); err != nil {
+		return model.NewAppError("Bleveengine.DeleteChannel", "bleveengine.delete_channel.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) IndexUser(user *model.User, teamsIds, channelsIds []string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	blvUser := BLVUserFromUserAndTeams(user, teamsIds, channelsIds)
+	if err := b.UserIndex.Index(blvUser.Id, blvUser); err != nil {
+		return model.NewAppError("Bleveengine.IndexUser", "bleveengine.index_user.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) {
+	if restrictedToChannels != nil && len(restrictedToChannels) == 0 {
+		return []string{}, []string{}, nil
+	}
+
+	// users in channel
+	var queries []query.Query
+	if term != "" {
+		termQ := bleve.NewPrefixQuery(strings.ToLower(term))
+		if options.AllowFullNames {
+			termQ.SetField("SuggestionsWithFullname")
+		} else {
+			termQ.SetField("SuggestionsWithoutFullname")
+		}
+		queries = append(queries, termQ)
+	}
+
+	channelIdQ := bleve.NewTermQuery(channelId)
+	channelIdQ.SetField("ChannelsIds")
+	queries = append(queries, channelIdQ)
+
+	query := bleve.NewConjunctionQuery(queries...)
+
+	uchanSearch := bleve.NewSearchRequest(query)
+	uchanSearch.Size = options.Limit
+	uchan, err := b.UserIndex.Search(uchanSearch)
+	if err != nil {
+		return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.uchan.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// users not in channel
+	boolQ := bleve.NewBooleanQuery()
+
+	if term != "" {
+		termQ := bleve.NewPrefixQuery(strings.ToLower(term))
+		if options.AllowFullNames {
+			termQ.SetField("SuggestionsWithFullname")
+		} else {
+			termQ.SetField("SuggestionsWithoutFullname")
+		}
+		boolQ.AddMust(termQ)
+	}
+
+	teamIdQ := bleve.NewTermQuery(teamId)
+	teamIdQ.SetField("TeamsIds")
+	boolQ.AddMust(teamIdQ)
+
+	outsideChannelIdQ := bleve.NewTermQuery(channelId)
+	outsideChannelIdQ.SetField("ChannelsIds")
+	boolQ.AddMustNot(outsideChannelIdQ)
+
+	if len(restrictedToChannels) > 0 {
+		restrictedChannelsQ := bleve.NewDisjunctionQuery()
+		for _, channelId := range restrictedToChannels {
+			restrictedChannelQ := bleve.NewTermQuery(channelId)
+			restrictedChannelsQ.AddQuery(restrictedChannelQ)
+		}
+		boolQ.AddMust(restrictedChannelsQ)
+	}
+
+	nuchanSearch := bleve.NewSearchRequest(boolQ)
+	nuchanSearch.Size = options.Limit
+	nuchan, err := b.UserIndex.Search(nuchanSearch)
+	if err != nil {
+		return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.nuchan.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	uchanIds := []string{}
+	for _, result := range uchan.Hits {
+		uchanIds = append(uchanIds, result.ID)
+	}
+
+	nuchanIds := []string{}
+	for _, result := range nuchan.Hits {
+		nuchanIds = append(nuchanIds, result.ID)
+	}
+
+	return uchanIds, nuchanIds, nil
+}
+
+func (b *BleveEngine) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) {
+	if restrictedToChannels != nil && len(restrictedToChannels) == 0 {
+		return []string{}, nil
+	}
+
+	var rootQ query.Query
+	if term == "" && teamId == "" && restrictedToChannels == nil {
+		rootQ = bleve.NewMatchAllQuery()
+	} else {
+		boolQ := bleve.NewBooleanQuery()
+
+		if term != "" {
+			termQ := bleve.NewPrefixQuery(strings.ToLower(term))
+			if options.AllowFullNames {
+				termQ.SetField("SuggestionsWithFullname")
+			} else {
+				termQ.SetField("SuggestionsWithoutFullname")
+			}
+			boolQ.AddMust(termQ)
+		}
+
+		if len(restrictedToChannels) > 0 {
+			// restricted channels are already filtered by team, so we
+			// can search only those matches
+			restrictedChannelsQ := []query.Query{}
+			for _, channelId := range restrictedToChannels {
+				channelIdQ := bleve.NewTermQuery(channelId)
+				channelIdQ.SetField("ChannelsIds")
+				restrictedChannelsQ = append(restrictedChannelsQ, channelIdQ)
+			}
+			boolQ.AddMust(bleve.NewDisjunctionQuery(restrictedChannelsQ...))
+		} else {
+			// this means that we only need to restrict by team
+			if teamId != "" {
+				teamIdQ := bleve.NewTermQuery(teamId)
+				teamIdQ.SetField("TeamsIds")
+				boolQ.AddMust(teamIdQ)
+			}
+		}
+
+		rootQ = boolQ
+	}
+
+	search := bleve.NewSearchRequest(rootQ)
+	search.Size = options.Limit
+	results, err := b.UserIndex.Search(search)
+	if err != nil {
+		return nil, model.NewAppError("Bleveengine.SearchUsersInTeam", "bleveengine.search_users_in_team.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	usersIds := []string{}
+	for _, r := range results.Hits {
+		usersIds = append(usersIds, r.ID)
+	}
+
+	return usersIds, nil
+}
+
+func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	if err := b.UserIndex.Delete(user.Id); err != nil {
+		return model.NewAppError("Bleveengine.DeleteUser", "bleveengine.delete_user.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	blvFile := BLVFileFromFileInfo(file, channelId)
+	if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil {
+		return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) {
+	channelQueries := []query.Query{}
+	for _, channel := range channels {
+		channelIdQ := bleve.NewTermQuery(channel.Id)
+		channelIdQ.SetField("ChannelId")
+		channelQueries = append(channelQueries, channelIdQ)
+	}
+	channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
+
+	var termQueries []query.Query
+	var notTermQueries []query.Query
+	var filters []query.Query
+	var notFilters []query.Query
+
+	for i, params := range searchParams {
+		var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
+		if searchParams[0].OrTerms {
+			termOperator = query.MatchQueryOperatorOr
+		}
+
+		// Date, channels and FromUsers filters come in all
+		// searchParams iteration, and as they are global to the
+		// query, we only need to process them once
+		if i == 0 {
+			if len(params.InChannels) > 0 {
+				inChannels := []query.Query{}
+				for _, channelId := range params.InChannels {
+					channelQ := bleve.NewTermQuery(channelId)
+					channelQ.SetField("ChannelId")
+					inChannels = append(inChannels, channelQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
+			}
+
+			if len(params.ExcludedChannels) > 0 {
+				excludedChannels := []query.Query{}
+				for _, channelId := range params.ExcludedChannels {
+					channelQ := bleve.NewTermQuery(channelId)
+					channelQ.SetField("ChannelId")
+					excludedChannels = append(excludedChannels, channelQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
+			}
+
+			if len(params.FromUsers) > 0 {
+				fromUsers := []query.Query{}
+				for _, userId := range params.FromUsers {
+					userQ := bleve.NewTermQuery(userId)
+					userQ.SetField("CreatorId")
+					fromUsers = append(fromUsers, userQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
+			}
+
+			if len(params.ExcludedUsers) > 0 {
+				excludedUsers := []query.Query{}
+				for _, userId := range params.ExcludedUsers {
+					userQ := bleve.NewTermQuery(userId)
+					userQ.SetField("CreatorId")
+					excludedUsers = append(excludedUsers, userQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
+			}
+
+			if len(params.Extensions) > 0 {
+				extensions := []query.Query{}
+				for _, extension := range params.Extensions {
+					extensionQ := bleve.NewTermQuery(extension)
+					extensionQ.SetField("Extension")
+					extensions = append(extensions, extensionQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(extensions...))
+			}
+
+			if len(params.ExcludedExtensions) > 0 {
+				excludedExtensions := []query.Query{}
+				for _, extension := range params.ExcludedExtensions {
+					extensionQ := bleve.NewTermQuery(extension)
+					extensionQ.SetField("Extension")
+					excludedExtensions = append(excludedExtensions, extensionQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...))
+			}
+
+			if params.OnDate != "" {
+				before, after := params.GetOnDateMillis()
+				beforeFloat64 := float64(before)
+				afterFloat64 := float64(after)
+				onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
+				onDateQ.SetField("CreateAt")
+				filters = append(filters, onDateQ)
+			} else {
+				if params.AfterDate != "" || params.BeforeDate != "" {
+					var min, max *float64
+					if params.AfterDate != "" {
+						minf := float64(params.GetAfterDateMillis())
+						min = &minf
+					}
+
+					if params.BeforeDate != "" {
+						maxf := float64(params.GetBeforeDateMillis())
+						max = &maxf
+					}
+
+					dateQ := bleve.NewNumericRangeQuery(min, max)
+					dateQ.SetField("CreateAt")
+					filters = append(filters, dateQ)
+				}
+
+				if params.ExcludedAfterDate != "" {
+					minf := float64(params.GetExcludedAfterDateMillis())
+					dateQ := bleve.NewNumericRangeQuery(&minf, nil)
+					dateQ.SetField("CreateAt")
+					notFilters = append(notFilters, dateQ)
+				}
+
+				if params.ExcludedBeforeDate != "" {
+					maxf := float64(params.GetExcludedBeforeDateMillis())
+					dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
+					dateQ.SetField("CreateAt")
+					notFilters = append(notFilters, dateQ)
+				}
+
+				if params.ExcludedDate != "" {
+					before, after := params.GetExcludedDateMillis()
+					beforef := float64(before)
+					afterf := float64(after)
+					onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
+					onDateQ.SetField("CreateAt")
+					notFilters = append(notFilters, onDateQ)
+				}
+			}
+		}
+
+		if params.Terms != "" {
+			terms := []string{}
+			for _, term := range strings.Split(params.Terms, " ") {
+				if strings.HasSuffix(term, "*") {
+					nameQ := bleve.NewWildcardQuery(term)
+					nameQ.SetField("Name")
+					contentQ := bleve.NewWildcardQuery(term)
+					contentQ.SetField("Content")
+					termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
+				} else {
+					terms = append(terms, term)
+				}
+			}
+
+			if len(terms) > 0 {
+				nameQ := bleve.NewMatchQuery(strings.Join(terms, " "))
+				nameQ.SetField("Name")
+				nameQ.SetOperator(termOperator)
+				contentQ := bleve.NewMatchQuery(strings.Join(terms, " "))
+				contentQ.SetField("Content")
+				contentQ.SetOperator(termOperator)
+				termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
+			}
+		}
+
+		if params.ExcludedTerms != "" {
+			nameQ := bleve.NewMatchQuery(params.ExcludedTerms)
+			nameQ.SetField("Name")
+			nameQ.SetOperator(termOperator)
+			contentQ := bleve.NewMatchQuery(params.ExcludedTerms)
+			contentQ.SetField("Content")
+			contentQ.SetOperator(termOperator)
+			notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
+		}
+	}
+
+	allTermsQ := bleve.NewBooleanQuery()
+	allTermsQ.AddMustNot(notTermQueries...)
+	if searchParams[0].OrTerms {
+		allTermsQ.AddShould(termQueries...)
+	} else {
+		allTermsQ.AddMust(termQueries...)
+	}
+
+	query := bleve.NewBooleanQuery()
+	query.AddMust(channelDisjunctionQ)
+
+	if len(termQueries) > 0 || len(notTermQueries) > 0 {
+		query.AddMust(allTermsQ)
+	}
+
+	if len(filters) > 0 {
+		query.AddMust(bleve.NewConjunctionQuery(filters...))
+	}
+	if len(notFilters) > 0 {
+		query.AddMustNot(notFilters...)
+	}
+
+	search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
+	search.SortBy([]string{"-CreateAt"})
+	results, err := b.FileIndex.Search(search)
+	if err != nil {
+		return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	fileIds := []string{}
+
+	for _, r := range results.Hits {
+		fileIds = append(fileIds, r.ID)
+	}
+
+	return fileIds, nil
+}
+
+func (b *BleveEngine) DeleteFile(fileID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	if err := b.FileIndex.Delete(fileID); err != nil {
+		return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
+	resultsCount := int64(0)
+
+	for {
+		// As we are deleting the files after fetching them, we need to keep
+		// From fixed always to 0
+		searchRequest.From = 0
+		searchRequest.Size = batchSize
+		results, err := b.FileIndex.Search(searchRequest)
+		if err != nil {
+			return -1, err
+		}
+		batch := b.FileIndex.NewBatch()
+		for _, file := range results.Hits {
+			batch.Delete(file.ID)
+		}
+		if err := b.FileIndex.Batch(batch); err != nil {
+			return -1, err
+		}
+		resultsCount += int64(results.Hits.Len())
+		if results.Hits.Len() < batchSize {
+			break
+		}
+	}
+
+	return resultsCount, nil
+}
+
+func (b *BleveEngine) DeleteUserFiles(userID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	query := bleve.NewTermQuery(userID)
+	query.SetField("CreatorId")
+	search := bleve.NewSearchRequest(query)
+	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeleteUserFiles",
+			"bleveengine.delete_user_files.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted))
+
+	return nil
+}
+
+func (b *BleveEngine) DeletePostFiles(postID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	query := bleve.NewTermQuery(postID)
+	query.SetField("PostId")
+	search := bleve.NewSearchRequest(query)
+	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeletePostFiles",
+			"bleveengine.delete_post_files.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int64("deleted", deleted))
+
+	return nil
+}
+
+func (b *BleveEngine) DeleteFilesBatch(endTime, limit int64) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	endTimeFloat := float64(endTime)
+	query := bleve.NewNumericRangeQuery(nil, &endTimeFloat)
+	query.SetField("CreateAt")
+	search := bleve.NewSearchRequestOptions(query, int(limit), 0, false)
+	search.SortBy([]string{"-CreateAt"})
+
+	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeleteFilesBatch",
+			"bleveengine.delete_files_batch.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Files in batch deleted", mlog.Int64("endTime", endTime), mlog.Int64("limit", limit), mlog.Int64("deleted", deleted))
+
+	return nil
+}

server/platform/services/searchengine/bleveengine/testlib.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"fmt"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func createPost(userId string, channelId string) *model.Post {
+	post := &model.Post{
+		Message:       model.NewRandomString(15),
+		ChannelId:     channelId,
+		PendingPostId: model.NewId() + ":" + fmt.Sprint(model.GetMillis()),
+		UserId:        userId,
+		CreateAt:      1000000,
+	}
+	post.PreSave()
+
+	return post
+}

server/platform/services/searchengine/interface.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchengine
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+type SearchEngineInterface interface {
+	Start() *model.AppError
+	Stop() *model.AppError
+	GetFullVersion() string
+	GetVersion() int
+	GetPlugins() []string
+	UpdateConfig(cfg *model.Config)
+	GetName() string
+	IsActive() bool
+	IsIndexingEnabled() bool
+	IsSearchEnabled() bool
+	IsAutocompletionEnabled() bool
+	IsIndexingSync() bool
+	IndexPost(post *model.Post, teamId string) *model.AppError
+	SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError)
+	DeletePost(post *model.Post) *model.AppError
+	DeleteChannelPosts(channelID string) *model.AppError
+	DeleteUserPosts(userID string) *model.AppError
+	// IndexChannel indexes a given channel. The userIDs are only populated
+	// for private channels.
+	IndexChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError
+	SearchChannels(teamId, userID, term string, isGuest bool) ([]string, *model.AppError)
+	DeleteChannel(channel *model.Channel) *model.AppError
+	IndexUser(user *model.User, teamsIds, channelsIds []string) *model.AppError
+	SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError)
+	SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError)
+	DeleteUser(user *model.User) *model.AppError
+	IndexFile(file *model.FileInfo, channelId string) *model.AppError
+	SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError)
+	DeleteFile(fileID string) *model.AppError
+	DeletePostFiles(postID string) *model.AppError
+	DeleteUserFiles(userID string) *model.AppError
+	DeleteFilesBatch(endTime, limit int64) *model.AppError
+	TestConfig(cfg *model.Config) *model.AppError
+	PurgeIndexes() *model.AppError
+	RefreshIndexes() *model.AppError
+	DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError
+}

server/platform/services/searchengine/mocks/SearchEngineInterface.go

@@ -0,0 +1,597 @@
+// Code generated by mockery v2.10.4. DO NOT EDIT.
+
+// Regenerate this file using `make searchengine-mocks`.
+
+package mocks
+
+import (
+	model "github.com/mattermost/mattermost-server/v6/model"
+	mock "github.com/stretchr/testify/mock"
+
+	time "time"
+)
+
+// SearchEngineInterface is an autogenerated mock type for the SearchEngineInterface type
+type SearchEngineInterface struct {
+	mock.Mock
+}
+
+// DataRetentionDeleteIndexes provides a mock function with given fields: cutoff
+func (_m *SearchEngineInterface) DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError {
+	ret := _m.Called(cutoff)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(time.Time) *model.AppError); ok {
+		r0 = rf(cutoff)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeleteChannel provides a mock function with given fields: channel
+func (_m *SearchEngineInterface) DeleteChannel(channel *model.Channel) *model.AppError {
+	ret := _m.Called(channel)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Channel) *model.AppError); ok {
+		r0 = rf(channel)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeleteChannelPosts provides a mock function with given fields: channelID
+func (_m *SearchEngineInterface) DeleteChannelPosts(channelID string) *model.AppError {
+	ret := _m.Called(channelID)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
+		r0 = rf(channelID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeleteFile provides a mock function with given fields: fileID
+func (_m *SearchEngineInterface) DeleteFile(fileID string) *model.AppError {
+	ret := _m.Called(fileID)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
+		r0 = rf(fileID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeleteFilesBatch provides a mock function with given fields: endTime, limit
+func (_m *SearchEngineInterface) DeleteFilesBatch(endTime int64, limit int64) *model.AppError {
+	ret := _m.Called(endTime, limit)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(int64, int64) *model.AppError); ok {
+		r0 = rf(endTime, limit)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeletePost provides a mock function with given fields: post
+func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError {
+	ret := _m.Called(post)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Post) *model.AppError); ok {
+		r0 = rf(post)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeletePostFiles provides a mock function with given fields: postID
+func (_m *SearchEngineInterface) DeletePostFiles(postID string) *model.AppError {
+	ret := _m.Called(postID)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
+		r0 = rf(postID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeleteUser provides a mock function with given fields: user
+func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError {
+	ret := _m.Called(user)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.User) *model.AppError); ok {
+		r0 = rf(user)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeleteUserFiles provides a mock function with given fields: userID
+func (_m *SearchEngineInterface) DeleteUserFiles(userID string) *model.AppError {
+	ret := _m.Called(userID)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
+		r0 = rf(userID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// DeleteUserPosts provides a mock function with given fields: userID
+func (_m *SearchEngineInterface) DeleteUserPosts(userID string) *model.AppError {
+	ret := _m.Called(userID)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
+		r0 = rf(userID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// GetFullVersion provides a mock function with given fields:
+func (_m *SearchEngineInterface) GetFullVersion() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// GetName provides a mock function with given fields:
+func (_m *SearchEngineInterface) GetName() string {
+	ret := _m.Called()
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func() string); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// GetPlugins provides a mock function with given fields:
+func (_m *SearchEngineInterface) GetPlugins() []string {
+	ret := _m.Called()
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func() []string); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	return r0
+}
+
+// GetVersion provides a mock function with given fields:
+func (_m *SearchEngineInterface) GetVersion() int {
+	ret := _m.Called()
+
+	var r0 int
+	if rf, ok := ret.Get(0).(func() int); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(int)
+	}
+
+	return r0
+}
+
+// IndexChannel provides a mock function with given fields: channel, userIDs, teamMemberIDs
+func (_m *SearchEngineInterface) IndexChannel(channel *model.Channel, userIDs []string, teamMemberIDs []string) *model.AppError {
+	ret := _m.Called(channel, userIDs, teamMemberIDs)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Channel, []string, []string) *model.AppError); ok {
+		r0 = rf(channel, userIDs, teamMemberIDs)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// IndexFile provides a mock function with given fields: file, channelId
+func (_m *SearchEngineInterface) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
+	ret := _m.Called(file, channelId)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.FileInfo, string) *model.AppError); ok {
+		r0 = rf(file, channelId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// IndexPost provides a mock function with given fields: post, teamId
+func (_m *SearchEngineInterface) IndexPost(post *model.Post, teamId string) *model.AppError {
+	ret := _m.Called(post, teamId)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Post, string) *model.AppError); ok {
+		r0 = rf(post, teamId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// IndexUser provides a mock function with given fields: user, teamsIds, channelsIds
+func (_m *SearchEngineInterface) IndexUser(user *model.User, teamsIds []string, channelsIds []string) *model.AppError {
+	ret := _m.Called(user, teamsIds, channelsIds)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.User, []string, []string) *model.AppError); ok {
+		r0 = rf(user, teamsIds, channelsIds)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// IsActive provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsActive() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// IsAutocompletionEnabled provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsAutocompletionEnabled() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// IsIndexingEnabled provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsIndexingEnabled() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// IsIndexingSync provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsIndexingSync() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// IsSearchEnabled provides a mock function with given fields:
+func (_m *SearchEngineInterface) IsSearchEnabled() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// PurgeIndexes provides a mock function with given fields:
+func (_m *SearchEngineInterface) PurgeIndexes() *model.AppError {
+	ret := _m.Called()
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func() *model.AppError); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// RefreshIndexes provides a mock function with given fields:
+func (_m *SearchEngineInterface) RefreshIndexes() *model.AppError {
+	ret := _m.Called()
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func() *model.AppError); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// SearchChannels provides a mock function with given fields: teamId, userID, term, isGuest
+func (_m *SearchEngineInterface) SearchChannels(teamId string, userID string, term string, isGuest bool) ([]string, *model.AppError) {
+	ret := _m.Called(teamId, userID, term, isGuest)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(string, string, string, bool) []string); ok {
+		r0 = rf(teamId, userID, term, isGuest)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, string, string, bool) *model.AppError); ok {
+		r1 = rf(teamId, userID, term, isGuest)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// SearchFiles provides a mock function with given fields: channels, searchParams, page, perPage
+func (_m *SearchEngineInterface) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, *model.AppError) {
+	ret := _m.Called(channels, searchParams, page, perPage)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(model.ChannelList, []*model.SearchParams, int, int) []string); ok {
+		r0 = rf(channels, searchParams, page, perPage)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(model.ChannelList, []*model.SearchParams, int, int) *model.AppError); ok {
+		r1 = rf(channels, searchParams, page, perPage)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// SearchPosts provides a mock function with given fields: channels, searchParams, page, perPage
+func (_m *SearchEngineInterface) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
+	ret := _m.Called(channels, searchParams, page, perPage)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(model.ChannelList, []*model.SearchParams, int, int) []string); ok {
+		r0 = rf(channels, searchParams, page, perPage)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 model.PostSearchMatches
+	if rf, ok := ret.Get(1).(func(model.ChannelList, []*model.SearchParams, int, int) model.PostSearchMatches); ok {
+		r1 = rf(channels, searchParams, page, perPage)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(model.PostSearchMatches)
+		}
+	}
+
+	var r2 *model.AppError
+	if rf, ok := ret.Get(2).(func(model.ChannelList, []*model.SearchParams, int, int) *model.AppError); ok {
+		r2 = rf(channels, searchParams, page, perPage)
+	} else {
+		if ret.Get(2) != nil {
+			r2 = ret.Get(2).(*model.AppError)
+		}
+	}
+
+	return r0, r1, r2
+}
+
+// SearchUsersInChannel provides a mock function with given fields: teamId, channelId, restrictedToChannels, term, options
+func (_m *SearchEngineInterface) SearchUsersInChannel(teamId string, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) {
+	ret := _m.Called(teamId, channelId, restrictedToChannels, term, options)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(string, string, []string, string, *model.UserSearchOptions) []string); ok {
+		r0 = rf(teamId, channelId, restrictedToChannels, term, options)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 []string
+	if rf, ok := ret.Get(1).(func(string, string, []string, string, *model.UserSearchOptions) []string); ok {
+		r1 = rf(teamId, channelId, restrictedToChannels, term, options)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).([]string)
+		}
+	}
+
+	var r2 *model.AppError
+	if rf, ok := ret.Get(2).(func(string, string, []string, string, *model.UserSearchOptions) *model.AppError); ok {
+		r2 = rf(teamId, channelId, restrictedToChannels, term, options)
+	} else {
+		if ret.Get(2) != nil {
+			r2 = ret.Get(2).(*model.AppError)
+		}
+	}
+
+	return r0, r1, r2
+}
+
+// SearchUsersInTeam provides a mock function with given fields: teamId, restrictedToChannels, term, options
+func (_m *SearchEngineInterface) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) {
+	ret := _m.Called(teamId, restrictedToChannels, term, options)
+
+	var r0 []string
+	if rf, ok := ret.Get(0).(func(string, []string, string, *model.UserSearchOptions) []string); ok {
+		r0 = rf(teamId, restrictedToChannels, term, options)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]string)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, []string, string, *model.UserSearchOptions) *model.AppError); ok {
+		r1 = rf(teamId, restrictedToChannels, term, options)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// Start provides a mock function with given fields:
+func (_m *SearchEngineInterface) Start() *model.AppError {
+	ret := _m.Called()
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func() *model.AppError); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// Stop provides a mock function with given fields:
+func (_m *SearchEngineInterface) Stop() *model.AppError {
+	ret := _m.Called()
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func() *model.AppError); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// TestConfig provides a mock function with given fields: cfg
+func (_m *SearchEngineInterface) TestConfig(cfg *model.Config) *model.AppError {
+	ret := _m.Called(cfg)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*model.Config) *model.AppError); ok {
+		r0 = rf(cfg)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// UpdateConfig provides a mock function with given fields: cfg
+func (_m *SearchEngineInterface) UpdateConfig(cfg *model.Config) {
+	_m.Called(cfg)
+}

server/platform/services/searchengine/searchengine.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchengine
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func NewBroker(cfg *model.Config) *Broker {
+	return &Broker{
+		cfg: cfg,
+	}
+}
+
+func (seb *Broker) RegisterElasticsearchEngine(es SearchEngineInterface) {
+	seb.ElasticsearchEngine = es
+}
+
+func (seb *Broker) RegisterBleveEngine(be SearchEngineInterface) {
+	seb.BleveEngine = be
+}
+
+type Broker struct {
+	cfg                 *model.Config
+	ElasticsearchEngine SearchEngineInterface
+	BleveEngine         SearchEngineInterface
+}
+
+func (seb *Broker) UpdateConfig(cfg *model.Config) *model.AppError {
+	seb.cfg = cfg
+	if seb.ElasticsearchEngine != nil {
+		seb.ElasticsearchEngine.UpdateConfig(cfg)
+	}
+
+	if seb.BleveEngine != nil {
+		seb.BleveEngine.UpdateConfig(cfg)
+	}
+
+	return nil
+}
+
+func (seb *Broker) GetActiveEngines() []SearchEngineInterface {
+	engines := []SearchEngineInterface{}
+	if seb.ElasticsearchEngine != nil && seb.ElasticsearchEngine.IsActive() {
+		engines = append(engines, seb.ElasticsearchEngine)
+	}
+	if seb.BleveEngine != nil && seb.BleveEngine.IsActive() {
+		engines = append(engines, seb.BleveEngine)
+	}
+	return engines
+}

server/platform/services/searchengine/utils.go

@@ -0,0 +1,44 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchengine
+
+import (
+	"regexp"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils"
+)
+
+var EmailRegex = regexp.MustCompile(`^[^\s"]+@[^\s"]+$`)
+
+func GetSuggestionInputsSplitBy(term, splitStr string) []string {
+	splitTerm := strings.Split(strings.ToLower(term), splitStr)
+	var initialSuggestionList []string
+	for i := range splitTerm {
+		initialSuggestionList = append(initialSuggestionList, strings.Join(splitTerm[i:], splitStr))
+	}
+
+	suggestionList := []string{}
+	// If splitStr is not an empty space, we create a suggestion with it at the beginning
+	if splitStr == " " {
+		suggestionList = initialSuggestionList
+	} else {
+		for i, suggestion := range initialSuggestionList {
+			if i == 0 {
+				suggestionList = append(suggestionList, suggestion)
+			} else {
+				suggestionList = append(suggestionList, splitStr+suggestion, suggestion)
+			}
+		}
+	}
+	return suggestionList
+}
+
+func GetSuggestionInputsSplitByMultiple(term string, splitStrs []string) []string {
+	suggestionList := []string{}
+	for _, splitStr := range splitStrs {
+		suggestionList = append(suggestionList, GetSuggestionInputsSplitBy(term, splitStr)...)
+	}
+	return utils.RemoveDuplicatesFromStringArray(suggestionList)
+}

server/platform/services/searchengine/utils_test.go

@@ -0,0 +1,57 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package searchengine
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestElasticsearchGetSuggestionsSplitBy(t *testing.T) {
+	testCases := []struct {
+		Name     string
+		Term     string
+		SplitStr string ``
+		Expected []string
+	}{
+		{
+			Name:     "Single string",
+			Term:     "string",
+			SplitStr: " ",
+			Expected: []string{"string"},
+		},
+		{
+			Name:     "String with spaces",
+			Term:     "String with spaces",
+			SplitStr: " ",
+			Expected: []string{"string with spaces", "with spaces", "spaces"},
+		},
+		{
+			Name:     "Username split by a dot",
+			Term:     "name.surname",
+			SplitStr: ".",
+			Expected: []string{"name.surname", ".surname", "surname"},
+		},
+		{
+			Name:     "String split by several dashes",
+			Term:     "one-two-three",
+			SplitStr: "-",
+			Expected: []string{"one-two-three", "-two-three", "two-three", "-three", "three"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			res := GetSuggestionInputsSplitBy(tc.Term, tc.SplitStr)
+			assert.ElementsMatch(t, res, tc.Expected)
+		})
+	}
+}
+
+func TestElasticsearchGetSuggestionsSplitByMultiple(t *testing.T) {
+	r1 := GetSuggestionInputsSplitByMultiple("String with user.name", []string{" ", "."})
+	expectedR1 := []string{"string with user.name", "with user.name", "user.name", ".name", "name"}
+	assert.ElementsMatch(t, r1, expectedR1)
+}

server/platform/services/sharedchannel/attachment.go

@@ -0,0 +1,167 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"sync"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+// postsToAttachments returns the file attachments for a slice of posts that need to be synchronized.
+func (scs *Service) shouldSyncAttachment(fi *model.FileInfo, rc *model.RemoteCluster) bool {
+	sca, err := scs.server.GetStore().SharedChannel().GetAttachment(fi.Id, rc.RemoteId)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "error fetching shared channel attachment",
+				mlog.String("file_id", fi.Id),
+				mlog.String("remote_id", rc.RemoteId),
+				mlog.Err(err),
+			)
+		}
+		// no record so sync is needed
+		return true
+	}
+
+	return sca.LastSyncAt < fi.UpdateAt
+}
+
+// sendAttachmentForRemote asynchronously sends a file attachment to a remote cluster.
+func (scs *Service) sendAttachmentForRemote(fi *model.FileInfo, post *model.Post, rc *model.RemoteCluster) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot update remote cluster for remote id %s; Remote Cluster Service not enabled", rc.RemoteId)
+	}
+
+	us := &model.UploadSession{
+		Id:        model.NewId(),
+		Type:      model.UploadTypeAttachment,
+		UserId:    post.UserId,
+		ChannelId: post.ChannelId,
+		Filename:  fi.Name,
+		FileSize:  fi.Size,
+		RemoteId:  rc.RemoteId,
+		ReqFileId: fi.Id,
+	}
+
+	payload, err := json.Marshal(us)
+	if err != nil {
+		return err
+	}
+
+	msg := model.NewRemoteClusterMsg(TopicUploadCreate, payload)
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	var usResp model.UploadSession
+	var respErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	// creating the upload session on the remote server needs to be done synchronously.
+	err = rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		defer wg.Done()
+		if err != nil {
+			respErr = err
+			return
+		}
+		if !resp.IsSuccess() {
+			respErr = errors.New(resp.Err)
+			return
+		}
+		respErr = json.Unmarshal(resp.Payload, &usResp)
+	})
+
+	if err != nil {
+		return fmt.Errorf("error sending create upload session to remote %s for post %s: %w", rc.RemoteId, post.Id, err)
+	}
+
+	wg.Wait()
+
+	if respErr != nil {
+		return fmt.Errorf("invalid create upload session response for remote %s and post %s: %w", rc.RemoteId, post.Id, respErr)
+	}
+
+	ctx2, cancel2 := context.WithTimeout(context.Background(), remotecluster.SendFileTimeout)
+	defer cancel2()
+
+	return rcs.SendFile(ctx2, &usResp, fi, rc, scs.app, func(us *model.UploadSession, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		if err != nil {
+			return // this means the response could not be parsed; already logged
+		}
+
+		if !resp.IsSuccess() {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "send file failed",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.String("err", resp.Err),
+			)
+			return
+		}
+
+		// response payload should be a model.FileInfo.
+		var fi model.FileInfo
+		if err2 := json.Unmarshal(resp.Payload, &fi); err2 != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "invalid file info response after send file",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.Err(err2),
+			)
+			return
+		}
+
+		// save file attachment record in SharedChannelAttachments table
+		sca := &model.SharedChannelAttachment{
+			FileId:   fi.Id,
+			RemoteId: rc.RemoteId,
+		}
+		if _, err2 := scs.server.GetStore().SharedChannel().UpsertAttachment(sca); err2 != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "error saving SharedChannelAttachment",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("uploadId", usResp.Id),
+				mlog.Err(err2),
+			)
+			return
+		}
+
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "send file successful",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("uploadId", usResp.Id),
+		)
+	})
+}
+
+// onReceiveUploadCreate is called when a message requesting to create an upload session is received.  An upload session is
+// created and the id returned in the response.
+func (scs *Service) onReceiveUploadCreate(msg model.RemoteClusterMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	var us model.UploadSession
+
+	if err := json.Unmarshal(msg.Payload, &us); err != nil {
+		return fmt.Errorf("invalid upload session request: %w", err)
+	}
+
+	// make sure channel is shared for the remote sender
+	if _, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(us.ChannelId, rc.RemoteId); err != nil {
+		return fmt.Errorf("could not validate upload session for remote: %w", err)
+	}
+
+	us.RemoteId = rc.RemoteId // don't let remotes try to impersonate each other
+
+	// create upload session.
+	usSaved, appErr := scs.app.CreateUploadSession(request.EmptyContext(scs.server.Log()), &us)
+	if appErr != nil {
+		return appErr
+	}
+
+	response.SetPayload(usSaved)
+	return nil
+}

server/platform/services/sharedchannel/channelinvite.go

@@ -0,0 +1,219 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+// channelInviteMsg represents an invitation for a remote cluster to start sharing a channel.
+type channelInviteMsg struct {
+	ChannelId            string            `json:"channel_id"`
+	TeamId               string            `json:"team_id"`
+	ReadOnly             bool              `json:"read_only"`
+	Name                 string            `json:"name"`
+	DisplayName          string            `json:"display_name"`
+	Header               string            `json:"header"`
+	Purpose              string            `json:"purpose"`
+	Type                 model.ChannelType `json:"type"`
+	DirectParticipantIDs []string          `json:"direct_participant_ids"`
+}
+
+type InviteOption func(msg *channelInviteMsg)
+
+func WithDirectParticipantID(participantID string) InviteOption {
+	return func(msg *channelInviteMsg) {
+		msg.DirectParticipantIDs = append(msg.DirectParticipantIDs, participantID)
+	}
+}
+
+// SendChannelInvite asynchronously sends a channel invite to a remote cluster. The remote cluster is
+// expected to create a new channel with the same channel id, and respond with status OK.
+// If an error occurs on the remote cluster then an ephemeral message is posted to in the channel for userId.
+func (scs *Service) SendChannelInvite(channel *model.Channel, userId string, rc *model.RemoteCluster, options ...InviteOption) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot invite remote cluster for channel id %s; Remote Cluster Service not enabled", channel.Id)
+	}
+
+	sc, err := scs.server.GetStore().SharedChannel().Get(channel.Id)
+	if err != nil {
+		return err
+	}
+
+	invite := channelInviteMsg{
+		ChannelId:   channel.Id,
+		TeamId:      rc.RemoteTeamId,
+		ReadOnly:    sc.ReadOnly,
+		Name:        sc.ShareName,
+		DisplayName: sc.ShareDisplayName,
+		Header:      sc.ShareHeader,
+		Purpose:     sc.SharePurpose,
+		Type:        channel.Type,
+	}
+
+	for _, option := range options {
+		option(&invite)
+	}
+
+	json, err := json.Marshal(invite)
+	if err != nil {
+		return err
+	}
+
+	msg := model.NewRemoteClusterMsg(TopicChannelInvite, json)
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	return rcs.SendMsg(ctx, msg, rc, func(msg model.RemoteClusterMsg, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		if err != nil || !resp.IsSuccess() {
+			scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("Error sending channel invite for %s: %s", rc.DisplayName, combineErrors(err, resp.Err)))
+			return
+		}
+
+		scr := &model.SharedChannelRemote{
+			ChannelId:         sc.ChannelId,
+			CreatorId:         userId,
+			RemoteId:          rc.RemoteId,
+			IsInviteAccepted:  true,
+			IsInviteConfirmed: true,
+		}
+		if _, err = scs.server.GetStore().SharedChannel().SaveRemote(scr); err != nil {
+			scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("Error confirming channel invite for %s: %v", rc.DisplayName, err))
+			return
+		}
+		scs.NotifyChannelChanged(sc.ChannelId)
+		scs.sendEphemeralPost(channel.Id, userId, fmt.Sprintf("`%s` has been added to channel.", rc.DisplayName))
+	})
+}
+
+func combineErrors(err error, serror string) string {
+	var sb strings.Builder
+	if err != nil {
+		sb.WriteString(err.Error())
+	}
+	if serror != "" {
+		if sb.Len() > 0 {
+			sb.WriteString("; ")
+		}
+		sb.WriteString(serror)
+	}
+	return sb.String()
+}
+
+func (scs *Service) onReceiveChannelInvite(msg model.RemoteClusterMsg, rc *model.RemoteCluster, _ *remotecluster.Response) error {
+	if len(msg.Payload) == 0 {
+		return nil
+	}
+
+	var invite channelInviteMsg
+
+	if err := json.Unmarshal(msg.Payload, &invite); err != nil {
+		return fmt.Errorf("invalid channel invite: %w", err)
+	}
+
+	scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Channel invite received",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("channel_id", invite.ChannelId),
+		mlog.String("channel_name", invite.Name),
+		mlog.String("team_id", invite.TeamId),
+	)
+
+	// create channel if it doesn't exist; the channel may already exist, such as if it was shared then unshared at some point.
+	channel, err := scs.server.GetStore().Channel().Get(invite.ChannelId, true)
+	if err != nil {
+		if channel, err = scs.handleChannelCreation(invite, rc); err != nil {
+			return err
+		}
+	}
+
+	if invite.ReadOnly {
+		if err := scs.makeChannelReadOnly(channel); err != nil {
+			return fmt.Errorf("cannot make channel readonly `%s`: %w", invite.ChannelId, err)
+		}
+	}
+
+	sharedChannel := &model.SharedChannel{
+		ChannelId:        channel.Id,
+		TeamId:           channel.TeamId,
+		Home:             false,
+		ReadOnly:         invite.ReadOnly,
+		ShareName:        channel.Name,
+		ShareDisplayName: channel.DisplayName,
+		SharePurpose:     channel.Purpose,
+		ShareHeader:      channel.Header,
+		CreatorId:        rc.CreatorId,
+		RemoteId:         rc.RemoteId,
+		Type:             channel.Type,
+	}
+
+	if _, err := scs.server.GetStore().SharedChannel().Save(sharedChannel); err != nil {
+		scs.app.PermanentDeleteChannel(request.EmptyContext(scs.server.Log()), channel)
+		return fmt.Errorf("cannot create shared channel (channel_id=%s): %w", invite.ChannelId, err)
+	}
+
+	sharedChannelRemote := &model.SharedChannelRemote{
+		Id:                model.NewId(),
+		ChannelId:         channel.Id,
+		CreatorId:         channel.CreatorId,
+		IsInviteAccepted:  true,
+		IsInviteConfirmed: true,
+		RemoteId:          rc.RemoteId,
+	}
+
+	if _, err := scs.server.GetStore().SharedChannel().SaveRemote(sharedChannelRemote); err != nil {
+		scs.app.PermanentDeleteChannel(request.EmptyContext(scs.server.Log()), channel)
+		scs.server.GetStore().SharedChannel().Delete(sharedChannel.ChannelId)
+		return fmt.Errorf("cannot create shared channel remote (channel_id=%s): %w", invite.ChannelId, err)
+	}
+	return nil
+}
+
+func (scs *Service) handleChannelCreation(invite channelInviteMsg, rc *model.RemoteCluster) (*model.Channel, error) {
+	if invite.Type == model.ChannelTypeDirect {
+		return scs.createDirectChannel(invite)
+	}
+
+	channelNew := &model.Channel{
+		Id:          invite.ChannelId,
+		TeamId:      invite.TeamId,
+		Type:        invite.Type,
+		DisplayName: invite.DisplayName,
+		Name:        invite.Name,
+		Header:      invite.Header,
+		Purpose:     invite.Purpose,
+		CreatorId:   rc.CreatorId,
+		Shared:      model.NewBool(true),
+	}
+
+	// check user perms?
+	channel, appErr := scs.app.CreateChannelWithUser(request.EmptyContext(scs.server.Log()), channelNew, rc.CreatorId)
+	if appErr != nil {
+		return nil, fmt.Errorf("cannot create channel `%s`: %w", invite.ChannelId, appErr)
+	}
+
+	return channel, nil
+}
+
+func (scs *Service) createDirectChannel(invite channelInviteMsg) (*model.Channel, error) {
+	if len(invite.DirectParticipantIDs) != 2 {
+		return nil, fmt.Errorf("cannot create direct channel `%s` insufficient participant count `%d`", invite.ChannelId, len(invite.DirectParticipantIDs))
+	}
+
+	channel, err := scs.app.GetOrCreateDirectChannel(request.EmptyContext(scs.server.Log()), invite.DirectParticipantIDs[0], invite.DirectParticipantIDs[1], model.WithID(invite.ChannelId))
+	if err != nil {
+		return nil, fmt.Errorf("cannot create direct channel `%s`: %w", invite.ChannelId, err)
+	}
+
+	return channel, nil
+}

server/platform/services/sharedchannel/channelinvite_test.go

@@ -0,0 +1,195 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+func TestOnReceiveChannelInvite(t *testing.T) {
+	t.Run("when msg payload is empty, it does nothing", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger, err := mlog.NewLogger()
+		require.NoError(t, err)
+		mockServer.On("Log").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+
+		remoteCluster := &model.RemoteCluster{}
+		msg := model.RemoteClusterMsg{}
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+		mockStore.AssertNotCalled(t, "Channel")
+	})
+
+	t.Run("when invitation prescribes a readonly channel, it does create a readonly channel", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger, err := mlog.NewLogger()
+		require.NoError(t, err)
+		mockServer.On("Log").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{Name: "test"}
+		invitation := channelInviteMsg{
+			ChannelId: model.NewId(),
+			TeamId:    model.NewId(),
+			ReadOnly:  true,
+			Type:      "0",
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		mockSharedChannelStore := mocks.SharedChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(channel, nil)
+		mockSharedChannelStore.On("Save", mock.Anything).Return(nil, nil)
+		mockSharedChannelStore.On("SaveRemote", mock.Anything).Return(nil, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+		mockStore.On("SharedChannel").Return(&mockSharedChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+		createPostPermission := model.ChannelModeratedPermissionsMap[model.PermissionCreatePost.Id]
+		createReactionPermission := model.ChannelModeratedPermissionsMap[model.PermissionAddReaction.Id]
+		updateMap := model.ChannelModeratedRolesPatch{
+			Guests:  model.NewBool(false),
+			Members: model.NewBool(false),
+		}
+
+		readonlyChannelModerations := []*model.ChannelModerationPatch{
+			{
+				Name:  &createPostPermission,
+				Roles: &updateMap,
+			},
+			{
+				Name:  &createReactionPermission,
+				Roles: &updateMap,
+			},
+		}
+		mockApp.On("PatchChannelModerationsForChannel", mock.Anything, channel, readonlyChannelModerations).Return(nil, nil)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+	})
+
+	t.Run("when invitation prescribes a readonly channel and readonly update fails, it returns an error", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger, err := mlog.NewLogger()
+		require.NoError(t, err)
+		mockServer.On("Log").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{Name: "test2"}
+		invitation := channelInviteMsg{
+			ChannelId: model.NewId(),
+			TeamId:    model.NewId(),
+			ReadOnly:  true,
+			Type:      "0",
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(channel, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+		appErr := model.NewAppError("foo", "bar", nil, "boom", http.StatusBadRequest)
+
+		mockApp.On("PatchChannelModerationsForChannel", mock.Anything, channel, mock.Anything).Return(nil, appErr)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.Error(t, err)
+		assert.Equal(t, fmt.Sprintf("cannot make channel readonly `%s`: foo: bar, boom", invitation.ChannelId), err.Error())
+	})
+
+	t.Run("when invitation prescribes a direct channel, it does create a direct channel", func(t *testing.T) {
+		mockServer := &MockServerIface{}
+		mockLogger, err := mlog.NewLogger()
+		require.NoError(t, err)
+		mockServer.On("Log").Return(mockLogger)
+		mockApp := &MockAppIface{}
+		scs := &Service{
+			server: mockServer,
+			app:    mockApp,
+		}
+
+		mockStore := &mocks.Store{}
+		remoteCluster := &model.RemoteCluster{Name: "test3", CreatorId: model.NewId()}
+		invitation := channelInviteMsg{
+			ChannelId:            model.NewId(),
+			TeamId:               model.NewId(),
+			ReadOnly:             false,
+			Type:                 model.ChannelTypeDirect,
+			DirectParticipantIDs: []string{model.NewId(), model.NewId()},
+		}
+		payload, err := json.Marshal(invitation)
+		require.NoError(t, err)
+
+		msg := model.RemoteClusterMsg{
+			Payload: payload,
+		}
+		mockChannelStore := mocks.ChannelStore{}
+		mockSharedChannelStore := mocks.SharedChannelStore{}
+		channel := &model.Channel{}
+
+		mockChannelStore.On("Get", invitation.ChannelId, true).Return(nil, errors.New("boom"))
+		mockSharedChannelStore.On("Save", mock.Anything).Return(nil, nil)
+		mockSharedChannelStore.On("SaveRemote", mock.Anything).Return(nil, nil)
+		mockStore.On("Channel").Return(&mockChannelStore)
+		mockStore.On("SharedChannel").Return(&mockSharedChannelStore)
+
+		mockServer = scs.server.(*MockServerIface)
+		mockServer.On("GetStore").Return(mockStore)
+
+		mockApp.On("GetOrCreateDirectChannel", mock.AnythingOfType("*request.Context"), invitation.DirectParticipantIDs[0], invitation.DirectParticipantIDs[1], mock.AnythingOfType("model.ChannelOption")).Return(channel, nil)
+		defer mockApp.AssertExpectations(t)
+
+		err = scs.onReceiveChannelInvite(msg, remoteCluster, nil)
+		require.NoError(t, err)
+	})
+}

server/platform/services/sharedchannel/mock_AppIface_test.go

@@ -0,0 +1,398 @@
+// Code generated by mockery v2.10.4. DO NOT EDIT.
+
+// Regenerate this file using `make sharedchannel-mocks`.
+
+package sharedchannel
+
+import (
+	filestore "github.com/mattermost/mattermost-server/v6/server/platform/shared/filestore"
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/mattermost/mattermost-server/v6/model"
+
+	request "github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+)
+
+// MockAppIface is an autogenerated mock type for the AppIface type
+type MockAppIface struct {
+	mock.Mock
+}
+
+// AddUserToChannel provides a mock function with given fields: c, user, channel, skipTeamMemberIntegrityCheck
+func (_m *MockAppIface) AddUserToChannel(c request.CTX, user *model.User, channel *model.Channel, skipTeamMemberIntegrityCheck bool) (*model.ChannelMember, *model.AppError) {
+	ret := _m.Called(c, user, channel, skipTeamMemberIntegrityCheck)
+
+	var r0 *model.ChannelMember
+	if rf, ok := ret.Get(0).(func(request.CTX, *model.User, *model.Channel, bool) *model.ChannelMember); ok {
+		r0 = rf(c, user, channel, skipTeamMemberIntegrityCheck)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.ChannelMember)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(request.CTX, *model.User, *model.Channel, bool) *model.AppError); ok {
+		r1 = rf(c, user, channel, skipTeamMemberIntegrityCheck)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// AddUserToTeamByTeamId provides a mock function with given fields: c, teamId, user
+func (_m *MockAppIface) AddUserToTeamByTeamId(c *request.Context, teamId string, user *model.User) *model.AppError {
+	ret := _m.Called(c, teamId, user)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*request.Context, string, *model.User) *model.AppError); ok {
+		r0 = rf(c, teamId, user)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// CreateChannelWithUser provides a mock function with given fields: c, channel, userId
+func (_m *MockAppIface) CreateChannelWithUser(c request.CTX, channel *model.Channel, userId string) (*model.Channel, *model.AppError) {
+	ret := _m.Called(c, channel, userId)
+
+	var r0 *model.Channel
+	if rf, ok := ret.Get(0).(func(request.CTX, *model.Channel, string) *model.Channel); ok {
+		r0 = rf(c, channel, userId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Channel)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(request.CTX, *model.Channel, string) *model.AppError); ok {
+		r1 = rf(c, channel, userId)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// CreatePost provides a mock function with given fields: c, post, channel, triggerWebhooks, setOnline
+func (_m *MockAppIface) CreatePost(c request.CTX, post *model.Post, channel *model.Channel, triggerWebhooks bool, setOnline bool) (*model.Post, *model.AppError) {
+	ret := _m.Called(c, post, channel, triggerWebhooks, setOnline)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(request.CTX, *model.Post, *model.Channel, bool, bool) *model.Post); ok {
+		r0 = rf(c, post, channel, triggerWebhooks, setOnline)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(request.CTX, *model.Post, *model.Channel, bool, bool) *model.AppError); ok {
+		r1 = rf(c, post, channel, triggerWebhooks, setOnline)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// CreateUploadSession provides a mock function with given fields: c, us
+func (_m *MockAppIface) CreateUploadSession(c request.CTX, us *model.UploadSession) (*model.UploadSession, *model.AppError) {
+	ret := _m.Called(c, us)
+
+	var r0 *model.UploadSession
+	if rf, ok := ret.Get(0).(func(request.CTX, *model.UploadSession) *model.UploadSession); ok {
+		r0 = rf(c, us)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.UploadSession)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(request.CTX, *model.UploadSession) *model.AppError); ok {
+		r1 = rf(c, us)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// DeletePost provides a mock function with given fields: c, postID, deleteByID
+func (_m *MockAppIface) DeletePost(c request.CTX, postID string, deleteByID string) (*model.Post, *model.AppError) {
+	ret := _m.Called(c, postID, deleteByID)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(request.CTX, string, string) *model.Post); ok {
+		r0 = rf(c, postID, deleteByID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(request.CTX, string, string) *model.AppError); ok {
+		r1 = rf(c, postID, deleteByID)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// DeleteReactionForPost provides a mock function with given fields: c, reaction
+func (_m *MockAppIface) DeleteReactionForPost(c *request.Context, reaction *model.Reaction) *model.AppError {
+	ret := _m.Called(c, reaction)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(*request.Context, *model.Reaction) *model.AppError); ok {
+		r0 = rf(c, reaction)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// FileReader provides a mock function with given fields: path
+func (_m *MockAppIface) FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
+	ret := _m.Called(path)
+
+	var r0 filestore.ReadCloseSeeker
+	if rf, ok := ret.Get(0).(func(string) filestore.ReadCloseSeeker); ok {
+		r0 = rf(path)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(filestore.ReadCloseSeeker)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
+		r1 = rf(path)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// GetOrCreateDirectChannel provides a mock function with given fields: c, userId, otherUserId, channelOptions
+func (_m *MockAppIface) GetOrCreateDirectChannel(c request.CTX, userId string, otherUserId string, channelOptions ...model.ChannelOption) (*model.Channel, *model.AppError) {
+	_va := make([]interface{}, len(channelOptions))
+	for _i := range channelOptions {
+		_va[_i] = channelOptions[_i]
+	}
+	var _ca []interface{}
+	_ca = append(_ca, c, userId, otherUserId)
+	_ca = append(_ca, _va...)
+	ret := _m.Called(_ca...)
+
+	var r0 *model.Channel
+	if rf, ok := ret.Get(0).(func(request.CTX, string, string, ...model.ChannelOption) *model.Channel); ok {
+		r0 = rf(c, userId, otherUserId, channelOptions...)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Channel)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(request.CTX, string, string, ...model.ChannelOption) *model.AppError); ok {
+		r1 = rf(c, userId, otherUserId, channelOptions...)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// GetProfileImage provides a mock function with given fields: user
+func (_m *MockAppIface) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) {
+	ret := _m.Called(user)
+
+	var r0 []byte
+	if rf, ok := ret.Get(0).(func(*model.User) []byte); ok {
+		r0 = rf(user)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]byte)
+		}
+	}
+
+	var r1 bool
+	if rf, ok := ret.Get(1).(func(*model.User) bool); ok {
+		r1 = rf(user)
+	} else {
+		r1 = ret.Get(1).(bool)
+	}
+
+	var r2 *model.AppError
+	if rf, ok := ret.Get(2).(func(*model.User) *model.AppError); ok {
+		r2 = rf(user)
+	} else {
+		if ret.Get(2) != nil {
+			r2 = ret.Get(2).(*model.AppError)
+		}
+	}
+
+	return r0, r1, r2
+}
+
+// InvalidateCacheForUser provides a mock function with given fields: userID
+func (_m *MockAppIface) InvalidateCacheForUser(userID string) {
+	_m.Called(userID)
+}
+
+// MentionsToTeamMembers provides a mock function with given fields: c, message, teamID
+func (_m *MockAppIface) MentionsToTeamMembers(c request.CTX, message string, teamID string) model.UserMentionMap {
+	ret := _m.Called(c, message, teamID)
+
+	var r0 model.UserMentionMap
+	if rf, ok := ret.Get(0).(func(request.CTX, string, string) model.UserMentionMap); ok {
+		r0 = rf(c, message, teamID)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(model.UserMentionMap)
+		}
+	}
+
+	return r0
+}
+
+// NotifySharedChannelUserUpdate provides a mock function with given fields: user
+func (_m *MockAppIface) NotifySharedChannelUserUpdate(user *model.User) {
+	_m.Called(user)
+}
+
+// PatchChannelModerationsForChannel provides a mock function with given fields: c, channel, channelModerationsPatch
+func (_m *MockAppIface) PatchChannelModerationsForChannel(c request.CTX, channel *model.Channel, channelModerationsPatch []*model.ChannelModerationPatch) ([]*model.ChannelModeration, *model.AppError) {
+	ret := _m.Called(c, channel, channelModerationsPatch)
+
+	var r0 []*model.ChannelModeration
+	if rf, ok := ret.Get(0).(func(request.CTX, *model.Channel, []*model.ChannelModerationPatch) []*model.ChannelModeration); ok {
+		r0 = rf(c, channel, channelModerationsPatch)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.ChannelModeration)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(request.CTX, *model.Channel, []*model.ChannelModerationPatch) *model.AppError); ok {
+		r1 = rf(c, channel, channelModerationsPatch)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// PermanentDeleteChannel provides a mock function with given fields: c, channel
+func (_m *MockAppIface) PermanentDeleteChannel(c request.CTX, channel *model.Channel) *model.AppError {
+	ret := _m.Called(c, channel)
+
+	var r0 *model.AppError
+	if rf, ok := ret.Get(0).(func(request.CTX, *model.Channel) *model.AppError); ok {
+		r0 = rf(c, channel)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.AppError)
+		}
+	}
+
+	return r0
+}
+
+// SaveReactionForPost provides a mock function with given fields: c, reaction
+func (_m *MockAppIface) SaveReactionForPost(c *request.Context, reaction *model.Reaction) (*model.Reaction, *model.AppError) {
+	ret := _m.Called(c, reaction)
+
+	var r0 *model.Reaction
+	if rf, ok := ret.Get(0).(func(*request.Context, *model.Reaction) *model.Reaction); ok {
+		r0 = rf(c, reaction)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Reaction)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*request.Context, *model.Reaction) *model.AppError); ok {
+		r1 = rf(c, reaction)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// SendEphemeralPost provides a mock function with given fields: c, userId, post
+func (_m *MockAppIface) SendEphemeralPost(c request.CTX, userId string, post *model.Post) *model.Post {
+	ret := _m.Called(c, userId, post)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(request.CTX, string, *model.Post) *model.Post); ok {
+		r0 = rf(c, userId, post)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	return r0
+}
+
+// UpdatePost provides a mock function with given fields: c, post, safeUpdate
+func (_m *MockAppIface) UpdatePost(c *request.Context, post *model.Post, safeUpdate bool) (*model.Post, *model.AppError) {
+	ret := _m.Called(c, post, safeUpdate)
+
+	var r0 *model.Post
+	if rf, ok := ret.Get(0).(func(*request.Context, *model.Post, bool) *model.Post); ok {
+		r0 = rf(c, post, safeUpdate)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Post)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(*request.Context, *model.Post, bool) *model.AppError); ok {
+		r1 = rf(c, post, safeUpdate)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}

server/platform/services/sharedchannel/mock_ServerIface_test.go

@@ -0,0 +1,118 @@
+// Code generated by mockery v2.10.4. DO NOT EDIT.
+
+// Regenerate this file using `make sharedchannel-mocks`.
+
+package sharedchannel
+
+import (
+	mlog "github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/mattermost/mattermost-server/v6/model"
+
+	remotecluster "github.com/mattermost/mattermost-server/v6/server/platform/services/remotecluster"
+
+	store "github.com/mattermost/mattermost-server/v6/server/channels/store"
+)
+
+// MockServerIface is an autogenerated mock type for the ServerIface type
+type MockServerIface struct {
+	mock.Mock
+}
+
+// AddClusterLeaderChangedListener provides a mock function with given fields: listener
+func (_m *MockServerIface) AddClusterLeaderChangedListener(listener func()) string {
+	ret := _m.Called(listener)
+
+	var r0 string
+	if rf, ok := ret.Get(0).(func(func()) string); ok {
+		r0 = rf(listener)
+	} else {
+		r0 = ret.Get(0).(string)
+	}
+
+	return r0
+}
+
+// Config provides a mock function with given fields:
+func (_m *MockServerIface) Config() *model.Config {
+	ret := _m.Called()
+
+	var r0 *model.Config
+	if rf, ok := ret.Get(0).(func() *model.Config); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Config)
+		}
+	}
+
+	return r0
+}
+
+// GetRemoteClusterService provides a mock function with given fields:
+func (_m *MockServerIface) GetRemoteClusterService() remotecluster.RemoteClusterServiceIFace {
+	ret := _m.Called()
+
+	var r0 remotecluster.RemoteClusterServiceIFace
+	if rf, ok := ret.Get(0).(func() remotecluster.RemoteClusterServiceIFace); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(remotecluster.RemoteClusterServiceIFace)
+		}
+	}
+
+	return r0
+}
+
+// GetStore provides a mock function with given fields:
+func (_m *MockServerIface) GetStore() store.Store {
+	ret := _m.Called()
+
+	var r0 store.Store
+	if rf, ok := ret.Get(0).(func() store.Store); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.Store)
+		}
+	}
+
+	return r0
+}
+
+// IsLeader provides a mock function with given fields:
+func (_m *MockServerIface) IsLeader() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// Log provides a mock function with given fields:
+func (_m *MockServerIface) Log() *mlog.Logger {
+	ret := _m.Called()
+
+	var r0 *mlog.Logger
+	if rf, ok := ret.Get(0).(func() *mlog.Logger); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*mlog.Logger)
+		}
+	}
+
+	return r0
+}
+
+// RemoveClusterLeaderChangedListener provides a mock function with given fields: id
+func (_m *MockServerIface) RemoveClusterLeaderChangedListener(id string) {
+	_m.Called(id)
+}

server/platform/services/sharedchannel/msg.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"encoding/json"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// syncMsg represents a change in content (post add/edit/delete, reaction add/remove, users).
+// It is sent to remote clusters as the payload of a `RemoteClusterMsg`.
+type syncMsg struct {
+	Id        string                 `json:"id"`
+	ChannelId string                 `json:"channel_id"`
+	Users     map[string]*model.User `json:"users,omitempty"`
+	Posts     []*model.Post          `json:"posts,omitempty"`
+	Reactions []*model.Reaction      `json:"reactions,omitempty"`
+}
+
+func newSyncMsg(channelID string) *syncMsg {
+	return &syncMsg{
+		Id:        model.NewId(),
+		ChannelId: channelID,
+	}
+}
+
+func (sm *syncMsg) ToJSON() ([]byte, error) {
+	b, err := json.Marshal(sm)
+	if err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func (sm *syncMsg) String() string {
+	json, err := sm.ToJSON()
+	if err != nil {
+		return ""
+	}
+	return string(json)
+}

server/platform/services/sharedchannel/permalink.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/i18n"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+var (
+	// Team name regex taken from model.IsValidTeamName
+	permaLinkRegex       = regexp.MustCompile(`https?://[0-9.\-A-Za-z]+/[a-z0-9]+([a-z\-0-9]+|(__)?)[a-z0-9]+/pl/([a-zA-Z0-9]+)`)
+	permaLinkSharedRegex = regexp.MustCompile(`https?://[0-9.\-A-Za-z]+/[a-z0-9]+([a-z\-0-9]+|(__)?)[a-z0-9]+/plshared/([a-zA-Z0-9]+)`)
+)
+
+const (
+	permalinkMarker = "plshared"
+)
+
+// processPermalinkToRemote processes all permalinks going towards a remote site.
+func (scs *Service) processPermalinkToRemote(p *model.Post) string {
+	var sent bool
+	return permaLinkRegex.ReplaceAllStringFunc(p.Message, func(msg string) string {
+		// Extract the postID (This is simple enough not to warrant full-blown URL parsing.)
+		lastSlash := strings.LastIndexByte(msg, '/')
+		postID := msg[lastSlash+1:]
+		opts := model.GetPostsOptions{
+			SkipFetchThreads: true,
+		}
+		postList, err := scs.server.GetStore().Post().Get(context.Background(), postID, opts, "", map[string]bool{})
+		if err != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceWarn, "Unable to get post during replacing permalinks", mlog.Err(err))
+			return msg
+		}
+		if len(postList.Order) == 0 {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceWarn, "No post found for permalink", mlog.String("postID", postID))
+			return msg
+		}
+
+		// If postID is for a different channel
+		if postList.Posts[postList.Order[0]].ChannelId != p.ChannelId {
+			// Send ephemeral message to OP (only once per message).
+			if !sent {
+				scs.sendEphemeralPost(p.ChannelId, p.UserId, i18n.T("sharedchannel.permalink.not_found"))
+				sent = true
+			}
+			// But don't modify msg
+			return msg
+		}
+
+		// Otherwise, modify pl to plshared as a marker to be replaced by remote sites
+		return strings.Replace(msg, "/pl/", "/"+permalinkMarker+"/", 1)
+	})
+}
+
+// processPermalinkFromRemote processes all permalinks coming from a remote site.
+func (scs *Service) processPermalinkFromRemote(p *model.Post, team *model.Team) string {
+	return permaLinkSharedRegex.ReplaceAllStringFunc(p.Message, func(remoteLink string) string {
+		// Extract host name
+		parsed, err := url.Parse(remoteLink)
+		if err != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceWarn, "Unable to parse the remote link during replacing permalinks", mlog.Err(err))
+			return remoteLink
+		}
+
+		// Replace with local SiteURL
+		parsed.Scheme = scs.siteURL.Scheme
+		parsed.Host = scs.siteURL.Host
+
+		// Replace team name with local team
+		teamEnd := strings.Index(parsed.Path, "/"+permalinkMarker)
+		parsed.Path = "/" + team.Name + parsed.Path[teamEnd:]
+
+		// Replace plshared with pl
+		return strings.Replace(parsed.String(), "/"+permalinkMarker+"/", "/pl/", 1)
+	})
+}

server/platform/services/sharedchannel/permalink_test.go

@@ -0,0 +1,112 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+func TestProcessPermalinkToRemote(t *testing.T) {
+	scs := &Service{
+		server: &MockServerIface{},
+		app:    &MockAppIface{},
+	}
+
+	mockStore := &mocks.Store{}
+	mockPostStore := mocks.PostStore{}
+	utils.TranslationsPreInit()
+
+	pl := &model.PostList{}
+	mockPostStore.On("Get", context.Background(), "postID", model.GetPostsOptions{SkipFetchThreads: true}, "", map[string]bool{}).Return(pl, nil)
+
+	mockStore.On("Post").Return(&mockPostStore)
+
+	mockServer := scs.server.(*MockServerIface)
+	mockServer.On("GetStore").Return(mockStore)
+	mockServer.On("Log").Return(mlog.NewLogger())
+
+	mockApp := scs.app.(*MockAppIface)
+	mockApp.On("SendEphemeralPost", mock.Anything, "user", mock.AnythingOfType("*model.Post")).Return(&model.Post{}).Times(1)
+	defer mockApp.AssertExpectations(t)
+
+	t.Run("same channel", func(t *testing.T) {
+		post := &model.Post{
+			Message:   "hello world https://comm.matt.com/team/pl/postID link",
+			ChannelId: "sourceChan",
+			UserId:    "user",
+		}
+
+		*pl = model.PostList{
+			Order: []string{"1"},
+			Posts: map[string]*model.Post{
+				"1": {
+					ChannelId: "sourceChan",
+					UserId:    "user",
+				},
+			},
+		}
+
+		out := scs.processPermalinkToRemote(post)
+		assert.Equal(t, "hello world https://comm.matt.com/team/plshared/postID link", out)
+	})
+
+	t.Run("different channel", func(t *testing.T) {
+		post := &model.Post{
+			Message:   "hello world https://comm.matt.com/team/pl/postID link https://comm.matt.com/team/pl/postID ",
+			ChannelId: "sourceChan",
+			UserId:    "user",
+		}
+
+		*pl = model.PostList{
+			Order: []string{"1"},
+			Posts: map[string]*model.Post{
+				"1": {
+					ChannelId: "otherChan",
+				},
+			},
+		}
+		out := scs.processPermalinkToRemote(post)
+		assert.Equal(t, "hello world https://comm.matt.com/team/pl/postID link https://comm.matt.com/team/pl/postID ", out)
+	})
+}
+
+func TestProcessPermalinkFromRemote(t *testing.T) {
+	t.Run("has match", func(t *testing.T) {
+		parsed, _ := url.Parse("http://mysite.com")
+		scs := &Service{
+			server:  &MockServerIface{},
+			siteURL: parsed,
+		}
+
+		out := scs.processPermalinkFromRemote(&model.Post{Message: "hello world https://comm.matt.com/team/plshared/postID link"},
+			&model.Team{Name: "myteam"})
+		assert.Equal(t,
+			"hello world http://mysite.com/myteam/pl/postID link",
+			out)
+	})
+
+	t.Run("does not match", func(t *testing.T) {
+		parsed, _ := url.Parse("http://mysite.com")
+		scs := &Service{
+			server:  &MockServerIface{},
+			siteURL: parsed,
+		}
+
+		out := scs.processPermalinkFromRemote(&model.Post{Message: "hello world https://comm.matt.com/team/pl/postID link"},
+			&model.Team{Name: "myteam"})
+		assert.Equal(t,
+			"hello world https://comm.matt.com/team/pl/postID link",
+			out)
+	})
+}

server/platform/services/sharedchannel/response.go

@@ -0,0 +1,16 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+type SyncResponse struct {
+	UsersLastUpdateAt int64    `json:"users_last_update_at"`
+	UserErrors        []string `json:"user_errors"`
+	UsersSyncd        []string `json:"users_syncd"`
+
+	PostsLastUpdateAt int64    `json:"posts_last_update_at"`
+	PostErrors        []string `json:"post_errors"`
+
+	ReactionsLastUpdateAt int64    `json:"reactions_last_update_at"`
+	ReactionErrors        []string `json:"reaction_errors"`
+}

server/platform/services/sharedchannel/service.go

@@ -0,0 +1,249 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/filestore"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	TopicSync                    = "sharedchannel_sync"
+	TopicChannelInvite           = "sharedchannel_invite"
+	TopicUploadCreate            = "sharedchannel_upload"
+	MaxRetries                   = 3
+	MaxPostsPerSync              = 12 // a bit more than one typical screenfull of posts
+	MaxUsersPerSync              = 25
+	NotifyRemoteOfflineThreshold = time.Second * 10
+	NotifyMinimumDelay           = time.Second * 2
+	MaxUpsertRetries             = 25
+	ProfileImageSyncTimeout      = time.Second * 5
+	KeyRemoteUsername            = "RemoteUsername"
+	KeyRemoteEmail               = "RemoteEmail"
+)
+
+// Mocks can be re-generated with `make sharedchannel-mocks`.
+type ServerIface interface {
+	Config() *model.Config
+	IsLeader() bool
+	AddClusterLeaderChangedListener(listener func()) string
+	RemoveClusterLeaderChangedListener(id string)
+	GetStore() store.Store
+	Log() *mlog.Logger
+	GetRemoteClusterService() remotecluster.RemoteClusterServiceIFace
+}
+
+type AppIface interface {
+	SendEphemeralPost(c request.CTX, userId string, post *model.Post) *model.Post
+	CreateChannelWithUser(c request.CTX, channel *model.Channel, userId string) (*model.Channel, *model.AppError)
+	GetOrCreateDirectChannel(c request.CTX, userId, otherUserId string, channelOptions ...model.ChannelOption) (*model.Channel, *model.AppError)
+	AddUserToChannel(c request.CTX, user *model.User, channel *model.Channel, skipTeamMemberIntegrityCheck bool) (*model.ChannelMember, *model.AppError)
+	AddUserToTeamByTeamId(c *request.Context, teamId string, user *model.User) *model.AppError
+	PermanentDeleteChannel(c request.CTX, channel *model.Channel) *model.AppError
+	CreatePost(c request.CTX, post *model.Post, channel *model.Channel, triggerWebhooks bool, setOnline bool) (savedPost *model.Post, err *model.AppError)
+	UpdatePost(c *request.Context, post *model.Post, safeUpdate bool) (*model.Post, *model.AppError)
+	DeletePost(c request.CTX, postID, deleteByID string) (*model.Post, *model.AppError)
+	SaveReactionForPost(c *request.Context, reaction *model.Reaction) (*model.Reaction, *model.AppError)
+	DeleteReactionForPost(c *request.Context, reaction *model.Reaction) *model.AppError
+	PatchChannelModerationsForChannel(c request.CTX, channel *model.Channel, channelModerationsPatch []*model.ChannelModerationPatch) ([]*model.ChannelModeration, *model.AppError)
+	CreateUploadSession(c request.CTX, us *model.UploadSession) (*model.UploadSession, *model.AppError)
+	FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
+	MentionsToTeamMembers(c request.CTX, message, teamID string) model.UserMentionMap
+	GetProfileImage(user *model.User) ([]byte, bool, *model.AppError)
+	InvalidateCacheForUser(userID string)
+	NotifySharedChannelUserUpdate(user *model.User)
+}
+
+// errNotFound allows checking against Store.ErrNotFound errors without making Store a dependency.
+type errNotFound interface {
+	IsErrNotFound() bool
+}
+
+// errInvalidInput allows checking against Store.ErrInvalidInput errors without making Store a dependency.
+type errInvalidInput interface {
+	InvalidInputInfo() (entity string, field string, value any)
+}
+
+// Service provides shared channel synchronization.
+type Service struct {
+	server       ServerIface
+	app          AppIface
+	changeSignal chan struct{}
+
+	// everything below guarded by `mux`
+	mux                       sync.RWMutex
+	active                    bool
+	leaderListenerId          string
+	connectionStateListenerId string
+	done                      chan struct{}
+	tasks                     map[string]syncTask
+	syncTopicListenerId       string
+	inviteTopicListenerId     string
+	uploadTopicListenerId     string
+	siteURL                   *url.URL
+}
+
+// NewSharedChannelService creates a RemoteClusterService instance.
+func NewSharedChannelService(server ServerIface, app AppIface) (*Service, error) {
+	service := &Service{
+		server:       server,
+		app:          app,
+		changeSignal: make(chan struct{}, 1),
+		tasks:        make(map[string]syncTask),
+	}
+	parsed, err := url.Parse(*server.Config().ServiceSettings.SiteURL)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse SiteURL: %w", err)
+	}
+	service.siteURL = parsed
+	return service, nil
+}
+
+// Start is called by the server on server start-up.
+func (scs *Service) Start() error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return errors.New("Shared Channel Service cannot activate: requires Remote Cluster Service")
+	}
+
+	scs.mux.Lock()
+	scs.leaderListenerId = scs.server.AddClusterLeaderChangedListener(scs.onClusterLeaderChange)
+	scs.syncTopicListenerId = rcs.AddTopicListener(TopicSync, scs.onReceiveSyncMessage)
+	scs.inviteTopicListenerId = rcs.AddTopicListener(TopicChannelInvite, scs.onReceiveChannelInvite)
+	scs.uploadTopicListenerId = rcs.AddTopicListener(TopicUploadCreate, scs.onReceiveUploadCreate)
+	scs.connectionStateListenerId = rcs.AddConnectionStateListener(scs.onConnectionStateChange)
+	scs.mux.Unlock()
+
+	scs.onClusterLeaderChange()
+
+	return nil
+}
+
+// Shutdown is called by the server on server shutdown.
+func (scs *Service) Shutdown() error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return errors.New("Shared Channel Service cannot shutdown: requires Remote Cluster Service")
+	}
+
+	scs.mux.Lock()
+	id := scs.leaderListenerId
+	rcs.RemoveTopicListener(scs.syncTopicListenerId)
+	scs.syncTopicListenerId = ""
+	rcs.RemoveTopicListener(scs.inviteTopicListenerId)
+	scs.inviteTopicListenerId = ""
+	rcs.RemoveConnectionStateListener(scs.connectionStateListenerId)
+	scs.connectionStateListenerId = ""
+	scs.mux.Unlock()
+
+	scs.server.RemoveClusterLeaderChangedListener(id)
+	scs.pause()
+	return nil
+}
+
+// Active determines whether the service is active on the node or not.
+func (scs *Service) Active() bool {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	return scs.active
+}
+
+func (scs *Service) sendEphemeralPost(channelId string, userId string, text string) {
+	ephemeral := &model.Post{
+		ChannelId: channelId,
+		Message:   text,
+		CreateAt:  model.GetMillis(),
+	}
+	scs.app.SendEphemeralPost(request.EmptyContext(scs.server.Log()), userId, ephemeral)
+}
+
+// onClusterLeaderChange is called whenever the cluster leader may have changed.
+func (scs *Service) onClusterLeaderChange() {
+	if scs.server.IsLeader() {
+		scs.resume()
+	} else {
+		scs.pause()
+	}
+}
+
+func (scs *Service) resume() {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	if scs.active {
+		return // already active
+	}
+
+	scs.active = true
+	scs.done = make(chan struct{})
+
+	go scs.syncLoop(scs.done)
+
+	scs.server.Log().Debug("Shared Channel Service active")
+}
+
+func (scs *Service) pause() {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	if !scs.active {
+		return // already inactive
+	}
+
+	scs.active = false
+	close(scs.done)
+	scs.done = nil
+
+	scs.server.Log().Debug("Shared Channel Service inactive")
+}
+
+// Makes the remote channel to be read-only(announcement mode, only admins can create posts and reactions).
+func (scs *Service) makeChannelReadOnly(channel *model.Channel) *model.AppError {
+	createPostPermission := model.ChannelModeratedPermissionsMap[model.PermissionCreatePost.Id]
+	createReactionPermission := model.ChannelModeratedPermissionsMap[model.PermissionAddReaction.Id]
+	updateMap := model.ChannelModeratedRolesPatch{
+		Guests:  model.NewBool(false),
+		Members: model.NewBool(false),
+	}
+
+	readonlyChannelModerations := []*model.ChannelModerationPatch{
+		{
+			Name:  &createPostPermission,
+			Roles: &updateMap,
+		},
+		{
+			Name:  &createReactionPermission,
+			Roles: &updateMap,
+		},
+	}
+
+	_, err := scs.app.PatchChannelModerationsForChannel(request.EmptyContext(scs.server.Log()), channel, readonlyChannelModerations)
+	return err
+}
+
+// onConnectionStateChange is called whenever the connection state of a remote cluster changes,
+// for example when one comes back online.
+func (scs *Service) onConnectionStateChange(rc *model.RemoteCluster, online bool) {
+	if online {
+		// when a previously offline remote comes back online force a sync.
+		scs.ForceSyncForRemote(rc)
+	}
+
+	scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Remote cluster connection status changed",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("remoteId", rc.RemoteId),
+		mlog.Bool("online", online),
+	)
+}

server/platform/services/sharedchannel/sync_recv.go

@@ -0,0 +1,399 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+func (scs *Service) onReceiveSyncMessage(msg model.RemoteClusterMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	if msg.Topic != TopicSync {
+		return fmt.Errorf("wrong topic, expected `%s`, got `%s`", TopicSync, msg.Topic)
+	}
+
+	if len(msg.Payload) == 0 {
+		return errors.New("empty sync message")
+	}
+
+	if scs.server.Log().IsLevelEnabled(mlog.LvlSharedChannelServiceMessagesInbound) {
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceMessagesInbound, "inbound message",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("msg", string(msg.Payload)),
+		)
+	}
+
+	var sm syncMsg
+
+	if err := json.Unmarshal(msg.Payload, &sm); err != nil {
+		return fmt.Errorf("invalid sync message: %w", err)
+	}
+	return scs.processSyncMessage(request.EmptyContext(scs.server.Log()), &sm, rc, response)
+}
+
+func (scs *Service) processSyncMessage(c request.CTX, syncMsg *syncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
+	var channel *model.Channel
+	var team *model.Team
+
+	var err error
+	syncResp := SyncResponse{
+		UserErrors:     make([]string, 0),
+		UsersSyncd:     make([]string, 0),
+		PostErrors:     make([]string, 0),
+		ReactionErrors: make([]string, 0),
+	}
+
+	scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Sync msg received",
+		mlog.String("remote", rc.Name),
+		mlog.String("channel_id", syncMsg.ChannelId),
+		mlog.Int("user_count", len(syncMsg.Users)),
+		mlog.Int("post_count", len(syncMsg.Posts)),
+		mlog.Int("reaction_count", len(syncMsg.Reactions)),
+	)
+
+	if channel, err = scs.server.GetStore().Channel().Get(syncMsg.ChannelId, true); err != nil {
+		// if the channel doesn't exist then none of these sync items are going to work.
+		return fmt.Errorf("channel not found processing sync message: %w", err)
+	}
+
+	// add/update users before posts
+	for _, user := range syncMsg.Users {
+		if userSaved, err := scs.upsertSyncUser(c, user, channel, rc); err != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync user",
+				mlog.String("remote", rc.Name),
+				mlog.String("channel_id", syncMsg.ChannelId),
+				mlog.String("user_id", user.Id),
+				mlog.Err(err))
+		} else {
+			syncResp.UsersSyncd = append(syncResp.UsersSyncd, userSaved.Id)
+			if syncResp.UsersLastUpdateAt < user.UpdateAt {
+				syncResp.UsersLastUpdateAt = user.UpdateAt
+			}
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "User upserted via sync",
+				mlog.String("remote", rc.Name),
+				mlog.String("channel_id", syncMsg.ChannelId),
+				mlog.String("user_id", user.Id),
+			)
+		}
+	}
+
+	for _, post := range syncMsg.Posts {
+		if syncMsg.ChannelId != post.ChannelId {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "ChannelId mismatch",
+				mlog.String("remote", rc.Name),
+				mlog.String("sm.ChannelId", syncMsg.ChannelId),
+				mlog.String("sm.Post.ChannelId", post.ChannelId),
+				mlog.String("PostId", post.Id),
+			)
+			syncResp.PostErrors = append(syncResp.PostErrors, post.Id)
+			continue
+		}
+
+		if channel.Type != model.ChannelTypeDirect && team == nil {
+			var err2 error
+			team, err2 = scs.server.GetStore().Channel().GetTeamForChannel(syncMsg.ChannelId)
+			if err2 != nil {
+				scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error getting Team for Channel",
+					mlog.String("ChannelId", post.ChannelId),
+					mlog.String("PostId", post.Id),
+					mlog.String("remote", rc.Name),
+					mlog.Err(err2),
+				)
+				syncResp.PostErrors = append(syncResp.PostErrors, post.Id)
+				continue
+			}
+		}
+
+		// process perma-links for remote
+		if team != nil {
+			post.Message = scs.processPermalinkFromRemote(post, team)
+		}
+
+		// add/update post
+		rpost, err := scs.upsertSyncPost(post, channel, rc)
+		if err != nil {
+			syncResp.PostErrors = append(syncResp.PostErrors, post.Id)
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync post",
+				mlog.String("post_id", post.Id),
+				mlog.String("channel_id", post.ChannelId),
+				mlog.String("remote", rc.Name),
+				mlog.Err(err),
+			)
+		} else if syncResp.PostsLastUpdateAt < rpost.UpdateAt {
+			syncResp.PostsLastUpdateAt = rpost.UpdateAt
+		}
+	}
+
+	// add/remove reactions
+	for _, reaction := range syncMsg.Reactions {
+		if _, err := scs.upsertSyncReaction(reaction, rc); err != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error upserting sync reaction",
+				mlog.String("remote", rc.Name),
+				mlog.String("user_id", reaction.UserId),
+				mlog.String("post_id", reaction.PostId),
+				mlog.String("emoji", reaction.EmojiName),
+				mlog.Int64("delete_at", reaction.DeleteAt),
+				mlog.Err(err),
+			)
+		} else {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Reaction upserted via sync",
+				mlog.String("remote", rc.Name),
+				mlog.String("user_id", reaction.UserId),
+				mlog.String("post_id", reaction.PostId),
+				mlog.String("emoji", reaction.EmojiName),
+				mlog.Int64("delete_at", reaction.DeleteAt),
+			)
+
+			if syncResp.ReactionsLastUpdateAt < reaction.UpdateAt {
+				syncResp.ReactionsLastUpdateAt = reaction.UpdateAt
+			}
+		}
+	}
+
+	response.SetPayload(syncResp)
+
+	return nil
+}
+
+func (scs *Service) upsertSyncUser(c request.CTX, user *model.User, channel *model.Channel, rc *model.RemoteCluster) (*model.User, error) {
+	var err error
+	if user.RemoteId == nil || *user.RemoteId == "" {
+		user.RemoteId = model.NewString(rc.RemoteId)
+	}
+
+	// Check if user already exists
+	euser, err := scs.server.GetStore().User().Get(context.Background(), user.Id)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return nil, fmt.Errorf("error checking sync user: %w", err)
+		}
+	}
+
+	var userSaved *model.User
+	if euser == nil {
+		if userSaved, err = scs.insertSyncUser(user, channel, rc); err != nil {
+			return nil, err
+		}
+	} else {
+		patch := &model.UserPatch{
+			Username:  &user.Username,
+			Nickname:  &user.Nickname,
+			FirstName: &user.FirstName,
+			LastName:  &user.LastName,
+			Email:     &user.Email,
+			Props:     user.Props,
+			Position:  &user.Position,
+			Locale:    &user.Locale,
+			Timezone:  user.Timezone,
+			RemoteId:  user.RemoteId,
+		}
+		if userSaved, err = scs.updateSyncUser(patch, euser, channel, rc); err != nil {
+			return nil, err
+		}
+	}
+
+	// Add user to team. We do this here regardless of whether the user was
+	// just created or patched since there are three steps to adding a user
+	// (insert rec, add to team, add to channel) and any one could fail.
+	// Instead of undoing what succeeded on any failure we simply do all steps each
+	// time. AddUserToChannel & AddUserToTeamByTeamId do not error if user was already
+	// added and exit quickly.
+	if err := scs.app.AddUserToTeamByTeamId(request.EmptyContext(scs.server.Log()), channel.TeamId, userSaved); err != nil {
+		return nil, fmt.Errorf("error adding sync user to Team: %w", err)
+	}
+
+	// add user to channel
+	if _, err := scs.app.AddUserToChannel(c, userSaved, channel, false); err != nil {
+		return nil, fmt.Errorf("error adding sync user to ChannelMembers: %w", err)
+	}
+	return userSaved, nil
+}
+
+func (scs *Service) insertSyncUser(user *model.User, channel *model.Channel, rc *model.RemoteCluster) (*model.User, error) {
+	var err error
+	var userSaved *model.User
+	var suffix string
+
+	// ensure the new user is created with system_user role and random password.
+	user = sanitizeUserForSync(user)
+
+	// save the original username and email in props (if not already done by another remote)
+	if _, ok := user.GetProp(KeyRemoteUsername); !ok {
+		user.SetProp(KeyRemoteUsername, user.Username)
+	}
+	if _, ok := user.GetProp(KeyRemoteEmail); !ok {
+		user.SetProp(KeyRemoteEmail, user.Email)
+	}
+
+	// Apply a suffix to the username until it is unique. Collisions will be quite
+	// rare since we are joining a username that is unique at a remote site with a unique
+	// name for that site. However we need to truncate the combined name to 64 chars and
+	// that might introduce a collision.
+	for i := 1; i <= MaxUpsertRetries; i++ {
+		if i > 1 {
+			suffix = strconv.FormatInt(int64(i), 10)
+		}
+
+		user.Username = mungUsername(user.Username, rc.Name, suffix, model.UserNameMaxLength)
+		user.Email = mungEmail(rc.Name, model.UserEmailMaxLength)
+
+		if userSaved, err = scs.server.GetStore().User().Save(user); err != nil {
+			e, ok := err.(errInvalidInput)
+			if !ok {
+				break
+			}
+			_, field, value := e.InvalidInputInfo()
+			if field == "email" || field == "username" {
+				// username or email collision; try again with different suffix
+				scs.server.Log().Log(mlog.LvlSharedChannelServiceWarn, "Collision inserting sync user",
+					mlog.String("field", field),
+					mlog.Any("value", value),
+					mlog.Int("attempt", i),
+					mlog.Err(err),
+				)
+			}
+		} else {
+			scs.app.NotifySharedChannelUserUpdate(userSaved)
+			return userSaved, nil
+		}
+	}
+	return nil, fmt.Errorf("error inserting sync user %s: %w", user.Id, err)
+}
+
+func (scs *Service) updateSyncUser(patch *model.UserPatch, user *model.User, channel *model.Channel, rc *model.RemoteCluster) (*model.User, error) {
+	var err error
+	var update *model.UserUpdate
+	var suffix string
+
+	// preserve existing real username/email since Patch will over-write them;
+	// the real username/email in props can be updated if they don't contain colons,
+	// meaning the update is coming from the user's origin server (not munged).
+	realUsername, _ := user.GetProp(KeyRemoteUsername)
+	realEmail, _ := user.GetProp(KeyRemoteEmail)
+
+	if patch.Username != nil && !strings.Contains(*patch.Username, ":") {
+		realUsername = *patch.Username
+	}
+	if patch.Email != nil && !strings.Contains(*patch.Email, ":") {
+		realEmail = *patch.Email
+	}
+
+	user.Patch(patch)
+	user = sanitizeUserForSync(user)
+	user.SetProp(KeyRemoteUsername, realUsername)
+	user.SetProp(KeyRemoteEmail, realEmail)
+
+	// Apply a suffix to the username until it is unique.
+	for i := 1; i <= MaxUpsertRetries; i++ {
+		if i > 1 {
+			suffix = strconv.FormatInt(int64(i), 10)
+		}
+		user.Username = mungUsername(user.Username, rc.Name, suffix, model.UserNameMaxLength)
+		user.Email = mungEmail(rc.Name, model.UserEmailMaxLength)
+
+		if update, err = scs.server.GetStore().User().Update(user, false); err != nil {
+			e, ok := err.(errInvalidInput)
+			if !ok {
+				break
+			}
+			_, field, value := e.InvalidInputInfo()
+			if field == "email" || field == "username" {
+				// username or email collision; try again with different suffix
+				scs.server.Log().Log(mlog.LvlSharedChannelServiceWarn, "Collision updating sync user",
+					mlog.String("field", field),
+					mlog.Any("value", value),
+					mlog.Int("attempt", i),
+					mlog.Err(err),
+				)
+			}
+		} else {
+			scs.app.InvalidateCacheForUser(update.New.Id)
+			scs.app.NotifySharedChannelUserUpdate(update.New)
+			return update.New, nil
+		}
+	}
+	return nil, fmt.Errorf("error updating sync user %s: %w", user.Id, err)
+}
+
+func (scs *Service) upsertSyncPost(post *model.Post, channel *model.Channel, rc *model.RemoteCluster) (*model.Post, error) {
+	var appErr *model.AppError
+
+	post.RemoteId = model.NewString(rc.RemoteId)
+
+	rpost, err := scs.server.GetStore().Post().GetSingle(post.Id, true)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return nil, fmt.Errorf("error checking sync post: %w", err)
+		}
+	}
+
+	if rpost == nil {
+		// post doesn't exist; create new one
+		rpost, appErr = scs.app.CreatePost(request.EmptyContext(scs.server.Log()), post, channel, true, true)
+		if appErr == nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Created sync post",
+				mlog.String("post_id", post.Id),
+				mlog.String("channel_id", post.ChannelId),
+			)
+		}
+	} else if post.DeleteAt > 0 {
+		// delete post
+		rpost, appErr = scs.app.DeletePost(request.EmptyContext(scs.server.Log()), post.Id, post.UserId)
+		if appErr == nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Deleted sync post",
+				mlog.String("post_id", post.Id),
+				mlog.String("channel_id", post.ChannelId),
+			)
+		}
+	} else if post.EditAt > rpost.EditAt || post.Message != rpost.Message {
+		// update post
+		rpost, appErr = scs.app.UpdatePost(request.EmptyContext(scs.server.Log()), post, false)
+		if appErr == nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Updated sync post",
+				mlog.String("post_id", post.Id),
+				mlog.String("channel_id", post.ChannelId),
+			)
+		}
+	} else {
+		// nothing to update
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Update to sync post ignored",
+			mlog.String("post_id", post.Id),
+			mlog.String("channel_id", post.ChannelId),
+		)
+	}
+
+	var rerr error
+	if appErr != nil {
+		rerr = errors.New(appErr.Error())
+	}
+	return rpost, rerr
+}
+
+func (scs *Service) upsertSyncReaction(reaction *model.Reaction, rc *model.RemoteCluster) (*model.Reaction, error) {
+	savedReaction := reaction
+	var appErr *model.AppError
+
+	reaction.RemoteId = model.NewString(rc.RemoteId)
+
+	if reaction.DeleteAt == 0 {
+		savedReaction, appErr = scs.app.SaveReactionForPost(request.EmptyContext(scs.server.Log()), reaction)
+	} else {
+		appErr = scs.app.DeleteReactionForPost(request.EmptyContext(scs.server.Log()), reaction)
+	}
+
+	var err error
+	if appErr != nil {
+		err = errors.New(appErr.Error())
+	}
+	return savedReaction, err
+}

server/platform/services/sharedchannel/sync_send.go

@@ -0,0 +1,437 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/i18n"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type syncTask struct {
+	id         string
+	channelID  string
+	remoteID   string
+	AddedAt    time.Time
+	retryCount int
+	retryMsg   *syncMsg
+	schedule   time.Time
+}
+
+func newSyncTask(channelID string, remoteID string, retryMsg *syncMsg) syncTask {
+	var retryID string
+	if retryMsg != nil {
+		retryID = retryMsg.Id
+	}
+
+	return syncTask{
+		id:        channelID + remoteID + retryID, // combination of ids to avoid duplicates
+		channelID: channelID,
+		remoteID:  remoteID, // empty means update all remote clusters
+		retryMsg:  retryMsg,
+		schedule:  time.Now(),
+	}
+}
+
+// incRetry increments the retry counter and returns true if MaxRetries not exceeded.
+func (st *syncTask) incRetry() bool {
+	st.retryCount++
+	return st.retryCount <= MaxRetries
+}
+
+// NotifyChannelChanged is called to indicate that a shared channel has been modified,
+// thus triggering an update to all remote clusters.
+func (scs *Service) NotifyChannelChanged(channelID string) {
+	if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
+		return
+	}
+
+	task := newSyncTask(channelID, "", nil)
+	task.schedule = time.Now().Add(NotifyMinimumDelay)
+	scs.addTask(task)
+}
+
+// NotifyUserProfileChanged is called to indicate that a user belonging to at least one
+// shared channel has modified their user profile (name, username, email, custom status, profile image)
+func (scs *Service) NotifyUserProfileChanged(userID string) {
+	if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
+		return
+	}
+
+	scusers, err := scs.server.GetStore().SharedChannel().GetUsersForUser(userID)
+	if err != nil {
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Failed to fetch shared channel users",
+			mlog.String("userID", userID),
+			mlog.Err(err),
+		)
+		return
+	}
+	if len(scusers) == 0 {
+		return
+	}
+
+	notified := make(map[string]struct{})
+
+	for _, user := range scusers {
+		// update every channel + remote combination they belong to.
+		// Redundant updates (ie. to same remote for multiple channels) will be
+		// filtered out.
+		combo := user.ChannelId + user.RemoteId
+		if _, ok := notified[combo]; ok {
+			continue
+		}
+		notified[combo] = struct{}{}
+		task := newSyncTask(user.ChannelId, user.RemoteId, nil)
+		task.schedule = time.Now().Add(NotifyMinimumDelay)
+		scs.addTask(task)
+	}
+}
+
+// ForceSyncForRemote causes all channels shared with the remote to be synchronized.
+func (scs *Service) ForceSyncForRemote(rc *model.RemoteCluster) {
+	if rcs := scs.server.GetRemoteClusterService(); rcs == nil {
+		return
+	}
+
+	// fetch all channels shared with this remote.
+	opts := model.SharedChannelRemoteFilterOpts{
+		RemoteId: rc.RemoteId,
+	}
+	scrs, err := scs.server.GetStore().SharedChannel().GetRemotes(opts)
+	if err != nil {
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Failed to fetch shared channel remotes",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("remoteId", rc.RemoteId),
+			mlog.Err(err),
+		)
+		return
+	}
+
+	for _, scr := range scrs {
+		task := newSyncTask(scr.ChannelId, rc.RemoteId, nil)
+		task.schedule = time.Now().Add(NotifyMinimumDelay)
+		scs.addTask(task)
+	}
+}
+
+// addTask adds or re-adds a task to the queue.
+func (scs *Service) addTask(task syncTask) {
+	task.AddedAt = time.Now()
+	scs.mux.Lock()
+	if _, ok := scs.tasks[task.id]; !ok {
+		scs.tasks[task.id] = task
+	}
+	scs.mux.Unlock()
+
+	// wake up the sync goroutine
+	select {
+	case scs.changeSignal <- struct{}{}:
+	default:
+		// that's ok, the sync routine is already busy
+	}
+}
+
+// syncLoop is called via a dedicated goroutine to wait for notifications of channel changes and
+// updates each remote based on those changes.
+func (scs *Service) syncLoop(done chan struct{}) {
+	// create a timer to periodically check the task queue, but only if there is
+	// a delayed task in the queue.
+	delay := time.NewTimer(NotifyMinimumDelay)
+	defer stopTimer(delay)
+
+	// wait for channel changed signal and update for oldest task.
+	for {
+		select {
+		case <-scs.changeSignal:
+			if wait := scs.doSync(); wait > 0 {
+				stopTimer(delay)
+				delay.Reset(wait)
+			}
+		case <-delay.C:
+			if wait := scs.doSync(); wait > 0 {
+				delay.Reset(wait)
+			}
+		case <-done:
+			return
+		}
+	}
+}
+
+func stopTimer(timer *time.Timer) {
+	timer.Stop()
+	select {
+	case <-timer.C:
+	default:
+	}
+}
+
+// doSync checks the task queue for any tasks to be processed and processes all that are ready.
+// If any delayed tasks remain in queue then the duration until the next scheduled task is returned.
+func (scs *Service) doSync() time.Duration {
+	var task syncTask
+	var ok bool
+	var shortestWait time.Duration
+
+	for {
+		task, ok, shortestWait = scs.removeOldestTask()
+		if !ok {
+			break
+		}
+		if err := scs.processTask(task); err != nil {
+			// put task back into map so it will update again
+			if task.incRetry() {
+				scs.addTask(task)
+			} else {
+				scs.server.Log().Error("Failed to synchronize shared channel",
+					mlog.String("channelId", task.channelID),
+					mlog.String("remoteId", task.remoteID),
+					mlog.Err(err),
+				)
+			}
+		}
+	}
+	return shortestWait
+}
+
+// removeOldestTask removes and returns the oldest task in the task map.
+// A task coming in via NotifyChannelChanged must stay in queue for at least
+// `NotifyMinimumDelay` to ensure we don't go nuts trying to sync during a bulk update.
+// If no tasks are available then false is returned.
+func (scs *Service) removeOldestTask() (syncTask, bool, time.Duration) {
+	scs.mux.Lock()
+	defer scs.mux.Unlock()
+
+	var oldestTask syncTask
+	var oldestKey string
+	var shortestWait time.Duration
+
+	for key, task := range scs.tasks {
+		// check if task is ready
+		if wait := time.Until(task.schedule); wait > 0 {
+			if wait < shortestWait || shortestWait == 0 {
+				shortestWait = wait
+			}
+			continue
+		}
+		// task is ready; check if it's the oldest ready task
+		if task.AddedAt.Before(oldestTask.AddedAt) || oldestTask.AddedAt.IsZero() {
+			oldestKey = key
+			oldestTask = task
+		}
+	}
+
+	if oldestKey != "" {
+		delete(scs.tasks, oldestKey)
+		return oldestTask, true, shortestWait
+	}
+	return oldestTask, false, shortestWait
+}
+
+// processTask updates one or more remote clusters with any new channel content.
+func (scs *Service) processTask(task syncTask) error {
+	var err error
+	var remotes []*model.RemoteCluster
+
+	if task.remoteID == "" {
+		filter := model.RemoteClusterQueryFilter{
+			InChannel:     task.channelID,
+			OnlyConfirmed: true,
+		}
+		remotes, err = scs.server.GetStore().RemoteCluster().GetAll(filter)
+		if err != nil {
+			return err
+		}
+	} else {
+		rc, err := scs.server.GetStore().RemoteCluster().Get(task.remoteID)
+		if err != nil {
+			return err
+		}
+		if !rc.IsOnline() {
+			return fmt.Errorf("Failed updating shared channel '%s' for offline remote cluster '%s'", task.channelID, rc.DisplayName)
+		}
+		remotes = []*model.RemoteCluster{rc}
+	}
+
+	for _, rc := range remotes {
+		rtask := task
+		rtask.remoteID = rc.RemoteId
+		if err := scs.syncForRemote(rtask, rc); err != nil {
+			// retry...
+			if rtask.incRetry() {
+				scs.addTask(rtask)
+			} else {
+				scs.server.Log().Error("Failed to synchronize shared channel for remote cluster",
+					mlog.String("channelId", rtask.channelID),
+					mlog.String("remote", rc.DisplayName),
+					mlog.Err(err),
+				)
+			}
+		}
+	}
+	return nil
+}
+
+func (scs *Service) handlePostError(postId string, task syncTask, rc *model.RemoteCluster) {
+	if task.retryMsg != nil && len(task.retryMsg.Posts) == 1 && task.retryMsg.Posts[0].Id == postId {
+		// this was a retry for specific post that failed previously. Try again if within MaxRetries.
+		if task.incRetry() {
+			scs.addTask(task)
+		} else {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "error syncing post",
+				mlog.String("remote", rc.DisplayName),
+				mlog.String("post_id", postId),
+			)
+		}
+		return
+	}
+
+	// this post failed as part of a group of posts. Retry as an individual post.
+	post, err := scs.server.GetStore().Post().GetSingle(postId, true)
+	if err != nil {
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "error fetching post for sync retry",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("post_id", postId),
+		)
+		return
+	}
+
+	syncMsg := newSyncMsg(task.channelID)
+	syncMsg.Posts = []*model.Post{post}
+
+	scs.addTask(newSyncTask(task.channelID, task.remoteID, syncMsg))
+}
+
+// notifyRemoteOffline creates an ephemeral post to the author for any posts created recently to remotes
+// that are offline.
+func (scs *Service) notifyRemoteOffline(posts []*model.Post, rc *model.RemoteCluster) {
+	// only send one ephemeral post per author.
+	notified := make(map[string]bool)
+
+	// range the slice in reverse so the newest posts are visited first; this ensures an ephemeral
+	// get added where it is mostly likely to be seen.
+	for i := len(posts) - 1; i >= 0; i-- {
+		post := posts[i]
+		if didNotify := notified[post.UserId]; didNotify {
+			continue
+		}
+
+		postCreateAt := model.GetTimeForMillis(post.CreateAt)
+
+		if post.DeleteAt == 0 && post.UserId != "" && time.Since(postCreateAt) < NotifyRemoteOfflineThreshold {
+			T := scs.getUserTranslations(post.UserId)
+			ephemeral := &model.Post{
+				ChannelId: post.ChannelId,
+				Message:   T("sharedchannel.cannot_deliver_post", map[string]any{"Remote": rc.DisplayName}),
+				CreateAt:  post.CreateAt + 1,
+			}
+			scs.app.SendEphemeralPost(request.EmptyContext(scs.server.Log()), post.UserId, ephemeral)
+
+			notified[post.UserId] = true
+		}
+	}
+}
+
+func (scs *Service) updateCursorForRemote(scrId string, rc *model.RemoteCluster, cursor model.GetPostsSinceForSyncCursor) {
+	if err := scs.server.GetStore().SharedChannel().UpdateRemoteCursor(scrId, cursor); err != nil {
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "error updating cursor for shared channel remote",
+			mlog.String("remote", rc.DisplayName),
+			mlog.Err(err),
+		)
+		return
+	}
+	scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "updated cursor for remote",
+		mlog.String("remote_id", rc.RemoteId),
+		mlog.String("remote", rc.DisplayName),
+		mlog.Int64("last_post_update_at", cursor.LastPostUpdateAt),
+		mlog.String("last_post_id", cursor.LastPostId),
+	)
+}
+
+func (scs *Service) getUserTranslations(userId string) i18n.TranslateFunc {
+	var locale string
+	user, err := scs.server.GetStore().User().Get(context.Background(), userId)
+	if err == nil {
+		locale = user.Locale
+	}
+
+	if locale == "" {
+		locale = model.DefaultLocale
+	}
+	return i18n.GetUserTranslations(locale)
+}
+
+// shouldUserSync determines if a user needs to be synchronized.
+// User should be synchronized if it has no entry in the SharedChannelUsers table for the specified channel,
+// or there is an entry but the LastSyncAt is less than user.UpdateAt
+func (scs *Service) shouldUserSync(user *model.User, channelID string, rc *model.RemoteCluster) (sync bool, syncImage bool, err error) {
+	// don't sync users with the remote they originated from.
+	if user.RemoteId != nil && *user.RemoteId == rc.RemoteId {
+		return false, false, nil
+	}
+
+	scu, err := scs.server.GetStore().SharedChannel().GetSingleUser(user.Id, channelID, rc.RemoteId)
+	if err != nil {
+		if _, ok := err.(errNotFound); !ok {
+			return false, false, err
+		}
+
+		// user not in the SharedChannelUsers table, so we must add them.
+		scu = &model.SharedChannelUser{
+			UserId:    user.Id,
+			RemoteId:  rc.RemoteId,
+			ChannelId: channelID,
+		}
+		if _, err = scs.server.GetStore().SharedChannel().SaveUser(scu); err != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error adding user to shared channel users",
+				mlog.String("remote_id", rc.RemoteId),
+				mlog.String("user_id", user.Id),
+				mlog.String("channel_id", user.Id),
+				mlog.Err(err),
+			)
+		}
+		return true, true, nil
+	}
+
+	return user.UpdateAt > scu.LastSyncAt, user.LastPictureUpdate > scu.LastSyncAt, nil
+}
+
+func (scs *Service) syncProfileImage(user *model.User, channelID string, rc *model.RemoteCluster) {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), ProfileImageSyncTimeout)
+	defer cancel()
+
+	rcs.SendProfileImage(ctx, user.Id, rc, scs.app, func(userId string, rc *model.RemoteCluster, resp *remotecluster.Response, err error) {
+		if resp.IsSuccess() {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Users profile image synchronized",
+				mlog.String("remote_id", rc.RemoteId),
+				mlog.String("user_id", user.Id),
+			)
+
+			if err2 := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(user.Id, channelID, rc.RemoteId); err2 != nil {
+				scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error updating users LastSyncTime after profile image update",
+					mlog.String("remote_id", rc.RemoteId),
+					mlog.String("user_id", user.Id),
+					mlog.Err(err2),
+				)
+			}
+			return
+		}
+
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error synchronizing users profile image",
+			mlog.String("remote_id", rc.RemoteId),
+			mlog.String("user_id", user.Id),
+			mlog.Err(err),
+		)
+	})
+}

server/platform/services/sharedchannel/sync_send_remote.go

@@ -0,0 +1,545 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/wiggin77/merror"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/remotecluster"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type sendSyncMsgResultFunc func(syncResp SyncResponse, err error)
+
+type attachment struct {
+	fi   *model.FileInfo
+	post *model.Post
+}
+
+type syncData struct {
+	task syncTask
+	rc   *model.RemoteCluster
+	scr  *model.SharedChannelRemote
+
+	users         map[string]*model.User
+	profileImages map[string]*model.User
+	posts         []*model.Post
+	reactions     []*model.Reaction
+	attachments   []attachment
+
+	resultRepeat     bool
+	resultNextCursor model.GetPostsSinceForSyncCursor
+}
+
+func newSyncData(task syncTask, rc *model.RemoteCluster, scr *model.SharedChannelRemote) *syncData {
+	return &syncData{
+		task:             task,
+		rc:               rc,
+		scr:              scr,
+		users:            make(map[string]*model.User),
+		profileImages:    make(map[string]*model.User),
+		resultNextCursor: model.GetPostsSinceForSyncCursor{LastPostUpdateAt: scr.LastPostUpdateAt, LastPostId: scr.LastPostId},
+	}
+}
+
+func (sd *syncData) isEmpty() bool {
+	return len(sd.users) == 0 && len(sd.profileImages) == 0 && len(sd.posts) == 0 && len(sd.reactions) == 0 && len(sd.attachments) == 0
+}
+
+func (sd *syncData) isCursorChanged() bool {
+	return sd.scr.LastPostUpdateAt != sd.resultNextCursor.LastPostUpdateAt || sd.scr.LastPostId != sd.resultNextCursor.LastPostId
+}
+
+// syncForRemote updates a remote cluster with any new posts/reactions for a specific
+// channel. If many changes are found, only the oldest X changes are sent and the channel
+// is re-added to the task map. This ensures no channels are starved for updates even if some
+// channels are very active.
+// Returning an error forces a retry on the task.
+func (scs *Service) syncForRemote(task syncTask, rc *model.RemoteCluster) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot update remote cluster %s for channel id %s; Remote Cluster Service not enabled", rc.Name, task.channelID)
+	}
+
+	scr, err := scs.server.GetStore().SharedChannel().GetRemoteByIds(task.channelID, rc.RemoteId)
+	if err != nil {
+		return err
+	}
+
+	// if this is retrying a failed msg, just send it again.
+	if task.retryMsg != nil {
+		sd := newSyncData(task, rc, scr)
+		sd.users = task.retryMsg.Users
+		sd.posts = task.retryMsg.Posts
+		sd.reactions = task.retryMsg.Reactions
+		return scs.sendSyncData(sd)
+	}
+
+	sd := newSyncData(task, rc, scr)
+
+	// schedule another sync if the repeat flag is set at some point.
+	defer func(rpt *bool) {
+		if *rpt {
+			scs.addTask(newSyncTask(task.channelID, task.remoteID, nil))
+		}
+	}(&sd.resultRepeat)
+
+	// fetch new posts or retry post.
+	if err := scs.fetchPostsForSync(sd); err != nil {
+		return fmt.Errorf("cannot fetch posts for sync %v: %w", sd, err)
+	}
+
+	if !rc.IsOnline() {
+		if len(sd.posts) != 0 {
+			scs.notifyRemoteOffline(sd.posts, rc)
+		}
+		sd.resultRepeat = false
+		return nil
+	}
+
+	// fetch users that have updated their user profile or image.
+	if err := scs.fetchUsersForSync(sd); err != nil {
+		return fmt.Errorf("cannot fetch users for sync %v: %w", sd, err)
+	}
+
+	// fetch reactions for posts
+	if err := scs.fetchReactionsForSync(sd); err != nil {
+		return fmt.Errorf("cannot fetch reactions for sync %v: %w", sd, err)
+	}
+
+	// fetch users associated with posts & reactions
+	if err := scs.fetchPostUsersForSync(sd); err != nil {
+		return fmt.Errorf("cannot fetch post users for sync %v: %w", sd, err)
+	}
+
+	// filter out any posts that don't need to be sent.
+	scs.filterPostsForSync(sd)
+
+	// fetch attachments for posts
+	if err := scs.fetchPostAttachmentsForSync(sd); err != nil {
+		return fmt.Errorf("cannot fetch post attachments for sync %v: %w", sd, err)
+	}
+
+	if sd.isEmpty() {
+		scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Not sending sync data; everything filtered out",
+			mlog.String("remote", rc.DisplayName),
+			mlog.String("channel_id", task.channelID),
+			mlog.Bool("repeat", sd.resultRepeat),
+		)
+		if sd.isCursorChanged() {
+			scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
+		}
+		return nil
+	}
+
+	scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Sending sync data",
+		mlog.String("remote", rc.DisplayName),
+		mlog.String("channel_id", task.channelID),
+		mlog.Bool("repeat", sd.resultRepeat),
+		mlog.Int("users", len(sd.users)),
+		mlog.Int("images", len(sd.profileImages)),
+		mlog.Int("posts", len(sd.posts)),
+		mlog.Int("reactions", len(sd.reactions)),
+		mlog.Int("attachments", len(sd.attachments)),
+	)
+
+	return scs.sendSyncData(sd)
+}
+
+// fetchUsersForSync populates the sync data with any channel users who updated their user profile
+// since the last sync.
+func (scs *Service) fetchUsersForSync(sd *syncData) error {
+	filter := model.GetUsersForSyncFilter{
+		ChannelID: sd.task.channelID,
+		Limit:     MaxUsersPerSync,
+	}
+	users, err := scs.server.GetStore().SharedChannel().GetUsersForSync(filter)
+	if err != nil {
+		return err
+	}
+
+	for _, u := range users {
+		if u.GetRemoteID() != sd.rc.RemoteId {
+			sd.users[u.Id] = u
+		}
+	}
+
+	filter.CheckProfileImage = true
+	usersImage, err := scs.server.GetStore().SharedChannel().GetUsersForSync(filter)
+	if err != nil {
+		return err
+	}
+
+	for _, u := range usersImage {
+		if u.GetRemoteID() != sd.rc.RemoteId {
+			sd.profileImages[u.Id] = u
+		}
+	}
+	return nil
+}
+
+// fetchPostsForSync populates the sync data with any new posts since the last sync.
+func (scs *Service) fetchPostsForSync(sd *syncData) error {
+	options := model.GetPostsSinceForSyncOptions{
+		ChannelId:      sd.task.channelID,
+		IncludeDeleted: true,
+	}
+	cursor := model.GetPostsSinceForSyncCursor{
+		LastPostUpdateAt: sd.scr.LastPostUpdateAt,
+		LastPostId:       sd.scr.LastPostId,
+	}
+
+	posts, nextCursor, err := scs.server.GetStore().Post().GetPostsSinceForSync(options, cursor, MaxPostsPerSync)
+	if err != nil {
+		return fmt.Errorf("could not fetch new posts for sync: %w", err)
+	}
+
+	// Append the posts individually, checking for root posts that might appear later in the list.
+	// This is due to the UpdateAt collision handling algorithm where the order of posts is not based
+	// on UpdateAt or CreateAt when the posts have the same UpdateAt value. Here we are guarding
+	// against a root post with the same UpdateAt (and probably the same CreateAt) appearing later
+	// in the list and must be sync'd before the child post. This is and edge case that likely only
+	// happens during load testing or bulk imports.
+	for _, p := range posts {
+		if p.RootId != "" {
+			root, err := scs.server.GetStore().Post().GetSingle(p.RootId, true)
+			if err == nil {
+				if (root.CreateAt >= cursor.LastPostUpdateAt || root.UpdateAt >= cursor.LastPostUpdateAt) && !containsPost(sd.posts, root) {
+					sd.posts = append(sd.posts, root)
+				}
+			}
+		}
+		sd.posts = append(sd.posts, p)
+	}
+
+	sd.resultNextCursor = nextCursor
+	sd.resultRepeat = len(posts) == MaxPostsPerSync
+	return nil
+}
+
+func containsPost(posts []*model.Post, post *model.Post) bool {
+	for _, p := range posts {
+		if p.Id == post.Id {
+			return true
+		}
+	}
+	return false
+}
+
+// fetchReactionsForSync populates the sync data with any new reactions since the last sync.
+func (scs *Service) fetchReactionsForSync(sd *syncData) error {
+	merr := merror.New()
+	for _, post := range sd.posts {
+		// any reactions originating from the remote cluster are filtered out
+		reactions, err := scs.server.GetStore().Reaction().GetForPostSince(post.Id, sd.scr.LastPostUpdateAt, sd.rc.RemoteId, true)
+		if err != nil {
+			merr.Append(fmt.Errorf("could not get reactions for post %s: %w", post.Id, err))
+			continue
+		}
+		sd.reactions = append(sd.reactions, reactions...)
+	}
+	return merr.ErrorOrNil()
+}
+
+// fetchPostUsersForSync populates the sync data with all users associated with posts.
+func (scs *Service) fetchPostUsersForSync(sd *syncData) error {
+	sc, err := scs.server.GetStore().SharedChannel().Get(sd.task.channelID)
+	if err != nil {
+		return fmt.Errorf("cannot determine teamID: %w", err)
+	}
+
+	type p2mm struct {
+		post       *model.Post
+		mentionMap model.UserMentionMap
+	}
+
+	userIDs := make(map[string]p2mm)
+
+	for _, reaction := range sd.reactions {
+		userIDs[reaction.UserId] = p2mm{}
+	}
+
+	for _, post := range sd.posts {
+		// add author
+		userIDs[post.UserId] = p2mm{}
+
+		// get mentions and users for each mention
+		mentionMap := scs.app.MentionsToTeamMembers(request.EmptyContext(scs.server.Log()), post.Message, sc.TeamId)
+		for _, userID := range mentionMap {
+			userIDs[userID] = p2mm{
+				post:       post,
+				mentionMap: mentionMap,
+			}
+		}
+	}
+
+	merr := merror.New()
+
+	for userID, v := range userIDs {
+		user, err := scs.server.GetStore().User().Get(context.Background(), userID)
+		if err != nil {
+			merr.Append(fmt.Errorf("could not get user %s: %w", userID, err))
+			continue
+		}
+
+		sync, syncImage, err2 := scs.shouldUserSync(user, sd.task.channelID, sd.rc)
+		if err2 != nil {
+			merr.Append(fmt.Errorf("could not check should sync user %s: %w", userID, err))
+			continue
+		}
+
+		if sync {
+			sd.users[user.Id] = user
+		}
+
+		if syncImage {
+			sd.profileImages[user.Id] = user
+		}
+
+		// if this was a mention then put the real username in place of the username+remotename, but only
+		// when sending to the remote that the user belongs to.
+		if v.post != nil && user.RemoteId != nil && *user.RemoteId == sd.rc.RemoteId {
+			fixMention(v.post, v.mentionMap, user)
+		}
+	}
+	return merr.ErrorOrNil()
+}
+
+// fetchPostAttachmentsForSync populates the sync data with any file attachments for new posts.
+func (scs *Service) fetchPostAttachmentsForSync(sd *syncData) error {
+	merr := merror.New()
+	for _, post := range sd.posts {
+		fis, err := scs.server.GetStore().FileInfo().GetForPost(post.Id, false, true, true)
+		if err != nil {
+			merr.Append(fmt.Errorf("could not get file attachment info for post %s: %w", post.Id, err))
+			continue
+		}
+
+		for _, fi := range fis {
+			if scs.shouldSyncAttachment(fi, sd.rc) {
+				sd.attachments = append(sd.attachments, attachment{fi: fi, post: post})
+			}
+		}
+	}
+	return merr.ErrorOrNil()
+}
+
+// filterPostsforSync removes any posts that do not need to sync.
+func (scs *Service) filterPostsForSync(sd *syncData) {
+	filtered := make([]*model.Post, 0, len(sd.posts))
+
+	for _, p := range sd.posts {
+		// Don't resend an existing post where only the reactions changed.
+		// Posts we must send:
+		//   - new posts (EditAt == 0)
+		//   - edited posts (EditAt >= LastPostUpdateAt)
+		//   - deleted posts (DeleteAt > 0)
+		if p.EditAt > 0 && p.EditAt < sd.scr.LastPostUpdateAt && p.DeleteAt == 0 {
+			continue
+		}
+
+		// Don't send a deleted post if it is just the original copy from an edit.
+		if p.DeleteAt > 0 && p.OriginalId != "" {
+			continue
+		}
+
+		// don't sync a post back to the remote it came from.
+		if p.GetRemoteID() == sd.rc.RemoteId {
+			continue
+		}
+
+		// parse out all permalinks in the message.
+		p.Message = scs.processPermalinkToRemote(p)
+
+		filtered = append(filtered, p)
+	}
+	sd.posts = filtered
+}
+
+// sendSyncData sends all the collected users, posts, reactions, images, and attachments to the
+// remote cluster.
+// The order of items sent is important: users -> attachments -> posts -> reactions -> profile images
+func (scs *Service) sendSyncData(sd *syncData) error {
+	merr := merror.New()
+
+	sanitizeSyncData(sd)
+
+	// send users
+	if len(sd.users) != 0 {
+		if err := scs.sendUserSyncData(sd); err != nil {
+			merr.Append(fmt.Errorf("cannot send user sync data: %w", err))
+		}
+	}
+
+	// send attachments
+	if len(sd.attachments) != 0 {
+		scs.sendAttachmentSyncData(sd)
+	}
+
+	// send posts
+	if len(sd.posts) != 0 {
+		if err := scs.sendPostSyncData(sd); err != nil {
+			merr.Append(fmt.Errorf("cannot send post sync data: %w", err))
+		}
+	} else if sd.isCursorChanged() {
+		scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
+	}
+
+	// send reactions
+	if len(sd.reactions) != 0 {
+		if err := scs.sendReactionSyncData(sd); err != nil {
+			merr.Append(fmt.Errorf("cannot send reaction sync data: %w", err))
+		}
+	}
+
+	// send user profile images
+	if len(sd.profileImages) != 0 {
+		scs.sendProfileImageSyncData(sd)
+	}
+
+	return merr.ErrorOrNil()
+}
+
+// sendUserSyncData sends the collected user updates to the remote cluster.
+func (scs *Service) sendUserSyncData(sd *syncData) error {
+	msg := newSyncMsg(sd.task.channelID)
+	msg.Users = sd.users
+
+	err := scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
+		for _, userID := range syncResp.UsersSyncd {
+			if err := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(userID, sd.task.channelID, sd.rc.RemoteId); err != nil {
+				scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Cannot update shared channel user LastSyncAt",
+					mlog.String("user_id", userID),
+					mlog.String("channel_id", sd.task.channelID),
+					mlog.String("remote_id", sd.rc.RemoteId),
+					mlog.Err(err),
+				)
+			}
+		}
+		if len(syncResp.UserErrors) != 0 {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Response indicates error for user(s) sync",
+				mlog.String("channel_id", sd.task.channelID),
+				mlog.String("remote_id", sd.rc.RemoteId),
+				mlog.Any("users", syncResp.UserErrors),
+			)
+		}
+	})
+	return err
+}
+
+// sendAttachmentSyncData sends the collected post updates to the remote cluster.
+func (scs *Service) sendAttachmentSyncData(sd *syncData) {
+	for _, a := range sd.attachments {
+		if err := scs.sendAttachmentForRemote(a.fi, a.post, sd.rc); err != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Cannot sync post attachment",
+				mlog.String("post_id", a.post.Id),
+				mlog.String("channel_id", sd.task.channelID),
+				mlog.String("remote_id", sd.rc.RemoteId),
+				mlog.Err(err),
+			)
+		}
+		// updating SharedChannelAttachments with LastSyncAt is already done.
+	}
+}
+
+// sendPostSyncData sends the collected post updates to the remote cluster.
+func (scs *Service) sendPostSyncData(sd *syncData) error {
+	msg := newSyncMsg(sd.task.channelID)
+	msg.Posts = sd.posts
+
+	return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
+		if len(syncResp.PostErrors) != 0 {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Response indicates error for post(s) sync",
+				mlog.String("channel_id", sd.task.channelID),
+				mlog.String("remote_id", sd.rc.RemoteId),
+				mlog.Any("posts", syncResp.PostErrors),
+			)
+
+			for _, postID := range syncResp.PostErrors {
+				scs.handlePostError(postID, sd.task, sd.rc)
+			}
+		}
+		scs.updateCursorForRemote(sd.scr.Id, sd.rc, sd.resultNextCursor)
+	})
+}
+
+// sendReactionSyncData sends the collected reaction updates to the remote cluster.
+func (scs *Service) sendReactionSyncData(sd *syncData) error {
+	msg := newSyncMsg(sd.task.channelID)
+	msg.Reactions = sd.reactions
+
+	return scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp SyncResponse, errResp error) {
+		if len(syncResp.ReactionErrors) != 0 {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Response indicates error for reactions(s) sync",
+				mlog.String("channel_id", sd.task.channelID),
+				mlog.String("remote_id", sd.rc.RemoteId),
+				mlog.Any("reaction_posts", syncResp.ReactionErrors),
+			)
+		}
+	})
+}
+
+// sendProfileImageSyncData sends the collected user profile image updates to the remote cluster.
+func (scs *Service) sendProfileImageSyncData(sd *syncData) {
+	for _, user := range sd.profileImages {
+		scs.syncProfileImage(user, sd.task.channelID, sd.rc)
+	}
+}
+
+// sendSyncMsgToRemote synchronously sends the sync message to the remote cluster.
+func (scs *Service) sendSyncMsgToRemote(msg *syncMsg, rc *model.RemoteCluster, f sendSyncMsgResultFunc) error {
+	rcs := scs.server.GetRemoteClusterService()
+	if rcs == nil {
+		return fmt.Errorf("cannot update remote cluster %s for channel id %s; Remote Cluster Service not enabled", rc.Name, msg.ChannelId)
+	}
+
+	b, err := json.Marshal(msg)
+	if err != nil {
+		return err
+	}
+	rcMsg := model.NewRemoteClusterMsg(TopicSync, b)
+
+	ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
+	defer cancel()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	err = rcs.SendMsg(ctx, rcMsg, rc, func(rcMsg model.RemoteClusterMsg, rc *model.RemoteCluster, rcResp *remotecluster.Response, errResp error) {
+		defer wg.Done()
+
+		var syncResp SyncResponse
+		if err2 := json.Unmarshal(rcResp.Payload, &syncResp); err2 != nil {
+			scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Invalid sync msg response from remote cluster",
+				mlog.String("remote", rc.Name),
+				mlog.String("channel_id", msg.ChannelId),
+				mlog.Err(err2),
+			)
+			return
+		}
+
+		if f != nil {
+			f(syncResp, errResp)
+		}
+	})
+
+	wg.Wait()
+	return err
+}
+
+func sanitizeSyncData(sd *syncData) {
+	for id, user := range sd.users {
+		sd.users[id] = sanitizeUserForSync(user)
+	}
+	for id, user := range sd.profileImages {
+		sd.profileImages[id] = sanitizeUserForSync(user)
+	}
+}

server/platform/services/sharedchannel/util.go

@@ -0,0 +1,101 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+// fixMention replaces any mentions in a post for the user with the user's real username.
+func fixMention(post *model.Post, mentionMap model.UserMentionMap, user *model.User) {
+	if post == nil || len(mentionMap) == 0 {
+		return
+	}
+
+	realUsername, ok := user.GetProp(KeyRemoteUsername)
+	if !ok {
+		return
+	}
+
+	// there may be more than one mention for each user so we have to walk the whole map.
+	for mention, id := range mentionMap {
+		if id == user.Id && strings.Contains(mention, ":") {
+			post.Message = strings.ReplaceAll(post.Message, "@"+mention, "@"+realUsername)
+		}
+	}
+}
+
+func sanitizeUserForSync(user *model.User) *model.User {
+	user.Password = model.NewId()
+	user.AuthData = nil
+	user.AuthService = ""
+	user.Roles = "system_user"
+	user.AllowMarketing = false
+	user.NotifyProps = model.StringMap{}
+	user.LastPasswordUpdate = 0
+	user.LastPictureUpdate = 0
+	user.FailedAttempts = 0
+	user.MfaActive = false
+	user.MfaSecret = ""
+
+	return user
+}
+
+// mungUsername creates a new username by combining username and remote cluster name, plus
+// a suffix to create uniqueness. If the resulting username exceeds the max length then
+// it is truncated and ellipses added.
+func mungUsername(username string, remotename string, suffix string, maxLen int) string {
+	if suffix != "" {
+		suffix = "~" + suffix
+	}
+
+	// If the username already contains a colon then another server already munged it.
+	// In that case we can split on the colon and use the existing remote name.
+	// We still need to re-mung with suffix in case of collision.
+	comps := strings.Split(username, ":")
+	if len(comps) >= 2 {
+		username = comps[0]
+		remotename = strings.Join(comps[1:], "")
+	}
+
+	var userEllipses string
+	var remoteEllipses string
+
+	// The remotename is allowed to use up to half the maxLen, and the username gets the remaining space.
+	// Username might have a suffix to account for, and remotename always has a preceding colon.
+	half := maxLen / 2
+
+	// If the remotename is less than half the maxLen, then the left over space can be given to
+	// the username.
+	extra := half - (len(remotename) + 1)
+	if extra < 0 {
+		extra = 0
+	}
+
+	truncUser := (len(username) + len(suffix)) - (half + extra)
+	if truncUser > 0 {
+		username = username[:len(username)-truncUser-3]
+		userEllipses = "..."
+	}
+
+	truncRemote := (len(remotename) + 1) - (maxLen - (len(username) + len(userEllipses) + len(suffix)))
+	if truncRemote > 0 {
+		remotename = remotename[:len(remotename)-truncRemote-3]
+		remoteEllipses = "..."
+	}
+
+	return fmt.Sprintf("%s%s%s:%s%s", username, suffix, userEllipses, remotename, remoteEllipses)
+}
+
+// mungEmail creates a unique email address using a UID and remote name.
+func mungEmail(remotename string, maxLen int) string {
+	s := fmt.Sprintf("%s@%s", model.NewId(), remotename)
+	if len(s) > maxLen {
+		s = s[:maxLen]
+	}
+	return s
+}

server/platform/services/sharedchannel/util_test.go

@@ -0,0 +1,76 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package sharedchannel
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func Test_mungUsername(t *testing.T) {
+	type args struct {
+		username   string
+		remotename string
+		suffix     string
+		maxLen     int
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"everything empty", args{username: "", remotename: "", suffix: "", maxLen: 64}, ":"},
+
+		{"no trunc, no suffix", args{username: "bart", remotename: "example.com", suffix: "", maxLen: 64}, "bart:example.com"},
+		{"no trunc, suffix", args{username: "bart", remotename: "example.com", suffix: "2", maxLen: 64}, "bart~2:example.com"},
+
+		{"trunc remote, no suffix", args{username: "bart", remotename: "example1234567890.com", suffix: "", maxLen: 24}, "bart:example123456789..."},
+		{"trunc remote, suffix", args{username: "bart", remotename: "example1234567890.com", suffix: "2", maxLen: 24}, "bart~2:example1234567..."},
+
+		{"trunc both, no suffix", args{username: R(24, "A"), remotename: R(24, "B"), suffix: "", maxLen: 24}, "AAAAAAAAA...:BBBBBBBB..."},
+		{"trunc both, suffix", args{username: R(24, "A"), remotename: R(24, "B"), suffix: "10", maxLen: 24}, "AAAAAA~10...:BBBBBBBB..."},
+
+		{"trunc user, no suffix", args{username: R(40, "A"), remotename: "abc", suffix: "", maxLen: 24}, "AAAAAAAAAAAAAAAAA...:abc"},
+		{"trunc user, suffix", args{username: R(40, "A"), remotename: "abc", suffix: "11", maxLen: 24}, "AAAAAAAAAAAAAA~11...:abc"},
+
+		{"trunc user, remote, no suffix", args{username: R(40, "A"), remotename: "abcdefghijk", suffix: "", maxLen: 24}, "AAAAAAAAA...:abcdefghijk"},
+		{"trunc user, remote, suffix", args{username: R(40, "A"), remotename: "abcdefghijk", suffix: "19", maxLen: 24}, "AAAAAA~19...:abcdefghijk"},
+
+		{"short user, long remote, no suffix", args{username: "bart", remotename: R(40, "B"), suffix: "", maxLen: 24}, "bart:BBBBBBBBBBBBBBBB..."},
+		{"long user, short remote, no suffix", args{username: R(40, "A"), remotename: "abc.com", suffix: "", maxLen: 24}, "AAAAAAAAAAAAA...:abc.com"},
+
+		{"short user, long remote, suffix", args{username: "bart", remotename: R(40, "B"), suffix: "12", maxLen: 24}, "bart~12:BBBBBBBBBBBBB..."},
+		{"long user, short remote, suffix", args{username: R(40, "A"), remotename: "abc.com", suffix: "12", maxLen: 24}, "AAAAAAAAAA~12...:abc.com"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := mungUsername(tt.args.username, tt.args.remotename, tt.args.suffix, tt.args.maxLen); got != tt.want {
+				t.Errorf("mungUsername() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_mungUsernameFuzz(t *testing.T) {
+	// ensure no index out of bounds panic for any combination
+	for i := 0; i < 70; i++ {
+		for j := 0; j < 70; j++ {
+			for k := 0; k < 3; k++ {
+				username := R(i, "A")
+				remotename := R(j, "B")
+				suffix := R(k, "1")
+
+				result := mungUsername(username, remotename, suffix, 64)
+				require.LessOrEqual(t, len(result), 64)
+			}
+		}
+	}
+}
+
+// R returns a string with the specified string repeated `count` times.
+func R(count int, s string) string {
+	return strings.Repeat(s, count)
+}

server/platform/services/slackimport/converters.go

@@ -0,0 +1,150 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package slackimport
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+func slackConvertTimeStamp(ts string) int64 {
+	timeString := strings.SplitN(ts, ".", 2)[0]
+
+	timeStamp, err := strconv.ParseInt(timeString, 10, 64)
+	if err != nil {
+		mlog.Warn("Slack Import: Bad timestamp detected.")
+		return 1
+	}
+	return timeStamp * 1000 // Convert to milliseconds
+}
+
+func slackConvertChannelName(channelName string, channelId string) string {
+	newName := strings.Trim(channelName, "_-")
+	if len(newName) == 1 {
+		return "slack-channel-" + newName
+	}
+
+	if isValidChannelNameCharacters(newName) {
+		return newName
+	}
+	return strings.ToLower(channelId)
+}
+
+func slackConvertUserMentions(users []slackUser, posts map[string][]slackPost) map[string][]slackPost {
+	var regexes = make(map[string]*regexp.Regexp, len(users))
+	for _, user := range users {
+		r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>")
+		if err != nil {
+			mlog.Warn("Slack Import: Unable to compile the @mention, matching regular expression for the Slack user.", mlog.String("user_name", user.Username), mlog.String("user_id", user.Id))
+			continue
+		}
+		regexes["@"+user.Username] = r
+	}
+
+	// Special cases.
+	regexes["@here"], _ = regexp.Compile(`<!here\|@here>`)
+	regexes["@channel"], _ = regexp.Compile("<!channel>")
+	regexes["@all"], _ = regexp.Compile("<!everyone>")
+
+	for channelName, channelPosts := range posts {
+		for postIdx, post := range channelPosts {
+			for mention, r := range regexes {
+				post.Text = r.ReplaceAllString(post.Text, mention)
+				posts[channelName][postIdx] = post
+			}
+		}
+	}
+
+	return posts
+}
+
+func slackConvertChannelMentions(channels []slackChannel, posts map[string][]slackPost) map[string][]slackPost {
+	var regexes = make(map[string]*regexp.Regexp, len(channels))
+	for _, channel := range channels {
+		r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>")
+		if err != nil {
+			mlog.Warn("Slack Import: Unable to compile the !channel, matching regular expression for the Slack channel.", mlog.String("channel_id", channel.Id), mlog.String("channel_name", channel.Name))
+			continue
+		}
+		regexes["~"+channel.Name] = r
+	}
+
+	for channelName, channelPosts := range posts {
+		for postIdx, post := range channelPosts {
+			for channelReplace, r := range regexes {
+				post.Text = r.ReplaceAllString(post.Text, channelReplace)
+				posts[channelName][postIdx] = post
+			}
+		}
+	}
+
+	return posts
+}
+
+func slackConvertPostsMarkup(posts map[string][]slackPost) map[string][]slackPost {
+	regexReplaceAllString := []struct {
+		regex *regexp.Regexp
+		rpl   string
+	}{
+		// URL
+		{
+			regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`),
+			"[$2]($1)",
+		},
+		// bold
+		{
+			regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`),
+			"$1**$2**",
+		},
+		// strikethrough
+		{
+			regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`),
+			"$1~~$2~~",
+		},
+		// single paragraph blockquote
+		// Slack converts > character to &gt;
+		{
+			regexp.MustCompile(`(?sm)^&gt;`),
+			">",
+		},
+	}
+
+	regexReplaceAllStringFunc := []struct {
+		regex *regexp.Regexp
+		fn    func(string) string
+	}{
+		// multiple paragraphs blockquotes
+		{
+			regexp.MustCompile(`(?sm)^>&gt;&gt;(.+)$`),
+			func(src string) string {
+				// remove >>> prefix, might have leading \n
+				prefixRegexp := regexp.MustCompile(`^([\n])?>&gt;&gt;(.*)`)
+				src = prefixRegexp.ReplaceAllString(src, "$1$2")
+				// append > to start of line
+				appendRegexp := regexp.MustCompile(`(?m)^`)
+				return appendRegexp.ReplaceAllString(src, ">$0")
+			},
+		},
+	}
+
+	for channelName, channelPosts := range posts {
+		for postIdx, post := range channelPosts {
+			result := post.Text
+
+			for _, rule := range regexReplaceAllString {
+				result = rule.regex.ReplaceAllString(result, rule.rpl)
+			}
+
+			for _, rule := range regexReplaceAllStringFunc {
+				result = rule.regex.ReplaceAllStringFunc(result, rule.fn)
+			}
+			posts[channelName][postIdx].Text = result
+		}
+	}
+
+	return posts
+}

server/platform/services/slackimport/main_test.go

@@ -0,0 +1,31 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package slackimport
+
+import (
+	"fmt"
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	prevDir, err := os.Getwd()
+	if err != nil {
+		panic("Failed to get current working directory: " + err.Error())
+	}
+
+	err = os.Chdir("../..")
+	if err != nil {
+		panic(fmt.Sprintf("Failed to set current working directory to %s: %s", "../..", err.Error()))
+	}
+
+	defer func() {
+		err := os.Chdir(prevDir)
+		if err != nil {
+			panic(fmt.Sprintf("Failed to restore current working directory to %s: %s", prevDir, err.Error()))
+		}
+	}()
+
+	os.Exit(m.Run())
+}

server/platform/services/slackimport/parsers.go

@@ -0,0 +1,50 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package slackimport
+
+import (
+	"encoding/json"
+	"io"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+func slackParseChannels(data io.Reader, channelType model.ChannelType) ([]slackChannel, error) {
+	decoder := json.NewDecoder(data)
+
+	var channels []slackChannel
+	if err := decoder.Decode(&channels); err != nil {
+		mlog.Warn("Slack Import: Error occurred when parsing some Slack channels. Import may work anyway.", mlog.Err(err))
+		return channels, err
+	}
+
+	for i := range channels {
+		channels[i].Type = channelType
+	}
+
+	return channels, nil
+}
+
+func slackParseUsers(data io.Reader) ([]slackUser, error) {
+	decoder := json.NewDecoder(data)
+
+	var users []slackUser
+	err := decoder.Decode(&users)
+	// This actually returns errors that are ignored.
+	// In this case it is erroring because of a null that Slack
+	// introduced. So we just return the users here.
+	return users, err
+}
+
+func slackParsePosts(data io.Reader) ([]slackPost, error) {
+	decoder := json.NewDecoder(data)
+
+	var posts []slackPost
+	if err := decoder.Decode(&posts); err != nil {
+		mlog.Warn("Slack Import: Error occurred when parsing some Slack posts. Import may work anyway.", mlog.Err(err))
+		return posts, err
+	}
+	return posts, nil
+}

server/platform/services/slackimport/slackimport.go

@@ -0,0 +1,831 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package slackimport
+
+import (
+	"archive/zip"
+	"bytes"
+	"errors"
+	"image"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store"
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/i18n"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type slackChannel struct {
+	Id      string          `json:"id"`
+	Name    string          `json:"name"`
+	Creator string          `json:"creator"`
+	Members []string        `json:"members"`
+	Purpose slackChannelSub `json:"purpose"`
+	Topic   slackChannelSub `json:"topic"`
+	Type    model.ChannelType
+}
+
+type slackChannelSub struct {
+	Value string `json:"value"`
+}
+
+type slackProfile struct {
+	FirstName string `json:"first_name"`
+	LastName  string `json:"last_name"`
+	Email     string `json:"email"`
+}
+
+type slackUser struct {
+	Id       string       `json:"id"`
+	Username string       `json:"name"`
+	Profile  slackProfile `json:"profile"`
+}
+
+type slackFile struct {
+	Id    string `json:"id"`
+	Title string `json:"title"`
+}
+
+type slackPost struct {
+	User        string                   `json:"user"`
+	BotId       string                   `json:"bot_id"`
+	BotUsername string                   `json:"username"`
+	Text        string                   `json:"text"`
+	TimeStamp   string                   `json:"ts"`
+	ThreadTS    string                   `json:"thread_ts"`
+	Type        string                   `json:"type"`
+	SubType     string                   `json:"subtype"`
+	Comment     *slackComment            `json:"comment"`
+	Upload      bool                     `json:"upload"`
+	File        *slackFile               `json:"file"`
+	Files       []*slackFile             `json:"files"`
+	Attachments []*model.SlackAttachment `json:"attachments"`
+}
+
+var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString
+
+const slackImportMaxFileSize = 1024 * 1024 * 70
+
+type slackComment struct {
+	User    string `json:"user"`
+	Comment string `json:"comment"`
+}
+
+// Actions provides the actions that needs to be used for import slack data
+type Actions struct {
+	UpdateActive           func(*model.User, bool) (*model.User, *model.AppError)
+	AddUserToChannel       func(request.CTX, *model.User, *model.Channel, bool) (*model.ChannelMember, *model.AppError)
+	JoinUserToTeam         func(*model.Team, *model.User, string) (*model.TeamMember, *model.AppError)
+	CreateDirectChannel    func(request.CTX, string, string, ...model.ChannelOption) (*model.Channel, *model.AppError)
+	CreateGroupChannel     func(request.CTX, []string) (*model.Channel, *model.AppError)
+	CreateChannel          func(*model.Channel, bool) (*model.Channel, *model.AppError)
+	DoUploadFile           func(time.Time, string, string, string, string, []byte) (*model.FileInfo, *model.AppError)
+	GenerateThumbnailImage func(image.Image, string, string)
+	GeneratePreviewImage   func(image.Image, string, string)
+	InvalidateAllCaches    func()
+	MaxPostSize            func() int
+	PrepareImage           func(fileData []byte) (image.Image, string, func(), error)
+}
+
+// SlackImporter is a service that allows to import slack dumps into mattermost
+type SlackImporter struct {
+	store   store.Store
+	actions Actions
+	config  *model.Config
+}
+
+// New creates a new SlackImporter service instance. It receive a store, a set of actions and the current config.
+// It is expected to be used right away and discarded after that
+func New(store store.Store, actions Actions, config *model.Config) *SlackImporter {
+	return &SlackImporter{
+		store:   store,
+		actions: actions,
+		config:  config,
+	}
+}
+
+func (si *SlackImporter) SlackImport(c request.CTX, fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
+	// Create log file
+	log := bytes.NewBufferString(i18n.T("api.slackimport.slack_import.log"))
+
+	zipreader, err := zip.NewReader(fileData, fileSize)
+	if err != nil || zipreader.File == nil {
+		log.WriteString(i18n.T("api.slackimport.slack_import.zip.app_error"))
+		return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, "", http.StatusBadRequest).Wrap(err), log
+	}
+
+	var channels []slackChannel
+	var publicChannels []slackChannel
+	var privateChannels []slackChannel
+	var groupChannels []slackChannel
+	var directChannels []slackChannel
+
+	var users []slackUser
+	posts := make(map[string][]slackPost)
+	uploads := make(map[string]*zip.File)
+	for _, file := range zipreader.File {
+		fileReader, err := file.Open()
+		if err != nil {
+			log.WriteString(i18n.T("api.slackimport.slack_import.open.app_error", map[string]any{"Filename": file.Name}))
+			return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]any{"Filename": file.Name}, "", http.StatusInternalServerError).Wrap(err), log
+		}
+		reader := utils.NewLimitedReaderWithError(fileReader, slackImportMaxFileSize)
+		if file.Name == "channels.json" {
+			publicChannels, err = slackParseChannels(reader, model.ChannelTypeOpen)
+			if errors.Is(err, utils.SizeLimitExceeded) {
+				log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name}))
+				continue
+			}
+			channels = append(channels, publicChannels...)
+		} else if file.Name == "dms.json" {
+			directChannels, err = slackParseChannels(reader, model.ChannelTypeDirect)
+			if errors.Is(err, utils.SizeLimitExceeded) {
+				log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name}))
+				continue
+			}
+			channels = append(channels, directChannels...)
+		} else if file.Name == "groups.json" {
+			privateChannels, err = slackParseChannels(reader, model.ChannelTypePrivate)
+			if errors.Is(err, utils.SizeLimitExceeded) {
+				log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name}))
+				continue
+			}
+			channels = append(channels, privateChannels...)
+		} else if file.Name == "mpims.json" {
+			groupChannels, err = slackParseChannels(reader, model.ChannelTypeGroup)
+			if errors.Is(err, utils.SizeLimitExceeded) {
+				log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name}))
+				continue
+			}
+			channels = append(channels, groupChannels...)
+		} else if file.Name == "users.json" {
+			users, err = slackParseUsers(reader)
+			if errors.Is(err, utils.SizeLimitExceeded) {
+				log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name}))
+				continue
+			}
+		} else {
+			spl := strings.Split(file.Name, "/")
+			if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
+				newposts, err := slackParsePosts(reader)
+				if errors.Is(err, utils.SizeLimitExceeded) {
+					log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name}))
+					continue
+				}
+				channel := spl[0]
+				if _, ok := posts[channel]; !ok {
+					posts[channel] = newposts
+				} else {
+					posts[channel] = append(posts[channel], newposts...)
+				}
+			} else if len(spl) == 3 && spl[0] == "__uploads" {
+				uploads[spl[1]] = file
+			}
+		}
+	}
+
+	posts = slackConvertUserMentions(users, posts)
+	posts = slackConvertChannelMentions(channels, posts)
+	posts = slackConvertPostsMarkup(posts)
+
+	addedUsers := si.slackAddUsers(teamID, users, log)
+	botUser := si.slackAddBotUser(teamID, log)
+
+	si.slackAddChannels(c, teamID, channels, posts, addedUsers, uploads, botUser, log)
+
+	if botUser != nil {
+		si.deactivateSlackBotUser(botUser)
+	}
+
+	si.actions.InvalidateAllCaches()
+
+	log.WriteString(i18n.T("api.slackimport.slack_import.notes"))
+	log.WriteString("=======\r\n\r\n")
+
+	log.WriteString(i18n.T("api.slackimport.slack_import.note1"))
+	log.WriteString(i18n.T("api.slackimport.slack_import.note2"))
+	log.WriteString(i18n.T("api.slackimport.slack_import.note3"))
+
+	return nil, log
+}
+
+func truncateRunes(s string, i int) string {
+	runes := []rune(s)
+	if len(runes) > i {
+		return string(runes[:i])
+	}
+	return s
+}
+
+func (si *SlackImporter) slackAddUsers(teamId string, slackusers []slackUser, importerLog *bytes.Buffer) map[string]*model.User {
+	// Log header
+	importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.created"))
+	importerLog.WriteString("===============\r\n\r\n")
+
+	addedUsers := make(map[string]*model.User)
+
+	// Need the team
+	team, err := si.store.Team().Get(teamId)
+	if err != nil {
+		importerLog.WriteString(i18n.T("api.slackimport.slack_import.team_fail"))
+		return addedUsers
+	}
+
+	for _, sUser := range slackusers {
+		firstName := sUser.Profile.FirstName
+		lastName := sUser.Profile.LastName
+		email := sUser.Profile.Email
+		if email == "" {
+			email = sUser.Username + "@example.com"
+			importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.missing_email_address", map[string]any{"Email": email, "Username": sUser.Username}))
+			mlog.Warn("Slack Import: User does not have an email address in the Slack export. Used username as a placeholder. The user should update their email address once logged in to the system.", mlog.String("user_email", email), mlog.String("user_name", sUser.Username))
+		}
+
+		password := model.NewId()
+
+		// Check for email conflict and use existing user if found
+		if existingUser, err := si.store.User().GetByEmail(email); err == nil {
+			addedUsers[sUser.Id] = existingUser
+			if _, err := si.actions.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil {
+				importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]any{"Email": existingUser.Email, "Username": existingUser.Username}))
+			} else {
+				importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.merge_existing", map[string]any{"Email": existingUser.Email, "Username": existingUser.Username}))
+			}
+			continue
+		}
+
+		email = strings.ToLower(email)
+		newUser := model.User{
+			Username:  sUser.Username,
+			FirstName: firstName,
+			LastName:  lastName,
+			Email:     email,
+			Password:  password,
+		}
+
+		mUser := si.oldImportUser(team, &newUser)
+		if mUser == nil {
+			importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.unable_import", map[string]any{"Username": sUser.Username}))
+			continue
+		}
+		addedUsers[sUser.Id] = mUser
+		importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.email_pwd", map[string]any{"Email": newUser.Email, "Password": password}))
+	}
+
+	return addedUsers
+}
+
+func (si *SlackImporter) slackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
+	team, err := si.store.Team().Get(teamId)
+	if err != nil {
+		log.WriteString(i18n.T("api.slackimport.slack_import.team_fail"))
+		return nil
+	}
+
+	password := model.NewId()
+	username := "slackimportuser_" + model.NewId()
+	email := username + "@localhost"
+
+	botUser := model.User{
+		Username:  username,
+		FirstName: "",
+		LastName:  "",
+		Email:     email,
+		Password:  password,
+	}
+
+	mUser := si.oldImportUser(team, &botUser)
+	if mUser == nil {
+		log.WriteString(i18n.T("api.slackimport.slack_add_bot_user.unable_import", map[string]any{"Username": username}))
+		return nil
+	}
+
+	log.WriteString(i18n.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]any{"Email": botUser.Email, "Password": password}))
+	return mUser
+}
+
+func (si *SlackImporter) slackAddPosts(teamId string, channel *model.Channel, posts []slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
+	sort.Slice(posts, func(i, j int) bool {
+		return slackConvertTimeStamp(posts[i].TimeStamp) < slackConvertTimeStamp(posts[j].TimeStamp)
+	})
+	threads := make(map[string]string)
+	for _, sPost := range posts {
+		switch {
+		case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
+			if sPost.User == "" {
+				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
+				continue
+			}
+			if users[sPost.User] == nil {
+				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
+			}
+			if sPost.Upload {
+				if sPost.File != nil {
+					if fileInfo, ok := si.slackUploadFile(sPost.File, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok {
+						newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
+					}
+				} else if sPost.Files != nil {
+					for _, file := range sPost.Files {
+						if fileInfo, ok := si.slackUploadFile(file, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok {
+							newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
+						}
+					}
+				}
+			}
+			// If post in thread
+			if sPost.ThreadTS != "" && sPost.ThreadTS != sPost.TimeStamp {
+				newPost.RootId = threads[sPost.ThreadTS]
+			}
+			postId := si.oldImportPost(&newPost)
+			// If post is thread starter
+			if sPost.ThreadTS == sPost.TimeStamp {
+				threads[sPost.ThreadTS] = postId
+			}
+		case sPost.Type == "message" && sPost.SubType == "file_comment":
+			if sPost.Comment == nil {
+				mlog.Debug("Slack Import: Unable to import the message as it has no comments.")
+				continue
+			}
+			if sPost.Comment.User == "" {
+				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
+				continue
+			}
+			if users[sPost.Comment.User] == nil {
+				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.Comment.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Comment.Comment,
+				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
+			}
+			si.oldImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "bot_message":
+			if botUser == nil {
+				mlog.Warn("Slack Import: Unable to import the bot message as the bot user does not exist.")
+				continue
+			}
+			if sPost.BotId == "" {
+				mlog.Warn("Slack Import: Unable to import bot message as the BotId field is missing.")
+				continue
+			}
+
+			props := make(model.StringInterface)
+			props["override_username"] = sPost.BotUsername
+			if len(sPost.Attachments) > 0 {
+				props["attachments"] = sPost.Attachments
+			}
+
+			post := &model.Post{
+				UserId:    botUser.Id,
+				ChannelId: channel.Id,
+				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
+				Message:   sPost.Text,
+				Type:      model.PostTypeSlackAttachment,
+			}
+
+			postId := si.oldImportIncomingWebhookPost(post, props)
+			// If post is thread starter
+			if sPost.ThreadTS == sPost.TimeStamp {
+				threads[sPost.ThreadTS] = postId
+			}
+		case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
+			if sPost.User == "" {
+				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
+				continue
+			}
+			if users[sPost.User] == nil {
+				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
+				continue
+			}
+
+			var postType string
+			if sPost.SubType == "channel_join" {
+				postType = model.PostTypeJoinChannel
+			} else {
+				postType = model.PostTypeLeaveChannel
+			}
+
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
+				Type:      postType,
+				Props: model.StringInterface{
+					"username": users[sPost.User].Username,
+				},
+			}
+			si.oldImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "me_message":
+			if sPost.User == "" {
+				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
+				continue
+			}
+			if users[sPost.User] == nil {
+				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   "*" + sPost.Text + "*",
+				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
+			}
+			postId := si.oldImportPost(&newPost)
+			// If post is thread starter
+			if sPost.ThreadTS == sPost.TimeStamp {
+				threads[sPost.ThreadTS] = postId
+			}
+		case sPost.Type == "message" && sPost.SubType == "channel_topic":
+			if sPost.User == "" {
+				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
+				continue
+			}
+			if users[sPost.User] == nil {
+				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
+				Type:      model.PostTypeHeaderChange,
+			}
+			si.oldImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "channel_purpose":
+			if sPost.User == "" {
+				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
+				continue
+			}
+			if users[sPost.User] == nil {
+				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
+				Type:      model.PostTypePurposeChange,
+			}
+			si.oldImportPost(&newPost)
+		case sPost.Type == "message" && sPost.SubType == "channel_name":
+			if sPost.User == "" {
+				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
+				continue
+			}
+			if users[sPost.User] == nil {
+				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
+				continue
+			}
+			newPost := model.Post{
+				UserId:    users[sPost.User].Id,
+				ChannelId: channel.Id,
+				Message:   sPost.Text,
+				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
+				Type:      model.PostTypeDisplaynameChange,
+			}
+			si.oldImportPost(&newPost)
+		default:
+			mlog.Warn(
+				"Slack Import: Unable to import the message as its type is not supported",
+				mlog.String("post_type", sPost.Type),
+				mlog.String("post_subtype", sPost.SubType),
+			)
+		}
+	}
+}
+
+func (si *SlackImporter) slackUploadFile(slackPostFile *slackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) {
+	if slackPostFile == nil {
+		mlog.Warn("Slack Import: Unable to attach the file to the post as the latter has no file section present in Slack export.")
+		return nil, false
+	}
+	file, ok := uploads[slackPostFile.Id]
+	if !ok {
+		mlog.Warn("Slack Import: Unable to import file as the file is missing from the Slack export zip file.", mlog.String("file_id", slackPostFile.Id))
+		return nil, false
+	}
+	openFile, err := file.Open()
+	if err != nil {
+		mlog.Warn("Slack Import: Unable to open the file from the Slack export.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err))
+		return nil, false
+	}
+	defer openFile.Close()
+
+	timestamp := utils.TimeFromMillis(slackConvertTimeStamp(slackTimestamp))
+	uploadedFile, err := si.oldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name))
+	if err != nil {
+		mlog.Warn("Slack Import: An error occurred when uploading file.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err))
+		return nil, false
+	}
+
+	return uploadedFile, true
+}
+
+func (si *SlackImporter) deactivateSlackBotUser(user *model.User) {
+	if _, err := si.actions.UpdateActive(user, false); err != nil {
+		mlog.Warn("Slack Import: Unable to deactivate the user account used for the bot.")
+	}
+}
+
+func (si *SlackImporter) addSlackUsersToChannel(c request.CTX, members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
+	for _, member := range members {
+		user, ok := users[member]
+		if !ok {
+			log.WriteString(i18n.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]any{"Username": "?"}))
+			continue
+		}
+		if _, err := si.actions.AddUserToChannel(c, user, channel, false); err != nil {
+			log.WriteString(i18n.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]any{"Username": user.Username}))
+		}
+	}
+}
+
+func slackSanitiseChannelProperties(channel model.Channel) model.Channel {
+	if utf8.RuneCountInString(channel.DisplayName) > model.ChannelDisplayNameMaxRunes {
+		mlog.Warn("Slack Import: Channel display name exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName))
+		channel.DisplayName = truncateRunes(channel.DisplayName, model.ChannelDisplayNameMaxRunes)
+	}
+
+	if len(channel.Name) > model.ChannelNameMaxLength {
+		mlog.Warn("Slack Import: Channel handle exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName))
+		channel.Name = channel.Name[0:model.ChannelNameMaxLength]
+	}
+
+	if utf8.RuneCountInString(channel.Purpose) > model.ChannelPurposeMaxRunes {
+		mlog.Warn("Slack Import: Channel purpose exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName))
+		channel.Purpose = truncateRunes(channel.Purpose, model.ChannelPurposeMaxRunes)
+	}
+
+	if utf8.RuneCountInString(channel.Header) > model.ChannelHeaderMaxRunes {
+		mlog.Warn("Slack Import: Channel header exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName))
+		channel.Header = truncateRunes(channel.Header, model.ChannelHeaderMaxRunes)
+	}
+
+	return channel
+}
+
+func (si *SlackImporter) slackAddChannels(c request.CTX, teamId string, slackchannels []slackChannel, posts map[string][]slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel {
+	// Write Header
+	importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.added"))
+	importerLog.WriteString("=================\r\n\r\n")
+
+	addedChannels := make(map[string]*model.Channel)
+	for _, sChannel := range slackchannels {
+		newChannel := model.Channel{
+			TeamId:      teamId,
+			Type:        sChannel.Type,
+			DisplayName: sChannel.Name,
+			Name:        slackConvertChannelName(sChannel.Name, sChannel.Id),
+			Purpose:     sChannel.Purpose.Value,
+			Header:      sChannel.Topic.Value,
+		}
+
+		// Direct message channels in Slack don't have a name so we set the id as name or else the messages won't get imported.
+		if newChannel.Type == model.ChannelTypeDirect {
+			sChannel.Name = sChannel.Id
+		}
+
+		newChannel = slackSanitiseChannelProperties(newChannel)
+
+		var mChannel *model.Channel
+		var err error
+		if mChannel, err = si.store.Channel().GetByName(teamId, sChannel.Name, true); err == nil {
+			// The channel already exists as an active channel. Merge with the existing one.
+			importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.merge", map[string]any{"DisplayName": newChannel.DisplayName}))
+		} else if _, nErr := si.store.Channel().GetDeletedByName(teamId, sChannel.Name); nErr == nil {
+			// The channel already exists but has been deleted. Generate a random string for the handle instead.
+			newChannel.Name = model.NewId()
+			newChannel = slackSanitiseChannelProperties(newChannel)
+		}
+
+		if mChannel == nil {
+			// Haven't found an existing channel to merge with. Try importing it as a new one.
+			mChannel = si.oldImportChannel(c, &newChannel, sChannel, users)
+			if mChannel == nil {
+				mlog.Warn("Slack Import: Unable to import Slack channel.", mlog.String("channel_display_name", newChannel.DisplayName))
+				importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.import_failed", map[string]any{"DisplayName": newChannel.DisplayName}))
+				continue
+			}
+		}
+
+		// Members for direct and group channels are added during the creation of the channel in the oldImportChannel function
+		if sChannel.Type == model.ChannelTypeOpen || sChannel.Type == model.ChannelTypePrivate {
+			si.addSlackUsersToChannel(c, sChannel.Members, users, mChannel, importerLog)
+		}
+		importerLog.WriteString(newChannel.DisplayName + "\r\n")
+		addedChannels[sChannel.Id] = mChannel
+		si.slackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser)
+	}
+
+	return addedChannels
+}
+
+//
+// -- Old SlackImport Functions --
+// Import functions are suitable for entering posts and users into the database without
+// some of the usual checks. (IsValid is still run)
+//
+
+func (si *SlackImporter) oldImportPost(post *model.Post) string {
+	// Workaround for empty messages, which may be the case if they are webhook posts.
+	firstIteration := true
+	firstPostId := ""
+	if post.RootId != "" {
+		firstPostId = post.RootId
+	}
+	maxPostSize := si.actions.MaxPostSize()
+	for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) {
+		var remainder string
+		if messageRuneCount > maxPostSize {
+			remainder = string(([]rune(post.Message))[maxPostSize:])
+			post.Message = truncateRunes(post.Message, maxPostSize)
+		} else {
+			remainder = ""
+		}
+
+		post.Hashtags, _ = model.ParseHashtags(post.Message)
+
+		post.RootId = firstPostId
+
+		_, err := si.store.Post().Save(post)
+		if err != nil {
+			mlog.Debug("Error saving post.", mlog.String("user_id", post.UserId), mlog.String("message", post.Message))
+		}
+
+		if firstIteration {
+			if firstPostId == "" {
+				firstPostId = post.Id
+			}
+			for _, fileId := range post.FileIds {
+				if err := si.store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); err != nil {
+					mlog.Error(
+						"Error attaching files to post.",
+						mlog.String("post_id", post.Id),
+						mlog.String("file_ids", strings.Join(post.FileIds, ",")),
+						mlog.String("user_id", post.UserId),
+						mlog.Err(err),
+					)
+				}
+			}
+			post.FileIds = nil
+		}
+
+		post.Id = ""
+		post.CreateAt++
+		post.Message = remainder
+		firstIteration = false
+	}
+	return firstPostId
+}
+
+func (si *SlackImporter) oldImportUser(team *model.Team, user *model.User) *model.User {
+	user.MakeNonNil()
+
+	user.Roles = model.SystemUserRoleId
+
+	ruser, nErr := si.store.User().Save(user)
+	if nErr != nil {
+		mlog.Debug("Error saving user.", mlog.Err(nErr))
+		return nil
+	}
+
+	if _, err := si.store.User().VerifyEmail(ruser.Id, ruser.Email); err != nil {
+		mlog.Warn("Failed to set email verified.", mlog.Err(err))
+	}
+
+	if _, err := si.actions.JoinUserToTeam(team, user, ""); err != nil {
+		mlog.Warn("Failed to join team when importing.", mlog.Err(err))
+	}
+
+	return ruser
+}
+
+func (si *SlackImporter) oldImportChannel(c request.CTX, channel *model.Channel, sChannel slackChannel, users map[string]*model.User) *model.Channel {
+	switch {
+	case channel.Type == model.ChannelTypeDirect:
+		if len(sChannel.Members) < 2 {
+			return nil
+		}
+		u1 := users[sChannel.Members[0]]
+		u2 := users[sChannel.Members[1]]
+		if u1 == nil || u2 == nil {
+			mlog.Warn("Either or both of user ids not found in users.json. Ignoring.", mlog.String("id1", sChannel.Members[0]), mlog.String("id2", sChannel.Members[1]))
+			return nil
+		}
+		sc, err := si.actions.CreateDirectChannel(c, u1.Id, u2.Id)
+		if err != nil {
+			return nil
+		}
+
+		return sc
+	// check if direct channel has less than 8 members and if not import as private channel instead
+	case channel.Type == model.ChannelTypeGroup && len(sChannel.Members) < 8:
+		members := make([]string, len(sChannel.Members))
+
+		for i := range sChannel.Members {
+			u := users[sChannel.Members[i]]
+			if u == nil {
+				mlog.Warn("User not found in users.json. Ignoring.", mlog.String("id", sChannel.Members[i]))
+				continue
+			}
+			members[i] = u.Id
+		}
+
+		creator := users[sChannel.Creator]
+		if creator == nil {
+			return nil
+		}
+		sc, err := si.actions.CreateGroupChannel(c, members)
+		if err != nil {
+			return nil
+		}
+
+		return sc
+	case channel.Type == model.ChannelTypeGroup:
+		channel.Type = model.ChannelTypePrivate
+		sc, err := si.actions.CreateChannel(channel, false)
+		if err != nil {
+			return nil
+		}
+
+		return sc
+	}
+
+	sc, err := si.store.Channel().Save(channel, *si.config.TeamSettings.MaxChannelsPerTeam)
+	if err != nil {
+		return nil
+	}
+
+	return sc
+}
+
+func (si *SlackImporter) oldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
+	buf := bytes.NewBuffer(nil)
+	io.Copy(buf, file)
+	data := buf.Bytes()
+
+	fileInfo, err := si.actions.DoUploadFile(timestamp, teamId, channelId, userId, fileName, data)
+	if err != nil {
+		return nil, err
+	}
+
+	if fileInfo.IsImage() && !fileInfo.IsSvg() {
+		img, imgType, release, err := si.actions.PrepareImage(data)
+		if err != nil {
+			return nil, err
+		}
+		defer release()
+		si.actions.GenerateThumbnailImage(img, imgType, fileInfo.ThumbnailPath)
+		si.actions.GeneratePreviewImage(img, imgType, fileInfo.PreviewPath)
+	}
+
+	return fileInfo, nil
+}
+
+func (si *SlackImporter) oldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) string {
+	linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`)
+	post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})")
+
+	post.AddProp("from_webhook", "true")
+
+	if _, ok := props["override_username"]; !ok {
+		post.AddProp("override_username", model.DefaultWebhookUsername)
+	}
+
+	if len(props) > 0 {
+		for key, val := range props {
+			if key == "attachments" {
+				if attachments, success := val.([]*model.SlackAttachment); success {
+					model.ParseSlackAttachment(post, attachments)
+				}
+			} else if key != "from_webhook" {
+				post.AddProp(key, val)
+			}
+		}
+	}
+
+	return si.oldImportPost(post)
+}

server/platform/services/slackimport/slackimport_test.go

@@ -0,0 +1,404 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package slackimport
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/app/request"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+func TestSlackConvertTimeStamp(t *testing.T) {
+	assert.EqualValues(t, slackConvertTimeStamp("1469785419.000033"), 1469785419000)
+}
+
+func TestSlackConvertChannelName(t *testing.T) {
+	for _, tc := range []struct {
+		nameInput string
+		idInput   string
+		output    string
+	}{
+		{"test-channel", "C0G08DLQH", "test-channel"},
+		{"_test_channel_", "C0G04DLQH", "test_channel"},
+		{"__test", "C0G07DLQH", "test"},
+		{"-t", "C0G06DLQH", "slack-channel-t"},
+		{"a", "C0G05DLQH", "slack-channel-a"},
+		{"случайный", "C0G05DLQD", "c0g05dlqd"},
+	} {
+		assert.Equal(t, slackConvertChannelName(tc.nameInput, tc.idInput), tc.output, "nameInput = %v", tc.nameInput)
+	}
+}
+
+func TestSlackConvertUserMentions(t *testing.T) {
+	users := []slackUser{
+		{Id: "U00000A0A", Username: "firstuser"},
+		{Id: "U00000B1B", Username: "seconduser"},
+	}
+
+	posts := map[string][]slackPost{
+		"test-channel": {
+			{
+				Text: "<!channel>: Hi guys.",
+			},
+			{
+				Text: "Calling <!here|@here>.",
+			},
+			{
+				Text: "Yo <!everyone>.",
+			},
+			{
+				Text: "Regular user test <@U00000B1B|seconduser> and <@U00000A0A>.",
+			},
+		},
+	}
+
+	expectedPosts := map[string][]slackPost{
+		"test-channel": {
+			{
+				Text: "@channel: Hi guys.",
+			},
+			{
+				Text: "Calling @here.",
+			},
+			{
+				Text: "Yo @all.",
+			},
+			{
+				Text: "Regular user test @seconduser and @firstuser.",
+			},
+		},
+	}
+
+	assert.Equal(t, expectedPosts, slackConvertUserMentions(users, posts))
+}
+
+func TestSlackConvertChannelMentions(t *testing.T) {
+	channels := []slackChannel{
+		{Id: "C000AA00A", Name: "one"},
+		{Id: "C000BB11B", Name: "two"},
+	}
+
+	posts := map[string][]slackPost{
+		"test-channel": {
+			{
+				Text: "Go to <#C000AA00A>.",
+			},
+			{
+				User: "U00000A0A",
+				Text: "Try <#C000BB11B|two> for this.",
+			},
+		},
+	}
+
+	expectedPosts := map[string][]slackPost{
+		"test-channel": {
+			{
+				Text: "Go to ~one.",
+			},
+			{
+				User: "U00000A0A",
+				Text: "Try ~two for this.",
+			},
+		},
+	}
+
+	assert.Equal(t, expectedPosts, slackConvertChannelMentions(channels, posts))
+}
+
+func openTestFile(t *testing.T, filename string) (*os.File, error) {
+	working, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	t.Log("working directory:", working)
+
+	path := filepath.Join("../tests", filename)
+	return os.Open(path)
+}
+
+func TestSlackParseChannels(t *testing.T) {
+	file, err := openTestFile(t, "slack-import-test-channels.json")
+	require.NoError(t, err)
+	defer file.Close()
+
+	channels, err := slackParseChannels(file, model.ChannelTypeOpen)
+	require.NoError(t, err)
+	assert.Equal(t, 6, len(channels))
+}
+
+func TestSlackParseDirectMessages(t *testing.T) {
+	file, err := openTestFile(t, "slack-import-test-direct-messages.json")
+	require.NoError(t, err)
+	defer file.Close()
+
+	channels, err := slackParseChannels(file, model.ChannelTypeDirect)
+	require.NoError(t, err)
+	assert.Equal(t, 4, len(channels))
+}
+
+func TestSlackParsePrivateChannels(t *testing.T) {
+	file, err := openTestFile(t, "slack-import-test-private-channels.json")
+	require.NoError(t, err)
+	defer file.Close()
+
+	channels, err := slackParseChannels(file, model.ChannelTypePrivate)
+	require.NoError(t, err)
+	assert.Equal(t, 1, len(channels))
+}
+
+func TestSlackParseGroupDirectMessages(t *testing.T) {
+	file, err := openTestFile(t, "slack-import-test-group-direct-messages.json")
+	require.NoError(t, err)
+	defer file.Close()
+
+	channels, err := slackParseChannels(file, model.ChannelTypeGroup)
+	require.NoError(t, err)
+	assert.Equal(t, 3, len(channels))
+}
+
+func TestSlackParseUsers(t *testing.T) {
+	file, err := openTestFile(t, "slack-import-test-users.json")
+	require.NoError(t, err)
+	defer file.Close()
+
+	users, err := slackParseUsers(file)
+	require.NoError(t, err)
+	assert.Equal(t, 11, len(users))
+}
+
+func TestSlackParsePosts(t *testing.T) {
+	file, err := openTestFile(t, "slack-import-test-posts.json")
+	require.NoError(t, err)
+	defer file.Close()
+
+	posts, err := slackParsePosts(file)
+	require.NoError(t, err)
+	assert.Equal(t, 9, len(posts))
+}
+
+func TestSlackParseMultipleAttachments(t *testing.T) {
+	file, err := openTestFile(t, "slack-import-test-posts.json")
+	require.NoError(t, err)
+	defer file.Close()
+
+	posts, err := slackParsePosts(file)
+	require.NoError(t, err)
+	assert.Equal(t, 2, len(posts[8].Files))
+}
+
+func TestSlackSanitiseChannelProperties(t *testing.T) {
+	c1 := model.Channel{
+		DisplayName: "display-name",
+		Name:        "name",
+		Purpose:     "The channel purpose",
+		Header:      "The channel header",
+	}
+
+	c1s := slackSanitiseChannelProperties(c1)
+	assert.Equal(t, c1, c1s)
+
+	c2 := model.Channel{
+		DisplayName: strings.Repeat("abcdefghij", 7),
+		Name:        strings.Repeat("abcdefghij", 7),
+		Purpose:     strings.Repeat("0123456789", 30),
+		Header:      strings.Repeat("0123456789", 120),
+	}
+
+	c2s := slackSanitiseChannelProperties(c2)
+	assert.Equal(t, model.Channel{
+		DisplayName: strings.Repeat("abcdefghij", 6) + "abcd",
+		Name:        strings.Repeat("abcdefghij", 6) + "abcd",
+		Purpose:     strings.Repeat("0123456789", 25),
+		Header:      strings.Repeat("0123456789", 102) + "0123",
+	}, c2s)
+}
+
+func TestSlackConvertPostsMarkup(t *testing.T) {
+	input := make(map[string][]slackPost)
+	input["test"] = []slackPost{
+		{
+			Text: "This message contains a link to <https://google.com|Google>.",
+		},
+		{
+			Text: "This message contains a mailto link to <mailto:me@example.com|me@example.com> in it.",
+		},
+		{
+			Text: "This message contains a *bold* word.",
+		},
+		{
+			Text: "This is not a * bold * word.",
+		},
+		{
+			Text: `There is *no bold word
+in this*.`,
+		},
+		{
+			Text: "*This* is not a*bold* word.*This* is a bold word, *and* this; *and* this too.",
+		},
+		{
+			Text: "This message contains a ~strikethrough~ word.",
+		},
+		{
+			Text: "This is not a ~ strikethrough ~ word.",
+		},
+		{
+			Text: `There is ~no strikethrough word
+in this~.`,
+		},
+		{
+			Text: "~This~ is not a~strikethrough~ word.~This~ is a strikethrough word, ~and~ this; ~and~ this too.",
+		},
+		{
+			Text: `This message contains multiple paragraphs blockquotes
+&gt;&gt;&gt;first
+second
+third`,
+		},
+		{
+			Text: `This message contains single paragraph blockquotes
+&gt;something
+&gt;another thing`,
+		},
+		{
+			Text: "This message has no > block quote",
+		},
+	}
+
+	expectedOutput := make(map[string][]slackPost)
+	expectedOutput["test"] = []slackPost{
+		{
+			Text: "This message contains a link to [Google](https://google.com).",
+		},
+		{
+			Text: "This message contains a mailto link to [me@example.com](mailto:me@example.com) in it.",
+		},
+		{
+			Text: "This message contains a **bold** word.",
+		},
+		{
+			Text: "This is not a * bold * word.",
+		},
+		{
+			Text: `There is *no bold word
+in this*.`,
+		},
+		{
+			Text: "**This** is not a*bold* word.**This** is a bold word, **and** this; **and** this too.",
+		},
+		{
+			Text: "This message contains a ~~strikethrough~~ word.",
+		},
+		{
+			Text: "This is not a ~ strikethrough ~ word.",
+		},
+		{
+			Text: `There is ~no strikethrough word
+in this~.`,
+		},
+		{
+			Text: "~~This~~ is not a~strikethrough~ word.~~This~~ is a strikethrough word, ~~and~~ this; ~~and~~ this too.",
+		},
+		{
+			Text: `This message contains multiple paragraphs blockquotes
+>first
+>second
+>third`,
+		},
+		{
+			Text: `This message contains single paragraph blockquotes
+>something
+>another thing`,
+		},
+		{
+			Text: "This message has no > block quote",
+		},
+	}
+
+	assert.Equal(t, expectedOutput, slackConvertPostsMarkup(input))
+}
+
+func TestOldImportChannel(t *testing.T) {
+	u1 := &model.User{
+		Id:       model.NewId(),
+		Username: "test-user-1",
+	}
+	u2 := &model.User{
+		Id:       model.NewId(),
+		Username: "test-user-2",
+	}
+	store := &mocks.Store{}
+	config := &model.Config{}
+	config.SetDefaults()
+	ctx := request.EmptyContext(nil)
+	ctx.SetLogger(mlog.CreateConsoleTestLogger(true, mlog.LvlDebug))
+
+	t.Run("No panic on direct channel", func(t *testing.T) {
+		// ch := th.CreateDmChannel(u1)
+		ch := &model.Channel{
+			Type: model.ChannelTypeDirect,
+			Name: "test-channel",
+		}
+		users := map[string]*model.User{
+			u2.Id: u2,
+		}
+		sCh := slackChannel{
+			Id:      "someid",
+			Members: []string{u1.Id, "randomID"},
+			Creator: "randomID2",
+		}
+
+		actions := Actions{}
+
+		importer := New(store, actions, config)
+		_ = importer.oldImportChannel(ctx, ch, sCh, users)
+	})
+
+	t.Run("No panic on direct channel with 1 member", func(t *testing.T) {
+		ch := &model.Channel{
+			Type: model.ChannelTypeDirect,
+			Name: "test-channel",
+		}
+		users := map[string]*model.User{
+			u1.Id: u1,
+		}
+		sCh := slackChannel{
+			Id:      "someid",
+			Members: []string{u1.Id},
+			Creator: "randomID2",
+		}
+
+		actions := Actions{}
+
+		importer := New(store, actions, config)
+		_ = importer.oldImportChannel(ctx, ch, sCh, users)
+	})
+
+	t.Run("No panic on group channel", func(t *testing.T) {
+		ch := &model.Channel{
+			Type: model.ChannelTypeGroup,
+			Name: "test-channel",
+		}
+		users := map[string]*model.User{
+			u1.Id: u1,
+		}
+		sCh := slackChannel{
+			Id:      "someid",
+			Members: []string{u1.Id},
+			Creator: "randomID2",
+		}
+		actions := Actions{}
+
+		importer := New(store, actions, config)
+		_ = importer.oldImportChannel(ctx, ch, sCh, users)
+	})
+}

server/platform/services/telemetry/mocks/ServerIface.go

@@ -0,0 +1,167 @@
+// Code generated by mockery v2.10.4. DO NOT EDIT.
+
+// Regenerate this file using `make telemetry-mocks`.
+
+package mocks
+
+import (
+	context "context"
+
+	httpservice "github.com/mattermost/mattermost-server/v6/server/platform/services/httpservice"
+	mock "github.com/stretchr/testify/mock"
+
+	model "github.com/mattermost/mattermost-server/v6/model"
+
+	plugin "github.com/mattermost/mattermost-server/v6/plugin"
+
+	product "github.com/mattermost/mattermost-server/v6/server/channels/product"
+)
+
+// ServerIface is an autogenerated mock type for the ServerIface type
+type ServerIface struct {
+	mock.Mock
+}
+
+// Config provides a mock function with given fields:
+func (_m *ServerIface) Config() *model.Config {
+	ret := _m.Called()
+
+	var r0 *model.Config
+	if rf, ok := ret.Get(0).(func() *model.Config); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Config)
+		}
+	}
+
+	return r0
+}
+
+// GetPluginsEnvironment provides a mock function with given fields:
+func (_m *ServerIface) GetPluginsEnvironment() *plugin.Environment {
+	ret := _m.Called()
+
+	var r0 *plugin.Environment
+	if rf, ok := ret.Get(0).(func() *plugin.Environment); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*plugin.Environment)
+		}
+	}
+
+	return r0
+}
+
+// GetRoleByName provides a mock function with given fields: _a0, _a1
+func (_m *ServerIface) GetRoleByName(_a0 context.Context, _a1 string) (*model.Role, *model.AppError) {
+	ret := _m.Called(_a0, _a1)
+
+	var r0 *model.Role
+	if rf, ok := ret.Get(0).(func(context.Context, string) *model.Role); ok {
+		r0 = rf(_a0, _a1)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Role)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(context.Context, string) *model.AppError); ok {
+		r1 = rf(_a0, _a1)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// GetSchemes provides a mock function with given fields: _a0, _a1, _a2
+func (_m *ServerIface) GetSchemes(_a0 string, _a1 int, _a2 int) ([]*model.Scheme, *model.AppError) {
+	ret := _m.Called(_a0, _a1, _a2)
+
+	var r0 []*model.Scheme
+	if rf, ok := ret.Get(0).(func(string, int, int) []*model.Scheme); ok {
+		r0 = rf(_a0, _a1, _a2)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).([]*model.Scheme)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, int, int) *model.AppError); ok {
+		r1 = rf(_a0, _a1, _a2)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// HTTPService provides a mock function with given fields:
+func (_m *ServerIface) HTTPService() httpservice.HTTPService {
+	ret := _m.Called()
+
+	var r0 httpservice.HTTPService
+	if rf, ok := ret.Get(0).(func() httpservice.HTTPService); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(httpservice.HTTPService)
+		}
+	}
+
+	return r0
+}
+
+// HooksManager provides a mock function with given fields:
+func (_m *ServerIface) HooksManager() *product.HooksManager {
+	ret := _m.Called()
+
+	var r0 *product.HooksManager
+	if rf, ok := ret.Get(0).(func() *product.HooksManager); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*product.HooksManager)
+		}
+	}
+
+	return r0
+}
+
+// IsLeader provides a mock function with given fields:
+func (_m *ServerIface) IsLeader() bool {
+	ret := _m.Called()
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func() bool); ok {
+		r0 = rf()
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// License provides a mock function with given fields:
+func (_m *ServerIface) License() *model.License {
+	ret := _m.Called()
+
+	var r0 *model.License
+	if rf, ok := ret.Get(0).(func() *model.License); ok {
+		r0 = rf()
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.License)
+		}
+	}
+
+	return r0
+}

server/platform/services/telemetry/telemetry_test.go

@@ -0,0 +1,725 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package telemetry
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin"
+	"github.com/mattermost/mattermost-server/v6/plugin/plugintest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/product"
+	storeMocks "github.com/mattermost/mattermost-server/v6/server/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v6/server/config"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/httpservice"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/telemetry/mocks"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type FakeConfigService struct {
+	cfg *model.Config
+}
+
+type testTelemetryPayload struct {
+	MessageId string
+	SentAt    time.Time
+	Batch     []struct {
+		MessageId  string
+		UserId     string
+		Event      string
+		Timestamp  time.Time
+		Properties map[string]any
+	}
+	Context struct {
+		Library struct {
+			Name    string
+			Version string
+		}
+	}
+}
+
+type testBatch struct {
+	MessageId  string
+	UserId     string
+	Event      string
+	Timestamp  time.Time
+	Properties map[string]any
+}
+
+func assertPayload(t *testing.T, actual testTelemetryPayload, event string, properties map[string]any) {
+	t.Helper()
+	assert.NotEmpty(t, actual.MessageId)
+	assert.False(t, actual.SentAt.IsZero())
+	if assert.Len(t, actual.Batch, 1) {
+		assert.NotEmpty(t, actual.Batch[0].MessageId, "message id should not be empty")
+		assert.Equal(t, testTelemetryID, actual.Batch[0].UserId)
+		if event != "" {
+			assert.Equal(t, event, actual.Batch[0].Event)
+		}
+		assert.False(t, actual.Batch[0].Timestamp.IsZero(), "batch timestamp should not be the zero value")
+		if properties != nil {
+			assert.Equal(t, properties, actual.Batch[0].Properties)
+		}
+	}
+	assert.Equal(t, "analytics-go", actual.Context.Library.Name)
+	assert.Equal(t, "3.3.0", actual.Context.Library.Version)
+}
+
+func collectBatches(t *testing.T, info *[]testBatch, pchan chan testTelemetryPayload) {
+	t.Helper()
+	for {
+		select {
+		case result := <-pchan:
+			assertPayload(t, result, "", nil)
+			*info = append(*info, result.Batch[0])
+		case <-time.After(time.Second * 1):
+			return
+		}
+	}
+}
+
+func makeTelemetryServiceAndReceiver(t *testing.T, cloudLicense bool) (*TelemetryService, chan testTelemetryPayload, *model.Config, func()) {
+
+	cfg := &model.Config{}
+	cfg.SetDefaults()
+	serverIfaceMock, storeMock, deferredAssertions, cleanUp := initializeMocks(cfg, cloudLicense)
+
+	testLogger, _ := mlog.NewLogger()
+	logCfg, _ := config.MloggerConfigFromLoggerConfig(&cfg.LogSettings, nil, config.GetLogFileLocation)
+	if errCfg := testLogger.ConfigureTargets(logCfg, nil); errCfg != nil {
+		panic("failed to configure test logger: " + errCfg.Error())
+	}
+
+	pchan := make(chan testTelemetryPayload, 100)
+	receiver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.NoError(t, err)
+
+		var p testTelemetryPayload
+		err = json.Unmarshal(body, &p)
+		require.NoError(t, err)
+
+		pchan <- p
+	}))
+
+	service, err := New(serverIfaceMock, storeMock, searchengine.NewBroker(cfg), testLogger, false)
+	require.NoError(t, err)
+
+	service.TelemetryID = testTelemetryID
+	service.rudderClient = nil
+	service.initRudder(receiver.URL, RudderKey)
+
+	// initializing rudder send a client identify message
+	select {
+	case identifyMessage := <-pchan:
+		assertPayload(t, identifyMessage, "", nil)
+	case <-time.After(time.Second * 1):
+		require.Fail(t, "Did not receive ID message")
+	}
+
+	return service, pchan, cfg, func() {
+		receiver.Close()
+		testLogger.Shutdown()
+		cleanUp()
+		deferredAssertions(t)
+	}
+}
+
+const testTelemetryID = "test-telemetry-id-12345"
+
+func (fcs *FakeConfigService) Config() *model.Config                                       { return fcs.cfg }
+func (fcs *FakeConfigService) AddConfigListener(f func(old, current *model.Config)) string { return "" }
+func (fcs *FakeConfigService) RemoveConfigListener(key string)                             {}
+func (fcs *FakeConfigService) AsymmetricSigningKey() *ecdsa.PrivateKey                     { return nil }
+
+func initializeMocks(cfg *model.Config, cloudLicense bool) (*mocks.ServerIface, *storeMocks.Store, func(t *testing.T), func()) {
+	serverIfaceMock := &mocks.ServerIface{}
+	logger, _ := mlog.NewLogger()
+
+	configService := &FakeConfigService{cfg}
+	serverIfaceMock.On("Config").Return(cfg)
+	serverIfaceMock.On("IsLeader").Return(true)
+
+	pluginDir, _ := os.MkdirTemp("", "")
+	webappPluginDir, _ := os.MkdirTemp("", "")
+	cleanUp := func() {
+		os.RemoveAll(pluginDir)
+		os.RemoveAll(webappPluginDir)
+	}
+	pluginsAPIMock := &plugintest.API{}
+	pluginEnv, _ := plugin.NewEnvironment(
+		func(m *model.Manifest) plugin.API { return pluginsAPIMock },
+		nil,
+		pluginDir, webappPluginDir,
+		false,
+		logger,
+		nil)
+	serverIfaceMock.On("GetPluginsEnvironment").Return(pluginEnv, nil)
+
+	if cloudLicense {
+		serverIfaceMock.On("License").Return(model.NewTestLicense("cloud"), nil)
+	} else {
+		serverIfaceMock.On("License").Return(model.NewTestLicense(), nil)
+	}
+	serverIfaceMock.On("GetRoleByName", context.Background(), "system_admin").Return(&model.Role{Permissions: []string{"sa-test1", "sa-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "system_user").Return(&model.Role{Permissions: []string{"su-test1", "su-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "system_user_manager").Return(&model.Role{Permissions: []string{"sum-test1", "sum-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "system_manager").Return(&model.Role{Permissions: []string{"sm-test1", "sm-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "system_read_only_admin").Return(&model.Role{Permissions: []string{"sra-test1", "sra-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "system_custom_group_admin").Return(&model.Role{Permissions: []string{"scga-test1", "scga-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "team_admin").Return(&model.Role{Permissions: []string{"ta-test1", "ta-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "team_user").Return(&model.Role{Permissions: []string{"tu-test1", "tu-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "team_guest").Return(&model.Role{Permissions: []string{"tg-test1", "tg-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "channel_admin").Return(&model.Role{Permissions: []string{"ca-test1", "ca-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "channel_user").Return(&model.Role{Permissions: []string{"cu-test1", "cu-test2"}}, nil)
+	serverIfaceMock.On("GetRoleByName", context.Background(), "channel_guest").Return(&model.Role{Permissions: []string{"cg-test1", "cg-test2"}}, nil)
+	serverIfaceMock.On("GetSchemes", "team", 0, 100).Return([]*model.Scheme{}, nil)
+	serverIfaceMock.On("HTTPService").Return(httpservice.MakeHTTPService(configService))
+	serverIfaceMock.On("HooksManager").Return(product.NewHooksManager(nil))
+
+	storeMock := &storeMocks.Store{}
+	storeMock.On("GetDbVersion", false).Return("5.24.0", nil)
+
+	systemStore := storeMocks.SystemStore{}
+	systemStore.On("Get").Return(make(model.StringMap), nil)
+	systemID := &model.System{Name: model.SystemTelemetryId, Value: "test"}
+	systemStore.On("InsertIfExists", mock.Anything).Return(systemID, nil)
+	systemStore.On("GetByName", model.AdvancedPermissionsMigrationKey).Return(nil, nil)
+	systemStore.On("GetByName", model.MigrationKeyAdvancedPermissionsPhase2).Return(nil, nil)
+
+	userStore := storeMocks.UserStore{}
+	userStore.On("Count", model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: true, ExcludeRegularUsers: false, TeamId: "", ViewRestrictions: nil}).Return(int64(10), nil)
+	userStore.On("Count", model.UserCountOptions{IncludeBotAccounts: true, IncludeDeleted: false, ExcludeRegularUsers: true, TeamId: "", ViewRestrictions: nil}).Return(int64(100), nil)
+	userStore.On("Count", model.UserCountOptions{Roles: []string{model.SystemManagerRoleId}}).Return(int64(5), nil)
+	userStore.On("Count", model.UserCountOptions{Roles: []string{model.SystemUserManagerRoleId}}).Return(int64(10), nil)
+	userStore.On("Count", model.UserCountOptions{Roles: []string{model.SystemReadOnlyAdminRoleId}}).Return(int64(15), nil)
+	userStore.On("Count", model.UserCountOptions{Roles: []string{model.SystemCustomGroupAdminRoleId}}).Return(int64(15), nil)
+	userStore.On("AnalyticsGetGuestCount").Return(int64(11), nil)
+	userStore.On("AnalyticsActiveCount", mock.Anything, model.UserCountOptions{IncludeBotAccounts: false, IncludeDeleted: false, ExcludeRegularUsers: false, TeamId: "", ViewRestrictions: nil}).Return(int64(5), nil)
+	userStore.On("AnalyticsGetInactiveUsersCount").Return(int64(8), nil)
+	userStore.On("AnalyticsGetSystemAdminCount").Return(int64(9), nil)
+
+	teamStore := storeMocks.TeamStore{}
+	teamStore.On("AnalyticsTeamCount", (*model.TeamSearch)(nil)).Return(int64(3), nil)
+	teamStore.On("GroupSyncedTeamCount").Return(int64(16), nil)
+
+	channelStore := storeMocks.ChannelStore{}
+	channelStore.On("AnalyticsTypeCount", "", model.ChannelTypeOpen).Return(int64(25), nil)
+	channelStore.On("AnalyticsTypeCount", "", model.ChannelTypePrivate).Return(int64(26), nil)
+	channelStore.On("AnalyticsTypeCount", "", model.ChannelTypeDirect).Return(int64(27), nil)
+	channelStore.On("AnalyticsDeletedTypeCount", "", model.ChannelTypeOpen).Return(int64(22), nil)
+	channelStore.On("AnalyticsDeletedTypeCount", "", model.ChannelTypePrivate).Return(int64(23), nil)
+	channelStore.On("GroupSyncedChannelCount").Return(int64(17), nil)
+
+	postStore := storeMocks.PostStore{}
+	postStore.On("AnalyticsPostCount", &model.PostCountOptions{}).Return(int64(1000), nil)
+	postStore.On("AnalyticsPostCountsByDay", &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: false, YesterdayOnly: true}).Return(model.AnalyticsRows{}, nil)
+	postStore.On("AnalyticsPostCountsByDay", &model.AnalyticsPostCountsOptions{TeamId: "", BotsOnly: true, YesterdayOnly: true}).Return(model.AnalyticsRows{}, nil)
+
+	commandStore := storeMocks.CommandStore{}
+	commandStore.On("AnalyticsCommandCount", "").Return(int64(15), nil)
+
+	webhookStore := storeMocks.WebhookStore{}
+	webhookStore.On("AnalyticsIncomingCount", "").Return(int64(16), nil)
+	webhookStore.On("AnalyticsOutgoingCount", "").Return(int64(17), nil)
+
+	groupStore := storeMocks.GroupStore{}
+	groupStore.On("GroupCount").Return(int64(25), nil)
+	groupStore.On("GroupTeamCount").Return(int64(26), nil)
+	groupStore.On("GroupChannelCount").Return(int64(27), nil)
+	groupStore.On("GroupMemberCount").Return(int64(32), nil)
+	groupStore.On("DistinctGroupMemberCount").Return(int64(22), nil)
+	groupStore.On("GroupCountWithAllowReference").Return(int64(13), nil)
+	groupStore.On("GroupCountBySource", model.GroupSourceCustom).Return(int64(10), nil)
+	groupStore.On("GroupCountBySource", model.GroupSourceLdap).Return(int64(2), nil)
+	groupStore.On("DistinctGroupMemberCountForSource", mock.AnythingOfType("model.GroupSource")).Return(int64(1), nil)
+
+	schemeStore := storeMocks.SchemeStore{}
+	schemeStore.On("CountByScope", "channel").Return(int64(8), nil)
+	schemeStore.On("CountByScope", "team").Return(int64(7), nil)
+	schemeStore.On("CountWithoutPermission", "channel", "create_post", model.RoleScopeChannel, model.RoleTypeUser).Return(int64(6), nil)
+	schemeStore.On("CountWithoutPermission", "channel", "create_post", model.RoleScopeChannel, model.RoleTypeGuest).Return(int64(7), nil)
+	schemeStore.On("CountWithoutPermission", "channel", "add_reaction", model.RoleScopeChannel, model.RoleTypeUser).Return(int64(8), nil)
+	schemeStore.On("CountWithoutPermission", "channel", "add_reaction", model.RoleScopeChannel, model.RoleTypeGuest).Return(int64(9), nil)
+	schemeStore.On("CountWithoutPermission", "channel", "manage_public_channel_members", model.RoleScopeChannel, model.RoleTypeUser).Return(int64(10), nil)
+	schemeStore.On("CountWithoutPermission", "channel", "use_channel_mentions", model.RoleScopeChannel, model.RoleTypeUser).Return(int64(11), nil)
+	schemeStore.On("CountWithoutPermission", "channel", "use_channel_mentions", model.RoleScopeChannel, model.RoleTypeGuest).Return(int64(12), nil)
+
+	storeMock.On("System").Return(&systemStore)
+	storeMock.On("User").Return(&userStore)
+	storeMock.On("Team").Return(&teamStore)
+	storeMock.On("Channel").Return(&channelStore)
+	storeMock.On("Post").Return(&postStore)
+	storeMock.On("Command").Return(&commandStore)
+	storeMock.On("Webhook").Return(&webhookStore)
+	storeMock.On("Group").Return(&groupStore)
+	storeMock.On("Scheme").Return(&schemeStore)
+
+	return serverIfaceMock, storeMock, func(t *testing.T) {
+		serverIfaceMock.AssertExpectations(t)
+		storeMock.AssertExpectations(t)
+		systemStore.AssertExpectations(t)
+		pluginsAPIMock.AssertExpectations(t)
+	}, cleanUp
+}
+
+func TestEnsureTelemetryID(t *testing.T) {
+	t.Run("test ID in database and does not run twice", func(t *testing.T) {
+		storeMock := &storeMocks.Store{}
+
+		systemStore := storeMocks.SystemStore{}
+		returnValue := &model.System{
+			Name:  model.SystemTelemetryId,
+			Value: "test",
+		}
+		systemStore.On("InsertIfExists", mock.AnythingOfType("*model.System")).Return(returnValue, nil).Once()
+
+		storeMock.On("System").Return(&systemStore)
+
+		serverIfaceMock := &mocks.ServerIface{}
+		cfg := &model.Config{}
+		cfg.SetDefaults()
+
+		testLogger, _ := mlog.NewLogger()
+
+		telemetryService, err := New(serverIfaceMock, storeMock, searchengine.NewBroker(cfg), testLogger, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, "test", telemetryService.TelemetryID)
+
+		telemetryService.ensureTelemetryID()
+		assert.Equal(t, "test", telemetryService.TelemetryID)
+
+		// No more calls to the store if we try to ensure it again
+		telemetryService.ensureTelemetryID()
+		assert.Equal(t, "test", telemetryService.TelemetryID)
+	})
+
+	t.Run("new test ID created", func(t *testing.T) {
+		storeMock := &storeMocks.Store{}
+
+		systemStore := storeMocks.SystemStore{}
+		returnValue := &model.System{
+			Name: model.SystemTelemetryId,
+		}
+
+		var generatedID string
+		systemStore.On("InsertIfExists", mock.AnythingOfType("*model.System")).Return(returnValue, nil).Once().Run(func(args mock.Arguments) {
+			s := args.Get(0).(*model.System)
+			returnValue.Value = s.Value
+			generatedID = s.Value
+		})
+		storeMock.On("System").Return(&systemStore)
+
+		serverIfaceMock := &mocks.ServerIface{}
+		cfg := &model.Config{}
+		cfg.SetDefaults()
+
+		testLogger, _ := mlog.NewLogger()
+
+		telemetryService, err := New(serverIfaceMock, storeMock, searchengine.NewBroker(cfg), testLogger, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, generatedID, telemetryService.TelemetryID)
+	})
+
+	t.Run("fail to save test ID", func(t *testing.T) {
+		storeMock := &storeMocks.Store{}
+
+		systemStore := storeMocks.SystemStore{}
+
+		insertError := errors.New("insert error")
+		systemStore.On("InsertIfExists", mock.AnythingOfType("*model.System")).Return(nil, insertError).Times(DBAccessAttempts)
+
+		storeMock.On("System").Return(&systemStore)
+
+		serverIfaceMock := &mocks.ServerIface{}
+		cfg := &model.Config{}
+		cfg.SetDefaults()
+
+		testLogger, _ := mlog.NewLogger()
+
+		_, err := New(serverIfaceMock, storeMock, searchengine.NewBroker(cfg), testLogger, false)
+		require.Error(t, err)
+	})
+}
+
+func TestPluginSetting(t *testing.T) {
+	settings := &model.PluginSettings{
+		Plugins: map[string]map[string]any{
+			"test": {
+				"foo": "bar",
+			},
+		},
+	}
+	assert.Equal(t, "bar", pluginSetting(settings, "test", "foo", "asd"))
+	assert.Equal(t, "asd", pluginSetting(settings, "test", "qwe", "asd"))
+}
+
+func TestPluginActivated(t *testing.T) {
+	states := map[string]*model.PluginState{
+		"foo": {
+			Enable: true,
+		},
+		"bar": {
+			Enable: false,
+		},
+	}
+	assert.True(t, pluginActivated(states, "foo"))
+	assert.False(t, pluginActivated(states, "bar"))
+	assert.False(t, pluginActivated(states, "none"))
+}
+
+const keyStorageBytes = "storage_bytes"
+
+func TestPluginVersion(t *testing.T) {
+	plugins := []*model.BundleInfo{
+		{
+			Manifest: &model.Manifest{
+				Id:      "test.plugin",
+				Version: "1.2.3",
+			},
+		},
+		{
+			Manifest: &model.Manifest{
+				Id:      "test.plugin2",
+				Version: "4.5.6",
+			},
+		},
+	}
+	assert.Equal(t, "1.2.3", pluginVersion(plugins, "test.plugin"))
+	assert.Equal(t, "4.5.6", pluginVersion(plugins, "test.plugin2"))
+	assert.Empty(t, pluginVersion(plugins, "unknown.plugin"))
+}
+
+func TestRudderTelemetry(t *testing.T) {
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	service, pchan, cfg, teardown := makeTelemetryServiceAndReceiver(t, false)
+	defer teardown()
+
+	marketplaceServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
+		res.WriteHeader(http.StatusOK)
+		json, err := json.Marshal([]*model.MarketplacePlugin{{
+			BaseMarketplacePlugin: &model.BaseMarketplacePlugin{
+				Manifest: &model.Manifest{
+					Id: "testplugin",
+				},
+			},
+		}})
+		require.NoError(t, err)
+		res.Write(json)
+	}))
+
+	defer marketplaceServer.Close()
+
+	collectInfo := func(info *[]string) {
+		t.Helper()
+		for {
+			select {
+			case result := <-pchan:
+				assertPayload(t, result, "", nil)
+				*info = append(*info, result.Batch[0].Event)
+			case <-time.After(time.Second * 1):
+				return
+			}
+		}
+	}
+
+	t.Run("Send", func(t *testing.T) {
+		testValue := "test-send-value-6789"
+		service.SendTelemetry("Testing Telemetry", map[string]any{
+			"hey": testValue,
+		})
+		select {
+		case result := <-pchan:
+			assertPayload(t, result, "Testing Telemetry", map[string]any{
+				"hey": testValue,
+			})
+		case <-time.After(time.Second * 1):
+			require.Fail(t, "Did not receive telemetry")
+		}
+	})
+
+	// Plugins remain disabled at this point
+	t.Run("SendDailyTelemetryPluginsDisabled", func(t *testing.T) {
+		service.sendDailyTelemetry(true)
+
+		var info []string
+		// Collect the info sent.
+		collectInfo(&info)
+
+		for _, item := range []string{
+			TrackConfigService,
+			TrackConfigTeam,
+			TrackConfigSQL,
+			TrackConfigLog,
+			TrackConfigNotificationLog,
+			TrackConfigFile,
+			TrackConfigRate,
+			TrackConfigEmail,
+			TrackConfigPrivacy,
+			TrackConfigOAuth,
+			TrackConfigLDAP,
+			TrackConfigCompliance,
+			TrackConfigLocalization,
+			TrackConfigSAML,
+			TrackConfigPassword,
+			TrackConfigCluster,
+			TrackConfigMetrics,
+			TrackConfigSupport,
+			TrackConfigNativeApp,
+			TrackConfigExperimental,
+			TrackConfigAnalytics,
+			TrackConfigPlugin,
+			TrackFeatureFlags,
+			TrackActivity,
+			TrackServer,
+			TrackConfigMessageExport,
+			TrackPlugins,
+		} {
+			require.Contains(t, info, item)
+		}
+	})
+
+	// Enable plugins for the remainder of the tests.
+	// th.Server.UpdateConfig(func(cfg *model.Config) { *cfg.PluginSettings.Enable = true })
+
+	t.Run("SendDailyTelemetry", func(t *testing.T) {
+		service.sendDailyTelemetry(true)
+
+		var info []string
+		// Collect the info sent.
+		collectInfo(&info)
+
+		for _, item := range []string{
+			TrackConfigService,
+			TrackConfigTeam,
+			TrackConfigSQL,
+			TrackConfigLog,
+			TrackConfigNotificationLog,
+			TrackConfigFile,
+			TrackConfigRate,
+			TrackConfigEmail,
+			TrackConfigPrivacy,
+			TrackConfigOAuth,
+			TrackConfigLDAP,
+			TrackConfigCompliance,
+			TrackConfigLocalization,
+			TrackConfigSAML,
+			TrackConfigPassword,
+			TrackConfigCluster,
+			TrackConfigMetrics,
+			TrackConfigSupport,
+			TrackConfigNativeApp,
+			TrackConfigExperimental,
+			TrackConfigAnalytics,
+			TrackConfigPlugin,
+			TrackFeatureFlags,
+			TrackActivity,
+			TrackServer,
+			TrackConfigMessageExport,
+			TrackPlugins,
+		} {
+			require.Contains(t, info, item)
+		}
+	})
+	t.Run("Telemetry for Marketplace plugins is returned", func(t *testing.T) {
+		service.trackPluginConfig(service.srv.Config(), marketplaceServer.URL)
+
+		var batches []testBatch
+		collectBatches(t, &batches, pchan)
+
+		for _, b := range batches {
+			if b.Event == TrackConfigPlugin {
+				assert.Contains(t, b.Properties, "enable_testplugin")
+				assert.Contains(t, b.Properties, "version_testplugin")
+
+				// Confirm known plugins are not present
+				assert.NotContains(t, b.Properties, "enable_jira")
+				assert.NotContains(t, b.Properties, "version_jira")
+			}
+		}
+	})
+
+	t.Run("Telemetry for known plugins is returned, if request to Marketplace fails", func(t *testing.T) {
+		service.trackPluginConfig(service.srv.Config(), "http://some.random.invalid.url")
+
+		var batches []testBatch
+		collectBatches(t, &batches, pchan)
+
+		for _, b := range batches {
+			if b.Event == TrackConfigPlugin {
+				assert.NotContains(t, b.Properties, "enable_testplugin")
+				assert.NotContains(t, b.Properties, "version_testplugin")
+
+				// Confirm known plugins are present
+				assert.Contains(t, b.Properties, "enable_jira")
+				assert.Contains(t, b.Properties, "version_jira")
+			}
+		}
+	})
+
+	t.Run("SendDailyTelemetryNoRudderKey", func(t *testing.T) {
+		if !strings.Contains(RudderKey, "placeholder") {
+			t.Skipf("Skipping telemetry on production builds")
+		}
+		service.sendDailyTelemetry(false)
+
+		select {
+		case <-pchan:
+			require.Fail(t, "Should not send telemetry when the rudder key is not set")
+		case <-time.After(time.Second * 1):
+			// Did not receive telemetry
+		}
+	})
+
+	t.Run("SendDailyTelemetryNonCloud", func(t *testing.T) {
+		if !strings.Contains(RudderKey, "placeholder") {
+			t.Skipf("Skipping telemetry on production builds")
+		}
+		service.sendDailyTelemetry(true)
+
+		var batches []testBatch
+		collectBatches(t, &batches, pchan)
+
+		var activityEvent testBatch
+		var found bool
+		for _, testBatch := range batches {
+			if testBatch.Event == TrackActivity {
+				activityEvent = testBatch
+				found = true
+				break
+			}
+		}
+		require.True(t, found, fmt.Sprintf("Expected to receive %q event, but received %q", TrackActivity, activityEvent.Event))
+
+		_, ok := activityEvent.Properties[keyStorageBytes]
+
+		require.False(t, ok, fmt.Sprintf("Expected non-cloud payload not to contain %q, got %+v", keyStorageBytes, activityEvent.Properties))
+	})
+
+	t.Run("SendDailyTelemetryDisabled", func(t *testing.T) {
+		if !strings.Contains(RudderKey, "placeholder") {
+			t.Skipf("Skipping telemetry on production builds")
+		}
+		*cfg.LogSettings.EnableDiagnostics = false
+		defer func() {
+			*cfg.LogSettings.EnableDiagnostics = true
+		}()
+
+		service.sendDailyTelemetry(true)
+
+		select {
+		case <-pchan:
+			require.Fail(t, "Should not send telemetry when they are disabled")
+		case <-time.After(time.Second * 1):
+			// Did not receive telemetry
+		}
+	})
+
+	t.Run("TestInstallationType", func(t *testing.T) {
+		os.Unsetenv(EnvVarInstallType)
+		service.sendDailyTelemetry(true)
+
+		var batches []testBatch
+		collectBatches(t, &batches, pchan)
+
+		for _, b := range batches {
+			if b.Event == TrackServer {
+				assert.Equal(t, b.Properties["installation_type"], "")
+			}
+		}
+
+		os.Setenv(EnvVarInstallType, "docker")
+		defer os.Unsetenv(EnvVarInstallType)
+
+		batches = []testBatch{}
+		collectBatches(t, &batches, pchan)
+
+		for _, b := range batches {
+			if b.Event == TrackServer {
+				assert.Equal(t, b.Properties["installation_type"], "docker")
+			}
+		}
+	})
+
+	t.Run("RudderConfigUsesConfigForValues", func(t *testing.T) {
+		if !strings.Contains(RudderKey, "placeholder") {
+			t.Skipf("Skipping telemetry on production builds")
+		}
+		os.Setenv("RudderKey", "abc123")
+		os.Setenv("RudderDataplaneURL", "arudderstackplace")
+		defer os.Unsetenv("RudderKey")
+		defer os.Unsetenv("RudderDataplaneURL")
+
+		config := service.getRudderConfig()
+
+		assert.Equal(t, "arudderstackplace", config.DataplaneURL)
+		assert.Equal(t, "abc123", config.RudderKey)
+	})
+}
+
+func TestRudderTelemetryCloud(t *testing.T) {
+	if !strings.Contains(RudderKey, "placeholder") {
+		t.Skipf("Skipping telemetry on production builds")
+	}
+	if testing.Short() {
+		t.SkipNow()
+	}
+
+	service, pchan, _, teardown := makeTelemetryServiceAndReceiver(t, true)
+	defer teardown()
+
+	fileInfoStore := storeMocks.FileInfoStore{}
+	mockBytes := int64(1000000000)
+	fileInfoStore.On("GetStorageUsage", true, false).Return(mockBytes, nil)
+	defer fileInfoStore.AssertExpectations(t)
+
+	service.dbStore.(*storeMocks.Store).On("FileInfo").Return(&fileInfoStore)
+	service.sendDailyTelemetry(true)
+
+	var batches []testBatch
+	collectBatches(t, &batches, pchan)
+
+	var activityEvent testBatch
+	var found bool
+	for _, batch := range batches {
+		if batch.Event == TrackActivity {
+			activityEvent = batch
+			found = true
+			break
+		}
+	}
+	require.True(t, found, fmt.Sprintf("Expected to receive %q event, but received %q: %+v", TrackActivity, activityEvent.Event, activityEvent))
+
+	storageBytes, ok := activityEvent.Properties[keyStorageBytes]
+
+	require.True(t, ok, fmt.Sprintf("Expected payload to contain %q", keyStorageBytes))
+	require.Equal(t, mockBytes, int64(storageBytes.(float64)), fmt.Sprintf("Expected storage usage of %d bytes", mockBytes))
+}
+
+func TestIsDefaultArray(t *testing.T) {
+	assert.True(t, isDefaultArray([]string{"one", "two"}, []string{"one", "two"}))
+	assert.False(t, isDefaultArray([]string{"one", "two"}, []string{"one", "two", "three"}))
+	assert.False(t, isDefaultArray([]string{"one", "two"}, []string{"one", "three"}))
+}

server/platform/services/timezones/default.go

@@ -0,0 +1,599 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package timezones
+
+var DefaultSupportedTimezones = []string{
+	"Africa/Abidjan",
+	"Africa/Accra",
+	"Africa/Addis_Ababa",
+	"Africa/Algiers",
+	"Africa/Asmara",
+	"Africa/Asmera",
+	"Africa/Bamako",
+	"Africa/Bangui",
+	"Africa/Banjul",
+	"Africa/Bissau",
+	"Africa/Blantyre",
+	"Africa/Brazzaville",
+	"Africa/Bujumbura",
+	"Africa/Cairo",
+	"Africa/Casablanca",
+	"Africa/Ceuta",
+	"Africa/Conakry",
+	"Africa/Dakar",
+	"Africa/Dar_es_Salaam",
+	"Africa/Djibouti",
+	"Africa/Douala",
+	"Africa/El_Aaiun",
+	"Africa/Freetown",
+	"Africa/Gaborone",
+	"Africa/Harare",
+	"Africa/Johannesburg",
+	"Africa/Juba",
+	"Africa/Kampala",
+	"Africa/Khartoum",
+	"Africa/Kigali",
+	"Africa/Kinshasa",
+	"Africa/Lagos",
+	"Africa/Libreville",
+	"Africa/Lome",
+	"Africa/Luanda",
+	"Africa/Lubumbashi",
+	"Africa/Lusaka",
+	"Africa/Malabo",
+	"Africa/Maputo",
+	"Africa/Maseru",
+	"Africa/Mbabane",
+	"Africa/Mogadishu",
+	"Africa/Monrovia",
+	"Africa/Nairobi",
+	"Africa/Ndjamena",
+	"Africa/Niamey",
+	"Africa/Nouakchott",
+	"Africa/Ouagadougou",
+	"Africa/Porto-Novo",
+	"Africa/Sao_Tome",
+	"Africa/Timbuktu",
+	"Africa/Tripoli",
+	"Africa/Tunis",
+	"Africa/Windhoek",
+	"America/Adak",
+	"America/Anchorage",
+	"America/Anguilla",
+	"America/Antigua",
+	"America/Araguaina",
+	"America/Argentina/Buenos_Aires",
+	"America/Argentina/Catamarca",
+	"America/Argentina/ComodRivadavia",
+	"America/Argentina/Cordoba",
+	"America/Argentina/Jujuy",
+	"America/Argentina/La_Rioja",
+	"America/Argentina/Mendoza",
+	"America/Argentina/Rio_Gallegos",
+	"America/Argentina/Salta",
+	"America/Argentina/San_Juan",
+	"America/Argentina/San_Luis",
+	"America/Argentina/Tucuman",
+	"America/Argentina/Ushuaia",
+	"America/Aruba",
+	"America/Asuncion",
+	"America/Atikokan",
+	"America/Atka",
+	"America/Bahia",
+	"America/Bahia_Banderas",
+	"America/Barbados",
+	"America/Belem",
+	"America/Belize",
+	"America/Blanc-Sablon",
+	"America/Boa_Vista",
+	"America/Bogota",
+	"America/Boise",
+	"America/Buenos_Aires",
+	"America/Cambridge_Bay",
+	"America/Campo_Grande",
+	"America/Cancun",
+	"America/Caracas",
+	"America/Catamarca",
+	"America/Cayenne",
+	"America/Cayman",
+	"America/Chicago",
+	"America/Chihuahua",
+	"America/Coral_Harbour",
+	"America/Cordoba",
+	"America/Costa_Rica",
+	"America/Creston",
+	"America/Cuiaba",
+	"America/Curacao",
+	"America/Danmarkshavn",
+	"America/Dawson",
+	"America/Dawson_Creek",
+	"America/Denver",
+	"America/Detroit",
+	"America/Dominica",
+	"America/Edmonton",
+	"America/Eirunepe",
+	"America/El_Salvador",
+	"America/Ensenada",
+	"America/Fort_Nelson",
+	"America/Fort_Wayne",
+	"America/Fortaleza",
+	"America/Glace_Bay",
+	"America/Godthab",
+	"America/Goose_Bay",
+	"America/Grand_Turk",
+	"America/Grenada",
+	"America/Guadeloupe",
+	"America/Guatemala",
+	"America/Guayaquil",
+	"America/Guyana",
+	"America/Halifax",
+	"America/Havana",
+	"America/Hermosillo",
+	"America/Indiana/Indianapolis",
+	"America/Indiana/Knox",
+	"America/Indiana/Marengo",
+	"America/Indiana/Petersburg",
+	"America/Indiana/Tell_City",
+	"America/Indiana/Vevay",
+	"America/Indiana/Vincennes",
+	"America/Indiana/Winamac",
+	"America/Indianapolis",
+	"America/Inuvik",
+	"America/Iqaluit",
+	"America/Jamaica",
+	"America/Jujuy",
+	"America/Juneau",
+	"America/Kentucky/Louisville",
+	"America/Kentucky/Monticello",
+	"America/Knox_IN",
+	"America/Kralendijk",
+	"America/La_Paz",
+	"America/Lima",
+	"America/Los_Angeles",
+	"America/Louisville",
+	"America/Lower_Princes",
+	"America/Maceio",
+	"America/Managua",
+	"America/Manaus",
+	"America/Marigot",
+	"America/Martinique",
+	"America/Matamoros",
+	"America/Mazatlan",
+	"America/Mendoza",
+	"America/Menominee",
+	"America/Merida",
+	"America/Metlakatla",
+	"America/Mexico_City",
+	"America/Miquelon",
+	"America/Moncton",
+	"America/Monterrey",
+	"America/Montevideo",
+	"America/Montreal",
+	"America/Montserrat",
+	"America/Nassau",
+	"America/New_York",
+	"America/Nipigon",
+	"America/Nome",
+	"America/Noronha",
+	"America/North_Dakota/Beulah",
+	"America/North_Dakota/Center",
+	"America/North_Dakota/New_Salem",
+	"America/Ojinaga",
+	"America/Panama",
+	"America/Pangnirtung",
+	"America/Paramaribo",
+	"America/Phoenix",
+	"America/Port-au-Prince",
+	"America/Port_of_Spain",
+	"America/Porto_Acre",
+	"America/Porto_Velho",
+	"America/Puerto_Rico",
+	"America/Punta_Arenas",
+	"America/Rainy_River",
+	"America/Rankin_Inlet",
+	"America/Recife",
+	"America/Regina",
+	"America/Resolute",
+	"America/Rio_Branco",
+	"America/Rosario",
+	"America/Santa_Isabel",
+	"America/Santarem",
+	"America/Santiago",
+	"America/Santo_Domingo",
+	"America/Sao_Paulo",
+	"America/Scoresbysund",
+	"America/Shiprock",
+	"America/Sitka",
+	"America/St_Barthelemy",
+	"America/St_Johns",
+	"America/St_Kitts",
+	"America/St_Lucia",
+	"America/St_Thomas",
+	"America/St_Vincent",
+	"America/Swift_Current",
+	"America/Tegucigalpa",
+	"America/Thule",
+	"America/Thunder_Bay",
+	"America/Tijuana",
+	"America/Toronto",
+	"America/Tortola",
+	"America/Vancouver",
+	"America/Virgin",
+	"America/Whitehorse",
+	"America/Winnipeg",
+	"America/Yakutat",
+	"America/Yellowknife",
+	"Antarctica/Casey",
+	"Antarctica/Davis",
+	"Antarctica/DumontDUrville",
+	"Antarctica/Macquarie",
+	"Antarctica/Mawson",
+	"Antarctica/McMurdo",
+	"Antarctica/Palmer",
+	"Antarctica/Rothera",
+	"Antarctica/South_Pole",
+	"Antarctica/Syowa",
+	"Antarctica/Troll",
+	"Antarctica/Vostok",
+	"Arctic/Longyearbyen",
+	"Asia/Aden",
+	"Asia/Almaty",
+	"Asia/Amman",
+	"Asia/Anadyr",
+	"Asia/Aqtau",
+	"Asia/Aqtobe",
+	"Asia/Ashgabat",
+	"Asia/Ashkhabad",
+	"Asia/Atyrau",
+	"Asia/Baghdad",
+	"Asia/Bahrain",
+	"Asia/Baku",
+	"Asia/Bangkok",
+	"Asia/Barnaul",
+	"Asia/Beirut",
+	"Asia/Bishkek",
+	"Asia/Brunei",
+	"Asia/Calcutta",
+	"Asia/Chita",
+	"Asia/Choibalsan",
+	"Asia/Chongqing",
+	"Asia/Chungking",
+	"Asia/Colombo",
+	"Asia/Dacca",
+	"Asia/Damascus",
+	"Asia/Dhaka",
+	"Asia/Dili",
+	"Asia/Dubai",
+	"Asia/Dushanbe",
+	"Asia/Famagusta",
+	"Asia/Gaza",
+	"Asia/Harbin",
+	"Asia/Hebron",
+	"Asia/Ho_Chi_Minh",
+	"Asia/Hong_Kong",
+	"Asia/Hovd",
+	"Asia/Irkutsk",
+	"Asia/Istanbul",
+	"Asia/Jakarta",
+	"Asia/Jayapura",
+	"Asia/Jerusalem",
+	"Asia/Kabul",
+	"Asia/Kamchatka",
+	"Asia/Karachi",
+	"Asia/Kashgar",
+	"Asia/Kathmandu",
+	"Asia/Katmandu",
+	"Asia/Khandyga",
+	"Asia/Kolkata",
+	"Asia/Krasnoyarsk",
+	"Asia/Kuala_Lumpur",
+	"Asia/Kuching",
+	"Asia/Kuwait",
+	"Asia/Macao",
+	"Asia/Macau",
+	"Asia/Magadan",
+	"Asia/Makassar",
+	"Asia/Manila",
+	"Asia/Muscat",
+	"Asia/Nicosia",
+	"Asia/Novokuznetsk",
+	"Asia/Novosibirsk",
+	"Asia/Omsk",
+	"Asia/Oral",
+	"Asia/Phnom_Penh",
+	"Asia/Pontianak",
+	"Asia/Pyongyang",
+	"Asia/Qatar",
+	"Asia/Qyzylorda",
+	"Asia/Rangoon",
+	"Asia/Riyadh",
+	"Asia/Saigon",
+	"Asia/Sakhalin",
+	"Asia/Samarkand",
+	"Asia/Seoul",
+	"Asia/Shanghai",
+	"Asia/Singapore",
+	"Asia/Srednekolymsk",
+	"Asia/Taipei",
+	"Asia/Tashkent",
+	"Asia/Tbilisi",
+	"Asia/Tehran",
+	"Asia/Tel_Aviv",
+	"Asia/Thimbu",
+	"Asia/Thimphu",
+	"Asia/Tokyo",
+	"Asia/Tomsk",
+	"Asia/Ujung_Pandang",
+	"Asia/Ulaanbaatar",
+	"Asia/Ulan_Bator",
+	"Asia/Urumqi",
+	"Asia/Ust-Nera",
+	"Asia/Vientiane",
+	"Asia/Vladivostok",
+	"Asia/Yakutsk",
+	"Asia/Yangon",
+	"Asia/Yekaterinburg",
+	"Asia/Yerevan",
+	"Atlantic/Azores",
+	"Atlantic/Bermuda",
+	"Atlantic/Canary",
+	"Atlantic/Cape_Verde",
+	"Atlantic/Faeroe",
+	"Atlantic/Faroe",
+	"Atlantic/Jan_Mayen",
+	"Atlantic/Madeira",
+	"Atlantic/Reykjavik",
+	"Atlantic/South_Georgia",
+	"Atlantic/St_Helena",
+	"Atlantic/Stanley",
+	"Australia/ACT",
+	"Australia/Adelaide",
+	"Australia/Brisbane",
+	"Australia/Broken_Hill",
+	"Australia/Canberra",
+	"Australia/Currie",
+	"Australia/Darwin",
+	"Australia/Eucla",
+	"Australia/Hobart",
+	"Australia/LHI",
+	"Australia/Lindeman",
+	"Australia/Lord_Howe",
+	"Australia/Melbourne",
+	"Australia/NSW",
+	"Australia/North",
+	"Australia/Perth",
+	"Australia/Queensland",
+	"Australia/South",
+	"Australia/Sydney",
+	"Australia/Tasmania",
+	"Australia/Victoria",
+	"Australia/West",
+	"Australia/Yancowinna",
+	"Brazil/Acre",
+	"Brazil/DeNoronha",
+	"Brazil/East",
+	"Brazil/West",
+	"CET",
+	"CST6CDT",
+	"Canada/Atlantic",
+	"Canada/Central",
+	"Canada/Eastern",
+	"Canada/Mountain",
+	"Canada/Newfoundland",
+	"Canada/Pacific",
+	"Canada/Saskatchewan",
+	"Canada/Yukon",
+	"Chile/Continental",
+	"Chile/EasterIsland",
+	"Cuba",
+	"EET",
+	"EST",
+	"EST5EDT",
+	"Egypt",
+	"Eire",
+	"Etc/GMT",
+	"Etc/GMT+0",
+	"Etc/GMT+1",
+	"Etc/GMT+10",
+	"Etc/GMT+11",
+	"Etc/GMT+12",
+	"Etc/GMT+2",
+	"Etc/GMT+3",
+	"Etc/GMT+4",
+	"Etc/GMT+5",
+	"Etc/GMT+6",
+	"Etc/GMT+7",
+	"Etc/GMT+8",
+	"Etc/GMT+9",
+	"Etc/GMT-0",
+	"Etc/GMT-1",
+	"Etc/GMT-10",
+	"Etc/GMT-11",
+	"Etc/GMT-12",
+	"Etc/GMT-13",
+	"Etc/GMT-14",
+	"Etc/GMT-2",
+	"Etc/GMT-3",
+	"Etc/GMT-4",
+	"Etc/GMT-5",
+	"Etc/GMT-6",
+	"Etc/GMT-7",
+	"Etc/GMT-8",
+	"Etc/GMT-9",
+	"Etc/GMT0",
+	"Etc/Greenwich",
+	"Etc/UCT",
+	"Etc/UTC",
+	"Etc/Universal",
+	"Etc/Zulu",
+	"Europe/Amsterdam",
+	"Europe/Andorra",
+	"Europe/Astrakhan",
+	"Europe/Athens",
+	"Europe/Belfast",
+	"Europe/Belgrade",
+	"Europe/Berlin",
+	"Europe/Bratislava",
+	"Europe/Brussels",
+	"Europe/Bucharest",
+	"Europe/Budapest",
+	"Europe/Busingen",
+	"Europe/Chisinau",
+	"Europe/Copenhagen",
+	"Europe/Dublin",
+	"Europe/Gibraltar",
+	"Europe/Guernsey",
+	"Europe/Helsinki",
+	"Europe/Isle_of_Man",
+	"Europe/Istanbul",
+	"Europe/Jersey",
+	"Europe/Kaliningrad",
+	"Europe/Kiev",
+	"Europe/Kirov",
+	"Europe/Lisbon",
+	"Europe/Ljubljana",
+	"Europe/London",
+	"Europe/Luxembourg",
+	"Europe/Madrid",
+	"Europe/Malta",
+	"Europe/Mariehamn",
+	"Europe/Minsk",
+	"Europe/Monaco",
+	"Europe/Moscow",
+	"Europe/Nicosia",
+	"Europe/Oslo",
+	"Europe/Paris",
+	"Europe/Podgorica",
+	"Europe/Prague",
+	"Europe/Riga",
+	"Europe/Rome",
+	"Europe/Samara",
+	"Europe/San_Marino",
+	"Europe/Sarajevo",
+	"Europe/Saratov",
+	"Europe/Simferopol",
+	"Europe/Skopje",
+	"Europe/Sofia",
+	"Europe/Stockholm",
+	"Europe/Tallinn",
+	"Europe/Tirane",
+	"Europe/Tiraspol",
+	"Europe/Ulyanovsk",
+	"Europe/Uzhgorod",
+	"Europe/Vaduz",
+	"Europe/Vatican",
+	"Europe/Vienna",
+	"Europe/Vilnius",
+	"Europe/Volgograd",
+	"Europe/Warsaw",
+	"Europe/Zagreb",
+	"Europe/Zaporozhye",
+	"Europe/Zurich",
+	"GB",
+	"GB-Eire",
+	"GMT",
+	"GMT+0",
+	"GMT-0",
+	"GMT0",
+	"Greenwich",
+	"HST",
+	"Hongkong",
+	"Iceland",
+	"Indian/Antananarivo",
+	"Indian/Chagos",
+	"Indian/Christmas",
+	"Indian/Cocos",
+	"Indian/Comoro",
+	"Indian/Kerguelen",
+	"Indian/Mahe",
+	"Indian/Maldives",
+	"Indian/Mauritius",
+	"Indian/Mayotte",
+	"Indian/Reunion",
+	"Iran",
+	"Israel",
+	"Jamaica",
+	"Japan",
+	"Kwajalein",
+	"Libya",
+	"MET",
+	"MST",
+	"MST7MDT",
+	"Mexico/BajaNorte",
+	"Mexico/BajaSur",
+	"Mexico/General",
+	"NZ",
+	"NZ-CHAT",
+	"Navajo",
+	"PRC",
+	"PST8PDT",
+	"Pacific/Apia",
+	"Pacific/Auckland",
+	"Pacific/Bougainville",
+	"Pacific/Chatham",
+	"Pacific/Chuuk",
+	"Pacific/Easter",
+	"Pacific/Efate",
+	"Pacific/Enderbury",
+	"Pacific/Fakaofo",
+	"Pacific/Fiji",
+	"Pacific/Funafuti",
+	"Pacific/Galapagos",
+	"Pacific/Gambier",
+	"Pacific/Guadalcanal",
+	"Pacific/Guam",
+	"Pacific/Honolulu",
+	"Pacific/Johnston",
+	"Pacific/Kiritimati",
+	"Pacific/Kosrae",
+	"Pacific/Kwajalein",
+	"Pacific/Majuro",
+	"Pacific/Marquesas",
+	"Pacific/Midway",
+	"Pacific/Nauru",
+	"Pacific/Niue",
+	"Pacific/Norfolk",
+	"Pacific/Noumea",
+	"Pacific/Pago_Pago",
+	"Pacific/Palau",
+	"Pacific/Pitcairn",
+	"Pacific/Pohnpei",
+	"Pacific/Ponape",
+	"Pacific/Port_Moresby",
+	"Pacific/Rarotonga",
+	"Pacific/Saipan",
+	"Pacific/Samoa",
+	"Pacific/Tahiti",
+	"Pacific/Tarawa",
+	"Pacific/Tongatapu",
+	"Pacific/Truk",
+	"Pacific/Wake",
+	"Pacific/Wallis",
+	"Pacific/Yap",
+	"Poland",
+	"Portugal",
+	"ROC",
+	"ROK",
+	"Singapore",
+	"Turkey",
+	"UCT",
+	"US/Alaska",
+	"US/Aleutian",
+	"US/Arizona",
+	"US/Central",
+	"US/East-Indiana",
+	"US/Eastern",
+	"US/Hawaii",
+	"US/Indiana-Starke",
+	"US/Michigan",
+	"US/Mountain",
+	"US/Pacific",
+	"US/Pacific-New",
+	"US/Samoa",
+	"UTC",
+	"Universal",
+	"W-SU",
+	"WET",
+	"Zulu",
+}

server/platform/services/timezones/timezones.go

@@ -0,0 +1,29 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package timezones
+
+type Timezones struct {
+	supportedZones []string
+}
+
+func New() *Timezones {
+	timezones := Timezones{}
+
+	timezones.supportedZones = DefaultSupportedTimezones
+
+	return &timezones
+}
+
+func (t *Timezones) GetSupported() []string {
+	return t.supportedZones
+}
+
+func DefaultUserTimezone() map[string]string {
+	defaultTimezone := make(map[string]string)
+	defaultTimezone["useAutomaticTimezone"] = "true"
+	defaultTimezone["automaticTimezone"] = ""
+	defaultTimezone["manualTimezone"] = ""
+
+	return defaultTimezone
+}

server/platform/services/timezones/timezones_test.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package timezones
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTimezoneConfig(t *testing.T) {
+	tz1 := New()
+	assert.NotEmpty(t, tz1.GetSupported())
+}
+
+func TestDefaultUserTimezone(t *testing.T) {
+	defaultTimezone := DefaultUserTimezone()
+	require.Equal(t, "true", defaultTimezone["useAutomaticTimezone"])
+	require.Empty(t, defaultTimezone["automaticTimezone"])
+	require.Empty(t, defaultTimezone["manualTimezone"])
+
+	defaultTimezone["useAutomaticTimezone"] = "false"
+	defaultTimezone["automaticTimezone"] = "EST"
+	defaultTimezone["manualTimezone"] = "AST"
+
+	defaultTimezone2 := DefaultUserTimezone()
+	require.Equal(t, "true", defaultTimezone2["useAutomaticTimezone"])
+	require.Empty(t, defaultTimezone2["automaticTimezone"])
+	require.Empty(t, defaultTimezone2["manualTimezone"])
+
+}

server/platform/services/tracing/tracing.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package tracing
+
+import (
+	"context"
+	"io"
+	"time"
+
+	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/uber/jaeger-client-go"
+	jaegercfg "github.com/uber/jaeger-client-go/config"
+	"github.com/uber/jaeger-client-go/zipkin"
+	"github.com/uber/jaeger-lib/metrics"
+
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+// Tracer is a wrapper around Jaeger OpenTracing client, used to properly de-initialize jaeger on exit
+type Tracer struct {
+	closer io.Closer
+}
+
+type LogrusAdapter struct {
+}
+
+// Error - logrus adapter for span errors
+func (LogrusAdapter) Error(msg string) {
+	mlog.Error(msg)
+}
+
+// Infof - logrus adapter for span info logging
+func (LogrusAdapter) Infof(msg string, args ...any) {
+	// we ignore Info messages from opentracing
+}
+
+// New instantiates Jaeger opentracing client with default options
+// To override the defaults use environment variables listed here: https://github.com/jaegertracing/jaeger-client-go/blob/master/config/config.go
+func New() (*Tracer, error) {
+	cfg := jaegercfg.Configuration{
+		Sampler: &jaegercfg.SamplerConfig{
+			Type:  jaeger.SamplerTypeConst,
+			Param: 1,
+		},
+		Reporter: &jaegercfg.ReporterConfig{
+			LogSpans: true,
+		},
+	}
+
+	zipkinPropagator := zipkin.NewZipkinB3HTTPHeaderPropagator()
+
+	closer, err := cfg.InitGlobalTracer(
+		"mattermost",
+		jaegercfg.Logger(LogrusAdapter{}),
+		jaegercfg.Metrics(metrics.NullFactory),
+		jaegercfg.Tag("serverStartTime", time.Now().UTC().Format(time.RFC3339)),
+		jaegercfg.Injector(opentracing.HTTPHeaders, zipkinPropagator),
+		jaegercfg.Extractor(opentracing.HTTPHeaders, zipkinPropagator),
+		jaegercfg.ZipkinSharedRPCSpan(true),
+	)
+	if err != nil {
+		return nil, err
+	}
+	mlog.Info("Opentracing initialized")
+	return &Tracer{
+		closer: closer,
+	}, nil
+}
+
+func (t *Tracer) Close() error {
+	return t.closer.Close()
+}
+
+func StartRootSpanByContext(ctx context.Context, operationName string) (opentracing.Span, context.Context) {
+	return opentracing.StartSpanFromContext(ctx, operationName)
+}
+
+func StartSpanWithParentByContext(ctx context.Context, operationName string) (opentracing.Span, context.Context) {
+	parentSpan := opentracing.SpanFromContext(ctx)
+
+	if parentSpan == nil {
+		return StartRootSpanByContext(ctx, operationName)
+	}
+
+	return opentracing.StartSpanFromContext(ctx, operationName, opentracing.ChildOf(parentSpan.Context()))
+}

server/platform/services/upgrader/errors.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package upgrader
+
+import (
+	"fmt"
+)
+
+// InvalidArch indicates that the current operating system or cpu architecture doesn't support upgrades
+type InvalidArch struct{}
+
+func NewInvalidArch() *InvalidArch {
+	return &InvalidArch{}
+}
+
+func (e *InvalidArch) Error() string {
+	return "invalid operating system or processor architecture"
+}
+
+// InvalidSignature indicates that the downloaded file doesn't have a valid signature.
+type InvalidSignature struct{}
+
+func NewInvalidSignature() *InvalidSignature {
+	return &InvalidSignature{}
+}
+
+func (e *InvalidSignature) Error() string {
+	return "invalid file signature"
+}
+
+// InvalidPermissions indicates that the file permissions doesn't allow to upgrade
+type InvalidPermissions struct {
+	ErrType            string
+	Path               string
+	FileUsername       string
+	MattermostUsername string
+}
+
+func NewInvalidPermissions(errType string, path string, mattermostUsername string, fileUsername string) *InvalidPermissions {
+	return &InvalidPermissions{
+		ErrType:            errType,
+		Path:               path,
+		FileUsername:       fileUsername,
+		MattermostUsername: mattermostUsername,
+	}
+}
+
+func (e *InvalidPermissions) Error() string {
+	return fmt.Sprintf("the user %s is unable to update the %s file", e.MattermostUsername, e.Path)
+}

server/platform/services/upgrader/pubkey.gpg

@@ -0,0 +1,18 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQENBFjZQxwBCAC6kNn3zDlq/aY83M9V7MHVPoK2jnZ3BfH7sA+ibQXsijCkPSR4
+5bCUJ9qVA4XKGK+cpO9vkolSNs10igCaaemaUZNB6ksu3gT737/SZcCAfRO+cLX7
+Q2la+jwTvu1YeT/M5xDZ1KHTFxsGskeIenz2rZHeuZwBl9qep34QszWtRX40eRts
+fl6WltLrepiExTp6NMZ50k+Em4JGM6CWBMo22ucy0jYjZXO5hEGb3o6NGiG+Dx2z
+b2J78LksCKGsSrn0F1rLJeA933bFL4g9ozv9asBlzmpgG77ESg6YE1N/Rh7WDzVA
+prIR0MuB5JjElASw5LDVxDV6RZsxEVQr7ETLABEBAAG0KU1hdHRlcm1vc3QgQnVp
+bGQgPGRldi1vcHNAbWF0dGVybW9zdC5jb20+iQFUBBMBCAA+AhsDBQsJCAcCBhUI
+CQoLAgQWAgMBAh4BAheAFiEEobMdRvDzoQsCzy1E+PLDF0R3SygFAmJOqWgFCQ03
+zUwACgkQ+PLDF0R3Syg/rQf8D5BgvVFnGuHDYNu2eiasZdfxmuhg1C7JGSLHqoCT
+SB/0SLLQyMeHsJLye/gbo3yhK8G9XYOm+obGF+NDxB0LtRaPv5Q6pIQYt88ZxOGA
+Kh6RG2DjYA5j410wYrN0mNzhudqnS2yZdyq215nEr7Z6l1T7L9OPcz0u0mF9RraQ
+nawzxbxc8mPuC5tMLTedViSkTYLgMY12TCSYhykseUIGrl/FBfMbmKwBHM52SZJh
+maBevuNymlFbODTciyE9Q7mJHkaamGKTXaa3Enlcf16oSoemawSBJuspaS0sZOW8
+dgi5l3V5YvfFvSk45axiZbnGYfN81G5mkSGAENSGSKVtMA==
+=kkvg
+-----END PGP PUBLIC KEY BLOCK-----

server/platform/services/upgrader/upgrader.go

@@ -0,0 +1,18 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+//go:build !linux
+// +build !linux
+
+package upgrader
+
+func CanIUpgradeToE0() error {
+	return &InvalidArch{}
+}
+
+func UpgradeToE0() error {
+	return &InvalidArch{}
+}
+
+func UpgradeToE0Status() (int64, error) {
+	return 0, &InvalidArch{}
+}

server/platform/services/upgrader/upgrader_linux.go

@@ -0,0 +1,368 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package upgrader
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	_ "embed"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/user"
+	"path"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"syscall"
+
+	"github.com/pkg/errors"
+	"golang.org/x/crypto/openpgp" //nolint:staticcheck
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+//go:embed pubkey.gpg
+var mattermostBuildPublicKeys []byte
+
+var (
+	upgradePercentage int64
+	m                 sync.Mutex
+	upgradeError      error
+	upgrading         int32
+)
+
+type writeCounter struct {
+	total int64
+	read  int64
+}
+
+func (wc *writeCounter) Write(p []byte) (int, error) {
+	n := len(p)
+	wc.read += int64(n)
+
+	if wc.total <= 0 {
+		// skip the percentage calculation for invalid totals
+		setUpgradePercentage(50)
+		return n, nil
+	}
+
+	percentage := (wc.read * 100) / wc.total
+	if percentage == 0 {
+		percentage = 1
+	} else if percentage >= 100 {
+		percentage = 99
+	}
+
+	setUpgradePercentage(percentage)
+
+	return n, nil
+}
+
+func getUpgradePercentage() int64 {
+	return atomic.LoadInt64(&upgradePercentage)
+}
+
+func setUpgradePercentage(to int64) {
+	atomic.StoreInt64(&upgradePercentage, to)
+}
+
+func getUpgradeError() error {
+	m.Lock()
+	defer m.Unlock()
+
+	return upgradeError
+}
+
+func setUpgradeError(err error) {
+	m.Lock()
+	defer m.Unlock()
+
+	upgradeError = err
+}
+
+func getCurrentVersionTgzURL() string {
+	version := model.CurrentVersion
+	if strings.HasPrefix(model.BuildNumber, version+"-rc") {
+		version = model.BuildNumber
+	}
+
+	return "https://releases.mattermost.com/" + version + "/mattermost-" + version + "-linux-amd64.tar.gz"
+}
+
+func verifySignature(filename string, sigfilename string, publicKey []byte) error {
+	keyring, err := openpgp.ReadArmoredKeyRing(bytes.NewReader(publicKey))
+	if err != nil {
+		mlog.Debug("Unable to load the public key to verify the file signature", mlog.Err(err))
+		return NewInvalidSignature()
+	}
+
+	mattermost_tar, err := os.Open(filename)
+	if err != nil {
+		mlog.Debug("Unable to open the Mattermost .tar file to verify the file signature", mlog.Err(err))
+		return NewInvalidSignature()
+	}
+
+	signature, err := os.Open(sigfilename)
+	if err != nil {
+		mlog.Debug("Unable to open the Mattermost .sig file verify the file signature", mlog.Err(err))
+		return NewInvalidSignature()
+	}
+
+	_, err = openpgp.CheckDetachedSignature(keyring, mattermost_tar, signature)
+	if err != nil {
+		mlog.Debug("Unable to verify the Mattermost file signature", mlog.Err(err))
+		return NewInvalidSignature()
+	}
+	return nil
+}
+
+func canIWriteTheExecutable() error {
+	executablePath, err := os.Executable()
+	if err != nil {
+		return errors.New("error getting the path of the executable")
+	}
+	executableInfo, err := os.Stat(path.Dir(executablePath))
+	if err != nil {
+		return errors.New("error getting the executable info")
+	}
+	stat, ok := executableInfo.Sys().(*syscall.Stat_t)
+	if !ok {
+		return errors.New("error getting the executable info")
+	}
+	fileUID := int(stat.Uid)
+	fileUser, err := user.LookupId(strconv.Itoa(fileUID))
+	if err != nil {
+		return errors.New("error getting the executable info")
+	}
+
+	mattermostUID := os.Getuid()
+	mattermostUser, err := user.LookupId(strconv.Itoa(mattermostUID))
+	if err != nil {
+		return errors.New("error getting the executable info")
+	}
+
+	mode := executableInfo.Mode()
+	if fileUID != mattermostUID && mode&(1<<1) == 0 && mode&(1<<7) == 0 {
+		return NewInvalidPermissions("invalid-user-and-permission", path.Dir(executablePath), mattermostUser.Username, fileUser.Username)
+	}
+
+	if fileUID != mattermostUID && mode&(1<<1) == 0 && mode&(1<<7) != 0 {
+		return NewInvalidPermissions("invalid-user", path.Dir(executablePath), mattermostUser.Username, fileUser.Username)
+	}
+
+	if fileUID == mattermostUID && mode&(1<<7) == 0 {
+		return NewInvalidPermissions("invalid-permission", path.Dir(executablePath), mattermostUser.Username, fileUser.Username)
+	}
+	return nil
+}
+
+func canIUpgrade() error {
+	if runtime.GOARCH != "amd64" {
+		return NewInvalidArch()
+	}
+	if runtime.GOOS != "linux" {
+		return NewInvalidArch()
+	}
+	return canIWriteTheExecutable()
+}
+
+func CanIUpgradeToE0() error {
+	if err := canIUpgrade(); err != nil {
+		return errors.Wrap(err, "unable to upgrade from TE to E0")
+	}
+	if model.BuildEnterpriseReady == "true" {
+		mlog.Warn("Unable to upgrade from TE to E0. The server is already running E0.")
+		return errors.New("you cannot upgrade your server from TE to E0 because you are already running Mattermost Enterprise Edition")
+	}
+	return nil
+}
+
+func UpgradeToE0() error {
+	if !atomic.CompareAndSwapInt32(&upgrading, 0, 1) {
+		mlog.Warn("Trying to upgrade while another upgrade is running")
+		return errors.New("another upgrade is already running")
+	}
+	defer atomic.CompareAndSwapInt32(&upgrading, 1, 0)
+
+	setUpgradePercentage(1)
+	setUpgradeError(nil)
+
+	executablePath, err := os.Executable()
+	if err != nil {
+		setUpgradeError(errors.New("error getting the executable path"))
+		mlog.Error("Unable to get the path of the Mattermost executable", mlog.Err(err))
+		setUpgradePercentage(0)
+		return err
+	}
+
+	filename, err := download(getCurrentVersionTgzURL())
+	if err != nil {
+		if filename != "" {
+			os.Remove(filename)
+		}
+		setUpgradeError(fmt.Errorf("error downloading the new Mattermost server binary file (percentage: %d)", getUpgradePercentage()))
+		mlog.Error("Unable to download the Mattermost server binary file", mlog.Int64("percentage", getUpgradePercentage()), mlog.String("url", getCurrentVersionTgzURL()), mlog.Err(err))
+		setUpgradePercentage(0)
+		return err
+	}
+	defer os.Remove(filename)
+
+	sigfilename, err := download(getCurrentVersionTgzURL() + ".sig")
+	if err != nil {
+		if sigfilename != "" {
+			os.Remove(sigfilename)
+		}
+		setUpgradeError(errors.New("error downloading the signature file of the new server"))
+		mlog.Error("Unable to download the signature file of the new Mattermost server", mlog.String("url", getCurrentVersionTgzURL()+".sig"), mlog.Err(err))
+		setUpgradePercentage(0)
+		return err
+	}
+	defer os.Remove(sigfilename)
+
+	err = verifySignature(filename, sigfilename, mattermostBuildPublicKeys)
+	if err != nil {
+		setUpgradeError(errors.New("unable to verify the signature of the downloaded file"))
+		mlog.Error("Unable to verify the signature of the downloaded file", mlog.Err(err))
+		setUpgradePercentage(0)
+		return err
+	}
+
+	err = extractBinary(executablePath, filename)
+	if err != nil {
+		setUpgradeError(err)
+		mlog.Error("Unable to extract the binary from the downloaded file", mlog.Err(err))
+		setUpgradePercentage(0)
+		return err
+	}
+
+	setUpgradePercentage(100)
+	return nil
+}
+
+func UpgradeToE0Status() (int64, error) {
+	return getUpgradePercentage(), getUpgradeError()
+}
+
+func download(url string) (string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		io.Copy(io.Discard, resp.Body)
+		return "", errors.Errorf("error downloading file %s: %s", url, resp.Status)
+	}
+
+	out, err := os.CreateTemp("", "*_mattermost.tar.gz")
+	if err != nil {
+		return "", err
+	}
+	defer out.Close()
+
+	counter := &writeCounter{total: resp.ContentLength}
+	_, err = io.Copy(out, io.TeeReader(resp.Body, counter))
+	if err != nil {
+		return "", err
+	}
+
+	return out.Name(), nil
+}
+
+func getFilePermissionsOrDefault(filename string, def os.FileMode) os.FileMode {
+	file, err := os.Open(filename)
+	if err != nil {
+		mlog.Warn("Unable to get the file permissions", mlog.String("filename", filename), mlog.Err(err))
+		return def
+	}
+	defer file.Close()
+
+	fileStats, err := file.Stat()
+	if err != nil {
+		mlog.Warn("Unable to get the file permissions", mlog.String("filename", filename), mlog.Err(err))
+		return def
+	}
+	return fileStats.Mode()
+}
+
+func extractBinary(executablePath string, filename string) error {
+	gzipStream, err := os.Open(filename)
+	if err != nil {
+		return err
+	}
+
+	uncompressedStream, err := gzip.NewReader(gzipStream)
+	if err != nil {
+		return err
+	}
+
+	tarReader := tar.NewReader(uncompressedStream)
+
+	for {
+		header, err := tarReader.Next()
+
+		if err == io.EOF {
+			return errors.New("unable to find the Mattermost binary in the downloaded version")
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if header.Typeflag == tar.TypeReg && header.Name == "mattermost/bin/mattermost" {
+			permissions := getFilePermissionsOrDefault(executablePath, 0755)
+			tmpFile, err := os.CreateTemp(path.Dir(executablePath), "*")
+			if err != nil {
+				return err
+			}
+			tmpFileName := tmpFile.Name()
+			os.Remove(tmpFileName)
+			err = os.Rename(executablePath, tmpFileName)
+			if err != nil {
+				return err
+			}
+			outFile, err := os.Create(executablePath)
+			if err != nil {
+				err2 := os.Rename(tmpFileName, executablePath)
+				if err2 != nil {
+					mlog.Fatal("Unable to restore the backup of the executable file. Restore the executable file manually.")
+					return errors.Wrap(err2, "critical error: unable to upgrade the binary or restore the old binary version. Please restore it manually")
+				}
+				return err
+			}
+			defer outFile.Close()
+			if _, err = io.Copy(outFile, tarReader); err != nil {
+				err2 := os.Remove(executablePath)
+				if err2 != nil {
+					mlog.Fatal("Unable to restore the backup of the executable file. Restore the executable file manually.")
+					return errors.Wrap(err2, "critical error: unable to upgrade the binary or restore the old binary version. Please restore it manually")
+				}
+
+				err2 = os.Rename(tmpFileName, executablePath)
+				if err2 != nil {
+					mlog.Fatal("Unable to restore the backup of the executable file. Restore the executable file manually.")
+					return errors.Wrap(err2, "critical error: unable to upgrade the binary or restore the old binary version. Please restore it manually")
+				}
+				return err
+			}
+			err = os.Remove(tmpFileName)
+			if err != nil {
+				mlog.Warn("Unable to clean up the binary backup file.", mlog.Err(err))
+			}
+			err = os.Chmod(executablePath, permissions)
+			if err != nil {
+				mlog.Warn("Unable to set the correct permissions for the file.", mlog.Err(err))
+			}
+			break
+		}
+	}
+	return nil
+}

server/platform/services/upgrader/upgrader_linux_test.go

@@ -0,0 +1,164 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package upgrader
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestCanIUpgradeToE0(t *testing.T) {
+	t.Run("when you are already in an enterprise build", func(t *testing.T) {
+		buildEnterprise := model.BuildEnterpriseReady
+		model.BuildEnterpriseReady = "true"
+		defer func() {
+			model.BuildEnterpriseReady = buildEnterprise
+		}()
+		require.Error(t, CanIUpgradeToE0())
+	})
+
+	t.Run("when you are not in an enterprise build", func(t *testing.T) {
+		buildEnterprise := model.BuildEnterpriseReady
+		model.BuildEnterpriseReady = "false"
+		defer func() {
+			model.BuildEnterpriseReady = buildEnterprise
+		}()
+		require.NoError(t, CanIUpgradeToE0())
+	})
+}
+
+func TestGetCurrentVersionTgzURL(t *testing.T) {
+	t.Run("get release version in regular version", func(t *testing.T) {
+		currentVersion := model.CurrentVersion
+		buildNumber := model.CurrentVersion
+		model.CurrentVersion = "5.22.0"
+		model.BuildNumber = "5.22.0"
+		defer func() {
+			model.CurrentVersion = currentVersion
+			model.BuildNumber = buildNumber
+		}()
+		require.Equal(t, "https://releases.mattermost.com/5.22.0/mattermost-5.22.0-linux-amd64.tar.gz", getCurrentVersionTgzURL())
+	})
+
+	t.Run("get release version in dev version", func(t *testing.T) {
+		currentVersion := model.CurrentVersion
+		buildNumber := model.CurrentVersion
+		model.CurrentVersion = "5.22.0"
+		model.BuildNumber = "5.22.0-dev"
+		defer func() {
+			model.CurrentVersion = currentVersion
+			model.BuildNumber = buildNumber
+		}()
+		require.Equal(t, "https://releases.mattermost.com/5.22.0/mattermost-5.22.0-linux-amd64.tar.gz", getCurrentVersionTgzURL())
+	})
+
+	t.Run("get release version in rc version", func(t *testing.T) {
+		currentVersion := model.CurrentVersion
+		buildNumber := model.CurrentVersion
+		model.CurrentVersion = "5.22.0"
+		model.BuildNumber = "5.22.0-rc2"
+		defer func() {
+			model.CurrentVersion = currentVersion
+			model.BuildNumber = buildNumber
+		}()
+		require.Equal(t, "https://releases.mattermost.com/5.22.0-rc2/mattermost-5.22.0-rc2-linux-amd64.tar.gz", getCurrentVersionTgzURL())
+	})
+}
+
+func TestExtractBinary(t *testing.T) {
+	t.Run("extract from empty file", func(t *testing.T) {
+		tmpMockTarGz, err := os.CreateTemp("", "mock_tgz")
+		require.NoError(t, err)
+		defer os.Remove(tmpMockTarGz.Name())
+		tmpMockTarGz.Close()
+
+		tmpMockExecutable, err := os.CreateTemp("", "mock_exe")
+		require.NoError(t, err)
+		defer os.Remove(tmpMockExecutable.Name())
+		tmpMockExecutable.Close()
+
+		extractBinary(tmpMockExecutable.Name(), tmpMockTarGz.Name())
+	})
+
+	t.Run("extract from empty tar.gz file", func(t *testing.T) {
+		tmpMockTarGz, err := os.CreateTemp("", "mock_tgz")
+		require.NoError(t, err)
+		defer os.Remove(tmpMockTarGz.Name())
+		gz := gzip.NewWriter(tmpMockTarGz)
+		tw := tar.NewWriter(gz)
+		tw.Close()
+		gz.Close()
+		tmpMockTarGz.Close()
+
+		tmpMockExecutable, err := os.CreateTemp("", "mock_exe")
+		require.NoError(t, err)
+		defer os.Remove(tmpMockExecutable.Name())
+		tmpMockExecutable.Close()
+
+		require.Error(t, extractBinary(tmpMockExecutable.Name(), tmpMockTarGz.Name()))
+	})
+
+	t.Run("extract from tar.gz without mattermost/bin/mattermost file", func(t *testing.T) {
+		tmpMockTarGz, err := os.CreateTemp("", "mock_tgz")
+		require.NoError(t, err)
+		defer os.Remove(tmpMockTarGz.Name())
+		gz := gzip.NewWriter(tmpMockTarGz)
+		tw := tar.NewWriter(gz)
+
+		tw.WriteHeader(&tar.Header{
+			Typeflag: tar.TypeReg,
+			Name:     "test-filename",
+			Size:     4,
+		})
+		tw.Write([]byte("test"))
+
+		gz.Close()
+		tmpMockTarGz.Close()
+
+		tmpMockExecutable, err := os.CreateTemp("", "mock_exe")
+		require.NoError(t, err)
+		defer os.Remove(tmpMockExecutable.Name())
+		tmpMockExecutable.Close()
+
+		require.Error(t, extractBinary(tmpMockExecutable.Name(), tmpMockTarGz.Name()))
+	})
+
+	t.Run("extract from tar.gz with mattermost/bin/mattermost file", func(t *testing.T) {
+		tmpMockTarGz, err := os.CreateTemp("", "mock_tgz")
+		require.NoError(t, err)
+		defer os.Remove(tmpMockTarGz.Name())
+		gz := gzip.NewWriter(tmpMockTarGz)
+		tw := tar.NewWriter(gz)
+
+		tw.WriteHeader(&tar.Header{
+			Typeflag: tar.TypeReg,
+			Name:     "mattermost/bin/mattermost",
+			Size:     4,
+		})
+		tw.Write([]byte("test"))
+
+		gz.Close()
+		tmpMockTarGz.Close()
+
+		tmpMockExecutable, err := os.CreateTemp("", "mock_exe")
+		require.NoError(t, err)
+		defer os.Remove(tmpMockExecutable.Name())
+		tmpMockExecutable.Close()
+
+		require.NoError(t, extractBinary(tmpMockExecutable.Name(), tmpMockTarGz.Name()))
+		tmpMockExecutableAfter, err := os.Open(tmpMockExecutable.Name())
+		require.NoError(t, err)
+		defer tmpMockExecutableAfter.Close()
+		bytes, err := io.ReadAll(tmpMockExecutableAfter)
+		require.NoError(t, err)
+		require.Equal(t, []byte("test"), bytes)
+	})
+}