Move cluster, webhub and store out of Server (#20899)

app/platform/busy.go

@@ -0,0 +1,155 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"encoding/json"
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+const (
+	TimestampFormat = "Mon Jan 2 15:04:05 -0700 MST 2006"
+)
+
+// Busy represents the busy state of the server. A server marked busy
+// will have non-critical services disabled. If a Cluster is provided
+// any changes will be propagated to each node.
+type Busy struct {
+	busy    int32 // protected via atomic for fast IsBusy calls
+	mux     sync.RWMutex
+	timer   *time.Timer
+	expires time.Time
+
+	cluster einterfaces.ClusterInterface
+}
+
+// NewBusy creates a new Busy instance with optional cluster which will
+// be notified of busy state changes.
+func NewBusy(cluster einterfaces.ClusterInterface) *Busy {
+	return &Busy{cluster: cluster}
+}
+
+// IsBusy returns true if the server has been marked as busy.
+func (b *Busy) IsBusy() bool {
+	if b == nil {
+		return false
+	}
+	return atomic.LoadInt32(&b.busy) != 0
+}
+
+// Set marks the server as busy for dur duration and notifies cluster nodes.
+func (b *Busy) Set(dur time.Duration) {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+
+	// minimum 1 second
+	if dur < (time.Second * 1) {
+		dur = time.Second * 1
+	}
+
+	b.setWithoutNotify(dur)
+
+	if b.cluster != nil {
+		sbs := &model.ServerBusyState{Busy: true, Expires: b.expires.Unix(), ExpiresTS: b.expires.UTC().Format(TimestampFormat)}
+		b.notifyServerBusyChange(sbs)
+	}
+}
+
+// must hold mutex
+func (b *Busy) setWithoutNotify(dur time.Duration) {
+	b.clearWithoutNotify()
+	atomic.StoreInt32(&b.busy, 1)
+	b.expires = time.Now().Add(dur)
+	b.timer = time.AfterFunc(dur, func() {
+		b.mux.Lock()
+		b.clearWithoutNotify()
+		b.mux.Unlock()
+	})
+}
+
+// ClearBusy marks the server as not busy and notifies cluster nodes.
+func (b *Busy) Clear() {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+
+	b.clearWithoutNotify()
+
+	if b.cluster != nil {
+		sbs := &model.ServerBusyState{Busy: false, Expires: time.Time{}.Unix(), ExpiresTS: ""}
+		b.notifyServerBusyChange(sbs)
+	}
+}
+
+// must hold mutex
+func (b *Busy) clearWithoutNotify() {
+	if b.timer != nil {
+		b.timer.Stop() // don't drain timer.C channel for AfterFunc timers.
+	}
+	b.timer = nil
+	b.expires = time.Time{}
+	atomic.StoreInt32(&b.busy, 0)
+}
+
+// Expires returns the expected time that the server
+// will be marked not busy. This expiry can be extended
+// via additional calls to SetBusy.
+func (b *Busy) Expires() time.Time {
+	b.mux.RLock()
+	defer b.mux.RUnlock()
+	return b.expires
+}
+
+// notifyServerBusyChange informs all cluster members of a server busy state change.
+func (b *Busy) notifyServerBusyChange(sbs *model.ServerBusyState) {
+	if b.cluster == nil {
+		return
+	}
+	buf, _ := json.Marshal(sbs)
+	msg := &model.ClusterMessage{
+		Event:            model.ClusterEventBusyStateChanged,
+		SendType:         model.ClusterSendReliable,
+		WaitForAllToSend: true,
+		Data:             buf,
+	}
+	b.cluster.SendClusterMessage(msg)
+}
+
+// ClusterEventChanged is called when a CLUSTER_EVENT_BUSY_STATE_CHANGED is received.
+func (b *Busy) ClusterEventChanged(sbs *model.ServerBusyState) {
+	b.mux.Lock()
+	defer b.mux.Unlock()
+
+	if sbs.Busy {
+		expires := time.Unix(sbs.Expires, 0)
+		dur := time.Until(expires)
+		if dur > 0 {
+			b.setWithoutNotify(dur)
+		}
+	} else {
+		b.clearWithoutNotify()
+	}
+}
+
+func (b *Busy) ToJSON() ([]byte, error) {
+	b.mux.RLock()
+	defer b.mux.RUnlock()
+
+	sbs := &model.ServerBusyState{
+		Busy:      atomic.LoadInt32(&b.busy) != 0,
+		Expires:   b.expires.Unix(),
+		ExpiresTS: b.expires.UTC().Format(TimestampFormat),
+	}
+	sbsJSON, jsonErr := json.Marshal(sbs)
+	if jsonErr != nil {
+		return []byte{}, fmt.Errorf("failed to encode server busy state to JSON: %w", jsonErr)
+	}
+
+	return sbsJSON, nil
+}

app/platform/busy_test.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestBusySet(t *testing.T) {
+	cluster := &ClusterMock{Busy: &Busy{}}
+	busy := NewBusy(cluster)
+
+	isNotBusy := func() bool {
+		return !busy.IsBusy()
+	}
+
+	require.False(t, busy.IsBusy())
+
+	busy.Set(time.Millisecond * 500)
+	require.True(t, busy.IsBusy())
+	require.True(t, compareBusyState(t, busy, cluster.Busy))
+
+	// should automatically expire after 500ms.
+	require.Eventually(t, isNotBusy, time.Second*15, time.Millisecond*100)
+	// allow a moment for cluster to sync.
+	require.Eventually(t, func() bool { return compareBusyState(t, busy, cluster.Busy) }, time.Second*15, time.Millisecond*20)
+
+	// test set after auto expiry.
+	busy.Set(time.Second * 30)
+	require.True(t, busy.IsBusy())
+	require.True(t, compareBusyState(t, busy, cluster.Busy))
+	expire := busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Second*10).Unix())
+
+	// test extending existing expiry
+	busy.Set(time.Minute * 5)
+	require.True(t, busy.IsBusy())
+	require.True(t, compareBusyState(t, busy, cluster.Busy))
+	expire = busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Minute*2).Unix())
+
+	busy.Clear()
+	require.False(t, busy.IsBusy())
+	require.True(t, compareBusyState(t, busy, cluster.Busy))
+}
+
+func TestBusyExpires(t *testing.T) {
+	cluster := &ClusterMock{Busy: &Busy{}}
+	busy := NewBusy(cluster)
+
+	isNotBusy := func() bool {
+		return !busy.IsBusy()
+	}
+
+	// get expiry before it is set
+	expire := busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+
+	// get expiry after it is set
+	busy.Set(time.Minute * 5)
+	expire = busy.Expires()
+	require.Greater(t, expire.Unix(), time.Now().Add(time.Minute*2).Unix())
+	require.True(t, compareBusyState(t, busy, cluster.Busy))
+
+	// get expiry after clear
+	busy.Clear()
+	expire = busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+	require.True(t, compareBusyState(t, busy, cluster.Busy))
+
+	// get expiry after auto-expire
+	busy.Set(time.Millisecond * 100)
+	require.Eventually(t, isNotBusy, time.Second*5, time.Millisecond*20)
+	expire = busy.Expires()
+	// should be time.Time zero value
+	require.Equal(t, time.Time{}.Unix(), expire.Unix())
+	// allow a moment for cluster to sync
+	require.Eventually(t, func() bool { return compareBusyState(t, busy, cluster.Busy) }, time.Second*15, time.Millisecond*20)
+}
+
+func TestBusyRace(t *testing.T) {
+	cluster := &ClusterMock{Busy: &Busy{}}
+	busy := NewBusy(cluster)
+
+	busy.Set(500 * time.Millisecond)
+
+	// We are sleeping in order to let the race trigger.
+	time.Sleep(time.Second)
+}
+
+func compareBusyState(t *testing.T, busy1 *Busy, busy2 *Busy) bool {
+	t.Helper()
+	if busy1.IsBusy() != busy2.IsBusy() {
+		busy1JSON, _ := busy1.ToJSON()
+		busy2JSON, _ := busy2.ToJSON()
+		t.Logf("busy1:%s;  busy2:%s\n", busy1JSON, busy2JSON)
+		return false
+	}
+	if busy1.Expires().Unix() != busy2.Expires().Unix() {
+		busy1JSON, _ := busy1.ToJSON()
+		busy2JSON, _ := busy2.ToJSON()
+		t.Logf("busy1:%s;  busy2:%s\n", busy1JSON, busy2JSON)
+		return false
+	}
+	return true
+}
+
+// ClusterMock simulates the busy state of a cluster.
+type ClusterMock struct {
+	Busy *Busy
+}
+
+func (c *ClusterMock) SendClusterMessage(msg *model.ClusterMessage) {
+	var sbs model.ServerBusyState
+	json.Unmarshal(msg.Data, &sbs)
+	c.Busy.ClusterEventChanged(&sbs)
+}
+
+func (c *ClusterMock) SendClusterMessageToNode(nodeID string, msg *model.ClusterMessage) error {
+	return nil
+}
+
+func (c *ClusterMock) StartInterNodeCommunication() {}
+func (c *ClusterMock) StopInterNodeCommunication()  {}
+func (c *ClusterMock) RegisterClusterMessageHandler(event model.ClusterEvent, crm einterfaces.ClusterMessageHandler) {
+}
+func (c *ClusterMock) GetClusterId() string                                       { return "cluster_mock" }
+func (c *ClusterMock) IsLeader() bool                                             { return false }
+func (c *ClusterMock) GetMyClusterInfo() *model.ClusterInfo                       { return nil }
+func (c *ClusterMock) GetClusterInfos() []*model.ClusterInfo                      { return nil }
+func (c *ClusterMock) NotifyMsg(buf []byte)                                       {}
+func (c *ClusterMock) GetClusterStats() ([]*model.ClusterStats, *model.AppError)  { return nil, nil }
+func (c *ClusterMock) GetLogs(page, perPage int) ([]string, *model.AppError)      { return nil, nil }
+func (c *ClusterMock) GetPluginStatuses() (model.PluginStatuses, *model.AppError) { return nil, nil }
+func (c *ClusterMock) ConfigChanged(previousConfig *model.Config, newConfig *model.Config, sendToOtherServer bool) *model.AppError {
+	return nil
+}
+func (c *ClusterMock) HealthScore() int { return 0 }

app/platform/cluster.go

@@ -3,16 +3,250 @@
 
 package platform
 
-import "github.com/mattermost/mattermost-server/v6/einterfaces"
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/v6/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/product"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/mattermost-server/v6/store"
+)
+
+// ensure cluster service wrapper implements `product.ClusterService`
+var _ product.ClusterService = (*PlatformService)(nil)
+
+// Ensure KV store wrapper implements `product.KVStoreService`
+var _ product.KVStoreService = (*PlatformService)(nil)
+
+func (ps *PlatformService) Cluster() einterfaces.ClusterInterface {
+	return ps.clusterIFace
+}
+
+func (ps *PlatformService) NewClusterDiscoveryService() *ClusterDiscoveryService {
+	ds := &ClusterDiscoveryService{
+		ClusterDiscovery: model.ClusterDiscovery{},
+		platform:         ps,
+		stop:             make(chan bool),
+	}
+
+	return ds
+}
 
 func (ps *PlatformService) IsLeader() bool {
-	if ps.License() != nil && *ps.Config().ClusterSettings.Enable && ps.cluster != nil {
-		return ps.cluster.IsLeader()
+	if ps.License() != nil && *ps.Config().ClusterSettings.Enable && ps.clusterIFace != nil {
+		return ps.clusterIFace.IsLeader()
 	}
 
 	return true
 }
 
-func (ps *PlatformService) SetCluster(impl einterfaces.ClusterInterface) {
-	ps.cluster = impl
+func (ps *PlatformService) SetCluster(impl einterfaces.ClusterInterface) { //nolint:unused
+	ps.clusterIFace = impl
+}
+
+func (ps *PlatformService) PublishPluginClusterEvent(productID string, ev model.PluginClusterEvent, opts model.PluginClusterEventSendOptions) error {
+	if ps.clusterIFace == nil {
+		return nil
+	}
+
+	msg := &model.ClusterMessage{
+		Event:            model.ClusterEventPluginEvent,
+		SendType:         opts.SendType,
+		WaitForAllToSend: false,
+		Props: map[string]string{
+			"ProductID": productID,
+			"EventID":   ev.Id,
+		},
+		Data: ev.Data,
+	}
+
+	// If TargetId is empty we broadcast to all other cluster nodes.
+	if opts.TargetId == "" {
+		ps.clusterIFace.SendClusterMessage(msg)
+	} else {
+		if err := ps.clusterIFace.SendClusterMessageToNode(opts.TargetId, msg); err != nil {
+			return fmt.Errorf("failed to send message to cluster node %q: %w", opts.TargetId, err)
+		}
+	}
+
+	return nil
+}
+
+func (ps *PlatformService) PublishWebSocketEvent(productID string, event string, payload map[string]any, broadcast *model.WebsocketBroadcast) {
+	ev := model.NewWebSocketEvent(fmt.Sprintf("custom_%v_%v", productID, event), "", "", "", nil, "")
+	ev = ev.SetBroadcast(broadcast).SetData(payload)
+	ps.Publish(ev)
+}
+
+func (ps *PlatformService) SetPluginKeyWithOptions(productID string, key string, value []byte, options model.PluginKVSetOptions) (bool, *model.AppError) {
+	if err := options.IsValid(); err != nil {
+		mlog.Debug("Failed to set plugin key value with options", mlog.String("plugin_id", productID), mlog.String("key", key), mlog.Err(err))
+		return false, err
+	}
+
+	updated, err := ps.Store.Plugin().SetWithOptions(productID, key, value, options)
+	if err != nil {
+		mlog.Error("Failed to set plugin key value with options", mlog.String("plugin_id", productID), mlog.String("key", key), mlog.Err(err))
+		var appErr *model.AppError
+		switch {
+		case errors.As(err, &appErr):
+			return false, appErr
+		default:
+			return false, model.NewAppError("SetPluginKeyWithOptions", "app.plugin_store.save.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+	}
+
+	// Clean up a previous entry using the hashed key, if it exists.
+	if err := ps.Store.Plugin().Delete(productID, getKeyHash(key)); err != nil {
+		mlog.Warn("Failed to clean up previously hashed plugin key value", mlog.String("plugin_id", productID), mlog.String("key", key), mlog.Err(err))
+	}
+
+	return updated, nil
+}
+
+func (ps *PlatformService) KVGet(productID, key string) ([]byte, *model.AppError) {
+	if kv, err := ps.Store.Plugin().Get(productID, key); err == nil {
+		return kv.Value, nil
+	} else if nfErr := new(store.ErrNotFound); !errors.As(err, &nfErr) {
+		mlog.Error("Failed to query plugin key value", mlog.String("plugin_id", productID), mlog.String("key", key), mlog.Err(err))
+		return nil, model.NewAppError("GetPluginKey", "app.plugin_store.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// Lookup using the hashed version of the key for keys written prior to v5.6.
+	if kv, err := ps.Store.Plugin().Get(productID, getKeyHash(key)); err == nil {
+		return kv.Value, nil
+	} else if nfErr := new(store.ErrNotFound); !errors.As(err, &nfErr) {
+		mlog.Error("Failed to query plugin key value using hashed key", mlog.String("plugin_id", productID), mlog.String("key", key), mlog.Err(err))
+		return nil, model.NewAppError("GetPluginKey", "app.plugin_store.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return nil, nil
+}
+
+func (ps *PlatformService) KVDelete(productID, key string) *model.AppError {
+	if err := ps.Store.Plugin().Delete(productID, getKeyHash(key)); err != nil {
+		ps.logger.Error("Failed to delete plugin key value", mlog.String("plugin_id", productID), mlog.String("key", key), mlog.Err(err))
+		return model.NewAppError("DeletePluginKey", "app.plugin_store.delete.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// Also delete the key without hashing
+	if err := ps.Store.Plugin().Delete(productID, key); err != nil {
+		ps.logger.Error("Failed to delete plugin key value using hashed key", mlog.String("plugin_id", productID), mlog.String("key", key), mlog.Err(err))
+		return model.NewAppError("DeletePluginKey", "app.plugin_store.delete.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return nil
+}
+
+func (ps *PlatformService) KVList(productID string, page, perPage int) ([]string, *model.AppError) {
+	data, err := ps.Store.Plugin().List(productID, page*perPage, perPage)
+	if err != nil {
+		ps.logger.Error("Failed to list plugin key values", mlog.Int("page", page), mlog.Int("perPage", perPage), mlog.Err(err))
+		return nil, model.NewAppError("ListPluginKeys", "app.plugin_store.list.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return data, nil
+}
+
+// Registers a given function to be called when the cluster leader may have changed. Returns a unique ID for the
+// listener which can later be used to remove it. If clustering is not enabled in this build, the callback will never
+// be called.
+func (ps *PlatformService) AddClusterLeaderChangedListener(listener func()) string {
+	id := model.NewId()
+	ps.clusterLeaderListeners.Store(id, listener)
+	return id
+}
+
+// Removes a listener function by the unique ID returned when AddConfigListener was called
+func (ps *PlatformService) RemoveClusterLeaderChangedListener(id string) {
+	ps.clusterLeaderListeners.Delete(id)
+}
+
+func (ps *PlatformService) InvokeClusterLeaderChangedListeners() {
+	ps.logger.Info("Cluster leader changed. Invoking ClusterLeaderChanged listeners.")
+	// This needs to be run in a separate goroutine otherwise a recursive lock happens
+	// because the listener function eventually ends up calling .IsLeader().
+	// Fixing this would require the changed event to pass the leader directly, but that
+	// requires a lot of work.
+	ps.Go(func() {
+		ps.clusterLeaderListeners.Range(func(_, listener any) bool {
+			listener.(func())()
+			return true
+		})
+	})
+}
+
+func (ps *PlatformService) Publish(message *model.WebSocketEvent) {
+	if ps.metricsImpl() != nil {
+		ps.metricsImpl().IncrementWebsocketEvent(message.EventType())
+	}
+
+	ps.PublishSkipClusterSend(message)
+
+	if ps.clusterIFace != nil {
+		data, err := message.ToJSON()
+		if err != nil {
+			mlog.Warn("Failed to encode message to JSON", mlog.Err(err))
+		}
+		cm := &model.ClusterMessage{
+			Event:    model.ClusterEventPublish,
+			SendType: model.ClusterSendBestEffort,
+			Data:     data,
+		}
+
+		if message.EventType() == model.WebsocketEventPosted ||
+			message.EventType() == model.WebsocketEventPostEdited ||
+			message.EventType() == model.WebsocketEventDirectAdded ||
+			message.EventType() == model.WebsocketEventGroupAdded ||
+			message.EventType() == model.WebsocketEventAddedToTeam ||
+			message.GetBroadcast().ReliableClusterSend {
+			cm.SendType = model.ClusterSendReliable
+		}
+
+		ps.clusterIFace.SendClusterMessage(cm)
+	}
+}
+
+func (ps *PlatformService) PublishSkipClusterSend(event *model.WebSocketEvent) {
+	if event.GetBroadcast().UserId != "" {
+		hub := ps.GetHubForUserId(event.GetBroadcast().UserId)
+		if hub != nil {
+			hub.Broadcast(event)
+		}
+	} else {
+		for _, hub := range ps.hubs {
+			hub.Broadcast(event)
+		}
+	}
+
+	// Notify shared channel sync service
+	ps.SharedChannelSyncHandler(event)
+}
+
+func (ps *PlatformService) ListPluginKeys(pluginID string, page, perPage int) ([]string, *model.AppError) {
+	data, err := ps.Store.Plugin().List(pluginID, page*perPage, perPage)
+	if err != nil {
+		mlog.Error("Failed to list plugin key values", mlog.Int("page", page), mlog.Int("perPage", perPage), mlog.Err(err))
+		return nil, model.NewAppError("ListPluginKeys", "app.plugin_store.list.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return data, nil
+}
+
+func (ps *PlatformService) DeletePluginKey(pluginID string, key string) *model.AppError {
+	if err := ps.Store.Plugin().Delete(pluginID, getKeyHash(key)); err != nil {
+		mlog.Error("Failed to delete plugin key value", mlog.String("plugin_id", pluginID), mlog.String("key", key), mlog.Err(err))
+		return model.NewAppError("DeletePluginKey", "app.plugin_store.delete.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// Also delete the key without hashing
+	if err := ps.Store.Plugin().Delete(pluginID, key); err != nil {
+		mlog.Error("Failed to delete plugin key value using hashed key", mlog.String("plugin_id", pluginID), mlog.String("key", key), mlog.Err(err))
+		return model.NewAppError("DeletePluginKey", "app.plugin_store.delete.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return nil
 }

app/platform/cluster_discovery.go

@@ -0,0 +1,77 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+const (
+	DiscoveryServiceWritePing = 60 * time.Second
+)
+
+type ClusterDiscoveryService struct {
+	model.ClusterDiscovery
+	platform *PlatformService
+	stop     chan bool
+}
+
+func (cds *ClusterDiscoveryService) Start() {
+	err := cds.platform.Store.ClusterDiscovery().Cleanup()
+	if err != nil {
+		mlog.Warn("ClusterDiscoveryService failed to cleanup the outdated cluster discovery information", mlog.Err(err))
+	}
+
+	exists, err := cds.platform.Store.ClusterDiscovery().Exists(&cds.ClusterDiscovery)
+	if err != nil {
+		mlog.Warn("ClusterDiscoveryService failed to check if row exists", mlog.String("ClusterDiscoveryID", cds.ClusterDiscovery.Id), mlog.Err(err))
+	} else if exists {
+		if _, err := cds.platform.Store.ClusterDiscovery().Delete(&cds.ClusterDiscovery); err != nil {
+			mlog.Warn("ClusterDiscoveryService failed to start clean", mlog.String("ClusterDiscoveryID", cds.ClusterDiscovery.Id), mlog.Err(err))
+		}
+	}
+
+	if err := cds.platform.Store.ClusterDiscovery().Save(&cds.ClusterDiscovery); err != nil {
+		mlog.Error("ClusterDiscoveryService failed to save", mlog.String("ClusterDiscoveryID", cds.ClusterDiscovery.Id), mlog.Err(err))
+		return
+	}
+
+	go func() {
+		mlog.Debug("ClusterDiscoveryService ping writer started", mlog.String("ClusterDiscoveryID", cds.ClusterDiscovery.Id))
+		ticker := time.NewTicker(DiscoveryServiceWritePing)
+		defer func() {
+			ticker.Stop()
+			if _, err := cds.platform.Store.ClusterDiscovery().Delete(&cds.ClusterDiscovery); err != nil {
+				mlog.Warn("ClusterDiscoveryService failed to cleanup", mlog.String("ClusterDiscoveryID", cds.ClusterDiscovery.Id), mlog.Err(err))
+			}
+			mlog.Debug("ClusterDiscoveryService ping writer stopped", mlog.String("ClusterDiscoveryID", cds.ClusterDiscovery.Id))
+		}()
+
+		for {
+			select {
+			case <-ticker.C:
+				if err := cds.platform.Store.ClusterDiscovery().SetLastPingAt(&cds.ClusterDiscovery); err != nil {
+					mlog.Error("ClusterDiscoveryService failed to write ping", mlog.String("ClusterDiscoveryID", cds.ClusterDiscovery.Id), mlog.Err(err))
+				}
+			case <-cds.stop:
+				return
+			}
+		}
+	}()
+}
+
+func (cds *ClusterDiscoveryService) Stop() {
+	cds.stop <- true
+}
+
+func (ps *PlatformService) GetClusterId() string {
+	if ps.Cluster() == nil {
+		return ""
+	}
+
+	return ps.Cluster().GetClusterId()
+}

app/platform/cluster_discovery_test.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestClusterDiscoveryService(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	ds := th.Service.NewClusterDiscoveryService()
+	ds.Type = model.CDSTypeApp
+	ds.ClusterName = "ClusterA"
+	ds.AutoFillHostname()
+
+	ds.Start()
+	time.Sleep(2 * time.Second)
+
+	ds.Stop()
+	time.Sleep(2 * time.Second)
+}

app/platform/cluster_handlers.go

@@ -0,0 +1,177 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"runtime/debug"
+
+	"github.com/mattermost/mattermost-server/v6/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+func (ps *PlatformService) RegisterClusterHandlers() {
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventPublish, ps.ClusterPublishHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventUpdateStatus, ps.ClusterUpdateStatusHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventInvalidateAllCaches, ps.ClusterInvalidateAllCachesHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForChannelMembersNotifyProps, ps.clusterInvalidateCacheForChannelMembersNotifyPropHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForChannelByName, ps.clusterInvalidateCacheForChannelByNameHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForUser, ps.clusterInvalidateCacheForUserHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventInvalidateCacheForUserTeams, ps.clusterInvalidateCacheForUserTeamsHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventBusyStateChanged, ps.clusterBusyStateChgHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventClearSessionCacheForUser, ps.clusterClearSessionCacheForUserHandler)
+	ps.clusterIFace.RegisterClusterMessageHandler(model.ClusterEventClearSessionCacheForAllUsers, ps.clusterClearSessionCacheForAllUsersHandler)
+
+	for e, h := range ps.additionalClusterHandlers {
+		ps.clusterIFace.RegisterClusterMessageHandler(e, h)
+	}
+}
+
+func (ps *PlatformService) RegisterClusterMessageHandler(ev model.ClusterEvent, h einterfaces.ClusterMessageHandler) {
+	ps.additionalClusterHandlers[ev] = h
+}
+
+// ClusterHandlersPreCheck checks whether the platform service is ready to handle cluster messages.
+func (ps *PlatformService) ClusterHandlersPreCheck() error {
+	if ps.Store == nil {
+		return fmt.Errorf("could not find store")
+	}
+
+	if ps.statusCache == nil {
+		return fmt.Errorf("could not find status cache")
+	}
+
+	return nil
+}
+
+func (ps *PlatformService) ClusterPublishHandler(msg *model.ClusterMessage) {
+	event, err := model.WebSocketEventFromJSON(bytes.NewReader(msg.Data))
+	if err != nil {
+		ps.logger.Warn("Failed to decode event from JSON", mlog.Err(err))
+		return
+	}
+
+	ps.PublishSkipClusterSend(event)
+}
+
+func (ps *PlatformService) ClusterUpdateStatusHandler(msg *model.ClusterMessage) {
+	var status model.Status
+	if jsonErr := json.Unmarshal(msg.Data, &status); jsonErr != nil {
+		ps.logger.Warn("Failed to decode status from JSON")
+	}
+
+	ps.statusCache.Set(status.UserId, status)
+}
+
+func (ps *PlatformService) ClusterInvalidateAllCachesHandler(msg *model.ClusterMessage) {
+	ps.InvalidateAllCachesSkipSend()
+}
+
+func (ps *PlatformService) clusterInvalidateCacheForChannelMembersNotifyPropHandler(msg *model.ClusterMessage) {
+	ps.invalidateCacheForChannelMembersNotifyPropsSkipClusterSend(string(msg.Data))
+}
+
+func (ps *PlatformService) clusterInvalidateCacheForChannelByNameHandler(msg *model.ClusterMessage) {
+	ps.invalidateCacheForChannelByNameSkipClusterSend(msg.Props["id"], msg.Props["name"])
+}
+
+func (ps *PlatformService) clusterInvalidateCacheForUserHandler(msg *model.ClusterMessage) {
+	ps.InvalidateCacheForUserSkipClusterSend(string(msg.Data))
+}
+
+func (ps *PlatformService) clusterInvalidateCacheForUserTeamsHandler(msg *model.ClusterMessage) {
+	ps.invalidateWebConnSessionCacheForUser(string(msg.Data))
+}
+
+func (ps *PlatformService) ClearSessionCacheForUserSkipClusterSend(userID string) {
+	ps.ClearUserSessionCacheLocal(userID)
+	ps.invalidateWebConnSessionCacheForUser(userID)
+}
+
+func (ps *PlatformService) ClearSessionCacheForAllUsersSkipClusterSend() {
+	ps.logger.Info("Purging sessions cache")
+	ps.ClearAllUsersSessionCacheLocal()
+}
+
+func (ps *PlatformService) clusterClearSessionCacheForUserHandler(msg *model.ClusterMessage) {
+	ps.ClearSessionCacheForUserSkipClusterSend(string(msg.Data))
+}
+
+func (ps *PlatformService) clusterClearSessionCacheForAllUsersHandler(msg *model.ClusterMessage) {
+	ps.ClearSessionCacheForAllUsersSkipClusterSend()
+}
+
+func (ps *PlatformService) clusterBusyStateChgHandler(msg *model.ClusterMessage) {
+	var sbs model.ServerBusyState
+	if jsonErr := json.Unmarshal(msg.Data, &sbs); jsonErr != nil {
+		mlog.Warn("Failed to decode server busy state from JSON", mlog.Err(jsonErr))
+	}
+
+	ps.Busy.ClusterEventChanged(&sbs)
+	if sbs.Busy {
+		ps.logger.Warn("server busy state activated via cluster event - non-critical services disabled", mlog.Int64("expires_sec", sbs.Expires))
+	} else {
+		ps.logger.Info("server busy state cleared via cluster event - non-critical services enabled")
+	}
+}
+
+func (ps *PlatformService) invalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelID string) {
+	ps.Store.Channel().InvalidateCacheForChannelMembersNotifyProps(channelID)
+}
+
+func (ps *PlatformService) invalidateCacheForChannelByNameSkipClusterSend(teamID, name string) {
+	if teamID == "" {
+		teamID = "dm"
+	}
+
+	ps.Store.Channel().InvalidateChannelByName(teamID, name)
+}
+
+func (ps *PlatformService) InvalidateCacheForUserSkipClusterSend(userID string) {
+	ps.Store.Channel().InvalidateAllChannelMembersForUser(userID)
+	ps.invalidateWebConnSessionCacheForUser(userID)
+}
+
+func (ps *PlatformService) invalidateWebConnSessionCacheForUser(userID string) {
+	hub := ps.GetHubForUserId(userID)
+	if hub != nil {
+		hub.InvalidateUser(userID)
+	}
+}
+
+func (ps *PlatformService) InvalidateAllCachesSkipSend() {
+	ps.logger.Info("Purging all caches")
+	ps.ClearAllUsersSessionCacheLocal()
+	ps.statusCache.Purge()
+	ps.Store.Team().ClearCaches()
+	ps.Store.Channel().ClearCaches()
+	ps.Store.User().ClearCaches()
+	ps.Store.Post().ClearCaches()
+	ps.Store.FileInfo().ClearCaches()
+	ps.Store.Webhook().ClearCaches()
+
+	linkCache.Purge()
+	ps.LoadLicense()
+}
+
+func (ps *PlatformService) InvalidateAllCaches() *model.AppError {
+	debug.FreeOSMemory()
+	ps.InvalidateAllCachesSkipSend()
+
+	if ps.clusterIFace != nil {
+
+		msg := &model.ClusterMessage{
+			Event:            model.ClusterEventInvalidateAllCaches,
+			SendType:         model.ClusterSendReliable,
+			WaitForAllToSend: true,
+		}
+
+		ps.clusterIFace.SendClusterMessage(msg)
+	}
+
+	return nil
+}

app/platform/config.go

@@ -4,24 +4,33 @@
 package platform
 
 import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/md5"
+	"crypto/rand"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 	"reflect"
+	"strconv"
 
 	"github.com/mattermost/mattermost-server/v6/config"
 	"github.com/mattermost/mattermost-server/v6/einterfaces"
 	"github.com/mattermost/mattermost-server/v6/model"
 	"github.com/mattermost/mattermost-server/v6/product"
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/mattermost-server/v6/store"
 )
 
 // ServiceConfig is used to initialize the PlatformService.
 // The mandatory fields will be checked during the initialization of the service.
 type ServiceConfig struct {
 	// Mandatory fields
-	ConfigStore  *config.Store
-	StartMetrics bool // TODO: find an elegant way to start/stop metrics server by default
+	ConfigStore *config.Store
+	Store       store.Store
 	// Optional fields
 	Metrics einterfaces.MetricsInterface
 	Cluster einterfaces.ClusterInterface
@@ -76,14 +85,14 @@ func (ps *PlatformService) SaveConfig(newCfg *model.Config, sendConfigChangeClus
 		return nil, nil, model.NewAppError("saveConfig", "app.save_config.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
 	}
 
-	if ps.serviceConfig.StartMetrics && *ps.Config().MetricsSettings.Enable {
+	if ps.startMetrics && *ps.Config().MetricsSettings.Enable {
 		ps.RestartMetrics()
 	} else {
 		ps.ShutdownMetrics()
 	}
 
-	if ps.cluster != nil {
-		err := ps.cluster.ConfigChanged(ps.configStore.RemoveEnvironmentOverrides(oldCfg),
+	if ps.clusterIFace != nil {
+		err := ps.clusterIFace.ConfigChanged(ps.configStore.RemoveEnvironmentOverrides(oldCfg),
 			ps.configStore.RemoveEnvironmentOverrides(newCfg), sendConfigChangeClusterMessage)
 		if err != nil {
 			return nil, nil, err
@@ -166,3 +175,214 @@ func (ps *PlatformService) HasConfigFile(name string) (bool, error) {
 func (ps *PlatformService) SetConfigReadOnlyFF(readOnly bool) {
 	ps.configStore.SetReadOnlyFF(readOnly)
 }
+
+func (ps *PlatformService) ClientConfigHash() string {
+	return ps.clientConfigHash.Load().(string)
+}
+
+func (ps *PlatformService) regenerateClientConfig() {
+	clientConfig := config.GenerateClientConfig(ps.Config(), ps.telemetryId, ps.License())
+	limitedClientConfig := config.GenerateLimitedClientConfig(ps.Config(), ps.telemetryId, ps.License())
+
+	if clientConfig["EnableCustomTermsOfService"] == "true" {
+		termsOfService, err := ps.Store.TermsOfService().GetLatest(true)
+		if err != nil {
+			mlog.Err(err)
+		} else {
+			clientConfig["CustomTermsOfServiceId"] = termsOfService.Id
+			limitedClientConfig["CustomTermsOfServiceId"] = termsOfService.Id
+		}
+	}
+
+	if key := ps.AsymmetricSigningKey(); key != nil {
+		der, _ := x509.MarshalPKIXPublicKey(&key.PublicKey)
+		clientConfig["AsymmetricSigningPublicKey"] = base64.StdEncoding.EncodeToString(der)
+		limitedClientConfig["AsymmetricSigningPublicKey"] = base64.StdEncoding.EncodeToString(der)
+	}
+
+	clientConfigJSON, _ := json.Marshal(clientConfig)
+	ps.clientConfig.Store(clientConfig)
+	ps.limitedClientConfig.Store(limitedClientConfig)
+	ps.clientConfigHash.Store(fmt.Sprintf("%x", md5.Sum(clientConfigJSON)))
+}
+
+// AsymmetricSigningKey will return a private key that can be used for asymmetric signing.
+func (ps *PlatformService) AsymmetricSigningKey() *ecdsa.PrivateKey {
+	if key := ps.asymmetricSigningKey.Load(); key != nil {
+		return key.(*ecdsa.PrivateKey)
+	}
+	return nil
+}
+
+// EnsureAsymmetricSigningKey ensures that an asymmetric signing key exists and future calls to
+// AsymmetricSigningKey will always return a valid signing key.
+func (ps *PlatformService) EnsureAsymmetricSigningKey() error {
+	if ps.AsymmetricSigningKey() != nil {
+		return nil
+	}
+
+	var key *model.SystemAsymmetricSigningKey
+
+	value, err := ps.Store.System().GetByName(model.SystemAsymmetricSigningKeyKey)
+	if err == nil {
+		if err := json.Unmarshal([]byte(value.Value), &key); err != nil {
+			return err
+		}
+	}
+
+	// If we don't already have a key, try to generate one.
+	if key == nil {
+		newECDSAKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+		if err != nil {
+			return err
+		}
+		newKey := &model.SystemAsymmetricSigningKey{
+			ECDSAKey: &model.SystemECDSAKey{
+				Curve: "P-256",
+				X:     newECDSAKey.X,
+				Y:     newECDSAKey.Y,
+				D:     newECDSAKey.D,
+			},
+		}
+		system := &model.System{
+			Name: model.SystemAsymmetricSigningKeyKey,
+		}
+		v, err := json.Marshal(newKey)
+		if err != nil {
+			return err
+		}
+		system.Value = string(v)
+		// If we were able to save the key, use it, otherwise log the error.
+		if err = ps.Store.System().Save(system); err != nil {
+			mlog.Warn("Failed to save AsymmetricSigningKey", mlog.Err(err))
+		} else {
+			key = newKey
+		}
+	}
+
+	// If we weren't able to save a new key above, another server must have beat us to it. Get the
+	// key from the database, and if that fails, error out.
+	if key == nil {
+		value, err := ps.Store.System().GetByName(model.SystemAsymmetricSigningKeyKey)
+		if err != nil {
+			return err
+		}
+
+		if err := json.Unmarshal([]byte(value.Value), &key); err != nil {
+			return err
+		}
+	}
+
+	var curve elliptic.Curve
+	switch key.ECDSAKey.Curve {
+	case "P-256":
+		curve = elliptic.P256()
+	default:
+		return fmt.Errorf("unknown curve: " + key.ECDSAKey.Curve)
+	}
+	ps.asymmetricSigningKey.Store(&ecdsa.PrivateKey{
+		PublicKey: ecdsa.PublicKey{
+			Curve: curve,
+			X:     key.ECDSAKey.X,
+			Y:     key.ECDSAKey.Y,
+		},
+		D: key.ECDSAKey.D,
+	})
+	ps.regenerateClientConfig()
+	return nil
+}
+
+// LimitedClientConfigWithComputed gets the configuration in a format suitable for sending to the client.
+func (ps *PlatformService) LimitedClientConfigWithComputed() map[string]string {
+	respCfg := map[string]string{}
+	for k, v := range ps.LimitedClientConfig() {
+		respCfg[k] = v
+	}
+
+	// These properties are not configurable, but nevertheless represent configuration expected
+	// by the client.
+	respCfg["NoAccounts"] = strconv.FormatBool(ps.IsFirstUserAccount())
+
+	return respCfg
+}
+
+// ClientConfigWithComputed gets the configuration in a format suitable for sending to the client.
+func (ps *PlatformService) ClientConfigWithComputed() map[string]string {
+	respCfg := map[string]string{}
+	for k, v := range ps.clientConfig.Load().(map[string]string) {
+		respCfg[k] = v
+	}
+
+	// These properties are not configurable, but nevertheless represent configuration expected
+	// by the client.
+	respCfg["NoAccounts"] = strconv.FormatBool(ps.IsFirstUserAccount())
+	respCfg["MaxPostSize"] = strconv.Itoa(ps.MaxPostSize())
+	respCfg["UpgradedFromTE"] = strconv.FormatBool(ps.isUpgradedFromTE())
+	respCfg["InstallationDate"] = ""
+	if installationDate, err := ps.GetSystemInstallDate(); err == nil {
+		respCfg["InstallationDate"] = strconv.FormatInt(installationDate, 10)
+	}
+	if ver, err := ps.Store.GetDBSchemaVersion(); err != nil {
+		mlog.Error("Could not get the schema version", mlog.Err(err))
+	} else {
+		respCfg["SchemaVersion"] = strconv.Itoa(ver)
+	}
+
+	return respCfg
+}
+
+func (ps *PlatformService) LimitedClientConfig() map[string]string {
+	return ps.limitedClientConfig.Load().(map[string]string)
+}
+
+func (ps *PlatformService) IsFirstUserAccount() bool {
+	cachedSessions, err := ps.sessionCache.Len()
+	if err != nil {
+		return false
+	}
+	if cachedSessions == 0 {
+		count, err := ps.Store.User().Count(model.UserCountOptions{IncludeDeleted: true})
+		if err != nil {
+			return false
+		}
+		if count <= 0 {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (ps *PlatformService) MaxPostSize() int {
+	maxPostSize := ps.Store.Post().GetMaxPostSize()
+	if maxPostSize == 0 {
+		return model.PostMessageMaxRunesV1
+	}
+
+	return maxPostSize
+}
+
+func (ps *PlatformService) isUpgradedFromTE() bool {
+	val, err := ps.Store.System().GetByName(model.SystemUpgradedFromTeId)
+	if err != nil {
+		return false
+	}
+	return val.Value == "true"
+}
+
+func (ps *PlatformService) GetSystemInstallDate() (int64, *model.AppError) {
+	systemData, err := ps.Store.System().GetByName(model.SystemInstallationDateKey)
+	if err != nil {
+		return 0, model.NewAppError("getSystemInstallDate", "app.system.get_by_name.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	value, err := strconv.ParseInt(systemData.Value, 10, 64)
+	if err != nil {
+		return 0, model.NewAppError("getSystemInstallDate", "app.system_install_date.parse_int.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return value, nil
+}
+
+func (ps *PlatformService) ClientConfig() map[string]string {
+	return ps.clientConfig.Load().(map[string]string)
+
+}

app/platform/config_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v6/einterfaces/mocks"
@@ -49,11 +50,10 @@ func TestConfigListener(t *testing.T) {
 }
 
 func TestConfigSave(t *testing.T) {
-	th := Setup(t)
-	defer th.TearDown()
-
 	cm := &mocks.ClusterInterface{}
-	th.Service.SetCluster(cm)
+	cm.On("SendClusterMessage", mock.AnythingOfType("*model.ClusterMessage")).Return(nil)
+	th := SetupWithCluster(t, cm)
+	defer th.TearDown()
 
 	t.Run("trigger a config changed event for the cluster", func(t *testing.T) {
 		oldCfg := th.Service.Config()
@@ -62,7 +62,6 @@ func TestConfigSave(t *testing.T) {
 
 		sanitizedOldCfg := th.Service.configStore.RemoveEnvironmentOverrides(oldCfg)
 		sanitizedNewCfg := th.Service.configStore.RemoveEnvironmentOverrides(newCfg)
-
 		cm.On("ConfigChanged", sanitizedOldCfg, sanitizedNewCfg, true).Return(nil)
 
 		_, _, appErr := th.Service.SaveConfig(newCfg, true)

app/platform/enterprise.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"github.com/mattermost/mattermost-server/v6/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/services/searchengine"
+)
+
+var clusterInterface func(*PlatformService) einterfaces.ClusterInterface
+
+func RegisterClusterInterface(f func(*PlatformService) einterfaces.ClusterInterface) {
+	clusterInterface = f
+}
+
+var elasticsearchInterface func(*PlatformService) searchengine.SearchEngineInterface
+
+func RegisterElasticsearchInterface(f func(*PlatformService) searchengine.SearchEngineInterface) {
+	elasticsearchInterface = f
+}
+
+var licenseInterface func(*PlatformService) einterfaces.LicenseInterface
+
+func RegisterLicenseInterface(f func(*PlatformService) einterfaces.LicenseInterface) {
+	licenseInterface = f
+}
+
+var metricsInterface func(*PlatformService, string, string) einterfaces.MetricsInterface
+
+func RegisterMetricsInterface(f func(*PlatformService, string, string) einterfaces.MetricsInterface) {
+	metricsInterface = f
+}

app/platform/errors.go

@@ -0,0 +1,24 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import "errors"
+
+var (
+	AcceptedDomainError       = errors.New("the email provided does not belong to an accepted domain")
+	VerifyUserError           = errors.New("could not update verify email field")
+	UserCountError            = errors.New("could not get the total number of the users.")
+	UserCreationDisabledError = errors.New("user creation is not allowed")
+	UserStoreIsEmptyError     = errors.New("could not check if the user store is empty")
+
+	GetTokenError            = errors.New("could not get token")
+	GetSessionError          = errors.New("could not get session")
+	DeleteTokenError         = errors.New("could not delete token")
+	DeleteSessionError       = errors.New("could not delete session")
+	DeleteAllAccessDataError = errors.New("could not delete all access data")
+
+	DefaultFontError   = errors.New("could not get default font")
+	UserInitialsError  = errors.New("could not get user initials")
+	ImageEncodingError = errors.New("could not encode image")
+)

app/platform/goroutines.go

@@ -0,0 +1,47 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import "sync/atomic"
+
+// Go creates a goroutine, but maintains a record of it to ensure that execution completes before
+// the server is shutdown.
+func (ps *PlatformService) Go(f func()) {
+	atomic.AddInt32(&ps.goroutineCount, 1)
+
+	go func() {
+		f()
+
+		atomic.AddInt32(&ps.goroutineCount, -1)
+		select {
+		case ps.goroutineExitSignal <- struct{}{}:
+		default:
+		}
+	}()
+}
+
+// WaitForGoroutines blocks until all goroutines created by App.Go exit.
+func (ps *PlatformService) WaitForGoroutines() {
+	for atomic.LoadInt32(&ps.goroutineCount) != 0 {
+		<-ps.goroutineExitSignal
+	}
+}
+
+func (ps *PlatformService) GoBuffered(f func()) {
+	ps.goroutineBuffered <- struct{}{}
+
+	atomic.AddInt32(&ps.goroutineCount, 1)
+
+	go func() {
+		f()
+
+		atomic.AddInt32(&ps.goroutineCount, -1)
+		select {
+		case ps.goroutineExitSignal <- struct{}{}:
+		default:
+		}
+
+		<-ps.goroutineBuffered
+	}()
+}

app/platform/helper_test.go

@@ -6,15 +6,53 @@ package platform
 import (
 	"io/ioutil"
 	"path/filepath"
+	"sync"
 	"testing"
 
 	"github.com/mattermost/mattermost-server/v6/config"
+	"github.com/mattermost/mattermost-server/v6/einterfaces"
 	"github.com/mattermost/mattermost-server/v6/model"
 	"github.com/mattermost/mattermost-server/v6/store"
+	"github.com/mattermost/mattermost-server/v6/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v6/testlib"
+	"github.com/stretchr/testify/mock"
 )
 
 type TestHelper struct {
 	Service *PlatformService
+	Suite   SuiteIFace
+
+	BasicTeam    *model.Team
+	BasicUser    *model.User
+	BasicUser2   *model.User
+	BasicChannel *model.Channel
+	// BasicPost    *model.Post
+
+	SystemAdminUser *model.User
+}
+
+var initBasicOnce sync.Once
+var userCache struct {
+	SystemAdminUser *model.User
+	BasicUser       *model.User
+	BasicUser2      *model.User
+}
+
+type mockSuite struct {
+}
+
+func (ms *mockSuite) SetStatusLastActivityAt(userID string, activityAt int64) {}
+func (ms *mockSuite) SetStatusOffline(userID string, manual bool)             {}
+func (ms *mockSuite) IsUserAway(lastActivityAt int64) bool                    { return false }
+func (ms *mockSuite) SetStatusOnline(userID string, manual bool)              {}
+func (ms *mockSuite) UpdateLastActivityAtIfNeeded(session model.Session)      {}
+func (ms *mockSuite) SetStatusAwayIfNeeded(userID string, manual bool)        {}
+func (ms *mockSuite) GetSession(token string) (*model.Session, *model.AppError) {
+	return &model.Session{}, nil
+}
+func (ms *mockSuite) RolesGrantPermission(roleNames []string, permissionId string) bool { return true }
+func (ms *mockSuite) UserCanSeeOtherUser(userID string, otherUserId string) (bool, *model.AppError) {
+	return true, nil
 }
 
 func Setup(tb testing.TB) *TestHelper {
@@ -29,6 +67,65 @@ func Setup(tb testing.TB) *TestHelper {
 	return setupTestHelper(dbStore, false, true, tb)
 }
 
+func (th *TestHelper) InitBasic() *TestHelper {
+	// create users once and cache them because password hashing is slow
+	initBasicOnce.Do(func() {
+		th.SystemAdminUser = th.CreateAdmin()
+		userCache.SystemAdminUser = th.SystemAdminUser.DeepCopy()
+
+		th.BasicUser = th.CreateUserOrGuest(false)
+		userCache.BasicUser = th.BasicUser.DeepCopy()
+
+		th.BasicUser2 = th.CreateUserOrGuest(false)
+		userCache.BasicUser2 = th.BasicUser2.DeepCopy()
+	})
+	// restore cached users
+	th.SystemAdminUser = userCache.SystemAdminUser.DeepCopy()
+	th.BasicUser = userCache.BasicUser.DeepCopy()
+	th.BasicUser2 = userCache.BasicUser2.DeepCopy()
+
+	users := []*model.User{th.SystemAdminUser, th.BasicUser, th.BasicUser2}
+	mainHelper.GetSQLStore().User().InsertUsers(users)
+
+	th.BasicTeam = th.CreateTeam()
+
+	// th.LinkUserToTeam(th.BasicUser, th.BasicTeam)
+	// th.LinkUserToTeam(th.BasicUser2, th.BasicTeam)
+	th.BasicChannel = th.CreateChannel(th.BasicTeam)
+	// th.BasicPost = th.CreatePost(th.BasicChannel)
+	return th
+}
+
+func SetupWithStoreMock(tb testing.TB) *TestHelper {
+	mockStore := testlib.GetMockStoreForSetupFunctions()
+	th := setupTestHelper(mockStore, false, false, tb)
+	statusMock := mocks.StatusStore{}
+	statusMock.On("UpdateExpiredDNDStatuses").Return([]*model.Status{}, nil)
+	statusMock.On("Get", "user1").Return(&model.Status{UserId: "user1", Status: model.StatusOnline}, nil)
+	statusMock.On("UpdateLastActivityAt", "user1", mock.Anything).Return(nil)
+	statusMock.On("SaveOrUpdate", mock.AnythingOfType("*model.Status")).Return(nil)
+	emptyMockStore := mocks.Store{}
+	emptyMockStore.On("Close").Return(nil)
+	emptyMockStore.On("Status").Return(&statusMock)
+	th.Service.Store = &emptyMockStore
+	return th
+}
+
+func SetupWithCluster(tb testing.TB, cluster einterfaces.ClusterInterface) *TestHelper {
+	if testing.Short() {
+		tb.SkipNow()
+	}
+	dbStore := mainHelper.GetStore()
+	dbStore.DropAllTables()
+	dbStore.MarkSystemRanUnitTests()
+	mainHelper.PreloadMigrations()
+
+	th := setupTestHelper(dbStore, true, true, tb)
+	th.Service.clusterIFace = cluster
+
+	return th
+}
+
 func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer bool, tb testing.TB) *TestHelper {
 	tempWorkspace, err := ioutil.TempDir("", "apptest")
 	if err != nil {
@@ -44,10 +141,14 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 	*memoryConfig.LogSettings.EnableSentry = false // disable error reporting during tests
 	*memoryConfig.AnnouncementSettings.AdminNoticesEnabled = false
 	*memoryConfig.AnnouncementSettings.UserNoticesEnabled = false
+	*memoryConfig.MetricsSettings.Enable = true
+	*memoryConfig.ServiceSettings.ListenAddress = ":0"
+	*memoryConfig.MetricsSettings.ListenAddress = ":0"
 	configStore.Set(memoryConfig)
 
 	ps, err := New(ServiceConfig{
 		ConfigStore: configStore,
+		Store:       dbStore,
 	})
 	if err != nil {
 		panic(err)
@@ -55,6 +156,7 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 
 	th := &TestHelper{
 		Service: ps,
+		Suite:   &mockSuite{},
 	}
 
 	// Share same configuration with app.TestHelper
@@ -79,9 +181,103 @@ func setupTestHelper(dbStore store.Store, enterprise bool, includeCacheLayer boo
 		th.Service.SetLicense(nil)
 	}
 
+	th.Service.HubStart(th.Suite)
+
 	return th
 }
 
 func (th *TestHelper) TearDown() {
-	// Add cleaning code here
+	th.Service.ShutdownMetrics()
+	th.Service.Shutdown()
+	th.Service.ShutdownConfig()
+}
+
+func (th *TestHelper) CreateTeam() *model.Team {
+	id := model.NewId()
+
+	team := &model.Team{
+		DisplayName: "dn_" + id,
+		Name:        "name" + id,
+		Email:       "success+" + id + "@simulator.amazonses.com",
+		Type:        model.TeamOpen,
+	}
+
+	var err error
+	if team, err = th.Service.Store.Team().Save(team); err != nil {
+		panic(err)
+	}
+	return team
+}
+
+func (th *TestHelper) CreateUserOrGuest(guest bool) *model.User {
+	id := model.NewId()
+
+	user := &model.User{
+		Email:         "success+" + id + "@simulator.amazonses.com",
+		Username:      "un_" + id,
+		Nickname:      "nn_" + id,
+		Password:      "Password1",
+		EmailVerified: true,
+		Roles:         model.SystemUserRoleId,
+	}
+
+	var err error
+	user, err = th.Service.Store.User().Save(user)
+	if err != nil {
+		panic(err)
+	}
+
+	return user
+}
+
+func (th *TestHelper) CreateAdmin() *model.User {
+	id := model.NewId()
+
+	user := &model.User{
+		Email:         "success+" + id + "@simulator.amazonses.com",
+		Username:      "un_" + id,
+		Nickname:      "nn_" + id,
+		Password:      "Password1",
+		EmailVerified: true,
+		Roles:         model.SystemAdminRoleId + " " + model.SystemUserRoleId,
+	}
+
+	var err error
+	user, err = th.Service.Store.User().Save(user)
+	if err != nil {
+		panic(err)
+	}
+
+	return user
+}
+
+type ChannelOption func(*model.Channel)
+
+func WithShared(v bool) ChannelOption {
+	return func(channel *model.Channel) {
+		channel.Shared = model.NewBool(v)
+	}
+}
+
+func (th *TestHelper) CreateChannel(team *model.Team, options ...ChannelOption) *model.Channel {
+	id := model.NewId()
+
+	channel := &model.Channel{
+		TeamId:      team.Id,
+		DisplayName: "dn_" + id,
+		Name:        "name" + id,
+		Type:        model.ChannelTypeOpen,
+	}
+
+	for _, option := range options {
+		option(channel)
+	}
+
+	var err error
+	channel, err = th.Service.Store.Channel().Save(channel, 999)
+	if err != nil {
+		panic(err)
+	}
+
+	return channel
 }

app/platform/license.go

@@ -0,0 +1,377 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/dgrijalva/jwt-go"
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/v6/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/jobs"
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/mattermost-server/v6/utils"
+)
+
+const (
+	LicenseEnv                = "MM_LICENSE"
+	LicenseRenewalURL         = "https://customers.mattermost.com/subscribe/renew"
+	JWTDefaultTokenExpiration = 7 * 24 * time.Hour // 7 days of expiration
+)
+
+var (
+	RequestTrialURL = "https://customers.mattermost.com/api/v1/trials"
+)
+
+// JWTClaims custom JWT claims with the needed information for the
+// renewal process
+type JWTClaims struct {
+	LicenseID   string `json:"license_id"`
+	ActiveUsers int64  `json:"active_users"`
+	jwt.StandardClaims
+}
+
+func (ps *PlatformService) LicenseManager() einterfaces.LicenseInterface {
+	return ps.licenseManager
+}
+
+func (ps *PlatformService) SetLicenseManager(impl einterfaces.LicenseInterface) {
+	ps.licenseManager = impl
+}
+
+func (ps *PlatformService) License() *model.License {
+	license, _ := ps.licenseValue.Load().(*model.License)
+	return license
+}
+
+func (ps *PlatformService) LoadLicense() {
+	// ENV var overrides all other sources of license.
+	licenseStr := os.Getenv(LicenseEnv)
+	if licenseStr != "" {
+		license, err := utils.LicenseValidator.LicenseFromBytes([]byte(licenseStr))
+		if err != nil {
+			ps.logger.Error("Failed to read license set in environment.", mlog.Err(err))
+			return
+		}
+
+		// skip the restrictions if license is a sanctioned trial
+		if !license.IsSanctionedTrial() && license.IsTrialLicense() {
+			canStartTrialLicense, err := ps.licenseManager.CanStartTrial()
+			if err != nil {
+				ps.logger.Error("Failed to validate trial eligibility.", mlog.Err(err))
+				return
+			}
+
+			if !canStartTrialLicense {
+				ps.logger.Info("Cannot start trial multiple times.")
+				return
+			}
+		}
+
+		if ps.ValidateAndSetLicenseBytes([]byte(licenseStr)) {
+			ps.logger.Info("License key from ENV is valid, unlocking enterprise features.")
+		}
+		return
+	}
+
+	licenseId := ""
+	props, nErr := ps.Store.System().Get()
+	if nErr == nil {
+		licenseId = props[model.SystemActiveLicenseId]
+	}
+
+	if !model.IsValidId(licenseId) {
+		// Lets attempt to load the file from disk since it was missing from the DB
+		license, licenseBytes := utils.GetAndValidateLicenseFileFromDisk(*ps.Config().ServiceSettings.LicenseFileLocation)
+
+		if license != nil {
+			if _, err := ps.SaveLicense(licenseBytes); err != nil {
+				ps.logger.Error("Failed to save license key loaded from disk.", mlog.Err(err))
+			} else {
+				licenseId = license.Id
+			}
+		}
+	}
+
+	record, nErr := ps.Store.License().Get(licenseId)
+	if nErr != nil {
+		ps.logger.Error("License key from https://mattermost.com required to unlock enterprise features.", mlog.Err(nErr))
+		ps.SetLicense(nil)
+		return
+	}
+
+	ps.ValidateAndSetLicenseBytes([]byte(record.Bytes))
+	ps.logger.Info("License key valid unlocking enterprise features.")
+}
+
+func (ps *PlatformService) SaveLicense(licenseBytes []byte) (*model.License, *model.AppError) {
+	success, licenseStr := utils.LicenseValidator.ValidateLicense(licenseBytes)
+	if !success {
+		return nil, model.NewAppError("addLicense", model.InvalidLicenseError, nil, "", http.StatusBadRequest)
+	}
+
+	var license model.License
+	if jsonErr := json.Unmarshal([]byte(licenseStr), &license); jsonErr != nil {
+		return nil, model.NewAppError("addLicense", "api.unmarshal_error", nil, "", http.StatusInternalServerError).Wrap(jsonErr)
+	}
+
+	uniqueUserCount, err := ps.Store.User().Count(model.UserCountOptions{})
+	if err != nil {
+		return nil, model.NewAppError("addLicense", "api.license.add_license.invalid_count.app_error", nil, "", http.StatusBadRequest).Wrap(err)
+	}
+
+	if uniqueUserCount > int64(*license.Features.Users) {
+		return nil, model.NewAppError("addLicense", "api.license.add_license.unique_users.app_error", map[string]any{"Users": *license.Features.Users, "Count": uniqueUserCount}, "", http.StatusBadRequest)
+	}
+
+	if license.IsExpired() {
+		return nil, model.NewAppError("addLicense", model.ExpiredLicenseError, nil, "", http.StatusBadRequest)
+	}
+
+	if *ps.Config().JobSettings.RunJobs && ps.Jobs != nil {
+		if err := ps.Jobs.StopWorkers(); err != nil && !errors.Is(err, jobs.ErrWorkersNotRunning) {
+			ps.logger.Warn("Stopping job server workers failed", mlog.Err(err))
+		}
+	}
+
+	if *ps.Config().JobSettings.RunScheduler && ps.Jobs != nil {
+		if err := ps.Jobs.StopSchedulers(); err != nil && !errors.Is(err, jobs.ErrSchedulersNotRunning) {
+			ps.logger.Error("Stopping job server schedulers failed", mlog.Err(err))
+		}
+	}
+
+	defer func() {
+		// restart job server workers - this handles the edge case where a license file is uploaded, but the job server
+		// doesn't start until the server is restarted, which prevents the 'run job now' buttons in system console from
+		// functioning as expected
+		if *ps.Config().JobSettings.RunJobs && ps.Jobs != nil {
+			if err := ps.Jobs.StartWorkers(); err != nil {
+				ps.logger.Error("Starting job server workers failed", mlog.Err(err))
+			}
+		}
+		if *ps.Config().JobSettings.RunScheduler && ps.Jobs != nil {
+			if err := ps.Jobs.StartSchedulers(); err != nil && !errors.Is(err, jobs.ErrSchedulersRunning) {
+				ps.logger.Error("Starting job server schedulers failed", mlog.Err(err))
+			}
+		}
+	}()
+
+	if ok := ps.SetLicense(&license); !ok {
+		return nil, model.NewAppError("addLicense", model.ExpiredLicenseError, nil, "", http.StatusBadRequest)
+	}
+
+	record := &model.LicenseRecord{}
+	record.Id = license.Id
+	record.Bytes = string(licenseBytes)
+
+	_, nErr := ps.Store.License().Save(record)
+	if nErr != nil {
+		ps.RemoveLicense()
+		var appErr *model.AppError
+		switch {
+		case errors.As(nErr, &appErr):
+			return nil, appErr
+		default:
+			return nil, model.NewAppError("addLicense", "api.license.add_license.save.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+		}
+	}
+
+	sysVar := &model.System{}
+	sysVar.Name = model.SystemActiveLicenseId
+	sysVar.Value = license.Id
+	if err := ps.Store.System().SaveOrUpdate(sysVar); err != nil {
+		ps.RemoveLicense()
+		return nil, model.NewAppError("addLicense", "api.license.add_license.save_active.app_error", nil, "", http.StatusInternalServerError)
+	}
+
+	ps.ReloadConfig()
+	ps.InvalidateAllCaches()
+
+	return &license, nil
+}
+
+func (ps *PlatformService) SetLicense(license *model.License) bool {
+	oldLicense := ps.licenseValue.Load()
+
+	defer func() {
+		for _, listener := range ps.licenseListeners {
+			if oldLicense == nil {
+				listener(nil, license)
+			} else {
+				listener(oldLicense.(*model.License), license)
+			}
+		}
+	}()
+
+	if license != nil {
+		license.Features.SetDefaults()
+
+		ps.licenseValue.Store(license)
+
+		ps.clientLicenseValue.Store(utils.GetClientLicense(license))
+		return true
+	}
+
+	ps.licenseValue.Store((*model.License)(nil))
+	ps.clientLicenseValue.Store(map[string]string(nil))
+
+	return false
+}
+
+func (ps *PlatformService) ValidateAndSetLicenseBytes(b []byte) bool {
+	if success, licenseStr := utils.LicenseValidator.ValidateLicense(b); success {
+		var license model.License
+		if jsonErr := json.Unmarshal([]byte(licenseStr), &license); jsonErr != nil {
+			ps.logger.Warn("Failed to decode license from JSON", mlog.Err(jsonErr))
+			return false
+		}
+		ps.SetLicense(&license)
+		return true
+	}
+
+	ps.logger.Warn("No valid enterprise license found")
+	return false
+}
+
+func (ps *PlatformService) SetClientLicense(m map[string]string) {
+	ps.clientLicenseValue.Store(m)
+}
+
+func (ps *PlatformService) ClientLicense() map[string]string {
+	if clientLicense, _ := ps.clientLicenseValue.Load().(map[string]string); clientLicense != nil {
+		return clientLicense
+	}
+	return map[string]string{"IsLicensed": "false"}
+}
+
+func (ps *PlatformService) RemoveLicense() *model.AppError {
+	if license, _ := ps.licenseValue.Load().(*model.License); license == nil {
+		return nil
+	}
+
+	ps.logger.Info("Remove license.", mlog.String("id", model.SystemActiveLicenseId))
+
+	sysVar := &model.System{}
+	sysVar.Name = model.SystemActiveLicenseId
+	sysVar.Value = ""
+
+	if err := ps.Store.System().SaveOrUpdate(sysVar); err != nil {
+		return model.NewAppError("RemoveLicense", "app.system.save.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	ps.SetLicense(nil)
+	ps.ReloadConfig()
+	ps.InvalidateAllCaches()
+
+	return nil
+}
+
+func (ps *PlatformService) AddLicenseListener(listener func(oldLicense, newLicense *model.License)) string {
+	id := model.NewId()
+	ps.licenseListeners[id] = listener
+	return id
+}
+
+func (ps *PlatformService) RemoveLicenseListener(id string) {
+	delete(ps.licenseListeners, id)
+}
+
+func (ps *PlatformService) GetSanitizedClientLicense() map[string]string {
+	return utils.GetSanitizedClientLicense(ps.ClientLicense())
+}
+
+// RequestTrialLicense request a trial license from the mattermost official license server
+func (ps *PlatformService) RequestTrialLicense(trialRequest *model.TrialLicenseRequest) *model.AppError {
+	trialRequestJSON, err := json.Marshal(trialRequest)
+	if err != nil {
+		return model.NewAppError("RequestTrialLicense", "api.unmarshal_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	resp, err := http.Post(RequestTrialURL, "application/json", bytes.NewBuffer(trialRequestJSON))
+	if err != nil {
+		return model.NewAppError("RequestTrialLicense", "api.license.request_trial_license.app_error", nil, "", http.StatusBadRequest).Wrap(err)
+	}
+	defer resp.Body.Close()
+
+	// CloudFlare sitting in front of the Customer Portal will block this request with a 451 response code in the event that the request originates from a country sanctioned by the U.S. Government.
+	if resp.StatusCode == http.StatusUnavailableForLegalReasons {
+		return model.NewAppError("RequestTrialLicense", "api.license.request_trial_license.embargoed", nil, "Request for trial license came from an embargoed country", http.StatusUnavailableForLegalReasons)
+	}
+
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		return model.NewAppError("RequestTrialLicense", "api.license.request_trial_license.app_error", nil,
+			fmt.Sprintf("Unexpected HTTP status code %q returned by server", resp.Status), http.StatusInternalServerError)
+	}
+
+	var licenseResponse map[string]string
+	err = json.NewDecoder(resp.Body).Decode(&licenseResponse)
+	if err != nil {
+		ps.logger.Warn("Error decoding license response", mlog.Err(err))
+	}
+
+	if _, ok := licenseResponse["license"]; !ok {
+		return model.NewAppError("RequestTrialLicense", "api.license.request_trial_license.app_error", nil, licenseResponse["message"], http.StatusBadRequest)
+	}
+
+	if _, err := ps.SaveLicense([]byte(licenseResponse["license"])); err != nil {
+		return err
+	}
+
+	ps.ReloadConfig()
+	ps.InvalidateAllCaches()
+
+	return nil
+}
+
+// GenerateRenewalToken returns a renewal token that expires after duration expiration
+func (ps *PlatformService) GenerateRenewalToken(expiration time.Duration) (string, *model.AppError) {
+	license := ps.License()
+	if license == nil {
+		return "", model.NewAppError("GenerateRenewalToken", "app.license.generate_renewal_token.no_license", nil, "", http.StatusBadRequest)
+	}
+
+	if *license.Features.Cloud {
+		return "", model.NewAppError("GenerateRenewalToken", "app.license.generate_renewal_token.bad_license", nil, "", http.StatusBadRequest)
+	}
+
+	activeUsers, err := ps.Store.User().Count(model.UserCountOptions{})
+	if err != nil {
+		return "", model.NewAppError("GenerateRenewalToken", "app.license.generate_renewal_token.app_error",
+			nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	expirationTime := time.Now().UTC().Add(expiration)
+	claims := &JWTClaims{
+		LicenseID:   license.Id,
+		ActiveUsers: activeUsers,
+		StandardClaims: jwt.StandardClaims{
+			ExpiresAt: expirationTime.Unix(),
+		},
+	}
+
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+	tokenString, err := token.SignedString([]byte(license.Customer.Email))
+	if err != nil {
+		return "", model.NewAppError("GenerateRenewalToken", "app.license.generate_renewal_token.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return tokenString, nil
+}
+
+// GenerateLicenseRenewalLink returns a link that points to the CWS where clients can renew license
+func (ps *PlatformService) GenerateLicenseRenewalLink() (string, string, *model.AppError) {
+	renewalToken, err := ps.GenerateRenewalToken(JWTDefaultTokenExpiration)
+	if err != nil {
+		return "", "", err
+	}
+	renewalLink := LicenseRenewalURL + "?token=" + renewalToken
+	return renewalLink, renewalToken, nil
+}

app/platform/license_test.go

@@ -0,0 +1,111 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestLoadLicense(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	th.Service.LoadLicense()
+	require.Nil(t, th.Service.License(), "shouldn't have a valid license")
+}
+
+func TestSaveLicense(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	b1 := []byte("junk")
+
+	_, err := th.Service.SaveLicense(b1)
+	require.NotNil(t, err, "shouldn't have saved license")
+}
+
+func TestRemoveLicense(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	err := th.Service.RemoveLicense()
+	require.Nil(t, err, "should have removed license")
+}
+
+func TestSetLicense(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	l1 := &model.License{}
+	l1.Features = &model.Features{}
+	l1.Customer = &model.Customer{}
+	l1.StartsAt = model.GetMillis() - 1000
+	l1.ExpiresAt = model.GetMillis() + 100000
+	ok := th.Service.SetLicense(l1)
+	require.True(t, ok, "license should have worked")
+
+	l3 := &model.License{}
+	l3.Features = &model.Features{}
+	l3.Customer = &model.Customer{}
+	l3.StartsAt = model.GetMillis() + 10000
+	l3.ExpiresAt = model.GetMillis() + 100000
+	ok = th.Service.SetLicense(l3)
+	require.True(t, ok, "license should have passed")
+}
+
+func TestGetSanitizedClientLicense(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	setLicense(th, nil)
+
+	m := th.Service.GetSanitizedClientLicense()
+
+	_, ok := m["Name"]
+	assert.False(t, ok)
+	_, ok = m["SkuName"]
+	assert.False(t, ok)
+	_, ok = m["SkuShortName"]
+	assert.False(t, ok)
+}
+
+func TestGenerateRenewalToken(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	t.Run("renewal token generated correctly", func(t *testing.T) {
+		setLicense(th, nil)
+		token, appErr := th.Service.GenerateRenewalToken(JWTDefaultTokenExpiration)
+		require.Nil(t, appErr)
+		require.NotEmpty(t, token)
+	})
+
+	t.Run("return error if there is no active license", func(t *testing.T) {
+		th.Service.SetLicense(nil)
+		_, appErr := th.Service.GenerateRenewalToken(JWTDefaultTokenExpiration)
+		require.NotNil(t, appErr)
+	})
+}
+
+func setLicense(th *TestHelper, customer *model.Customer) {
+	l1 := &model.License{}
+	l1.Features = &model.Features{}
+	if customer != nil {
+		l1.Customer = customer
+	} else {
+		l1.Customer = &model.Customer{}
+		l1.Customer.Name = "TestName"
+		l1.Customer.Email = "test@example.com"
+	}
+	l1.SkuName = "SKU NAME"
+	l1.SkuShortName = "SKU SHORT NAME"
+	l1.StartsAt = model.GetMillis() - 1000
+	l1.ExpiresAt = model.GetMillis() + 100000
+	th.Service.SetLicense(l1)
+}

app/platform/link_cache.go

@@ -0,0 +1,25 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/services/cache"
+)
+
+const LinkCacheSize = 10000
+const LinkCacheDuration = 1 * time.Hour
+
+var linkCache = cache.NewLRU(cache.LRUOptions{
+	Size: LinkCacheSize,
+})
+
+func PurgeLinkCache() {
+	linkCache.Purge()
+}
+
+func LinkCache() cache.Cache {
+	return linkCache
+}

app/platform/log.go

@@ -7,6 +7,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"os"
 	"time"
 
 	"github.com/mattermost/mattermost-server/v6/config"
@@ -14,6 +17,10 @@ import (
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
 )
 
+func (ps *PlatformService) Log() mlog.LoggerIFace {
+	return ps.logger
+}
+
 func (ps *PlatformService) ReconfigureLogger() error {
 	return ps.initLogging()
 }
@@ -82,11 +89,11 @@ func (ps *PlatformService) NotificationsLogger() *mlog.Logger {
 }
 
 func (ps *PlatformService) EnableLoggingMetrics() {
-	if ps.metrics == nil || ps.metrics.metricsImpl == nil {
+	if ps.metrics == nil || ps.metricsImpl() == nil {
 		return
 	}
 
-	ps.logger.SetMetricsCollector(ps.metrics.metricsImpl.GetLoggerMetricsCollector(), mlog.DefaultMetricsUpdateFreqMillis)
+	ps.logger.SetMetricsCollector(ps.metricsImpl().GetLoggerMetricsCollector(), mlog.DefaultMetricsUpdateFreqMillis)
 
 	// logging config needs to be reloaded when metrics collector is added or changed.
 	if err := ps.initLogging(); err != nil {
@@ -115,3 +122,75 @@ func (ps *PlatformService) RemoveUnlicensedLogTargets(license *model.License) {
 		return ti.Type != "*targets.Writer" && ti.Type != "*targets.File"
 	})
 }
+
+func (ps *PlatformService) GetLogsSkipSend(page, perPage int) ([]string, *model.AppError) {
+	var lines []string
+
+	if *ps.Config().LogSettings.EnableFile {
+		ps.Log().Flush()
+		logFile := config.GetLogFileLocation(*ps.Config().LogSettings.FileLocation)
+		file, err := os.Open(logFile)
+		if err != nil {
+			return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		defer file.Close()
+
+		var newLine = []byte{'\n'}
+		var lineCount int
+		const searchPos = -1
+		b := make([]byte, 1)
+		var endOffset int64 = 0
+
+		// if the file exists and it's last byte is '\n' - skip it
+		var stat os.FileInfo
+		if stat, err = os.Stat(logFile); err == nil {
+			if _, err = file.ReadAt(b, stat.Size()-1); err == nil && b[0] == newLine[0] {
+				endOffset = -1
+			}
+		}
+		lineEndPos, err := file.Seek(endOffset, io.SeekEnd)
+		if err != nil {
+			return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+		for {
+			pos, err := file.Seek(searchPos, io.SeekCurrent)
+			if err != nil {
+				return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			_, err = file.ReadAt(b, pos)
+			if err != nil {
+				return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			if b[0] == newLine[0] || pos == 0 {
+				lineCount++
+				if lineCount > page*perPage {
+					line := make([]byte, lineEndPos-pos)
+					_, err := file.ReadAt(line, pos)
+					if err != nil {
+						return nil, model.NewAppError("getLogs", "api.admin.file_read_error", nil, "", http.StatusInternalServerError).Wrap(err)
+					}
+					lines = append(lines, string(line))
+				}
+				if pos == 0 {
+					break
+				}
+				lineEndPos = pos
+			}
+
+			if len(lines) == perPage {
+				break
+			}
+		}
+
+		for i, j := 0, len(lines)-1; i < j; i, j = i+1, j-1 {
+			lines[i], lines[j] = lines[j], lines[i]
+		}
+	} else {
+		lines = append(lines, "")
+	}
+
+	return lines, nil
+}

app/platform/metrics.go

@@ -35,6 +35,14 @@ type platformMetrics struct {
 	cfgFn func() *model.Config
 }
 
+func (ps *PlatformService) metricsImpl() einterfaces.MetricsInterface {
+	if ps.metrics == nil {
+		return nil
+	}
+
+	return ps.metrics.metricsImpl
+}
+
 // resetMetrics resets the metrics server. Clears the metrics if the metrics are disabled by the config.
 func (ps *PlatformService) resetMetrics(metricsImpl einterfaces.MetricsInterface, cfgFn func() *model.Config) error {
 	if !*cfgFn().MetricsSettings.Enable {
@@ -173,12 +181,13 @@ func (ps *PlatformService) HandleMetrics(route string, h http.Handler) {
 }
 
 func (ps *PlatformService) RestartMetrics() error {
-	return ps.resetMetrics(ps.serviceConfig.Metrics, ps.serviceConfig.ConfigStore.Get)
+	return ps.resetMetrics(ps.serviceConfig.Metrics, ps.configStore.Get)
 }
 
 func (ps *PlatformService) Metrics() einterfaces.MetricsInterface {
 	if ps.metrics == nil {
 		return nil
 	}
-	return ps.metrics.metricsImpl
+
+	return ps.metricsImpl()
 }

app/platform/mocks/SuiteIFace.go

@@ -0,0 +1,131 @@
+// Code generated by mockery v2.14.0. DO NOT EDIT.
+
+// Regenerate this file using `make platform-mocks`.
+
+package mocks
+
+import (
+	model "github.com/mattermost/mattermost-server/v6/model"
+	mock "github.com/stretchr/testify/mock"
+)
+
+// SuiteIFace is an autogenerated mock type for the SuiteIFace type
+type SuiteIFace struct {
+	mock.Mock
+}
+
+// GetSession provides a mock function with given fields: token
+func (_m *SuiteIFace) GetSession(token string) (*model.Session, *model.AppError) {
+	ret := _m.Called(token)
+
+	var r0 *model.Session
+	if rf, ok := ret.Get(0).(func(string) *model.Session); ok {
+		r0 = rf(token)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Session)
+		}
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string) *model.AppError); ok {
+		r1 = rf(token)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+// IsUserAway provides a mock function with given fields: lastActivityAt
+func (_m *SuiteIFace) IsUserAway(lastActivityAt int64) bool {
+	ret := _m.Called(lastActivityAt)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(int64) bool); ok {
+		r0 = rf(lastActivityAt)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// RolesGrantPermission provides a mock function with given fields: roleNames, permissionId
+func (_m *SuiteIFace) RolesGrantPermission(roleNames []string, permissionId string) bool {
+	ret := _m.Called(roleNames, permissionId)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func([]string, string) bool); ok {
+		r0 = rf(roleNames, permissionId)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	return r0
+}
+
+// SetStatusAwayIfNeeded provides a mock function with given fields: userID, manual
+func (_m *SuiteIFace) SetStatusAwayIfNeeded(userID string, manual bool) {
+	_m.Called(userID, manual)
+}
+
+// SetStatusLastActivityAt provides a mock function with given fields: userID, activityAt
+func (_m *SuiteIFace) SetStatusLastActivityAt(userID string, activityAt int64) {
+	_m.Called(userID, activityAt)
+}
+
+// SetStatusOffline provides a mock function with given fields: userID, manual
+func (_m *SuiteIFace) SetStatusOffline(userID string, manual bool) {
+	_m.Called(userID, manual)
+}
+
+// SetStatusOnline provides a mock function with given fields: userID, manual
+func (_m *SuiteIFace) SetStatusOnline(userID string, manual bool) {
+	_m.Called(userID, manual)
+}
+
+// UpdateLastActivityAtIfNeeded provides a mock function with given fields: session
+func (_m *SuiteIFace) UpdateLastActivityAtIfNeeded(session model.Session) {
+	_m.Called(session)
+}
+
+// UserCanSeeOtherUser provides a mock function with given fields: userID, otherUserId
+func (_m *SuiteIFace) UserCanSeeOtherUser(userID string, otherUserId string) (bool, *model.AppError) {
+	ret := _m.Called(userID, otherUserId)
+
+	var r0 bool
+	if rf, ok := ret.Get(0).(func(string, string) bool); ok {
+		r0 = rf(userID, otherUserId)
+	} else {
+		r0 = ret.Get(0).(bool)
+	}
+
+	var r1 *model.AppError
+	if rf, ok := ret.Get(1).(func(string, string) *model.AppError); ok {
+		r1 = rf(userID, otherUserId)
+	} else {
+		if ret.Get(1) != nil {
+			r1 = ret.Get(1).(*model.AppError)
+		}
+	}
+
+	return r0, r1
+}
+
+type mockConstructorTestingTNewSuiteIFace interface {
+	mock.TestingT
+	Cleanup(func())
+}
+
+// NewSuiteIFace creates a new instance of SuiteIFace. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+func NewSuiteIFace(t mockConstructorTestingTNewSuiteIFace) *SuiteIFace {
+	mock := &SuiteIFace{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
+}

app/platform/options.go

@@ -0,0 +1,106 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/v6/config"
+	"github.com/mattermost/mattermost-server/v6/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/mattermost-server/v6/store"
+	"github.com/mattermost/mattermost-server/v6/store/localcachelayer"
+)
+
+type Option func(ps *PlatformService) error
+
+// By default, the app will use the store specified by the configuration. This allows you to
+// construct an app with a different store.
+//
+// The override parameter must be either a store.Store or func(App) store.Store().
+func StoreOverride(override any) Option {
+	return func(ps *PlatformService) error {
+		switch o := override.(type) {
+		case store.Store:
+			ps.newStore = func() (store.Store, error) {
+				return o, nil
+			}
+			return nil
+
+		case func(*PlatformService) store.Store:
+			ps.newStore = func() (store.Store, error) {
+				return o(ps), nil
+			}
+			return nil
+
+		default:
+			return errors.New("invalid StoreOverride")
+		}
+	}
+}
+
+func StoreOverrideWithCache(override store.Store) Option {
+	return func(ps *PlatformService) error {
+		ps.newStore = func() (store.Store, error) {
+			lcl, err := localcachelayer.NewLocalCacheLayer(override, ps.metricsImpl(), ps.clusterIFace, ps.cacheProvider)
+			if err != nil {
+				return nil, err
+			}
+			return lcl, nil
+		}
+
+		return nil
+	}
+}
+
+// Config applies the given config dsn, whether a path to config.json
+// or a database connection string. It receives as well a set of
+// custom defaults that will be applied for any unset property of the
+// config loaded from the dsn on top of the normal defaults
+func Config(dsn string, readOnly bool, configDefaults *model.Config) Option {
+	return func(ps *PlatformService) error {
+		configStore, err := config.NewStoreFromDSN(dsn, readOnly, configDefaults, true)
+		if err != nil {
+			return fmt.Errorf("failed to apply Config option: %w", err)
+		}
+
+		ps.configStore = configStore
+
+		return nil
+	}
+}
+
+// ConfigStore applies the given config store, typically to replace the traditional sources with a memory store for testing.
+func ConfigStore(configStore *config.Store) Option {
+	return func(ps *PlatformService) error {
+		ps.configStore = configStore
+
+		return nil
+	}
+}
+
+func StartMetrics() Option {
+	return func(ps *PlatformService) error {
+		ps.startMetrics = true
+		return nil
+	}
+}
+
+func SetLogger(logger *mlog.Logger) Option {
+	return func(ps *PlatformService) error {
+		ps.SetLogger(logger)
+
+		return nil
+	}
+}
+
+func SetCluster(cluster einterfaces.ClusterInterface) Option {
+	return func(ps *PlatformService) error {
+		ps.clusterIFace = cluster
+		return nil
+	}
+}

app/platform/searchengine.go

@@ -0,0 +1,87 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+func (ps *PlatformService) StartSearchEngine() (string, string) {
+	if ps.SearchEngine.ElasticsearchEngine != nil && ps.SearchEngine.ElasticsearchEngine.IsActive() {
+		ps.Go(func() {
+			if err := ps.SearchEngine.ElasticsearchEngine.Start(); err != nil {
+				ps.Log().Error(err.Error())
+			}
+		})
+	}
+
+	configListenerId := ps.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) {
+		if ps.SearchEngine == nil {
+			return
+		}
+		ps.SearchEngine.UpdateConfig(newConfig)
+
+		if ps.SearchEngine.ElasticsearchEngine != nil && !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing {
+			ps.Go(func() {
+				if err := ps.SearchEngine.ElasticsearchEngine.Start(); err != nil {
+					mlog.Error(err.Error())
+				}
+			})
+		} else if ps.SearchEngine.ElasticsearchEngine != nil && *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing {
+			ps.Go(func() {
+				if err := ps.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
+					mlog.Error(err.Error())
+				}
+			})
+		} else if ps.SearchEngine.ElasticsearchEngine != nil && *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionURL != *newConfig.ElasticsearchSettings.ConnectionURL || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff {
+			ps.Go(func() {
+				if *oldConfig.ElasticsearchSettings.EnableIndexing {
+					if err := ps.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
+						mlog.Error(err.Error())
+					}
+					if err := ps.SearchEngine.ElasticsearchEngine.Start(); err != nil {
+						mlog.Error(err.Error())
+					}
+				}
+			})
+		}
+	})
+
+	licenseListenerId := ps.AddLicenseListener(func(oldLicense, newLicense *model.License) {
+		if ps.SearchEngine == nil {
+			return
+		}
+		if oldLicense == nil && newLicense != nil {
+			if ps.SearchEngine.ElasticsearchEngine != nil && ps.SearchEngine.ElasticsearchEngine.IsActive() {
+				ps.Go(func() {
+					if err := ps.SearchEngine.ElasticsearchEngine.Start(); err != nil {
+						mlog.Error(err.Error())
+					}
+				})
+			}
+		} else if oldLicense != nil && newLicense == nil {
+			if ps.SearchEngine.ElasticsearchEngine != nil {
+				ps.Go(func() {
+					if err := ps.SearchEngine.ElasticsearchEngine.Stop(); err != nil {
+						mlog.Error(err.Error())
+					}
+				})
+			}
+		}
+	})
+
+	return configListenerId, licenseListenerId
+}
+
+func (ps *PlatformService) StopSearchEngine() {
+	ps.RemoveConfigListener(ps.searchConfigListenerId)
+	ps.RemoveLicenseListener(ps.searchLicenseListenerId)
+	if ps.SearchEngine != nil && ps.SearchEngine.ElasticsearchEngine != nil && ps.SearchEngine.ElasticsearchEngine.IsActive() {
+		ps.SearchEngine.ElasticsearchEngine.Stop()
+	}
+	if ps.SearchEngine != nil && ps.SearchEngine.BleveEngine != nil && ps.SearchEngine.BleveEngine.IsActive() {
+		ps.SearchEngine.BleveEngine.Stop()
+	}
+}

app/platform/server_license.go

@@ -1,19 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.txt for license information.
-
-package platform
-
-import (
-	"github.com/mattermost/mattermost-server/v6/model"
-)
-
-// License returns the license stored in the server struct.
-// This should be removed with MM-45839
-func (ps *PlatformService) License() *model.License {
-	license, _ := ps.licenseValue.Load().(*model.License)
-	return license
-}
-
-func (ps *PlatformService) SetLicense(license *model.License) {
-	ps.licenseValue.Store(license)
-}

app/platform/service.go

@@ -5,57 +5,293 @@ package platform
 
 import (
 	"fmt"
+	"hash/maphash"
+	"net/http"
+	"runtime"
 	"sync"
 	"sync/atomic"
 
 	"github.com/mattermost/mattermost-server/v6/app/featureflag"
 	"github.com/mattermost/mattermost-server/v6/config"
 	"github.com/mattermost/mattermost-server/v6/einterfaces"
+	"github.com/mattermost/mattermost-server/v6/jobs"
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin"
+	"github.com/mattermost/mattermost-server/v6/services/cache"
+	"github.com/mattermost/mattermost-server/v6/services/searchengine"
+	"github.com/mattermost/mattermost-server/v6/services/searchengine/bleveengine"
 	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/mattermost-server/v6/store"
+	"github.com/mattermost/mattermost-server/v6/store/localcachelayer"
+	"github.com/mattermost/mattermost-server/v6/store/retrylayer"
+	"github.com/mattermost/mattermost-server/v6/store/searchlayer"
+	"github.com/mattermost/mattermost-server/v6/store/sqlstore"
+	"github.com/mattermost/mattermost-server/v6/store/timerlayer"
 )
 
 // PlatformService is the service for the platform related tasks. It is
 // responsible for non-entity related functionalities that are required
 // by a product such as database access, configuration access, licensing etc.
 type PlatformService struct {
-	serviceConfig ServiceConfig
+	sqlStore *sqlstore.SqlStore
+	Store    store.Store
+	newStore func() (store.Store, error)
+
+	WebSocketRouter *WebSocketRouter
+
+	serviceConfig *ServiceConfig
 	configStore   *config.Store
 
+	cacheProvider cache.Provider
+	statusCache   cache.Cache
+	sessionCache  cache.Cache
+	sessionPool   sync.Pool
+
+	asymmetricSigningKey atomic.Value
+	clientConfig         atomic.Value
+	clientConfigHash     atomic.Value
+	limitedClientConfig  atomic.Value
+
 	logger              *mlog.Logger
 	notificationsLogger *mlog.Logger
 
-	metrics *platformMetrics
+	startMetrics bool
+	metrics      *platformMetrics
 
 	featureFlagSynchronizerMutex sync.Mutex
 	featureFlagSynchronizer      *featureflag.Synchronizer
 	featureFlagStop              chan struct{}
 	featureFlagStopped           chan struct{}
 
-	licenseValue atomic.Value
-	telemetryId  string
+	licenseValue       atomic.Value
+	clientLicenseValue atomic.Value
+	licenseListeners   map[string]func(*model.License, *model.License)
+	licenseManager     einterfaces.LicenseInterface
 
-	cluster einterfaces.ClusterInterface
+	telemetryId       string
+	configListenerId  string
+	licenseListenerId string
+
+	clusterLeaderListeners sync.Map
+	clusterIFace           einterfaces.ClusterInterface
+	Busy                   *Busy
+
+	SearchEngine            *searchengine.Broker
+	searchConfigListenerId  string
+	searchLicenseListenerId string
+
+	Jobs *jobs.JobServer
+
+	hubs     []*Hub
+	hashSeed maphash.Seed
+
+	goroutineCount      int32
+	goroutineExitSignal chan struct{}
+	goroutineBuffered   chan struct{}
+
+	additionalClusterHandlers map[model.ClusterEvent]einterfaces.ClusterMessageHandler
+	sharedChannelService      SharedChannelServiceIFace
+
+	pluginEnv *plugin.Environment
 }
 
 // New creates a new PlatformService.
-func New(sc ServiceConfig) (*PlatformService, error) {
+func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 	if err := sc.validate(); err != nil {
 		return nil, err
 	}
 
+	// Step 0: Create the PlatformService.
+	// ConfigStore is and should be handled on a upper level.
 	ps := &PlatformService{
-		serviceConfig: sc,
-		configStore:   sc.ConfigStore,
-		cluster:       sc.Cluster,
+		serviceConfig:       &sc,
+		Store:               sc.Store,
+		configStore:         sc.ConfigStore,
+		clusterIFace:        sc.Cluster,
+		hashSeed:            maphash.MakeSeed(),
+		goroutineExitSignal: make(chan struct{}, 1),
+		goroutineBuffered:   make(chan struct{}, runtime.NumCPU()),
+		WebSocketRouter: &WebSocketRouter{
+			handlers: make(map[string]webSocketHandler),
+		},
+		sessionPool: sync.Pool{
+			New: func() any {
+				return &model.Session{}
+			},
+		},
+		licenseListeners:          map[string]func(*model.License, *model.License){},
+		additionalClusterHandlers: map[model.ClusterEvent]einterfaces.ClusterMessageHandler{},
 	}
 
+	// Step 1: Cache provider.
+	// At the moment we only have this implementation
+	// in the future the cache provider will be built based on the loaded config
+	ps.cacheProvider = cache.NewProvider()
+	if err2 := ps.cacheProvider.Connect(); err2 != nil {
+		return nil, fmt.Errorf("unable to connect to cache provider: %w", err2)
+	}
+
+	// Apply options, some of the options overrides the default config actually.
+	for _, option := range options {
+		if err := option(ps); err != nil {
+			return nil, fmt.Errorf("failed to apply option: %w", err)
+		}
+	}
+
+	// Step 2: Start logging.
 	if err := ps.initLogging(); err != nil {
 		return nil, fmt.Errorf("failed to initialize logging: %w", err)
 	}
 
-	if err := ps.resetMetrics(sc.Metrics, ps.configStore.Get); err != nil {
+	// This is called after initLogging() to avoid a race condition.
+	mlog.Info("Server is initializing...", mlog.String("go_version", runtime.Version()))
+
+	// Step 3: Search Engine
+	searchEngine := searchengine.NewBroker(ps.Config())
+	bleveEngine := bleveengine.NewBleveEngine(ps.Config())
+	if err := bleveEngine.Start(); err != nil {
 		return nil, err
 	}
+	searchEngine.RegisterBleveEngine(bleveEngine)
+	ps.SearchEngine = searchEngine
+
+	// Step 4: Init Enterprise
+	// Depends on step 3 (s.SearchEngine must be non-nil)
+	ps.initEnterprise()
+
+	// Step 5: Store.
+	// Depends on Step 1 (config), 4 (metrics, cluster) and 5 (cacheProvider).
+	if ps.newStore == nil {
+		ps.newStore = func() (store.Store, error) {
+			ps.sqlStore = sqlstore.New(ps.Config().SqlSettings, ps.Metrics())
+
+			lcl, err2 := localcachelayer.NewLocalCacheLayer(
+				retrylayer.New(ps.sqlStore),
+				ps.Metrics(),
+				ps.clusterIFace,
+				ps.cacheProvider,
+			)
+			if err2 != nil {
+				return nil, fmt.Errorf("cannot create local cache layer: %w", err2)
+			}
+
+			searchStore := searchlayer.NewSearchLayer(
+				lcl,
+				ps.SearchEngine,
+				ps.Config(),
+			)
+
+			ps.AddConfigListener(func(prevCfg, cfg *model.Config) {
+				searchStore.UpdateConfig(cfg)
+			})
+
+			license := ps.License()
+			ps.sqlStore.UpdateLicense(license)
+			ps.AddLicenseListener(func(oldLicense, newLicense *model.License) {
+				ps.sqlStore.UpdateLicense(newLicense)
+			})
+
+			return timerlayer.New(
+				searchStore,
+				ps.Metrics(),
+			), nil
+		}
+	}
+
+	var err error
+	ps.Store, err = ps.newStore()
+	if err != nil {
+		return nil, fmt.Errorf("cannot create store: %w", err)
+	}
+
+	// Needed before loading license
+	ps.statusCache, err = ps.cacheProvider.NewCache(&cache.CacheOptions{
+		Size:           model.StatusCacheSize,
+		Striped:        true,
+		StripedBuckets: maxInt(runtime.NumCPU()-1, 1),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to create status cache: %w", err)
+	}
+
+	ps.sessionCache, err = ps.cacheProvider.NewCache(&cache.CacheOptions{
+		Size:           model.SessionCacheSize,
+		Striped:        true,
+		StripedBuckets: maxInt(runtime.NumCPU()-1, 1),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("could not create session cache: %w", err)
+	}
+
+	if model.BuildEnterpriseReady == "true" {
+		ps.LoadLicense()
+	}
+
+	if metricsInterface != nil {
+		sc.Metrics = metricsInterface(ps, *ps.configStore.Get().SqlSettings.DriverName, *ps.configStore.Get().SqlSettings.DataSource)
+	}
+
+	if ps.startMetrics {
+		if err = ps.resetMetrics(sc.Metrics, ps.configStore.Get); err != nil {
+			return nil, err
+		}
+	}
+
+	if err = ps.EnsureAsymmetricSigningKey(); err != nil {
+		return nil, fmt.Errorf("unable to ensure asymmetric signing key: %w", err)
+	}
+
+	ps.Busy = NewBusy(ps.clusterIFace)
+
+	ps.configListenerId = ps.AddConfigListener(func(_, _ *model.Config) {
+		ps.regenerateClientConfig()
+
+		message := model.NewWebSocketEvent(model.WebsocketEventConfigChanged, "", "", "", nil, "")
+
+		message.Add("config", ps.ClientConfigWithComputed())
+		ps.Go(func() {
+			ps.Publish(message)
+		})
+
+		if err = ps.ReconfigureLogger(); err != nil {
+			mlog.Error("Error re-configuring logging after config change", mlog.Err(err))
+			return
+		}
+	})
+	ps.licenseListenerId = ps.AddLicenseListener(func(oldLicense, newLicense *model.License) {
+		ps.regenerateClientConfig()
+
+		message := model.NewWebSocketEvent(model.WebsocketEventLicenseChanged, "", "", "", nil, "")
+		message.Add("license", ps.GetSanitizedClientLicense())
+		ps.Go(func() {
+			ps.Publish(message)
+		})
+
+	})
+
+	// Enable developer settings if this is a "dev" build
+	if model.BuildNumber == "dev" {
+		ps.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.EnableDeveloper = true })
+	}
+
+	ps.AddLicenseListener(func(oldLicense, newLicense *model.License) {
+		if (oldLicense == nil && newLicense == nil) || !ps.startMetrics {
+			return
+		}
+
+		if oldLicense != nil && newLicense != nil && *oldLicense.Features.Metrics == *newLicense.Features.Metrics {
+			return
+		}
+
+		if err := ps.RestartMetrics(); err != nil {
+			ps.logger.Error("Failed to reset metrics server", mlog.Err(err))
+		}
+	})
+
+	ps.SearchEngine.UpdateConfig(ps.Config())
+	searchConfigListenerId, searchLicenseListenerId := ps.StartSearchEngine()
+	ps.searchConfigListenerId = searchConfigListenerId
+	ps.searchLicenseListenerId = searchLicenseListenerId
 
 	return ps, nil
 }
@@ -69,6 +305,8 @@ func (ps *PlatformService) ShutdownMetrics() error {
 }
 
 func (ps *PlatformService) ShutdownConfig() error {
+	ps.RemoveConfigListener(ps.configListenerId)
+
 	if ps.configStore != nil {
 		err := ps.configStore.Close()
 		if err != nil {
@@ -86,3 +324,90 @@ func (ps *PlatformService) SetTelemetryId(id string) {
 func (ps *PlatformService) SetLogger(logger *mlog.Logger) {
 	ps.logger = logger
 }
+
+func (ps *PlatformService) initEnterprise() {
+	if clusterInterface != nil && ps.clusterIFace == nil {
+		ps.clusterIFace = clusterInterface(ps)
+	}
+
+	if elasticsearchInterface != nil {
+		ps.SearchEngine.RegisterElasticsearchEngine(elasticsearchInterface(ps))
+	}
+
+	if licenseInterface != nil {
+		ps.licenseManager = licenseInterface(ps)
+	}
+}
+
+func (ps *PlatformService) TotalWebsocketConnections() int {
+	// This method is only called after the hub is initialized.
+	// Therefore, no mutex is needed to protect s.hubs.
+	count := int64(0)
+	for _, hub := range ps.hubs {
+		count = count + atomic.LoadInt64(&hub.connectionCount)
+	}
+
+	return int(count)
+}
+
+func (ps *PlatformService) Shutdown() error {
+	ps.HubStop()
+
+	ps.RemoveLicenseListener(ps.licenseListenerId)
+
+	if ps.Store != nil {
+		ps.Store.Close()
+	}
+
+	if ps.cacheProvider != nil {
+		if err := ps.cacheProvider.Close(); err != nil {
+			return fmt.Errorf("unable to cleanly shutdown cache: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (ps *PlatformService) CacheProvider() cache.Provider {
+	return ps.cacheProvider
+}
+
+func (ps *PlatformService) StatusCache() cache.Cache {
+	return ps.statusCache
+}
+
+// SetSqlStore is used for plugin testing
+func (ps *PlatformService) SetSqlStore(s *sqlstore.SqlStore) {
+	ps.sqlStore = s
+}
+
+func (ps *PlatformService) SetSharedChannelService(s SharedChannelServiceIFace) {
+	ps.sharedChannelService = s
+}
+
+func (ps *PlatformService) SetPluginsEnvironment(env *plugin.Environment) {
+	ps.pluginEnv = env
+}
+
+// GetPluginStatuses meant to be used by cluster implementation
+func (ps *PlatformService) GetPluginStatuses() (model.PluginStatuses, *model.AppError) {
+	if ps.pluginEnv == nil {
+		return nil, model.NewAppError("GetPluginStatuses", "app.plugin.disabled.app_error", nil, "", http.StatusNotImplemented)
+	}
+
+	pluginStatuses, err := ps.pluginEnv.Statuses()
+	if err != nil {
+		return nil, model.NewAppError("GetPluginStatuses", "app.plugin.get_statuses.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// Add our cluster ID
+	for _, status := range pluginStatuses {
+		if ps.Cluster() != nil {
+			status.ClusterId = ps.Cluster().GetClusterId()
+		} else {
+			status.ClusterId = ""
+		}
+	}
+
+	return pluginStatuses, nil
+}

app/platform/service_test.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"os"
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v6/config"
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/store/storetest"
+	"github.com/stretchr/testify/require"
+)
+
+func TestReadReplicaDisabledBasedOnLicense(t *testing.T) {
+	cfg := model.Config{}
+	cfg.SetDefaults()
+	driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
+	if driverName == "" {
+		driverName = model.DatabaseDriverPostgres
+	}
+	dsn := ""
+	if driverName == model.DatabaseDriverPostgres {
+		dsn = os.Getenv("TEST_DATABASE_POSTGRESQL_DSN")
+	} else {
+		dsn = os.Getenv("TEST_DATABASE_MYSQL_DSN")
+	}
+	cfg.SqlSettings = *storetest.MakeSqlSettings(driverName, false)
+	if dsn != "" {
+		cfg.SqlSettings.DataSource = &dsn
+	}
+	cfg.SqlSettings.DataSourceReplicas = []string{*cfg.SqlSettings.DataSource}
+	cfg.SqlSettings.DataSourceSearchReplicas = []string{*cfg.SqlSettings.DataSource}
+
+	t.Run("Read Replicas with no License", func(t *testing.T) {
+		configStore := config.NewTestMemoryStore()
+		configStore.Set(&cfg)
+		ps, err := New(ServiceConfig{
+			ConfigStore: configStore,
+		})
+		require.NoError(t, err)
+		require.Same(t, ps.sqlStore.GetMasterX(), ps.sqlStore.GetReplicaX())
+		require.Len(t, ps.Config().SqlSettings.DataSourceReplicas, 1)
+	})
+
+	t.Run("Read Replicas With License", func(t *testing.T) {
+		configStore := config.NewTestMemoryStore()
+		configStore.Set(&cfg)
+		ps, err := New(ServiceConfig{
+			ConfigStore: configStore,
+		}, func(ps *PlatformService) error {
+			ps.licenseValue.Store(model.NewTestLicense())
+			return nil
+		})
+		require.NoError(t, err)
+		require.NotSame(t, ps.sqlStore.GetMasterX(), ps.sqlStore.GetReplicaX())
+		require.Len(t, ps.Config().SqlSettings.DataSourceReplicas, 1)
+	})
+
+	t.Run("Search Replicas with no License", func(t *testing.T) {
+		configStore := config.NewTestMemoryStore()
+		configStore.Set(&cfg)
+		ps, err := New(ServiceConfig{
+			ConfigStore: configStore,
+		})
+		require.NoError(t, err)
+		require.Same(t, ps.sqlStore.GetMasterX(), ps.sqlStore.GetSearchReplicaX())
+		require.Len(t, ps.Config().SqlSettings.DataSourceSearchReplicas, 1)
+	})
+
+	t.Run("Search Replicas With License", func(t *testing.T) {
+		configStore := config.NewTestMemoryStore()
+		configStore.Set(&cfg)
+		ps, err := New(ServiceConfig{
+			ConfigStore: configStore,
+		}, func(ps *PlatformService) error {
+			ps.licenseValue.Store(model.NewTestLicense())
+			return nil
+		})
+		require.NoError(t, err)
+		require.NotSame(t, ps.sqlStore.GetMasterX(), ps.sqlStore.GetSearchReplicaX())
+		require.Len(t, ps.Config().SqlSettings.DataSourceSearchReplicas, 1)
+	})
+}

app/platform/session.go

@@ -0,0 +1,262 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/mattermost-server/v6/store/sqlstore"
+)
+
+func (ps *PlatformService) ReturnSessionToPool(session *model.Session) {
+	if session != nil {
+		session.Id = ""
+		ps.sessionPool.Put(session)
+	}
+}
+
+func (ps *PlatformService) CreateSession(session *model.Session) (*model.Session, error) {
+	session.Token = ""
+
+	session, err := ps.Store.Session().Save(session)
+	if err != nil {
+		return nil, err
+	}
+
+	ps.AddSessionToCache(session)
+
+	return session, nil
+}
+
+func (ps *PlatformService) GetSessionContext(ctx context.Context, token string) (*model.Session, error) {
+	return ps.Store.Session().Get(ctx, token)
+}
+
+func (ps *PlatformService) GetSessions(userID string) ([]*model.Session, error) {
+	return ps.Store.Session().GetSessions(userID)
+}
+
+func (ps *PlatformService) AddSessionToCache(session *model.Session) {
+	ps.sessionCache.SetWithExpiry(session.Token, session, time.Duration(int64(*ps.Config().ServiceSettings.SessionCacheInMinutes))*time.Minute)
+}
+
+func (ps *PlatformService) SessionCacheLength() int {
+	if l, err := ps.sessionCache.Len(); err == nil {
+		return l
+	}
+	return 0
+}
+
+func (ps *PlatformService) ClearUserSessionCacheLocal(userID string) {
+	if keys, err := ps.sessionCache.Keys(); err == nil {
+		var session *model.Session
+		for _, key := range keys {
+			if err := ps.sessionCache.Get(key, &session); err == nil {
+				if session.UserId == userID {
+					ps.sessionCache.Remove(key)
+					if m := ps.metricsImpl(); m != nil {
+						m.IncrementMemCacheInvalidationCounterSession()
+					}
+				}
+			}
+		}
+	}
+}
+
+func (ps *PlatformService) ClearAllUsersSessionCacheLocal() {
+	ps.sessionCache.Purge()
+}
+
+func (ps *PlatformService) ClearUserSessionCache(userID string) {
+	ps.ClearUserSessionCacheLocal(userID)
+
+	if ps.clusterIFace != nil {
+		msg := &model.ClusterMessage{
+			Event:    model.ClusterEventClearSessionCacheForUser,
+			SendType: model.ClusterSendReliable,
+			Data:     []byte(userID),
+		}
+		ps.clusterIFace.SendClusterMessage(msg)
+	}
+}
+
+func (ps *PlatformService) ClearAllUsersSessionCache() {
+	ps.ClearAllUsersSessionCacheLocal()
+
+	if ps.clusterIFace != nil {
+		msg := &model.ClusterMessage{
+			Event:    model.ClusterEventClearSessionCacheForAllUsers,
+			SendType: model.ClusterSendReliable,
+		}
+		ps.clusterIFace.SendClusterMessage(msg)
+	}
+}
+
+func (ps *PlatformService) GetSession(token string) (*model.Session, error) {
+	var session = ps.sessionPool.Get().(*model.Session)
+	if err := ps.sessionCache.Get(token, session); err == nil {
+		if m := ps.metricsImpl(); m != nil {
+			m.IncrementMemCacheHitCounterSession()
+		}
+	} else {
+		if m := ps.metricsImpl(); m != nil {
+			m.IncrementMemCacheMissCounterSession()
+		}
+	}
+
+	if session.Id != "" {
+		return session, nil
+	}
+
+	return ps.GetSessionContext(sqlstore.WithMaster(context.Background()), token)
+}
+
+func (ps *PlatformService) GetSessionByID(sessionID string) (*model.Session, error) {
+	return ps.Store.Session().Get(context.Background(), sessionID)
+}
+
+func (ps *PlatformService) RevokeSessionsFromAllUsers() error {
+	// revoke tokens before sessions so they can't be used to relogin
+	nErr := ps.Store.OAuth().RemoveAllAccessData()
+	if nErr != nil {
+		return fmt.Errorf("%s: %w", nErr.Error(), DeleteAllAccessDataError)
+	}
+	err := ps.Store.Session().RemoveAllSessions()
+	if err != nil {
+		return err
+	}
+
+	ps.ClearAllUsersSessionCache()
+	return nil
+}
+
+func (ps *PlatformService) RevokeSessionsForDeviceId(userID string, deviceID string, currentSessionId string) error {
+	sessions, err := ps.Store.Session().GetSessions(userID)
+	if err != nil {
+		return err
+	}
+	for _, session := range sessions {
+		if session.DeviceId == deviceID && session.Id != currentSessionId {
+			mlog.Debug("Revoking sessionId for userId. Re-login with the same device Id", mlog.String("session_id", session.Id), mlog.String("user_id", userID))
+			if err := ps.RevokeSession(session); err != nil {
+				mlog.Warn("Could not revoke session for device", mlog.String("device_id", deviceID), mlog.Err(err))
+			}
+		}
+	}
+
+	return nil
+}
+
+func (ps *PlatformService) RevokeSession(session *model.Session) error {
+	if session.IsOAuth {
+		if err := ps.RevokeAccessToken(session.Token); err != nil {
+			return err
+		}
+	} else {
+		if err := ps.Store.Session().Remove(session.Id); err != nil {
+			return fmt.Errorf("%s: %w", err.Error(), DeleteSessionError)
+		}
+	}
+
+	ps.ClearUserSessionCache(session.UserId)
+
+	return nil
+}
+
+func (ps *PlatformService) RevokeAccessToken(token string) error {
+	session, _ := ps.GetSession(token)
+
+	defer ps.ReturnSessionToPool(session)
+
+	schan := make(chan error, 1)
+	go func() {
+		schan <- ps.Store.Session().Remove(token)
+		close(schan)
+	}()
+
+	if _, err := ps.Store.OAuth().GetAccessData(token); err != nil {
+		return fmt.Errorf("%s: %w", err.Error(), GetTokenError)
+	}
+
+	if err := ps.Store.OAuth().RemoveAccessData(token); err != nil {
+		return fmt.Errorf("%s: %w", err.Error(), DeleteTokenError)
+	}
+
+	if err := <-schan; err != nil {
+		return fmt.Errorf("%s: %w", err.Error(), DeleteSessionError)
+	}
+
+	if session != nil {
+		ps.ClearUserSessionCache(session.UserId)
+	}
+
+	return nil
+}
+
+// SetSessionExpireInHours sets the session's expiry the specified number of hours
+// relative to either the session creation date or the current time, depending
+// on the `ExtendSessionOnActivity` config setting.
+func (ps *PlatformService) SetSessionExpireInHours(session *model.Session, hours int) {
+	if session.CreateAt == 0 || *ps.Config().ServiceSettings.ExtendSessionLengthWithActivity {
+		session.ExpiresAt = model.GetMillis() + (1000 * 60 * 60 * int64(hours))
+	} else {
+		session.ExpiresAt = session.CreateAt + (1000 * 60 * 60 * int64(hours))
+	}
+}
+
+func (ps *PlatformService) ExtendSessionExpiry(session *model.Session, newExpiry int64) error {
+	if err := ps.Store.Session().UpdateExpiresAt(session.Id, newExpiry); err != nil {
+		return err
+	}
+
+	// Update local cache. No need to invalidate cache for cluster as the session cache timeout
+	// ensures each node will get an extended expiry within the next 10 minutes.
+	// Worst case is another node may generate a redundant expiry update.
+	session.ExpiresAt = newExpiry
+	ps.AddSessionToCache(session)
+
+	return nil
+}
+
+func (ps *PlatformService) UpdateSessionsIsGuest(userID string, isGuest bool) error {
+	sessions, err := ps.GetSessions(userID)
+	if err != nil {
+		return err
+	}
+
+	for _, session := range sessions {
+		session.AddProp(model.SessionPropIsGuest, fmt.Sprintf("%t", isGuest))
+		err := ps.Store.Session().UpdateProps(session)
+		if err != nil {
+			mlog.Warn("Unable to update isGuest session", mlog.Err(err))
+			continue
+		}
+		ps.AddSessionToCache(session)
+	}
+	return nil
+}
+
+func (ps *PlatformService) RevokeAllSessions(userID string) error {
+	sessions, err := ps.Store.Session().GetSessions(userID)
+	if err != nil {
+		return fmt.Errorf("%s: %w", err.Error(), GetSessionError)
+	}
+	for _, session := range sessions {
+		if session.IsOAuth {
+			ps.RevokeAccessToken(session.Token)
+		} else {
+			if err := ps.Store.Session().Remove(session.Id); err != nil {
+				return fmt.Errorf("%s: %w", err.Error(), DeleteSessionError)
+			}
+		}
+	}
+
+	ps.ClearUserSessionCache(userID)
+
+	return nil
+}

app/platform/session_test.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	dayInMillis = 86400000
+	grace       = 5 * 1000
+	thirtyDays  = dayInMillis * 30
+)
+
+func TestCache(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	session := &model.Session{
+		Id:     model.NewId(),
+		Token:  model.NewId(),
+		UserId: model.NewId(),
+	}
+
+	session2 := &model.Session{
+		Id:     model.NewId(),
+		Token:  model.NewId(),
+		UserId: model.NewId(),
+	}
+
+	th.Service.sessionCache.SetWithExpiry(session.Token, session, 5*time.Minute)
+	th.Service.sessionCache.SetWithExpiry(session2.Token, session2, 5*time.Minute)
+
+	keys, err := th.Service.sessionCache.Keys()
+	require.NoError(t, err)
+	require.NotEmpty(t, keys)
+
+	th.Service.ClearUserSessionCache(session.UserId)
+
+	rkeys, err := th.Service.sessionCache.Keys()
+	require.NoError(t, err)
+	require.Lenf(t, rkeys, len(keys)-1, "should have one less: %d - %d != 1", len(keys), len(rkeys))
+	require.NotEmpty(t, rkeys)
+
+	th.Service.ClearAllUsersSessionCache()
+
+	rkeys, err = th.Service.sessionCache.Keys()
+	require.NoError(t, err)
+	require.Empty(t, rkeys)
+}
+
+func TestSetSessionExpireInHours(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	now := model.GetMillis()
+	createAt := now - (dayInMillis * 20)
+
+	tests := []struct {
+		name   string
+		extend bool
+		create bool
+		days   int
+		want   int64
+	}{
+		{name: "zero days, extend", extend: true, create: true, days: 0, want: now},
+		{name: "zero days, extend", extend: true, create: false, days: 0, want: now},
+		{name: "zero days, no extend", extend: false, create: true, days: 0, want: createAt},
+		{name: "zero days, no extend", extend: false, create: false, days: 0, want: now},
+		{name: "thirty days, extend", extend: true, create: true, days: 30, want: now + thirtyDays},
+		{name: "thirty days, extend", extend: true, create: false, days: 30, want: now + thirtyDays},
+		{name: "thirty days, no extend", extend: false, create: true, days: 30, want: createAt + thirtyDays},
+		{name: "thirty days, no extend", extend: false, create: false, days: 30, want: now + thirtyDays},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			th.Service.UpdateConfig(func(cfg *model.Config) {
+				*cfg.ServiceSettings.ExtendSessionLengthWithActivity = tt.extend
+			})
+			var create int64
+			if tt.create {
+				create = createAt
+			}
+
+			session := &model.Session{
+				CreateAt:  create,
+				ExpiresAt: model.GetMillis() + dayInMillis,
+			}
+			th.Service.SetSessionExpireInHours(session, tt.days*24)
+
+			// must be within 5 seconds of expected time.
+			require.GreaterOrEqual(t, session.ExpiresAt, tt.want-grace)
+			require.LessOrEqual(t, session.ExpiresAt, tt.want+grace)
+		})
+	}
+}
+
+func TestOAuthRevokeAccessToken(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	err := th.Service.RevokeAccessToken(model.NewRandomString(16))
+	require.Error(t, err, "Should have failed due to an incorrect token")
+
+	session := &model.Session{}
+	session.CreateAt = model.GetMillis()
+	session.UserId = model.NewId()
+	session.Token = model.NewId()
+	session.Roles = model.SystemUserRoleId
+	th.Service.SetSessionExpireInHours(session, 24)
+
+	session, _ = th.Service.CreateSession(session)
+	err = th.Service.RevokeAccessToken(session.Token)
+	require.Error(t, err, "Should have failed does not have an access token")
+
+	accessData := &model.AccessData{}
+	accessData.Token = session.Token
+	accessData.UserId = session.UserId
+	accessData.RedirectUri = "http://example.com"
+	accessData.ClientId = model.NewId()
+	accessData.ExpiresAt = session.ExpiresAt
+
+	_, nErr := th.Service.Store.OAuth().SaveAccessData(accessData)
+	require.NoError(t, nErr)
+
+	err = th.Service.RevokeAccessToken(accessData.Token)
+	require.NoError(t, err)
+}

app/platform/shared_channel_notifier.go

@@ -0,0 +1,147 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/services/sharedchannel"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+var sharedChannelEventsForSync model.StringArray = []string{
+	model.WebsocketEventPosted,
+	model.WebsocketEventPostEdited,
+	model.WebsocketEventPostDeleted,
+	model.WebsocketEventReactionAdded,
+	model.WebsocketEventReactionRemoved,
+}
+
+var sharedChannelEventsForInvitation model.StringArray = []string{
+	model.WebsocketEventDirectAdded,
+}
+
+// SharedChannelSyncHandler is called when a websocket event is received by a cluster node.
+// Only on the leader node it will notify the sync service to perform necessary updates to the remote for the given
+// shared channel.
+func (ps *PlatformService) SharedChannelSyncHandler(event *model.WebSocketEvent) {
+	syncService := ps.sharedChannelService
+	if syncService == nil {
+		return
+	}
+	if isEligibleForEvents(syncService, event, sharedChannelEventsForSync) {
+		err := handleContentSync(ps, syncService, event)
+		if err != nil {
+			mlog.Warn(
+				err.Error(),
+				mlog.String("event", event.EventType()),
+				mlog.String("action", "content_sync"),
+			)
+		}
+	} else if isEligibleForEvents(syncService, event, sharedChannelEventsForInvitation) {
+		err := handleInvitation(ps, syncService, event)
+		if err != nil {
+			mlog.Warn(
+				err.Error(),
+				mlog.String("event", event.EventType()),
+				mlog.String("action", "invitation"),
+			)
+		}
+	}
+}
+
+func isEligibleForEvents(syncService SharedChannelServiceIFace, event *model.WebSocketEvent, events model.StringArray) bool {
+	return syncServiceEnabled(syncService) &&
+		eventHasChannel(event) &&
+		events.Contains(event.EventType())
+}
+
+func eventHasChannel(event *model.WebSocketEvent) bool {
+	return event.GetBroadcast() != nil &&
+		event.GetBroadcast().ChannelId != ""
+}
+
+func syncServiceEnabled(syncService SharedChannelServiceIFace) bool {
+	return syncService != nil &&
+		syncService.Active()
+}
+
+func handleContentSync(ps *PlatformService, syncService SharedChannelServiceIFace, event *model.WebSocketEvent) error {
+	channel, err := findChannel(ps, event.GetBroadcast().ChannelId)
+	if err != nil {
+		return err
+	}
+
+	if channel != nil && channel.IsShared() {
+		syncService.NotifyChannelChanged(channel.Id)
+	}
+
+	return nil
+}
+
+func handleInvitation(ps *PlatformService, syncService SharedChannelServiceIFace, event *model.WebSocketEvent) error {
+	channel, err := findChannel(ps, event.GetBroadcast().ChannelId)
+	if err != nil {
+		return err
+	}
+
+	if channel == nil || !channel.IsShared() {
+		return nil
+	}
+
+	creator, err := getUserFromEvent(ps, event, "creator_id")
+	if err != nil {
+		return err
+	}
+	// This is a termination condition, since on the other end when we are processing
+	// the invite we are re-triggering a model.WEBSOCKET_EVENT_DIRECT_ADDED, which will call this handler.
+	// When the creator is remote, it means that this is a DM that was not originated from the current server
+	// and therefore we do not need to do anything.
+	if creator == nil || creator.IsRemote() {
+		return nil
+	}
+
+	participant, err := getUserFromEvent(ps, event, "teammate_id")
+	if err != nil {
+		return err
+	}
+
+	if participant == nil {
+		return nil
+	}
+
+	rc, err := ps.Store.RemoteCluster().Get(*participant.RemoteId)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("couldn't find remote cluster %s, for creating shared channel invitation for a DM", *participant.RemoteId))
+	}
+
+	return syncService.SendChannelInvite(channel, creator.Id, rc, sharedchannel.WithDirectParticipantID(creator.Id), sharedchannel.WithDirectParticipantID(participant.Id))
+}
+
+func getUserFromEvent(ps *PlatformService, event *model.WebSocketEvent, key string) (*model.User, error) {
+	userID, ok := event.GetData()[key].(string)
+	if !ok || userID == "" {
+		return nil, fmt.Errorf("received websocket message that is eligible for sending an invitation but message does not have `%s` present", key)
+	}
+
+	user, err := ps.Store.User().Get(context.Background(), userID)
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't find user for creating shared channel invitation for a DM")
+	}
+
+	return user, nil
+}
+
+func findChannel(server *PlatformService, channelId string) (*model.Channel, error) {
+	channel, err := server.Store.Channel().Get(channelId, true)
+	if err != nil {
+		return nil, errors.Wrap(err, "received websocket message that is eligible for shared channel sync but channel does not exist")
+	}
+
+	return channel, nil
+}

app/platform/shared_channel_notifier_test.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestServerSyncSharedChannelHandler(t *testing.T) {
+	t.Run("sync service inactive, it does nothing", func(t *testing.T) {
+		th := SetupWithStoreMock(t)
+		defer th.TearDown()
+
+		mockService := NewMockSharedChannelService(nil)
+		mockService.active = false
+		th.Service.SetSharedChannelService(mockService)
+
+		th.Service.SharedChannelSyncHandler(&model.WebSocketEvent{})
+		assert.Empty(t, mockService.channelNotifications)
+	})
+
+	t.Run("sync service active and broadcast envelope has ineligible event, it does nothing", func(t *testing.T) {
+		th := Setup(t).InitBasic()
+		defer th.TearDown()
+
+		mockService := NewMockSharedChannelService(nil)
+		mockService.active = true
+		th.Service.SetSharedChannelService(mockService)
+
+		channel := th.CreateChannel(th.BasicTeam, WithShared(true))
+		websocketEvent := model.NewWebSocketEvent(model.WebsocketEventAddedToTeam, model.NewId(), channel.Id, "", nil, "")
+
+		th.Service.SharedChannelSyncHandler(websocketEvent)
+		assert.Empty(t, mockService.channelNotifications)
+	})
+
+	t.Run("sync service active and broadcast envelope has eligible event but channel does not exist, it does nothing", func(t *testing.T) {
+		th := Setup(t).InitBasic()
+		defer th.TearDown()
+
+		mockService := NewMockSharedChannelService(nil)
+		mockService.active = true
+		th.Service.SetSharedChannelService(mockService)
+
+		websocketEvent := model.NewWebSocketEvent(model.WebsocketEventPosted, model.NewId(), model.NewId(), "", nil, "")
+
+		th.Service.SharedChannelSyncHandler(websocketEvent)
+		assert.Empty(t, mockService.channelNotifications)
+	})
+
+	t.Run("sync service active when received eligible event, it triggers a shared channel content sync", func(t *testing.T) {
+		th := Setup(t).InitBasic()
+		defer th.TearDown()
+
+		mockService := NewMockSharedChannelService(nil)
+		mockService.active = true
+		th.Service.SetSharedChannelService(mockService)
+
+		channel := th.CreateChannel(th.BasicTeam, WithShared(true))
+		websocketEvent := model.NewWebSocketEvent(model.WebsocketEventPosted, th.BasicTeam.Id, channel.Id, "", nil, "")
+
+		th.Service.SharedChannelSyncHandler(websocketEvent)
+		require.Len(t, mockService.channelNotifications, 1)
+		assert.Equal(t, channel.Id, mockService.channelNotifications[0])
+	})
+}

app/platform/shared_channel_service_iface.go

@@ -0,0 +1,72 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/services/sharedchannel"
+)
+
+// SharedChannelServiceIFace is the interface to the shared channel service
+type SharedChannelServiceIFace interface {
+	Shutdown() error
+	Start() error
+	NotifyChannelChanged(channelId string)
+	NotifyUserProfileChanged(userID string)
+	SendChannelInvite(channel *model.Channel, userId string, rc *model.RemoteCluster, options ...sharedchannel.InviteOption) error
+	Active() bool
+}
+
+type MockOptionSharedChannelService func(service *mockSharedChannelService)
+
+func MockOptionSharedChannelServiceWithActive(active bool) MockOptionSharedChannelService {
+	return func(mrcs *mockSharedChannelService) {
+		mrcs.active = active
+	}
+}
+
+func NewMockSharedChannelService(service SharedChannelServiceIFace, options ...MockOptionSharedChannelService) *mockSharedChannelService {
+	mrcs := &mockSharedChannelService{service, true, []string{}, []string{}, 0}
+	for _, option := range options {
+		option(mrcs)
+	}
+	return mrcs
+}
+
+type mockSharedChannelService struct {
+	SharedChannelServiceIFace
+	active                   bool
+	channelNotifications     []string
+	userProfileNotifications []string
+	numInvitations           int
+}
+
+func (mrcs *mockSharedChannelService) NotifyChannelChanged(channelId string) {
+	mrcs.channelNotifications = append(mrcs.channelNotifications, channelId)
+}
+
+func (mrcs *mockSharedChannelService) NotifyUserProfileChanged(userId string) {
+	mrcs.userProfileNotifications = append(mrcs.userProfileNotifications, userId)
+}
+
+func (mrcs *mockSharedChannelService) Shutdown() error {
+	return nil
+}
+
+func (mrcs *mockSharedChannelService) Start() error {
+	return nil
+}
+
+func (mrcs *mockSharedChannelService) Active() bool {
+	return mrcs.active
+}
+
+func (mrcs *mockSharedChannelService) SendChannelInvite(channel *model.Channel, userId string, rc *model.RemoteCluster, options ...sharedchannel.InviteOption) error {
+	mrcs.numInvitations += 1
+	return nil
+}
+
+func (mrcs *mockSharedChannelService) NumInvitations() int {
+	return mrcs.numInvitations
+}

app/platform/status.go

@@ -0,0 +1,214 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+	"github.com/mattermost/mattermost-server/v6/store"
+)
+
+func (ps *PlatformService) AddStatusCacheSkipClusterSend(status *model.Status) {
+	ps.statusCache.Set(status.UserId, status)
+}
+
+func (ps *PlatformService) AddStatusCache(status *model.Status) {
+	ps.AddStatusCacheSkipClusterSend(status)
+
+	if ps.Cluster() != nil {
+		statusJSON, err := json.Marshal(status)
+		if err != nil {
+			ps.logger.Warn("Failed to encode status to JSON", mlog.Err(err))
+		}
+		msg := &model.ClusterMessage{
+			Event:    model.ClusterEventUpdateStatus,
+			SendType: model.ClusterSendBestEffort,
+			Data:     statusJSON,
+		}
+		ps.Cluster().SendClusterMessage(msg)
+	}
+}
+
+func (ps *PlatformService) GetAllStatuses() map[string]*model.Status {
+	if !*ps.Config().ServiceSettings.EnableUserStatuses {
+		return map[string]*model.Status{}
+	}
+
+	statusMap := map[string]*model.Status{}
+	if userIDs, err := ps.statusCache.Keys(); err == nil {
+		for _, userID := range userIDs {
+			status := ps.GetStatusFromCache(userID)
+			if status != nil {
+				statusMap[userID] = status
+			}
+		}
+	}
+	return statusMap
+}
+
+func (ps *PlatformService) GetStatusesByIds(userIDs []string) (map[string]any, *model.AppError) {
+	if !*ps.Config().ServiceSettings.EnableUserStatuses {
+		return map[string]any{}, nil
+	}
+
+	statusMap := map[string]any{}
+	metrics := ps.Metrics()
+
+	missingUserIds := []string{}
+	for _, userID := range userIDs {
+		var status *model.Status
+		if err := ps.statusCache.Get(userID, &status); err == nil {
+			statusMap[userID] = status.Status
+			if metrics != nil {
+				metrics.IncrementMemCacheHitCounter("Status")
+			}
+		} else {
+			missingUserIds = append(missingUserIds, userID)
+			if metrics != nil {
+				metrics.IncrementMemCacheMissCounter("Status")
+			}
+		}
+	}
+
+	if len(missingUserIds) > 0 {
+		statuses, err := ps.Store.Status().GetByIds(missingUserIds)
+		if err != nil {
+			return nil, model.NewAppError("GetStatusesByIds", "app.status.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		for _, s := range statuses {
+			ps.AddStatusCacheSkipClusterSend(s)
+			statusMap[s.UserId] = s.Status
+		}
+
+	}
+
+	// For the case where the user does not have a row in the Status table and cache
+	for _, userID := range missingUserIds {
+		if _, ok := statusMap[userID]; !ok {
+			statusMap[userID] = model.StatusOffline
+		}
+	}
+
+	return statusMap, nil
+}
+
+// GetUserStatusesByIds used by apiV4
+func (ps *PlatformService) GetUserStatusesByIds(userIDs []string) ([]*model.Status, *model.AppError) {
+	if !*ps.Config().ServiceSettings.EnableUserStatuses {
+		return []*model.Status{}, nil
+	}
+
+	var statusMap []*model.Status
+	metrics := ps.Metrics()
+
+	missingUserIds := []string{}
+	for _, userID := range userIDs {
+		var status *model.Status
+		if err := ps.statusCache.Get(userID, &status); err == nil {
+			statusMap = append(statusMap, status)
+			if metrics != nil {
+				metrics.IncrementMemCacheHitCounter("Status")
+			}
+		} else {
+			missingUserIds = append(missingUserIds, userID)
+			if metrics != nil {
+				metrics.IncrementMemCacheMissCounter("Status")
+			}
+		}
+	}
+
+	if len(missingUserIds) > 0 {
+		statuses, err := ps.Store.Status().GetByIds(missingUserIds)
+		if err != nil {
+			return nil, model.NewAppError("GetUserStatusesByIds", "app.status.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		for _, s := range statuses {
+			ps.AddStatusCacheSkipClusterSend(s)
+		}
+
+		statusMap = append(statusMap, statuses...)
+
+	}
+
+	// For the case where the user does not have a row in the Status table and cache
+	// remove the existing ids from missingUserIds and then create a offline state for the missing ones
+	// This also return the status offline for the non-existing Ids in the system
+	for i := 0; i < len(missingUserIds); i++ {
+		missingUserId := missingUserIds[i]
+		for _, userMap := range statusMap {
+			if missingUserId == userMap.UserId {
+				missingUserIds = append(missingUserIds[:i], missingUserIds[i+1:]...)
+				i--
+				break
+			}
+		}
+	}
+	for _, userID := range missingUserIds {
+		statusMap = append(statusMap, &model.Status{UserId: userID, Status: "offline"})
+	}
+
+	return statusMap, nil
+}
+
+func (ps *PlatformService) BroadcastStatus(status *model.Status) {
+	if ps.Busy.IsBusy() {
+		// this is considered a non-critical service and will be disabled when server busy.
+		return
+	}
+	event := model.NewWebSocketEvent(model.WebsocketEventStatusChange, "", "", status.UserId, nil, "")
+	event.Add("status", status.Status)
+	event.Add("user_id", status.UserId)
+	ps.Publish(event)
+}
+
+func (ps *PlatformService) SaveAndBroadcastStatus(status *model.Status) {
+	ps.AddStatusCache(status)
+
+	if err := ps.Store.Status().SaveOrUpdate(status); err != nil {
+		mlog.Warn("Failed to save status", mlog.String("user_id", status.UserId), mlog.Err(err))
+	}
+
+	ps.BroadcastStatus(status)
+}
+
+func (ps *PlatformService) GetStatusFromCache(userID string) *model.Status {
+	var status *model.Status
+	if err := ps.statusCache.Get(userID, &status); err == nil {
+		statusCopy := &model.Status{}
+		*statusCopy = *status
+		return statusCopy
+	}
+
+	return nil
+}
+
+func (ps *PlatformService) GetStatus(userID string) (*model.Status, *model.AppError) {
+	if !*ps.Config().ServiceSettings.EnableUserStatuses {
+		return &model.Status{}, nil
+	}
+
+	status := ps.GetStatusFromCache(userID)
+	if status != nil {
+		return status, nil
+	}
+
+	status, err := ps.Store.Status().Get(userID)
+	if err != nil {
+		var nfErr *store.ErrNotFound
+		switch {
+		case errors.As(err, &nfErr):
+			return nil, model.NewAppError("GetStatus", "app.status.get.missing.app_error", nil, "", http.StatusNotFound).Wrap(err)
+		default:
+			return nil, model.NewAppError("GetStatus", "app.status.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+	}
+
+	return status, nil
+}

app/platform/status_test.go

@@ -0,0 +1,39 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func TestSaveStatus(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	user := th.BasicUser
+
+	for _, statusString := range []string{
+		model.StatusOnline,
+		model.StatusAway,
+		model.StatusDnd,
+		model.StatusOffline,
+	} {
+		t.Run(statusString, func(t *testing.T) {
+			status := &model.Status{
+				UserId: user.Id,
+				Status: statusString,
+			}
+
+			th.Service.SaveAndBroadcastStatus(status)
+
+			after, err := th.Service.GetStatus(user.Id)
+			require.Nil(t, err, "failed to get status after save: %v", err)
+			require.Equal(t, statusString, after.Status, "failed to save status, got %v, expected %v", after.Status, statusString)
+		})
+	}
+}

app/platform/utils.go

@@ -0,0 +1,22 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"crypto/sha256"
+	"encoding/base64"
+)
+
+func getKeyHash(key string) string {
+	hash := sha256.New()
+	hash.Write([]byte(key))
+	return base64.StdEncoding.EncodeToString(hash.Sum(nil))
+}
+
+func maxInt(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}

app/platform/web_conn.go

@@ -0,0 +1,836 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/vmihailenco/msgpack/v5"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin"
+	"github.com/mattermost/mattermost-server/v6/shared/i18n"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+const (
+	sendQueueSize          = 256
+	sendSlowWarn           = (sendQueueSize * 50) / 100
+	sendFullWarn           = (sendQueueSize * 95) / 100
+	writeWaitTime          = 30 * time.Second
+	pongWaitTime           = 100 * time.Second
+	pingInterval           = (pongWaitTime * 6) / 10
+	authCheckInterval      = 5 * time.Second
+	webConnMemberCacheTime = 1000 * 60 * 30 // 30 minutes
+	deadQueueSize          = 128            // Approximated from /proc/sys/net/core/wmem_default / 2048 (avg msg size)
+)
+
+const (
+	reconnectFound    = "success"
+	reconnectNotFound = "failure"
+	reconnectLossless = "lossless"
+)
+
+const websocketMessagePluginPrefix = "custom_"
+
+type pluginWSPostedHook struct {
+	connectionID string
+	userID       string
+	req          *model.WebSocketRequest
+}
+
+type WebConnConfig struct {
+	WebSocket    *websocket.Conn
+	Session      model.Session
+	TFunc        i18n.TranslateFunc
+	Locale       string
+	ConnectionID string
+	Active       bool
+	ReuseCount   int
+
+	// These aren't necessary to be exported to api layer.
+	sequence         int
+	activeQueue      chan model.WebSocketMessage
+	deadQueue        []*model.WebSocketEvent
+	deadQueuePointer int
+}
+
+// WebConn represents a single websocket connection to a user.
+// It contains all the necessary state to manage sending/receiving data to/from
+// a websocket.
+type WebConn struct {
+	sessionExpiresAt   int64 // This should stay at the top for 64-bit alignment of 64-bit words accessed atomically
+	Platform           *PlatformService
+	Suite              SuiteIFace
+	PluginsEnvironment func() *plugin.Environment
+	WebSocket          *websocket.Conn
+	T                  i18n.TranslateFunc
+	Locale             string
+	Sequence           int64
+	UserId             string
+
+	allChannelMembers         map[string]string
+	lastAllChannelMembersTime int64
+	lastUserActivityAt        int64
+	send                      chan model.WebSocketMessage
+	// deadQueue behaves like a queue of a finite size
+	// which is used to store all messages that are sent via the websocket.
+	// It basically acts as the user-space socket buffer, and is used
+	// to resuscitate any messages that might have got lost when the connection is broken.
+	// It is implemented by using a circular buffer to keep it fast.
+	deadQueue []*model.WebSocketEvent
+	// Pointer which indicates the next slot to insert.
+	// It is only to be incremented during writing or clearing the queue.
+	deadQueuePointer int
+	// active indicates whether there is an open websocket connection attached
+	// to this webConn or not.
+	// It is not used as an atomic, because there is no need to.
+	// So do not use this outside the web hub.
+	active bool
+	// reuseCount indicates how many times this connection has been reused.
+	// This is used to differentiate between a fresh connection and
+	// a reused connection.
+	// It's theoretically possible for this number to wrap around. But we
+	// leave that as an edge-case.
+	reuseCount   int
+	sessionToken atomic.Value
+	session      atomic.Value
+	connectionID atomic.Value
+	endWritePump chan struct{}
+	pumpFinished chan struct{}
+	pluginPosted chan pluginWSPostedHook
+}
+
+// CheckConnResult indicates whether a connectionID was present in the hub or not.
+// And if so, contains the active and dead queue details.
+type CheckConnResult struct {
+	ConnectionID     string
+	UserID           string
+	ActiveQueue      chan model.WebSocketMessage
+	DeadQueue        []*model.WebSocketEvent
+	DeadQueuePointer int
+	ReuseCount       int
+}
+
+// PopulateWebConnConfig checks if the connection id already exists in the hub,
+// and if so, accordingly populates the other fields of the webconn.
+func (ps *PlatformService) PopulateWebConnConfig(s *model.Session, cfg *WebConnConfig, seqVal string) (*WebConnConfig, error) {
+	if !model.IsValidId(cfg.ConnectionID) {
+		return nil, fmt.Errorf("invalid connection id: %s", cfg.ConnectionID)
+	}
+
+	// This does not handle reconnect requests across nodes in a cluster.
+	// It falls back to the non-reliable case in that scenario.
+	res := ps.CheckWebConn(s.UserId, cfg.ConnectionID)
+	if res == nil {
+		// If the connection is not present, then we assume either timeout,
+		// or server restart. In that case, we set a new one.
+		cfg.ConnectionID = model.NewId()
+	} else {
+		// Connection is present, we get the active queue, dead queue
+		cfg.activeQueue = res.ActiveQueue
+		cfg.deadQueue = res.DeadQueue
+		cfg.deadQueuePointer = res.DeadQueuePointer
+		cfg.Active = false
+		cfg.ReuseCount = res.ReuseCount
+		// Now we get the sequence number
+		if seqVal == "" {
+			// Sequence_number must be sent with connection id.
+			// A client must be either non-compliant or fully compliant.
+			return nil, errors.New("sequence number not present in websocket request")
+		}
+		var err error
+		cfg.sequence, err = strconv.Atoi(seqVal)
+		if err != nil || cfg.sequence < 0 {
+			return nil, fmt.Errorf("invalid sequence number %s in query param: %v", seqVal, err)
+		}
+	}
+	return cfg, nil
+}
+
+// NewWebConn returns a new WebConn instance.
+func (ps *PlatformService) NewWebConn(cfg *WebConnConfig, suite SuiteIFace, envFn func() *plugin.Environment) *WebConn {
+	if cfg.Session.UserId != "" {
+		ps.Go(func() {
+			suite.SetStatusOnline(cfg.Session.UserId, false)
+			suite.UpdateLastActivityAtIfNeeded(cfg.Session)
+		})
+	}
+
+	// Disable TCP_NO_DELAY for higher throughput
+	var tcpConn *net.TCPConn
+	switch conn := cfg.WebSocket.UnderlyingConn().(type) {
+	case *net.TCPConn:
+		tcpConn = conn
+	case *tls.Conn:
+		newConn, ok := conn.NetConn().(*net.TCPConn)
+		if ok {
+			tcpConn = newConn
+		}
+	}
+
+	if tcpConn != nil {
+		err := tcpConn.SetNoDelay(false)
+		if err != nil {
+			mlog.Warn("Error in setting NoDelay socket opts", mlog.Err(err))
+		}
+	}
+
+	if cfg.activeQueue == nil {
+		cfg.activeQueue = make(chan model.WebSocketMessage, sendQueueSize)
+	}
+
+	if cfg.deadQueue == nil {
+		cfg.deadQueue = make([]*model.WebSocketEvent, deadQueueSize)
+	}
+
+	wc := &WebConn{
+		Platform:           ps,
+		Suite:              suite,
+		PluginsEnvironment: envFn,
+		send:               cfg.activeQueue,
+		deadQueue:          cfg.deadQueue,
+		deadQueuePointer:   cfg.deadQueuePointer,
+		Sequence:           int64(cfg.sequence),
+		WebSocket:          cfg.WebSocket,
+		lastUserActivityAt: model.GetMillis(),
+		UserId:             cfg.Session.UserId,
+		T:                  cfg.TFunc,
+		Locale:             cfg.Locale,
+		active:             cfg.Active,
+		reuseCount:         cfg.ReuseCount,
+		endWritePump:       make(chan struct{}),
+		pumpFinished:       make(chan struct{}),
+		pluginPosted:       make(chan pluginWSPostedHook, 10),
+	}
+
+	wc.SetSession(&cfg.Session)
+	wc.SetSessionToken(cfg.Session.Token)
+	wc.SetSessionExpiresAt(cfg.Session.ExpiresAt)
+	wc.SetConnectionID(cfg.ConnectionID)
+
+	if pluginsEnvironment := wc.PluginsEnvironment(); pluginsEnvironment != nil {
+		wc.Platform.Go(func() {
+			pluginsEnvironment.RunMultiPluginHook(func(hooks plugin.Hooks) bool {
+				hooks.OnWebSocketConnect(wc.GetConnectionID(), wc.UserId)
+				return true
+			}, plugin.OnWebSocketConnectID)
+		})
+	}
+
+	return wc
+}
+
+func (wc *WebConn) pluginPostedConsumer(wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	for msg := range wc.pluginPosted {
+		if pluginsEnvironment := wc.PluginsEnvironment(); pluginsEnvironment != nil {
+			pluginsEnvironment.RunMultiPluginHook(func(hooks plugin.Hooks) bool {
+				hooks.WebSocketMessageHasBeenPosted(msg.connectionID, msg.userID, msg.req)
+				return true
+			}, plugin.WebSocketMessageHasBeenPostedID)
+		}
+	}
+}
+
+// Close closes the WebConn.
+func (wc *WebConn) Close() {
+	wc.WebSocket.Close()
+	<-wc.pumpFinished
+}
+
+// GetSessionExpiresAt returns the time at which the session expires.
+func (wc *WebConn) GetSessionExpiresAt() int64 {
+	return atomic.LoadInt64(&wc.sessionExpiresAt)
+}
+
+// SetSessionExpiresAt sets the time at which the session expires.
+func (wc *WebConn) SetSessionExpiresAt(v int64) {
+	atomic.StoreInt64(&wc.sessionExpiresAt, v)
+}
+
+// GetSessionToken returns the session token of the connection.
+func (wc *WebConn) GetSessionToken() string {
+	return wc.sessionToken.Load().(string)
+}
+
+// SetSessionToken sets the session token of the connection.
+func (wc *WebConn) SetSessionToken(v string) {
+	wc.sessionToken.Store(v)
+}
+
+// SetConnectionID sets the connection id of the connection.
+func (wc *WebConn) SetConnectionID(id string) {
+	wc.connectionID.Store(id)
+}
+
+// GetConnectionID returns the connection id of the connection.
+func (wc *WebConn) GetConnectionID() string {
+	return wc.connectionID.Load().(string)
+}
+
+// areAllInactive returns whether all of the connections
+// are inactive or not.
+func areAllInactive(conns []*WebConn) bool {
+	for _, conn := range conns {
+		if conn.active {
+			return false
+		}
+	}
+	return true
+}
+
+// GetSession returns the session of the connection.
+func (wc *WebConn) GetSession() *model.Session {
+	return wc.session.Load().(*model.Session)
+}
+
+// SetSession sets the session of the connection.
+func (wc *WebConn) SetSession(v *model.Session) {
+	if v != nil {
+		v = v.DeepCopy()
+	}
+
+	wc.session.Store(v)
+}
+
+// Pump starts the WebConn instance. After this, the websocket
+// is ready to send/receive messages.
+func (wc *WebConn) Pump() {
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		wc.writePump()
+	}()
+
+	wg.Add(1)
+	go wc.pluginPostedConsumer(&wg)
+
+	wc.readPump()
+	close(wc.endWritePump)
+	close(wc.pluginPosted)
+	wg.Wait()
+	wc.Platform.HubUnregister(wc)
+	close(wc.pumpFinished)
+
+	if pluginsEnvironment := wc.PluginsEnvironment(); pluginsEnvironment != nil {
+		wc.Platform.Go(func() {
+			pluginsEnvironment.RunMultiPluginHook(func(hooks plugin.Hooks) bool {
+				hooks.OnWebSocketDisconnect(wc.GetConnectionID(), wc.UserId)
+				return true
+			}, plugin.OnWebSocketDisconnectID)
+		})
+	}
+}
+
+func (wc *WebConn) readPump() {
+	defer func() {
+		wc.WebSocket.Close()
+	}()
+	wc.WebSocket.SetReadLimit(model.SocketMaxMessageSizeKb)
+	wc.WebSocket.SetReadDeadline(time.Now().Add(pongWaitTime))
+	wc.WebSocket.SetPongHandler(func(string) error {
+		if err := wc.WebSocket.SetReadDeadline(time.Now().Add(pongWaitTime)); err != nil {
+			return err
+		}
+		if wc.IsAuthenticated() {
+			wc.Platform.Go(func() {
+				wc.Suite.SetStatusAwayIfNeeded(wc.UserId, false)
+			})
+		}
+		return nil
+	})
+
+	for {
+		msgType, rd, err := wc.WebSocket.NextReader()
+		if err != nil {
+			wc.logSocketErr("websocket.NextReader", err)
+			return
+		}
+
+		var decoder interface {
+			Decode(v any) error
+		}
+		if msgType == websocket.TextMessage {
+			decoder = json.NewDecoder(rd)
+		} else {
+			decoder = msgpack.NewDecoder(rd)
+		}
+		var req model.WebSocketRequest
+		if err = decoder.Decode(&req); err != nil {
+			wc.logSocketErr("websocket.Decode", err)
+			return
+		}
+
+		// Messages which actions are prefixed with the plugin prefix
+		// should only be dispatched to the plugins
+		if !strings.HasPrefix(req.Action, websocketMessagePluginPrefix) {
+			wc.Platform.WebSocketRouter.ServeWebSocket(wc, &req)
+		}
+
+		clonedReq, err := req.Clone()
+		if err != nil {
+			wc.logSocketErr("websocket.cloneRequest", err)
+			continue
+		}
+
+		wc.pluginPosted <- pluginWSPostedHook{wc.GetConnectionID(), wc.UserId, clonedReq}
+	}
+}
+
+func (wc *WebConn) writePump() {
+	ticker := time.NewTicker(pingInterval)
+	authTicker := time.NewTicker(authCheckInterval)
+
+	defer func() {
+		ticker.Stop()
+		authTicker.Stop()
+		wc.WebSocket.Close()
+	}()
+
+	if wc.Sequence != 0 {
+		if ok, index := wc.isInDeadQueue(wc.Sequence); ok {
+			if err := wc.drainDeadQueue(index); err != nil {
+				wc.logSocketErr("websocket.drainDeadQueue", err)
+				return
+			}
+			if m := wc.Platform.metricsImpl(); m != nil {
+				m.IncrementWebsocketReconnectEvent(reconnectFound)
+			}
+		} else if wc.hasMsgLoss() {
+			// If the seq number is not in dead queue, but it was supposed to be,
+			// then generate a different connection ID,
+			// and set sequence to 0, and clear dead queue.
+			wc.clearDeadQueue()
+			wc.SetConnectionID(model.NewId())
+			wc.Sequence = 0
+
+			// Send hello message
+			msg := wc.createHelloMessage()
+			wc.addToDeadQueue(msg)
+			if err := wc.writeMessage(msg); err != nil {
+				wc.logSocketErr("websocket.sendHello", err)
+				return
+			}
+			if m := wc.Platform.metricsImpl(); m != nil {
+				m.IncrementWebsocketReconnectEvent(reconnectNotFound)
+			}
+		} else {
+			if m := wc.Platform.metricsImpl(); m != nil {
+				m.IncrementWebsocketReconnectEvent(reconnectLossless)
+			}
+		}
+	}
+
+	var buf bytes.Buffer
+	// 2k is seen to be a good heuristic under which 98.5% of message sizes remain.
+	buf.Grow(1024 * 2)
+	enc := json.NewEncoder(&buf)
+
+	for {
+		select {
+		case msg, ok := <-wc.send:
+			if !ok {
+				wc.writeMessageBuf(websocket.CloseMessage, []byte{})
+				return
+			}
+
+			evt, evtOk := msg.(*model.WebSocketEvent)
+
+			buf.Reset()
+			var err error
+			if evtOk {
+				evt = evt.SetSequence(wc.Sequence)
+				err = evt.Encode(enc)
+				wc.Sequence++
+			} else {
+				err = enc.Encode(msg)
+			}
+			if err != nil {
+				mlog.Warn("Error in encoding websocket message", mlog.Err(err))
+				continue
+			}
+
+			if len(wc.send) >= sendFullWarn {
+				logData := []mlog.Field{
+					mlog.String("user_id", wc.UserId),
+					mlog.String("type", msg.EventType()),
+					mlog.Int("size", buf.Len()),
+				}
+				if evtOk {
+					logData = append(logData, mlog.String("channel_id", evt.GetBroadcast().ChannelId))
+				}
+
+				mlog.Warn("websocket.full", logData...)
+			}
+
+			if evtOk {
+				wc.addToDeadQueue(evt)
+			}
+
+			if err := wc.writeMessageBuf(websocket.TextMessage, buf.Bytes()); err != nil {
+				wc.logSocketErr("websocket.send", err)
+				return
+			}
+
+			if m := wc.Platform.metricsImpl(); m != nil {
+				m.IncrementWebSocketBroadcast(msg.EventType())
+			}
+		case <-ticker.C:
+			if err := wc.writeMessageBuf(websocket.PingMessage, []byte{}); err != nil {
+				wc.logSocketErr("websocket.ticker", err)
+				return
+			}
+
+		case <-wc.endWritePump:
+			return
+
+		case <-authTicker.C:
+			if wc.GetSessionToken() == "" {
+				mlog.Debug("websocket.authTicker: did not authenticate", mlog.Any("ip_address", wc.WebSocket.RemoteAddr()))
+				return
+			}
+			authTicker.Stop()
+		}
+	}
+}
+
+// writeMessageBuf is a helper utility that wraps the write to the socket
+// along with setting the write deadline.
+func (wc *WebConn) writeMessageBuf(msgType int, data []byte) error {
+	wc.WebSocket.SetWriteDeadline(time.Now().Add(writeWaitTime))
+	return wc.WebSocket.WriteMessage(msgType, data)
+}
+
+func (wc *WebConn) writeMessage(msg *model.WebSocketEvent) error {
+	// We don't use the encoder from the write pump because it's unwieldy to pass encoders
+	// around, and this is only called during initialization of the webConn.
+	var buf bytes.Buffer
+	err := msg.Encode(json.NewEncoder(&buf))
+	if err != nil {
+		mlog.Warn("Error in encoding websocket message", mlog.Err(err))
+		return nil
+	}
+	wc.Sequence++
+
+	return wc.writeMessageBuf(websocket.TextMessage, buf.Bytes())
+}
+
+// addToDeadQueue appends a message to the dead queue.
+func (wc *WebConn) addToDeadQueue(msg *model.WebSocketEvent) {
+	wc.deadQueue[wc.deadQueuePointer] = msg
+	wc.deadQueuePointer = (wc.deadQueuePointer + 1) % deadQueueSize
+}
+
+// hasMsgLoss indicates whether the next wanted sequence is right after
+// the latest element in the dead queue, which would mean there is no message loss.
+func (wc *WebConn) hasMsgLoss() bool {
+	var index int
+	// deadQueuePointer = 0 means either no msg written or the pointer
+	// has rolled over to its starting position.
+	if wc.deadQueuePointer == 0 {
+		// If last entry is nil, it means no msg is written.
+		if wc.deadQueue[deadQueueSize-1] == nil {
+			return false
+		}
+		// If it's not nil, that means it has rolled over to start, and we
+		// check the last position.
+		index = deadQueueSize - 1
+	} else { // deadQueuePointer != 0 means it's somewhere in the middle.
+		index = wc.deadQueuePointer - 1
+	}
+
+	if wc.deadQueue[index].GetSequence() == wc.Sequence-1 {
+		return false
+	}
+	return true
+}
+
+// isInDeadQueue checks whether a given sequence number is in the dead queue or not.
+// And if it is, it returns that index.
+func (wc *WebConn) isInDeadQueue(seq int64) (bool, int) {
+	// Can be optimized to traverse backwards from deadQueuePointer
+	// Hopefully, traversing 128 elements is not too much overhead.
+	for i := 0; i < deadQueueSize; i++ {
+		elem := wc.deadQueue[i]
+		if elem == nil {
+			return false, 0
+		}
+
+		if elem.GetSequence() == seq {
+			return true, i
+		}
+	}
+	return false, 0
+}
+
+func (wc *WebConn) clearDeadQueue() {
+	for i := 0; i < deadQueueSize; i++ {
+		if wc.deadQueue[i] == nil {
+			break
+		}
+		wc.deadQueue[i] = nil
+	}
+	wc.deadQueuePointer = 0
+}
+
+// drainDeadQueue will write all messages from a given index to the socket.
+// It is called with the assumption that the item with wc.Sequence is present
+// in it, because otherwise it would have been cleared from WebConn.
+func (wc *WebConn) drainDeadQueue(index int) error {
+	if wc.deadQueue[0] == nil {
+		// Empty queue
+		return nil
+	}
+
+	// This means pointer hasn't rolled over.
+	if wc.deadQueue[wc.deadQueuePointer] == nil {
+		// Clear till the end of queue.
+		for i := index; i < wc.deadQueuePointer; i++ {
+			if err := wc.writeMessage(wc.deadQueue[i]); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+
+	// We go on until next sequence number is smaller than previous one.
+	// Which means it has rolled over.
+	currPtr := index
+	for {
+		if err := wc.writeMessage(wc.deadQueue[currPtr]); err != nil {
+			return err
+		}
+		oldSeq := wc.deadQueue[currPtr].GetSequence() // TODO: possibly move this
+		currPtr = (currPtr + 1) % deadQueueSize       // to for loop condition
+		newSeq := wc.deadQueue[currPtr].GetSequence()
+		if oldSeq > newSeq {
+			break
+		}
+	}
+	return nil
+}
+
+// InvalidateCache resets all internal data of the WebConn.
+func (wc *WebConn) InvalidateCache() {
+	wc.allChannelMembers = nil
+	wc.lastAllChannelMembersTime = 0
+	wc.SetSession(nil)
+	wc.SetSessionExpiresAt(0)
+}
+
+// IsAuthenticated returns whether the given WebConn is authenticated or not.
+func (wc *WebConn) IsAuthenticated() bool {
+	// Check the expiry to see if we need to check for a new session
+	if wc.GetSessionExpiresAt() < model.GetMillis() {
+		if wc.GetSessionToken() == "" {
+			return false
+		}
+
+		session, err := wc.Suite.GetSession(wc.GetSessionToken())
+		if err != nil {
+			if err.StatusCode >= http.StatusBadRequest && err.StatusCode < http.StatusInternalServerError {
+				mlog.Debug("Invalid session.", mlog.Err(err))
+			} else {
+				mlog.Error("Could not get session", mlog.String("session_token", wc.GetSessionToken()), mlog.Err(err))
+			}
+
+			wc.SetSessionToken("")
+			wc.SetSession(nil)
+			wc.SetSessionExpiresAt(0)
+			return false
+		}
+
+		wc.SetSession(session)
+		wc.SetSessionExpiresAt(session.ExpiresAt)
+	}
+
+	return true
+}
+
+func (wc *WebConn) createHelloMessage() *model.WebSocketEvent {
+	ee := wc.Platform.LicenseManager() != nil
+
+	msg := model.NewWebSocketEvent(model.WebsocketEventHello, "", "", wc.UserId, nil, "")
+	msg.Add("server_version", fmt.Sprintf("%v.%v.%v.%v", model.CurrentVersion,
+		model.BuildNumber,
+		wc.Platform.ClientConfigHash(),
+		ee))
+	msg.Add("connection_id", wc.connectionID.Load())
+	return msg
+}
+
+func (wc *WebConn) ShouldSendEventToGuest(msg *model.WebSocketEvent) bool {
+	var userID string
+	var canSee bool
+
+	switch msg.EventType() {
+	case model.WebsocketEventUserUpdated:
+		user, ok := msg.GetData()["user"].(*model.User)
+		if !ok {
+			mlog.Debug("webhub.shouldSendEvent: user not found in message", mlog.Any("user", msg.GetData()["user"]))
+			return false
+		}
+		userID = user.Id
+	case model.WebsocketEventNewUser:
+		userID = msg.GetData()["user_id"].(string)
+	default:
+		return true
+	}
+
+	canSee, err := wc.Suite.UserCanSeeOtherUser(wc.UserId, userID)
+	if err != nil {
+		mlog.Error("webhub.shouldSendEvent.", mlog.Err(err))
+		return false
+	}
+
+	return canSee
+}
+
+// ShouldSendEvent returns whether the message should be sent or not.
+func (wc *WebConn) ShouldSendEvent(msg *model.WebSocketEvent) bool {
+	// IMPORTANT: Do not send event if WebConn does not have a session
+	if !wc.IsAuthenticated() {
+		return false
+	}
+
+	// When the pump starts to get slow we'll drop non-critical
+	// messages. We should skip those frames before they are
+	// queued to wc.send buffered channel.
+	if len(wc.send) >= sendSlowWarn {
+		switch msg.EventType() {
+		case model.WebsocketEventTyping,
+			model.WebsocketEventStatusChange,
+			model.WebsocketEventChannelViewed:
+			mlog.Warn(
+				"websocket.slow: dropping message",
+				mlog.String("user_id", wc.UserId),
+				mlog.String("type", msg.EventType()),
+			)
+			return false
+		}
+	}
+
+	// If the event contains sanitized data, only send to users that don't have permission to
+	// see sensitive data. Prevents admin clients from receiving events with bad data
+	var hasReadPrivateDataPermission *bool
+	if msg.GetBroadcast().ContainsSanitizedData {
+		hasReadPrivateDataPermission = model.NewBool(wc.Suite.RolesGrantPermission(wc.GetSession().GetUserRoles(), model.PermissionManageSystem.Id))
+
+		if *hasReadPrivateDataPermission {
+			return false
+		}
+	}
+
+	// If the event contains sensitive data, only send to users with permission to see it
+	if msg.GetBroadcast().ContainsSensitiveData {
+		if hasReadPrivateDataPermission == nil {
+			hasReadPrivateDataPermission = model.NewBool(wc.Suite.RolesGrantPermission(wc.GetSession().GetUserRoles(), model.PermissionManageSystem.Id))
+		}
+
+		if !*hasReadPrivateDataPermission {
+			return false
+		}
+	}
+
+	// If the event is destined to a specific connection
+	if msg.GetBroadcast().ConnectionId != "" {
+		return wc.GetConnectionID() == msg.GetBroadcast().ConnectionId
+	}
+
+	// If the event is destined to a specific user
+	if msg.GetBroadcast().UserId != "" {
+		return wc.UserId == msg.GetBroadcast().UserId
+	}
+
+	if wc.GetConnectionID() == msg.GetBroadcast().OmitConnectionId {
+		return false
+	}
+
+	// if the user is omitted don't send the message
+	if len(msg.GetBroadcast().OmitUsers) > 0 {
+		if _, ok := msg.GetBroadcast().OmitUsers[wc.UserId]; ok {
+			return false
+		}
+	}
+
+	// Only report events to users who are in the channel for the event
+	if msg.GetBroadcast().ChannelId != "" {
+		if model.GetMillis()-wc.lastAllChannelMembersTime > webConnMemberCacheTime {
+			wc.allChannelMembers = nil
+			wc.lastAllChannelMembersTime = 0
+		}
+
+		if wc.allChannelMembers == nil {
+			result, err := wc.Platform.Store.Channel().GetAllChannelMembersForUser(wc.UserId, false, false)
+			if err != nil {
+				mlog.Error("webhub.shouldSendEvent.", mlog.Err(err))
+				return false
+			}
+			wc.allChannelMembers = result
+			wc.lastAllChannelMembersTime = model.GetMillis()
+		}
+
+		if _, ok := wc.allChannelMembers[msg.GetBroadcast().ChannelId]; ok {
+			return true
+		}
+		return false
+	}
+
+	// Only report events to users who are in the team for the event
+	if msg.GetBroadcast().TeamId != "" {
+		return wc.isMemberOfTeam(msg.GetBroadcast().TeamId)
+	}
+
+	if wc.GetSession().Props[model.SessionPropIsGuest] == "true" {
+		return wc.ShouldSendEventToGuest(msg)
+	}
+
+	return true
+}
+
+// IsMemberOfTeam returns whether the user of the WebConn
+// is a member of the given teamID or not.
+func (wc *WebConn) isMemberOfTeam(teamID string) bool {
+	currentSession := wc.GetSession()
+
+	if currentSession == nil || currentSession.Token == "" {
+		session, err := wc.Suite.GetSession(wc.GetSessionToken())
+		if err != nil {
+			if err.StatusCode >= http.StatusBadRequest && err.StatusCode < http.StatusInternalServerError {
+				mlog.Debug("Invalid session.", mlog.Err(err))
+			} else {
+				mlog.Error("Could not get session", mlog.String("session_token", wc.GetSessionToken()), mlog.Err(err))
+			}
+			return false
+		}
+		wc.SetSession(session)
+		currentSession = session
+	}
+
+	return currentSession.GetTeamByTeamId(teamID) != nil
+}
+
+func (wc *WebConn) logSocketErr(source string, err error) {
+	// browsers will appear as CloseNoStatusReceived
+	if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseNoStatusReceived) {
+		mlog.Debug(source+": client side closed socket", mlog.String("user_id", wc.UserId))
+	} else {
+		mlog.Debug(source+": closing websocket", mlog.String("user_id", wc.UserId), mlog.Err(err))
+	}
+}

app/platform/web_conn_test.go

@@ -0,0 +1,228 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"bytes"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gorilla/websocket"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin"
+)
+
+func TestWebConnAddDeadQueue(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	wc := th.Service.NewWebConn(&WebConnConfig{
+		WebSocket: &websocket.Conn{},
+	}, th.Suite, func() *plugin.Environment { return nil })
+
+	for i := 0; i < 2; i++ {
+		msg := &model.WebSocketEvent{}
+		msg = msg.SetSequence(int64(i))
+		wc.addToDeadQueue(msg)
+	}
+
+	for i := 0; i < 2; i++ {
+		assert.Equal(t, int64(i), wc.deadQueue[i].GetSequence())
+	}
+
+	// Should push out the first two elements
+	for i := 0; i < deadQueueSize; i++ {
+		msg := &model.WebSocketEvent{}
+		msg = msg.SetSequence(int64(i + 2))
+		wc.addToDeadQueue(msg)
+	}
+	for i := 0; i < deadQueueSize; i++ {
+		assert.Equal(t, int64(i+2), wc.deadQueue[(i+2)%deadQueueSize].GetSequence())
+	}
+}
+
+func TestWebConnIsInDeadQueue(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	wc := th.Service.NewWebConn(&WebConnConfig{
+		WebSocket: &websocket.Conn{},
+	}, th.Suite, func() *plugin.Environment { return nil })
+
+	var i int
+	for ; i < 2; i++ {
+		msg := &model.WebSocketEvent{}
+		msg = msg.SetSequence(int64(i))
+		wc.addToDeadQueue(msg)
+	}
+
+	wc.Sequence = int64(0)
+	ok, ind := wc.isInDeadQueue(wc.Sequence)
+	assert.True(t, ok)
+	assert.Equal(t, 0, ind)
+	assert.True(t, wc.hasMsgLoss())
+	wc.Sequence = int64(1)
+	ok, ind = wc.isInDeadQueue(wc.Sequence)
+	assert.True(t, ok)
+	assert.Equal(t, 1, ind)
+	assert.True(t, wc.hasMsgLoss())
+	wc.Sequence = int64(2)
+	ok, ind = wc.isInDeadQueue(wc.Sequence)
+	assert.False(t, ok)
+	assert.Equal(t, 0, ind)
+	assert.False(t, wc.hasMsgLoss())
+
+	for ; i < deadQueueSize+2; i++ {
+		msg := &model.WebSocketEvent{}
+		msg = msg.SetSequence(int64(i))
+		wc.addToDeadQueue(msg)
+	}
+
+	wc.Sequence = int64(129)
+	ok, ind = wc.isInDeadQueue(wc.Sequence)
+	assert.True(t, ok)
+	assert.Equal(t, 1, ind)
+	wc.Sequence = int64(128)
+	ok, ind = wc.isInDeadQueue(wc.Sequence)
+	assert.True(t, ok)
+	assert.Equal(t, 0, ind)
+	wc.Sequence = int64(2)
+	ok, ind = wc.isInDeadQueue(wc.Sequence)
+	assert.True(t, ok)
+	assert.Equal(t, 2, ind)
+	assert.True(t, wc.hasMsgLoss())
+	wc.Sequence = int64(0)
+	ok, ind = wc.isInDeadQueue(wc.Sequence)
+	assert.False(t, ok)
+	assert.Equal(t, 0, ind)
+	wc.Sequence = int64(130)
+	ok, ind = wc.isInDeadQueue(wc.Sequence)
+	assert.False(t, ok)
+	assert.Equal(t, 0, ind)
+	assert.False(t, wc.hasMsgLoss())
+}
+
+func TestWebConnClearDeadQueue(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	wc := th.Service.NewWebConn(&WebConnConfig{
+		WebSocket: &websocket.Conn{},
+	}, th.Suite, func() *plugin.Environment { return nil })
+
+	var i int
+	for ; i < 2; i++ {
+		msg := &model.WebSocketEvent{}
+		msg = msg.SetSequence(int64(i))
+		wc.addToDeadQueue(msg)
+	}
+
+	wc.clearDeadQueue()
+
+	assert.Equal(t, 0, wc.deadQueuePointer)
+}
+
+func TestWebConnDrainDeadQueue(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	var dialConn = func(t *testing.T, th *TestHelper, addr net.Addr) *WebConn {
+		d := websocket.Dialer{}
+		c, _, err := d.Dial("ws://"+addr.String()+"/ws", nil)
+		require.NoError(t, err)
+
+		cfg := &WebConnConfig{
+			WebSocket: c,
+		}
+		return th.Service.NewWebConn(cfg, th.Suite, func() *plugin.Environment { return nil })
+	}
+
+	t.Run("Empty Queue", func(t *testing.T) {
+		var handler = func(t *testing.T) http.HandlerFunc {
+			return func(w http.ResponseWriter, req *http.Request) {
+				upgrader := &websocket.Upgrader{}
+				conn, err := upgrader.Upgrade(w, req, nil)
+				cnt := 0
+				for err == nil {
+					_, _, err = conn.ReadMessage()
+					cnt++
+				}
+				assert.Equal(t, 1, cnt)
+				if _, ok := err.(*websocket.CloseError); !ok {
+					require.NoError(t, err)
+				}
+			}
+		}
+		s := httptest.NewServer(handler(t))
+		defer s.Close()
+
+		wc := dialConn(t, th, s.Listener.Addr())
+		defer wc.WebSocket.Close()
+		wc.clearDeadQueue()
+
+		err := wc.drainDeadQueue(0)
+		require.NoError(t, err)
+	})
+
+	var handler = func(t *testing.T, seqNum int64, limit int) http.HandlerFunc {
+		return func(w http.ResponseWriter, req *http.Request) {
+			upgrader := &websocket.Upgrader{}
+			conn, err := upgrader.Upgrade(w, req, nil)
+			var buf []byte
+			i := seqNum
+			for err == nil {
+				_, buf, err = conn.ReadMessage()
+				if err != nil && len(buf) > 0 {
+					ev, jsonErr := model.WebSocketEventFromJSON(bytes.NewReader(buf))
+					require.NoError(t, jsonErr)
+					require.LessOrEqual(t, int(i), limit)
+					assert.Equal(t, i, ev.GetSequence())
+					i++
+				}
+			}
+			if _, ok := err.(*websocket.CloseError); !ok {
+				require.NoError(t, err)
+			}
+		}
+	}
+
+	run := func(seqNum int64, limit int) {
+		s := httptest.NewServer(handler(t, seqNum, limit))
+		defer s.Close()
+
+		wc := dialConn(t, th, s.Listener.Addr())
+		defer wc.WebSocket.Close()
+
+		for i := 0; i < limit; i++ {
+			msg := model.NewWebSocketEvent("", "", "", "", map[string]bool{}, "")
+			msg = msg.SetSequence(int64(i))
+			wc.addToDeadQueue(msg)
+		}
+		wc.Sequence = seqNum
+		ok, index := wc.isInDeadQueue(wc.Sequence)
+		require.True(t, ok)
+
+		err := wc.drainDeadQueue(index)
+		require.NoError(t, err)
+	}
+
+	t.Run("Half-full Queue", func(t *testing.T) {
+		t.Run("Middle", func(t *testing.T) { run(int64(2), 10) })
+		t.Run("Beginning", func(t *testing.T) { run(int64(0), 10) })
+		t.Run("End", func(t *testing.T) { run(int64(9), 10) })
+		t.Run("Full", func(t *testing.T) { run(int64(deadQueueSize-1), deadQueueSize) })
+	})
+
+	t.Run("Cycled Queue", func(t *testing.T) {
+		t.Run("First un-overwritten", func(t *testing.T) { run(int64(10), deadQueueSize+10) })
+		t.Run("End", func(t *testing.T) { run(int64(127), deadQueueSize+10) })
+		t.Run("Cycled End", func(t *testing.T) { run(int64(137), deadQueueSize+10) })
+		t.Run("Overwritten First", func(t *testing.T) { run(int64(128), deadQueueSize+10) })
+	})
+}

app/platform/web_hub.go

@@ -0,0 +1,666 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"hash/maphash"
+	"runtime"
+	"runtime/debug"
+	"strconv"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+const (
+	broadcastQueueSize         = 4096
+	inactiveConnReaperInterval = 5 * time.Minute
+)
+
+type SuiteIFace interface {
+	SetStatusLastActivityAt(userID string, activityAt int64)
+	SetStatusOffline(userID string, manual bool)
+	IsUserAway(lastActivityAt int64) bool
+	SetStatusOnline(userID string, manual bool)
+	UpdateLastActivityAtIfNeeded(session model.Session)
+	SetStatusAwayIfNeeded(userID string, manual bool)
+	GetSession(token string) (*model.Session, *model.AppError)
+	RolesGrantPermission(roleNames []string, permissionId string) bool
+	UserCanSeeOtherUser(userID string, otherUserId string) (bool, *model.AppError)
+}
+
+type webConnActivityMessage struct {
+	userID       string
+	sessionToken string
+	activityAt   int64
+}
+
+type webConnDirectMessage struct {
+	conn *WebConn
+	msg  model.WebSocketMessage
+}
+
+type webConnSessionMessage struct {
+	userID       string
+	sessionToken string
+	isRegistered chan bool
+}
+
+type webConnCheckMessage struct {
+	userID       string
+	connectionID string
+	result       chan *CheckConnResult
+}
+
+// Hub is the central place to manage all websocket connections in the server.
+// It handles different websocket events and sending messages to individual
+// user connections.
+type Hub struct {
+	// connectionCount should be kept first.
+	// See https://github.com/mattermost/mattermost-server/pull/7281
+	connectionCount int64
+	platform        *PlatformService
+	connectionIndex int
+	register        chan *WebConn
+	unregister      chan *WebConn
+	broadcast       chan *model.WebSocketEvent
+	stop            chan struct{}
+	didStop         chan struct{}
+	invalidateUser  chan string
+	activity        chan *webConnActivityMessage
+	directMsg       chan *webConnDirectMessage
+	explicitStop    bool
+	checkRegistered chan *webConnSessionMessage
+	checkConn       chan *webConnCheckMessage
+}
+
+// newWebHub creates a new Hub.
+func newWebHub(ps *PlatformService) *Hub {
+	return &Hub{
+		platform:        ps,
+		register:        make(chan *WebConn),
+		unregister:      make(chan *WebConn),
+		broadcast:       make(chan *model.WebSocketEvent, broadcastQueueSize),
+		stop:            make(chan struct{}),
+		didStop:         make(chan struct{}),
+		invalidateUser:  make(chan string),
+		activity:        make(chan *webConnActivityMessage),
+		directMsg:       make(chan *webConnDirectMessage),
+		checkRegistered: make(chan *webConnSessionMessage),
+		checkConn:       make(chan *webConnCheckMessage),
+	}
+}
+
+// HubStart starts all the hubs.
+func (ps *PlatformService) HubStart(suite SuiteIFace) {
+	// Total number of hubs is twice the number of CPUs.
+	numberOfHubs := runtime.NumCPU() * 2
+	ps.logger.Info("Starting websocket hubs", mlog.Int("number_of_hubs", numberOfHubs))
+
+	hubs := make([]*Hub, numberOfHubs)
+
+	for i := 0; i < numberOfHubs; i++ {
+		hubs[i] = newWebHub(ps)
+		hubs[i].connectionIndex = i
+		hubs[i].Start(suite)
+	}
+	// Assigning to the hubs slice without any mutex is fine because it is only assigned once
+	// during the start of the program and always read from after that.
+	ps.hubs = hubs
+}
+
+func (ps *PlatformService) InvalidateCacheForWebhook(webhookID string) {
+	ps.Store.Webhook().InvalidateWebhookCache(webhookID)
+}
+
+// HubStop stops all the hubs.
+func (ps *PlatformService) HubStop() {
+	ps.logger.Info("stopping websocket hub connections")
+
+	for _, hub := range ps.hubs {
+		hub.Stop()
+	}
+}
+
+// GetHubForUserId returns the hub for a given user id.
+func (ps *PlatformService) GetHubForUserId(userID string) *Hub {
+	// TODO: check if caching the userID -> hub mapping
+	// is worth the memory tradeoff.
+	// https://mattermost.atlassian.net/browse/MM-26629.
+	var hash maphash.Hash
+	hash.SetSeed(ps.hashSeed)
+	hash.Write([]byte(userID))
+	index := hash.Sum64() % uint64(len(ps.hubs))
+
+	return ps.hubs[int(index)]
+}
+
+// HubRegister registers a connection to a hub.
+func (ps *PlatformService) HubRegister(webConn *WebConn) {
+	hub := ps.GetHubForUserId(webConn.UserId)
+	if hub != nil {
+		if metrics := ps.metricsImpl(); metrics != nil {
+			metrics.IncrementWebSocketBroadcastUsersRegistered(strconv.Itoa(hub.connectionIndex), 1)
+		}
+		hub.Register(webConn)
+	}
+}
+
+// HubUnregister unregisters a connection from a hub.
+func (ps *PlatformService) HubUnregister(webConn *WebConn) {
+	hub := ps.GetHubForUserId(webConn.UserId)
+	if hub != nil {
+		if metrics := ps.metricsImpl(); metrics != nil {
+			metrics.DecrementWebSocketBroadcastUsersRegistered(strconv.Itoa(hub.connectionIndex), 1)
+		}
+		hub.Unregister(webConn)
+	}
+}
+
+func (ps *PlatformService) InvalidateCacheForChannel(channel *model.Channel) {
+	ps.Store.Channel().InvalidateChannel(channel.Id)
+	ps.invalidateCacheForChannelByNameSkipClusterSend(channel.TeamId, channel.Name)
+
+	if ps.clusterIFace != nil {
+		nameMsg := &model.ClusterMessage{
+			Event:    model.ClusterEventInvalidateCacheForChannelByName,
+			SendType: model.ClusterSendBestEffort,
+			Props:    make(map[string]string),
+		}
+
+		nameMsg.Props["name"] = channel.Name
+		if channel.TeamId == "" {
+			nameMsg.Props["id"] = "dm"
+		} else {
+			nameMsg.Props["id"] = channel.TeamId
+		}
+
+		ps.clusterIFace.SendClusterMessage(nameMsg)
+	}
+}
+
+func (ps *PlatformService) InvalidateCacheForChannelMembers(channelID string) {
+	ps.Store.User().InvalidateProfilesInChannelCache(channelID)
+	ps.Store.Channel().InvalidateMemberCount(channelID)
+	ps.Store.Channel().InvalidateGuestCount(channelID)
+}
+
+func (ps *PlatformService) InvalidateCacheForChannelMembersNotifyProps(channelID string) {
+	ps.invalidateCacheForChannelMembersNotifyPropsSkipClusterSend(channelID)
+
+	if ps.clusterIFace != nil {
+		msg := &model.ClusterMessage{
+			Event:    model.ClusterEventInvalidateCacheForChannelMembersNotifyProps,
+			SendType: model.ClusterSendBestEffort,
+			Data:     []byte(channelID),
+		}
+		ps.clusterIFace.SendClusterMessage(msg)
+	}
+}
+
+func (ps *PlatformService) InvalidateCacheForChannelPosts(channelID string) {
+	ps.Store.Channel().InvalidatePinnedPostCount(channelID)
+	ps.Store.Post().InvalidateLastPostTimeCache(channelID)
+}
+
+func (ps *PlatformService) InvalidateCacheForUser(userID string) {
+	ps.InvalidateCacheForUserSkipClusterSend(userID)
+
+	ps.Store.User().InvalidateProfilesInChannelCacheByUser(userID)
+	ps.Store.User().InvalidateProfileCacheForUser(userID)
+
+	if ps.clusterIFace != nil {
+		msg := &model.ClusterMessage{
+			Event:    model.ClusterEventInvalidateCacheForUser,
+			SendType: model.ClusterSendBestEffort,
+			Data:     []byte(userID),
+		}
+		ps.clusterIFace.SendClusterMessage(msg)
+	}
+}
+
+func (ps *PlatformService) InvalidateCacheForUserTeams(userID string) {
+	ps.invalidateWebConnSessionCacheForUser(userID)
+	ps.Store.Team().InvalidateAllTeamIdsForUser(userID)
+
+	if ps.clusterIFace != nil {
+		msg := &model.ClusterMessage{
+			Event:    model.ClusterEventInvalidateCacheForUserTeams,
+			SendType: model.ClusterSendBestEffort,
+			Data:     []byte(userID),
+		}
+		ps.clusterIFace.SendClusterMessage(msg)
+	}
+}
+
+// UpdateWebConnUserActivity sets the LastUserActivityAt of the hub for the given session.
+func (ps *PlatformService) UpdateWebConnUserActivity(session model.Session, activityAt int64) {
+	hub := ps.GetHubForUserId(session.UserId)
+	if hub != nil {
+		hub.UpdateActivity(session.UserId, session.Token, activityAt)
+	}
+}
+
+// SessionIsRegistered determines if a specific session has been registered
+func (ps *PlatformService) SessionIsRegistered(session model.Session) bool {
+	hub := ps.GetHubForUserId(session.UserId)
+	if hub != nil {
+		return hub.IsRegistered(session.UserId, session.Token)
+	}
+
+	return false
+}
+
+func (ps *PlatformService) CheckWebConn(userID, connectionID string) *CheckConnResult {
+	hub := ps.GetHubForUserId(userID)
+	if hub != nil {
+		return hub.CheckConn(userID, connectionID)
+	}
+	return nil
+}
+
+// Register registers a connection to the hub.
+func (h *Hub) Register(webConn *WebConn) {
+	select {
+	case h.register <- webConn:
+	case <-h.stop:
+	}
+}
+
+// Unregister unregisters a connection from the hub.
+func (h *Hub) Unregister(webConn *WebConn) {
+	select {
+	case h.unregister <- webConn:
+	case <-h.stop:
+	}
+}
+
+// Determines if a user's session is registered a connection from the hub.
+func (h *Hub) IsRegistered(userID, sessionToken string) bool {
+	ws := &webConnSessionMessage{
+		userID:       userID,
+		sessionToken: sessionToken,
+		isRegistered: make(chan bool),
+	}
+	select {
+	case h.checkRegistered <- ws:
+		return <-ws.isRegistered
+	case <-h.stop:
+	}
+	return false
+}
+
+func (h *Hub) CheckConn(userID, connectionID string) *CheckConnResult {
+	req := &webConnCheckMessage{
+		userID:       userID,
+		connectionID: connectionID,
+		result:       make(chan *CheckConnResult),
+	}
+	select {
+	case h.checkConn <- req:
+		return <-req.result
+	case <-h.stop:
+	}
+	return nil
+}
+
+// Broadcast broadcasts the message to all connections in the hub.
+func (h *Hub) Broadcast(message *model.WebSocketEvent) {
+	// XXX: The hub nil check is because of the way we setup our tests. We call
+	// `app.NewServer()` which returns a server, but only after that, we call
+	// `wsapi.Init()` to initialize the hub.  But in the `NewServer` call
+	// itself proceeds to broadcast some messages happily.  This needs to be
+	// fixed once the wsapi cyclic dependency with server/app goes away.
+	// And possibly, we can look into doing the hub initialization inside
+	// NewServer itself.
+	if h != nil && message != nil {
+		if metrics := h.platform.metricsImpl(); metrics != nil {
+			metrics.IncrementWebSocketBroadcastBufferSize(strconv.Itoa(h.connectionIndex), 1)
+		}
+		select {
+		case h.broadcast <- message:
+		case <-h.stop:
+		}
+	}
+}
+
+// InvalidateUser invalidates the cache for the given user.
+func (h *Hub) InvalidateUser(userID string) {
+	select {
+	case h.invalidateUser <- userID:
+	case <-h.stop:
+	}
+}
+
+// UpdateActivity sets the LastUserActivityAt field for the connection
+// of the user.
+func (h *Hub) UpdateActivity(userID, sessionToken string, activityAt int64) {
+	select {
+	case h.activity <- &webConnActivityMessage{
+		userID:       userID,
+		sessionToken: sessionToken,
+		activityAt:   activityAt,
+	}:
+	case <-h.stop:
+	}
+}
+
+// SendMessage sends the given message to the given connection.
+func (h *Hub) SendMessage(conn *WebConn, msg model.WebSocketMessage) {
+	select {
+	case h.directMsg <- &webConnDirectMessage{
+		conn: conn,
+		msg:  msg,
+	}:
+	case <-h.stop:
+	}
+}
+
+// Stop stops the hub.
+func (h *Hub) Stop() {
+	close(h.stop)
+	<-h.didStop
+}
+
+// Start starts the hub.
+func (h *Hub) Start(suite SuiteIFace) {
+	var doStart func()
+	var doRecoverableStart func()
+	var doRecover func()
+
+	doStart = func() {
+		mlog.Debug("Hub is starting", mlog.Int("index", h.connectionIndex))
+
+		ticker := time.NewTicker(inactiveConnReaperInterval)
+		defer ticker.Stop()
+
+		connIndex := newHubConnectionIndex(inactiveConnReaperInterval)
+
+		for {
+			select {
+			case webSessionMessage := <-h.checkRegistered:
+				conns := connIndex.ForUser(webSessionMessage.userID)
+				var isRegistered bool
+				for _, conn := range conns {
+					if !conn.active {
+						continue
+					}
+					if conn.GetSessionToken() == webSessionMessage.sessionToken {
+						isRegistered = true
+					}
+				}
+				webSessionMessage.isRegistered <- isRegistered
+			case req := <-h.checkConn:
+				var res *CheckConnResult
+				conn := connIndex.RemoveInactiveByConnectionID(req.userID, req.connectionID)
+				if conn != nil {
+					res = &CheckConnResult{
+						ConnectionID:     req.connectionID,
+						UserID:           req.userID,
+						ActiveQueue:      conn.send,
+						DeadQueue:        conn.deadQueue,
+						DeadQueuePointer: conn.deadQueuePointer,
+						ReuseCount:       conn.reuseCount + 1,
+					}
+				}
+				req.result <- res
+			case <-ticker.C:
+				connIndex.RemoveInactiveConnections()
+			case webConn := <-h.register:
+				// Mark the current one as active.
+				// There is no need to check if it was inactive or not,
+				// we will anyways need to make it active.
+				webConn.active = true
+
+				connIndex.Add(webConn)
+				atomic.StoreInt64(&h.connectionCount, int64(connIndex.AllActive()))
+
+				if webConn.IsAuthenticated() && webConn.reuseCount == 0 {
+					// The hello message should only be sent when the reuseCount is 0.
+					// i.e in server restart, or long timeout, or fresh connection case.
+					// In case of seq number not found in dead queue, it is handled by
+					// the webconn write pump.
+					webConn.send <- webConn.createHelloMessage()
+				}
+			case webConn := <-h.unregister:
+				// If already removed (via queue full), then removing again becomes a noop.
+				// But if not removed, mark inactive.
+				webConn.active = false
+
+				atomic.StoreInt64(&h.connectionCount, int64(connIndex.AllActive()))
+
+				if webConn.UserId == "" {
+					continue
+				}
+
+				conns := connIndex.ForUser(webConn.UserId)
+				if len(conns) == 0 || areAllInactive(conns) {
+					h.platform.Go(func() {
+						suite.SetStatusOffline(webConn.UserId, false)
+					})
+					continue
+				}
+				var latestActivity int64 = 0
+				for _, conn := range conns {
+					if !conn.active {
+						continue
+					}
+					if conn.lastUserActivityAt > latestActivity {
+						latestActivity = conn.lastUserActivityAt
+					}
+				}
+
+				if suite.IsUserAway(latestActivity) {
+					h.platform.Go(func() {
+						suite.SetStatusLastActivityAt(webConn.UserId, latestActivity)
+					})
+				}
+			case userID := <-h.invalidateUser:
+				for _, webConn := range connIndex.ForUser(userID) {
+					webConn.InvalidateCache()
+				}
+			case activity := <-h.activity:
+				for _, webConn := range connIndex.ForUser(activity.userID) {
+					if !webConn.active {
+						continue
+					}
+					if webConn.GetSessionToken() == activity.sessionToken {
+						webConn.lastUserActivityAt = activity.activityAt
+					}
+				}
+			case directMsg := <-h.directMsg:
+				if !connIndex.Has(directMsg.conn) {
+					continue
+				}
+				select {
+				case directMsg.conn.send <- directMsg.msg:
+				default:
+					mlog.Error("webhub.broadcast: cannot send, closing websocket for user", mlog.String("user_id", directMsg.conn.UserId))
+					close(directMsg.conn.send)
+					connIndex.Remove(directMsg.conn)
+				}
+			case msg := <-h.broadcast:
+				if metrics := h.platform.metricsImpl(); metrics != nil {
+					metrics.DecrementWebSocketBroadcastBufferSize(strconv.Itoa(h.connectionIndex), 1)
+				}
+				msg = msg.PrecomputeJSON()
+				broadcast := func(webConn *WebConn) {
+					if !connIndex.Has(webConn) {
+						return
+					}
+					if webConn.ShouldSendEvent(msg) {
+						select {
+						case webConn.send <- msg:
+						default:
+							mlog.Error("webhub.broadcast: cannot send, closing websocket for user", mlog.String("user_id", webConn.UserId))
+							close(webConn.send)
+							connIndex.Remove(webConn)
+						}
+					}
+				}
+
+				if connID := msg.GetBroadcast().ConnectionId; connID != "" {
+					if webConn := connIndex.byConnectionId[connID]; webConn != nil {
+						broadcast(webConn)
+						continue
+					}
+				} else if msg.GetBroadcast().UserId != "" {
+					candidates := connIndex.ForUser(msg.GetBroadcast().UserId)
+					for _, webConn := range candidates {
+						broadcast(webConn)
+					}
+					continue
+				}
+
+				candidates := connIndex.All()
+				for webConn := range candidates {
+					broadcast(webConn)
+				}
+			case <-h.stop:
+				for webConn := range connIndex.All() {
+					webConn.Close()
+					suite.SetStatusOffline(webConn.UserId, false)
+				}
+
+				h.explicitStop = true
+				close(h.didStop)
+
+				return
+			}
+		}
+	}
+
+	doRecoverableStart = func() {
+		defer doRecover()
+		doStart()
+	}
+
+	doRecover = func() {
+		if !h.explicitStop {
+			if r := recover(); r != nil {
+				mlog.Error("Recovering from Hub panic.", mlog.Any("panic", r))
+			} else {
+				mlog.Error("Webhub stopped unexpectedly. Recovering.")
+			}
+
+			mlog.Error(string(debug.Stack()))
+
+			go doRecoverableStart()
+		}
+	}
+
+	go doRecoverableStart()
+}
+
+// hubConnectionIndex provides fast addition, removal, and iteration of web connections.
+// It requires 3 functionalities which need to be very fast:
+// - check if a connection exists or not.
+// - get all connections for a given userID.
+// - get all connections.
+type hubConnectionIndex struct {
+	// byUserId stores the list of connections for a given userID
+	byUserId map[string][]*WebConn
+	// byConnection serves the dual purpose of storing the index of the webconn
+	// in the value of byUserId map, and also to get all connections.
+	byConnection   map[*WebConn]int
+	byConnectionId map[string]*WebConn
+	// staleThreshold is the limit beyond which inactive connections
+	// will be deleted.
+	staleThreshold time.Duration
+}
+
+func newHubConnectionIndex(interval time.Duration) *hubConnectionIndex {
+	return &hubConnectionIndex{
+		byUserId:       make(map[string][]*WebConn),
+		byConnection:   make(map[*WebConn]int),
+		byConnectionId: make(map[string]*WebConn),
+		staleThreshold: interval,
+	}
+}
+
+func (i *hubConnectionIndex) Add(wc *WebConn) {
+	i.byUserId[wc.UserId] = append(i.byUserId[wc.UserId], wc)
+	i.byConnection[wc] = len(i.byUserId[wc.UserId]) - 1
+	i.byConnectionId[wc.GetConnectionID()] = wc
+}
+
+func (i *hubConnectionIndex) Remove(wc *WebConn) {
+	wc.Platform.ReturnSessionToPool(wc.GetSession())
+
+	userConnIndex, ok := i.byConnection[wc]
+	if !ok {
+		return
+	}
+
+	// get the conn slice.
+	userConnections := i.byUserId[wc.UserId]
+	// get the last connection.
+	last := userConnections[len(userConnections)-1]
+	// set the slot that we are trying to remove to be the last connection.
+	userConnections[userConnIndex] = last
+	// remove the last connection from the slice.
+	i.byUserId[wc.UserId] = userConnections[:len(userConnections)-1]
+	// set the index of the connection that was moved to the new index.
+	i.byConnection[last] = userConnIndex
+
+	delete(i.byConnection, wc)
+	delete(i.byConnectionId, wc.GetConnectionID())
+}
+
+func (i *hubConnectionIndex) Has(wc *WebConn) bool {
+	_, ok := i.byConnection[wc]
+	return ok
+}
+
+// ForUser returns all connections for a user ID.
+func (i *hubConnectionIndex) ForUser(id string) []*WebConn {
+	return i.byUserId[id]
+}
+
+// All returns the full webConn index.
+func (i *hubConnectionIndex) All() map[*WebConn]int {
+	return i.byConnection
+}
+
+// RemoveInactiveByConnectionID removes an inactive connection for the given
+// userID and connectionID.
+func (i *hubConnectionIndex) RemoveInactiveByConnectionID(userID, connectionID string) *WebConn {
+	// To handle empty sessions.
+	if userID == "" {
+		return nil
+	}
+	for _, conn := range i.ForUser(userID) {
+		if conn.GetConnectionID() == connectionID && !conn.active {
+			i.Remove(conn)
+			return conn
+		}
+	}
+	return nil
+}
+
+// RemoveInactiveConnections removes all inactive connections whose lastUserActivityAt
+// exceeded staleThreshold.
+func (i *hubConnectionIndex) RemoveInactiveConnections() {
+	now := model.GetMillis()
+	for conn := range i.byConnection {
+		if !conn.active && now-conn.lastUserActivityAt > i.staleThreshold.Milliseconds() {
+			i.Remove(conn)
+		}
+	}
+}
+
+// AllActive returns the number of active connections.
+// This is only called during register/unregister so we can take
+// a bit of perf hit here.
+func (i *hubConnectionIndex) AllActive() int {
+	cnt := 0
+	for conn := range i.byConnection {
+		if conn.active {
+			cnt++
+		}
+	}
+	return cnt
+}

app/platform/web_hub_test.go

@@ -0,0 +1,561 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	platform_mocks "github.com/mattermost/mattermost-server/v6/app/platform/mocks"
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/plugin"
+	"github.com/mattermost/mattermost-server/v6/shared/i18n"
+	"github.com/mattermost/mattermost-server/v6/store/storetest/mocks"
+	"github.com/mattermost/mattermost-server/v6/testlib"
+)
+
+func dummyWebsocketHandler(t *testing.T) http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		upgrader := &websocket.Upgrader{
+			ReadBufferSize:  1024,
+			WriteBufferSize: 1024,
+		}
+		conn, err := upgrader.Upgrade(w, req, nil)
+		for err == nil {
+			_, _, err = conn.ReadMessage()
+		}
+		if _, ok := err.(*websocket.CloseError); !ok {
+			require.NoError(t, err)
+		}
+	}
+}
+
+func registerDummyWebConn(t *testing.T, th *TestHelper, addr net.Addr, session *model.Session) *WebConn {
+	d := websocket.Dialer{}
+	c, _, err := d.Dial("ws://"+addr.String()+"/ws", nil)
+	require.NoError(t, err)
+
+	cfg := &WebConnConfig{
+		WebSocket: c,
+		Session:   *session,
+		TFunc:     i18n.IdentityTfunc(),
+		Locale:    "en",
+	}
+	wc := th.Service.NewWebConn(cfg, th.Suite, func() *plugin.Environment { return nil })
+	th.Service.HubRegister(wc)
+	go wc.Pump()
+	return wc
+}
+
+func TestHubStopWithMultipleConnections(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	s := httptest.NewServer(dummyWebsocketHandler(t))
+	defer s.Close()
+
+	session, err := th.Service.CreateSession(&model.Session{
+		UserId: th.BasicUser.Id,
+	})
+	require.NoError(t, err)
+
+	th.Service.HubStart(th.Suite)
+	wc1 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+	wc2 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+	wc3 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+	defer wc1.Close()
+	defer wc2.Close()
+	defer wc3.Close()
+}
+
+// TestHubStopRaceCondition verifies that attempts to use the hub after it has shutdown does not
+// block the caller indefinitely.
+func TestHubStopRaceCondition(t *testing.T) {
+	th := Setup(t).InitBasic()
+	// We do not call TearDown because th.TearDown shuts down the hub again. And hub close is not idempotent.
+	// Making it idempotent is not really important to the server because close only happens once.
+	// So we just use this quick hack for the test.
+	s := httptest.NewServer(dummyWebsocketHandler(t))
+
+	session, err := th.Service.CreateSession(&model.Session{
+		UserId: th.BasicUser.Id,
+	})
+	require.NoError(t, err)
+
+	th.Service.HubStart(th.Suite)
+	wc1 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+	defer wc1.Close()
+
+	hub := th.Service.hubs[0]
+	th.Service.HubStop()
+
+	done := make(chan bool)
+	go func() {
+		wc4 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+		wc5 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+		hub.Register(wc4)
+		hub.Register(wc5)
+
+		hub.UpdateActivity("userId", "sessionToken", 0)
+
+		for i := 0; i <= broadcastQueueSize; i++ {
+			hub.Broadcast(model.NewWebSocketEvent("", "", "", "", nil, ""))
+		}
+
+		hub.InvalidateUser("userId")
+		hub.Unregister(wc4)
+		hub.Unregister(wc5)
+		close(done)
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(15 * time.Second):
+		require.FailNow(t, "hub call did not return within 15 seconds after stop")
+	}
+}
+
+func TestHubSessionRevokeRace(t *testing.T) {
+	th := SetupWithStoreMock(t)
+	defer th.TearDown()
+
+	sess1 := &model.Session{
+		Id:             "id1",
+		UserId:         "user1",
+		DeviceId:       "",
+		Token:          "sesstoken",
+		ExpiresAt:      model.GetMillis() + 300000,
+		LastActivityAt: 10000,
+	}
+
+	mockStore := th.Service.Store.(*mocks.Store)
+
+	mockUserStore := mocks.UserStore{}
+	mockUserStore.On("Count", mock.Anything).Return(int64(10), nil)
+	mockUserStore.On("GetUnreadCount", mock.AnythingOfType("string"), mock.AnythingOfType("bool")).Return(int64(1), nil)
+	mockPostStore := mocks.PostStore{}
+	mockPostStore.On("GetMaxPostSize").Return(65535, nil)
+	mockSystemStore := mocks.SystemStore{}
+	mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil)
+	mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil)
+	mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil)
+
+	mockSessionStore := mocks.SessionStore{}
+	mockSessionStore.On("UpdateLastActivityAt", "id1", mock.Anything).Return(nil)
+	mockSessionStore.On("Save", mock.AnythingOfType("*model.Session")).Return(sess1, nil)
+	mockSessionStore.On("Get", mock.Anything, "id1").Return(sess1, nil)
+	mockSessionStore.On("Remove", "id1").Return(nil)
+
+	mockStatusStore := mocks.StatusStore{}
+	mockStatusStore.On("Get", "user1").Return(&model.Status{UserId: "user1", Status: model.StatusOnline}, nil)
+	mockStatusStore.On("UpdateLastActivityAt", "user1", mock.Anything).Return(nil)
+	mockStatusStore.On("SaveOrUpdate", mock.AnythingOfType("*model.Status")).Return(nil)
+
+	mockOAuthStore := mocks.OAuthStore{}
+	mockStore.On("Session").Return(&mockSessionStore)
+	mockStore.On("OAuth").Return(&mockOAuthStore)
+	mockStore.On("Status").Return(&mockStatusStore)
+	mockStore.On("User").Return(&mockUserStore)
+	mockStore.On("Post").Return(&mockPostStore)
+	mockStore.On("System").Return(&mockSystemStore)
+	mockStore.On("GetDBSchemaVersion").Return(1, nil)
+
+	// This needs to be false for the condition to trigger
+	th.Service.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ServiceSettings.ExtendSessionLengthWithActivity = false
+	})
+
+	s := httptest.NewServer(dummyWebsocketHandler(t))
+	defer s.Close()
+
+	session, err := th.Service.CreateSession(&model.Session{
+		UserId: "testid",
+	})
+	require.NoError(t, err)
+
+	wc1 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+	hub := th.Service.GetHubForUserId(wc1.UserId)
+
+	done := make(chan bool)
+
+	time.Sleep(time.Second)
+	// We override the LastActivityAt which happens in NewWebConn.
+	// This is needed to call RevokeSessionById which triggers the race.
+	th.Service.AddSessionToCache(sess1)
+
+	go func() {
+		for i := 0; i <= broadcastQueueSize; i++ {
+			hub.Broadcast(model.NewWebSocketEvent("", "teamID", "", "", nil, ""))
+		}
+		close(done)
+	}()
+
+	// This call should happen _after_ !wc.IsAuthenticated() and _before_wc.isMemberOfTeam().
+	// There's no guarantee this will happen. But that's out best bet to trigger this race.
+	wc1.InvalidateCache()
+
+	for i := 0; i < 10; i++ {
+		// If broadcast buffer has not emptied,
+		// we sleep for a second and check again
+		if len(hub.broadcast) > 0 {
+			time.Sleep(time.Second)
+			continue
+		}
+	}
+	if len(hub.broadcast) > 0 {
+		require.Fail(t, "hub is deadlocked")
+	}
+}
+
+func TestHubConnIndex(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	connIndex := newHubConnectionIndex(1 * time.Second)
+
+	// User1
+	wc1 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   model.NewId(),
+	}
+	wc1.SetConnectionID(model.NewId())
+	wc1.SetSession(&model.Session{})
+
+	// User2
+	wc2 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   model.NewId(),
+	}
+	wc2.SetConnectionID(model.NewId())
+	wc2.SetSession(&model.Session{})
+
+	wc3 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   wc2.UserId,
+	}
+	wc3.SetConnectionID(model.NewId())
+	wc3.SetSession(&model.Session{})
+
+	wc4 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   wc2.UserId,
+	}
+	wc4.SetConnectionID(model.NewId())
+	wc4.SetSession(&model.Session{})
+
+	connIndex.Add(wc1)
+	connIndex.Add(wc2)
+	connIndex.Add(wc3)
+	connIndex.Add(wc4)
+
+	t.Run("Basic", func(t *testing.T) {
+		assert.True(t, connIndex.Has(wc1))
+		assert.True(t, connIndex.Has(wc2))
+
+		assert.ElementsMatch(t, connIndex.ForUser(wc2.UserId), []*WebConn{wc2, wc3, wc4})
+		assert.ElementsMatch(t, connIndex.ForUser(wc1.UserId), []*WebConn{wc1})
+		assert.True(t, connIndex.Has(wc2))
+		assert.True(t, connIndex.Has(wc1))
+		assert.Len(t, connIndex.All(), 4)
+	})
+
+	t.Run("RemoveMiddleUser2", func(t *testing.T) {
+		connIndex.Remove(wc3) // Remove from middle from user2
+
+		assert.ElementsMatch(t, connIndex.ForUser(wc2.UserId), []*WebConn{wc2, wc4})
+		assert.ElementsMatch(t, connIndex.ForUser(wc1.UserId), []*WebConn{wc1})
+		assert.True(t, connIndex.Has(wc2))
+		assert.False(t, connIndex.Has(wc3))
+		assert.True(t, connIndex.Has(wc4))
+		assert.Len(t, connIndex.All(), 3)
+	})
+
+	t.Run("RemoveUser1", func(t *testing.T) {
+		connIndex.Remove(wc1) // Remove sole connection from user1
+
+		assert.ElementsMatch(t, connIndex.ForUser(wc2.UserId), []*WebConn{wc2, wc4})
+		assert.ElementsMatch(t, connIndex.ForUser(wc1.UserId), []*WebConn{})
+		assert.Len(t, connIndex.All(), 2)
+		assert.False(t, connIndex.Has(wc1))
+		assert.True(t, connIndex.Has(wc2))
+	})
+
+	t.Run("RemoveEndUser2", func(t *testing.T) {
+		connIndex.Remove(wc4) // Remove from end from user2
+
+		assert.ElementsMatch(t, connIndex.ForUser(wc2.UserId), []*WebConn{wc2})
+		assert.ElementsMatch(t, connIndex.ForUser(wc1.UserId), []*WebConn{})
+		assert.True(t, connIndex.Has(wc2))
+		assert.False(t, connIndex.Has(wc3))
+		assert.False(t, connIndex.Has(wc4))
+		assert.Len(t, connIndex.All(), 1)
+	})
+}
+
+func TestHubConnIndexByConnectionId(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	connIndex := newHubConnectionIndex(1 * time.Second)
+
+	// User1
+	wc1ID := model.NewId()
+	wc1 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   model.NewId(),
+	}
+	wc1.SetConnectionID(wc1ID)
+	wc1.SetSession(&model.Session{})
+
+	// User2
+	wc2ID := model.NewId()
+	wc2 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   model.NewId(),
+	}
+	wc2.SetConnectionID(wc2ID)
+	wc2.SetSession(&model.Session{})
+
+	wc3ID := model.NewId()
+	wc3 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   wc2.UserId,
+	}
+	wc3.SetConnectionID(wc3ID)
+	wc3.SetSession(&model.Session{})
+
+	t.Run("no connections", func(t *testing.T) {
+		assert.False(t, connIndex.Has(wc1))
+		assert.False(t, connIndex.Has(wc2))
+		assert.False(t, connIndex.Has(wc3))
+		assert.Empty(t, connIndex.byConnectionId)
+	})
+
+	t.Run("adding", func(t *testing.T) {
+		connIndex.Add(wc1)
+		connIndex.Add(wc3)
+
+		assert.Len(t, connIndex.byConnectionId, 2)
+		assert.Equal(t, wc1, connIndex.byConnectionId[wc1ID])
+		assert.Equal(t, wc3, connIndex.byConnectionId[wc3ID])
+		assert.Equal(t, (*WebConn)(nil), connIndex.byConnectionId[wc2ID])
+	})
+
+	t.Run("removing", func(t *testing.T) {
+		connIndex.Remove(wc3)
+
+		assert.Len(t, connIndex.byConnectionId, 1)
+		assert.Equal(t, wc1, connIndex.byConnectionId[wc1ID])
+		assert.Equal(t, (*WebConn)(nil), connIndex.byConnectionId[wc3ID])
+		assert.Equal(t, (*WebConn)(nil), connIndex.byConnectionId[wc2ID])
+	})
+}
+
+func TestHubConnIndexInactive(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	connIndex := newHubConnectionIndex(2 * time.Second)
+
+	// User1
+	wc1 := &WebConn{
+		Platform: th.Service,
+		UserId:   model.NewId(),
+		active:   true,
+	}
+	wc1.SetConnectionID("conn1")
+	wc1.SetSession(&model.Session{})
+
+	// User2
+	wc2 := &WebConn{
+		Platform: th.Service,
+		UserId:   model.NewId(),
+		active:   true,
+	}
+	wc2.SetConnectionID("conn2")
+	wc2.SetSession(&model.Session{})
+
+	wc3 := &WebConn{
+		Platform: th.Service,
+		UserId:   wc2.UserId,
+		active:   false,
+	}
+	wc3.SetConnectionID("conn3")
+	wc3.SetSession(&model.Session{})
+
+	connIndex.Add(wc1)
+	connIndex.Add(wc2)
+	connIndex.Add(wc3)
+
+	assert.Nil(t, connIndex.RemoveInactiveByConnectionID(wc2.UserId, "conn2"))
+	assert.NotNil(t, connIndex.RemoveInactiveByConnectionID(wc2.UserId, "conn3"))
+	assert.Nil(t, connIndex.RemoveInactiveByConnectionID(wc1.UserId, "conn3"))
+	assert.False(t, connIndex.Has(wc3))
+	assert.Len(t, connIndex.ForUser(wc2.UserId), 1)
+
+	wc3.lastUserActivityAt = model.GetMillis()
+	connIndex.Add(wc3)
+	connIndex.RemoveInactiveConnections()
+	assert.True(t, connIndex.Has(wc3))
+	assert.Len(t, connIndex.ForUser(wc2.UserId), 2)
+	assert.Len(t, connIndex.All(), 3)
+
+	wc3.lastUserActivityAt = model.GetMillis() - (time.Minute).Milliseconds()
+	connIndex.RemoveInactiveConnections()
+	assert.False(t, connIndex.Has(wc3))
+	assert.Len(t, connIndex.ForUser(wc2.UserId), 1)
+	assert.Len(t, connIndex.All(), 2)
+}
+
+func TestReliableWebSocketSend(t *testing.T) {
+	testCluster := &testlib.FakeClusterInterface{}
+
+	th := SetupWithCluster(t, testCluster)
+	defer th.TearDown()
+
+	ev := model.NewWebSocketEvent("test_unreliable_event", "", "", "", nil, "")
+	ev = ev.SetBroadcast(&model.WebsocketBroadcast{})
+	th.Service.Publish(ev)
+	ev2 := model.NewWebSocketEvent("test_reliable_event", "", "", "", nil, "")
+
+	ev2 = ev2.SetBroadcast(&model.WebsocketBroadcast{
+		ReliableClusterSend: true,
+	})
+	th.Service.Publish(ev2)
+
+	messages := testCluster.GetMessages()
+
+	evJSON, err := ev.ToJSON()
+	require.NoError(t, err)
+	ev2JSON, err := ev2.ToJSON()
+	require.NoError(t, err)
+
+	require.Contains(t, messages, &model.ClusterMessage{
+		Event:    model.ClusterEventPublish,
+		Data:     evJSON,
+		SendType: model.ClusterSendBestEffort,
+	})
+	require.Contains(t, messages, &model.ClusterMessage{
+		Event:    model.ClusterEventPublish,
+		Data:     ev2JSON,
+		SendType: model.ClusterSendReliable,
+	})
+}
+
+func TestHubIsRegistered(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	session, err := th.Service.CreateSession(&model.Session{
+		UserId: th.BasicUser.Id,
+	})
+	require.NoError(t, err)
+
+	mockSuite := &platform_mocks.SuiteIFace{}
+	mockSuite.On("SetStatusOnline", th.BasicUser.Id, false).Return()
+	mockSuite.On("UpdateLastActivityAtIfNeeded", *session).Return()
+	mockSuite.On("GetSession", session.Token).Return(session, nil)
+	mockSuite.On("IsUserAway", mock.Anything).Return(false)
+	mockSuite.On("SetStatusOffline", th.BasicUser.Id, false).Return()
+
+	th.Suite = mockSuite
+
+	s := httptest.NewServer(dummyWebsocketHandler(t))
+	defer s.Close()
+
+	th.Service.HubStart(th.Suite)
+	wc1 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+	wc2 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+	wc3 := registerDummyWebConn(t, th, s.Listener.Addr(), session)
+	defer wc1.Close()
+	defer wc2.Close()
+	defer wc3.Close()
+
+	session1 := wc1.session.Load().(*model.Session)
+
+	assert.True(t, th.Service.SessionIsRegistered(*session1))
+	assert.True(t, th.Service.SessionIsRegistered(*wc2.session.Load().(*model.Session)))
+	assert.True(t, th.Service.SessionIsRegistered(*wc3.session.Load().(*model.Session)))
+
+	session4, err := th.Service.CreateSession(&model.Session{
+		UserId: th.BasicUser2.Id,
+	})
+	require.NoError(t, err)
+	assert.False(t, th.Service.SessionIsRegistered(*session4))
+}
+
+// Always run this with -benchtime=0.1s
+// See: https://github.com/golang/go/issues/27217.
+func BenchmarkHubConnIndex(b *testing.B) {
+	th := Setup(b).InitBasic()
+	defer th.TearDown()
+	connIndex := newHubConnectionIndex(1 * time.Second)
+
+	// User1
+	wc1 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   model.NewId(),
+	}
+
+	// User2
+	wc2 := &WebConn{
+		Platform: th.Service,
+		Suite:    th.Suite,
+		UserId:   model.NewId(),
+	}
+	b.ResetTimer()
+	b.Run("Add", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			connIndex.Add(wc1)
+			connIndex.Add(wc2)
+
+			b.StopTimer()
+			connIndex.Remove(wc1)
+			connIndex.Remove(wc2)
+			b.StartTimer()
+		}
+	})
+
+	b.Run("Remove", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			b.StopTimer()
+			connIndex.Add(wc1)
+			connIndex.Add(wc2)
+			b.StartTimer()
+
+			connIndex.Remove(wc1)
+			connIndex.Remove(wc2)
+		}
+	})
+}
+
+var hubSink *Hub
+
+func BenchmarkGetHubForUserId(b *testing.B) {
+	th := Setup(b).InitBasic()
+	defer th.TearDown()
+
+	th.Service.HubStart(th.Suite)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		hubSink = th.Service.GetHubForUserId(th.BasicUser.Id)
+	}
+}

app/platform/websocket_router.go

@@ -0,0 +1,113 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package platform
+
+import (
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/shared/i18n"
+	"github.com/mattermost/mattermost-server/v6/shared/mlog"
+)
+
+type webSocketHandler interface {
+	ServeWebSocket(*WebConn, *model.WebSocketRequest)
+}
+
+type WebSocketRouter struct {
+	handlers map[string]webSocketHandler
+}
+
+func (wr *WebSocketRouter) Handle(action string, handler webSocketHandler) {
+	wr.handlers[action] = handler
+}
+
+func (wr *WebSocketRouter) ServeWebSocket(conn *WebConn, r *model.WebSocketRequest) {
+	if r.Action == "" {
+		err := model.NewAppError("ServeWebSocket", "api.web_socket_router.no_action.app_error", nil, "", http.StatusBadRequest)
+		returnWebSocketError(conn.Platform, conn, r, err)
+		return
+	}
+
+	if r.Seq <= 0 {
+		err := model.NewAppError("ServeWebSocket", "api.web_socket_router.bad_seq.app_error", nil, "", http.StatusBadRequest)
+		returnWebSocketError(conn.Platform, conn, r, err)
+		return
+	}
+
+	if r.Action == model.WebsocketAuthenticationChallenge {
+		if conn.GetSessionToken() != "" {
+			return
+		}
+
+		token, ok := r.Data["token"].(string)
+		if !ok {
+			conn.WebSocket.Close()
+			return
+		}
+
+		session, err := conn.Suite.GetSession(token)
+		if err != nil {
+			conn.WebSocket.Close()
+			return
+		}
+		conn.SetSession(session)
+		conn.SetSessionToken(session.Token)
+		conn.UserId = session.UserId
+
+		conn.Platform.HubRegister(conn)
+
+		conn.Platform.Go(func() {
+			conn.Suite.SetStatusOnline(session.UserId, false)
+			conn.Suite.UpdateLastActivityAtIfNeeded(*session)
+		})
+
+		resp := model.NewWebSocketResponse(model.StatusOk, r.Seq, nil)
+		hub := conn.Platform.GetHubForUserId(conn.UserId)
+		if hub == nil {
+			return
+		}
+		hub.SendMessage(conn, resp)
+
+		return
+	}
+
+	if !conn.IsAuthenticated() {
+		err := model.NewAppError("ServeWebSocket", "api.web_socket_router.not_authenticated.app_error", nil, "", http.StatusUnauthorized)
+		returnWebSocketError(conn.Platform, conn, r, err)
+		return
+	}
+
+	handler, ok := wr.handlers[r.Action]
+	if !ok {
+		err := model.NewAppError("ServeWebSocket", "api.web_socket_router.bad_action.app_error", nil, "", http.StatusInternalServerError)
+		returnWebSocketError(conn.Platform, conn, r, err)
+		return
+	}
+
+	handler.ServeWebSocket(conn, r)
+}
+
+func returnWebSocketError(ps *PlatformService, conn *WebConn, r *model.WebSocketRequest, err *model.AppError) {
+	logF := mlog.Error
+	if err.StatusCode >= http.StatusBadRequest && err.StatusCode < http.StatusInternalServerError {
+		logF = mlog.Debug
+	}
+	logF(
+		"websocket routing error.",
+		mlog.Int64("seq", r.Seq),
+		mlog.String("user_id", conn.UserId),
+		mlog.String("system_message", err.SystemMessage(i18n.T)),
+		mlog.Err(err),
+	)
+
+	hub := ps.GetHubForUserId(conn.UserId)
+	if hub == nil {
+		return
+	}
+
+	err.DetailedError = ""
+	errorResp := model.NewWebSocketError(r.Seq, err)
+	hub.SendMessage(conn, errorResp)
+}