MM-62751: [Shared Channels] Allow remote users to be discoverable in the create DM/GM modal (#30918)

Этот коммит содержится в:
catalintomai
2025-06-13 16:51:12 +02:00
коммит произвёл GitHub
родитель 476b46d1d7
Коммит c46ed6c681
24 изменённых файлов: 2133 добавлений и 38 удалений

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@@ -0,0 +1,248 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package app
import (
"encoding/json"
"io"
"net/http"
"sync/atomic"
"testing"
"time"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/services/remotecluster"
"github.com/mattermost/mattermost/server/v8/platform/services/sharedchannel"
"github.com/stretchr/testify/require"
)
// writeOKResponse writes a standard OK JSON response in the format expected by remotecluster
func writeOKResponse(w http.ResponseWriter) {
response := &remotecluster.Response{
Status: "OK",
Err: "",
}
// Set empty sync response as payload
syncResp := &model.SyncResponse{}
_ = response.SetPayload(syncResp)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
respBytes, _ := json.Marshal(response)
_, _ = w.Write(respBytes)
}
// SelfReferentialSyncHandler handles incoming sync messages for self-referential tests.
// It processes the messages, updates cursors, and returns proper responses.
type SelfReferentialSyncHandler struct {
t *testing.T
service *sharedchannel.Service
selfCluster *model.RemoteCluster
syncMessageCount *int32
// Callbacks for capturing sync data
OnIndividualSync func(userId string, messageNumber int32)
OnBatchSync func(userIds []string, messageNumber int32)
OnGlobalUserSync func(userIds []string, messageNumber int32)
}
// NewSelfReferentialSyncHandler creates a new handler for processing sync messages in tests
func NewSelfReferentialSyncHandler(t *testing.T, service *sharedchannel.Service, selfCluster *model.RemoteCluster) *SelfReferentialSyncHandler {
count := int32(0)
return &SelfReferentialSyncHandler{
t: t,
service: service,
selfCluster: selfCluster,
syncMessageCount: &count,
}
}
// HandleRequest processes incoming HTTP requests for the test server.
// This handler includes common remote cluster endpoints to simulate a real remote cluster:
// - /api/v4/remotecluster/msg: Main sync message endpoint
// - /api/v4/remotecluster/ping: Ping endpoint to maintain online status (prevents offline after 5 minutes)
// - /api/v4/remotecluster/confirm_invite: Invitation confirmation endpoint
func (h *SelfReferentialSyncHandler) HandleRequest(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/v4/remotecluster/msg":
currentCall := atomic.AddInt32(h.syncMessageCount, 1)
// Read and process the sync message
body, _ := io.ReadAll(r.Body)
// The message is wrapped in a RemoteClusterFrame
var frame model.RemoteClusterFrame
err := json.Unmarshal(body, &frame)
if err == nil {
// Process the message to update cursor
response := &remotecluster.Response{}
processErr := h.service.OnReceiveSyncMessageForTesting(frame.Msg, h.selfCluster, response)
if processErr != nil {
response.Status = "ERROR"
response.Err = processErr.Error()
} else {
// Success - build a proper sync response
response.Status = "OK"
response.Err = ""
var syncMsg model.SyncMsg
if unmarshalErr := json.Unmarshal(frame.Msg.Payload, &syncMsg); unmarshalErr == nil {
syncResp := &model.SyncResponse{}
// Handle global user sync
if len(syncMsg.Users) > 0 {
userIds := make([]string, 0, len(syncMsg.Users))
for userId := range syncMsg.Users {
userIds = append(userIds, userId)
syncResp.UsersSyncd = append(syncResp.UsersSyncd, userId)
}
if h.OnGlobalUserSync != nil {
h.OnGlobalUserSync(userIds, currentCall)
}
}
_ = response.SetPayload(syncResp)
}
}
// Send the proper response
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
respBytes, _ := json.Marshal(response)
_, _ = w.Write(respBytes)
return
}
writeOKResponse(w)
case "/api/v4/remotecluster/ping":
writeOKResponse(w)
case "/api/v4/remotecluster/confirm_invite":
writeOKResponse(w)
default:
writeOKResponse(w)
}
}
// GetSyncMessageCount returns the current count of sync messages received
func (h *SelfReferentialSyncHandler) GetSyncMessageCount() int32 {
return atomic.LoadInt32(h.syncMessageCount)
}
// GetUsersFromSyncMsg extracts user IDs from a sync message
func GetUsersFromSyncMsg(msg model.SyncMsg) []string {
var userIds []string
// Extract from users field
for userId := range msg.Users {
userIds = append(userIds, userId)
}
return userIds
}
// EnsureCleanState ensures a clean test state by removing all shared channels, remote clusters,
// and extra team/channel members. This helps prevent state pollution between tests.
func EnsureCleanState(t *testing.T, th *TestHelper, ss store.Store) {
t.Helper()
// Clear all shared channels and remotes from previous tests
allSharedChannels, _ := ss.SharedChannel().GetAll(0, 1000, model.SharedChannelFilterOpts{})
for _, sc := range allSharedChannels {
// Delete all remotes for this channel
remotes, _ := ss.SharedChannel().GetRemotes(0, 100, model.SharedChannelRemoteFilterOpts{ChannelId: sc.ChannelId})
for _, remote := range remotes {
_, _ = ss.SharedChannel().DeleteRemote(remote.Id)
}
// Delete the shared channel
_, _ = ss.SharedChannel().Delete(sc.ChannelId)
}
// Delete all remote clusters
allRemoteClusters, _ := ss.RemoteCluster().GetAll(0, 1000, model.RemoteClusterQueryFilter{})
for _, rc := range allRemoteClusters {
_, _ = ss.RemoteCluster().Delete(rc.RemoteId)
}
// Remove all channel members from test channels (except the basic team/channel setup)
channels, _ := ss.Channel().GetAll(th.BasicTeam.Id)
for _, channel := range channels {
// Skip direct message and group channels, and skip the default channels
if channel.Type != model.ChannelTypeDirect && channel.Type != model.ChannelTypeGroup &&
channel.Id != th.BasicChannel.Id {
members, _ := ss.Channel().GetMembers(channel.Id, 0, 10000)
for _, member := range members {
_ = ss.Channel().RemoveMember(th.Context, channel.Id, member.UserId)
}
}
}
// Remove all users from teams except the basic test users
teams, _ := ss.Team().GetAll()
for _, team := range teams {
if team.Id == th.BasicTeam.Id {
members, _ := ss.Team().GetMembers(team.Id, 0, 10000, nil)
for _, member := range members {
// Keep only the basic test users
if member.UserId != th.BasicUser.Id && member.UserId != th.BasicUser2.Id &&
member.UserId != th.SystemAdminUser.Id {
_ = ss.Team().RemoveMember(th.Context, team.Id, member.UserId)
}
}
}
}
// Get all active users and deactivate non-basic ones
options := &model.UserGetOptions{
Page: 0,
PerPage: 200,
Active: true,
}
users, _ := ss.User().GetAllProfiles(options)
for _, user := range users {
// Keep only the basic test users active
if user.Id != th.BasicUser.Id && user.Id != th.BasicUser2.Id &&
user.Id != th.SystemAdminUser.Id {
// Deactivate the user (soft delete)
user.DeleteAt = model.GetMillis()
_, _ = ss.User().Update(th.Context, user, true)
}
}
// Verify cleanup is complete
require.Eventually(t, func() bool {
sharedChannels, _ := ss.SharedChannel().GetAll(0, 1000, model.SharedChannelFilterOpts{})
remoteClusters, _ := ss.RemoteCluster().GetAll(0, 1000, model.RemoteClusterQueryFilter{})
return len(sharedChannels) == 0 && len(remoteClusters) == 0
}, 2*time.Second, 100*time.Millisecond, "Failed to clean up shared channels and remote clusters")
// Reset batch size to default to ensure test isolation
defaultBatchSize := 20
th.App.UpdateConfig(func(cfg *model.Config) {
cfg.ConnectedWorkspacesSettings.GlobalUserSyncBatchSize = &defaultBatchSize
})
// Ensure services are running and ready
scsInterface := th.App.Srv().GetSharedChannelSyncService()
if scs, ok := scsInterface.(*sharedchannel.Service); ok {
require.Eventually(t, func() bool {
return scs.Active()
}, 2*time.Second, 100*time.Millisecond, "Shared channel service should be active")
}
rcService := th.App.Srv().GetRemoteClusterService()
if rcService != nil {
if rc, ok := rcService.(*remotecluster.Service); ok {
rc.SetActive(true)
}
require.Eventually(t, func() bool {
return rcService.Active()
}, 2*time.Second, 100*time.Millisecond, "Remote cluster service should be active")
}
}

Просмотреть файл

@@ -273,6 +273,8 @@ channels/db/migrations/mysql/000137_update_attribute_view.down.sql
channels/db/migrations/mysql/000137_update_attribute_view.up.sql
channels/db/migrations/mysql/000138_add_default_category_name_to_channel.down.sql
channels/db/migrations/mysql/000138_add_default_category_name_to_channel.up.sql
channels/db/migrations/mysql/000139_remoteclusters_add_last_global_user_sync_at.down.sql
channels/db/migrations/mysql/000139_remoteclusters_add_last_global_user_sync_at.up.sql
channels/db/migrations/postgres/000001_create_teams.down.sql
channels/db/migrations/postgres/000001_create_teams.up.sql
channels/db/migrations/postgres/000002_create_team_members.down.sql
@@ -547,3 +549,5 @@ channels/db/migrations/postgres/000137_update_attribute_view.down.sql
channels/db/migrations/postgres/000137_update_attribute_view.up.sql
channels/db/migrations/postgres/000138_add_default_category_name_to_channel.down.sql
channels/db/migrations/postgres/000138_add_default_category_name_to_channel.up.sql
channels/db/migrations/postgres/000139_remoteclusters_add_last_global_user_sync_at.down.sql
channels/db/migrations/postgres/000139_remoteclusters_add_last_global_user_sync_at.up.sql

Просмотреть файл

@@ -0,0 +1,14 @@
SET @preparedStatement = (SELECT IF(
(
SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'RemoteClusters'
AND COLUMN_NAME = 'LastGlobalUserSyncAt'
) > 0,
'ALTER TABLE RemoteClusters DROP COLUMN LastGlobalUserSyncAt',
'SELECT 1'
));
PREPARE alterIfExists FROM @preparedStatement;
EXECUTE alterIfExists;
DEALLOCATE PREPARE alterIfExists;

Просмотреть файл

@@ -0,0 +1,14 @@
SET @preparedStatement = (SELECT IF(
(
SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'RemoteClusters'
AND COLUMN_NAME = 'LastGlobalUserSyncAt'
) > 0,
'SELECT 1',
'ALTER TABLE RemoteClusters ADD COLUMN LastGlobalUserSyncAt bigint DEFAULT 0'
));
PREPARE alterIfNotExists FROM @preparedStatement;
EXECUTE alterIfNotExists;
DEALLOCATE PREPARE alterIfNotExists;

Просмотреть файл

@@ -0,0 +1 @@
ALTER TABLE remoteclusters DROP COLUMN IF EXISTS lastglobalusersyncat;

Просмотреть файл

@@ -0,0 +1 @@
ALTER TABLE remoteclusters ADD COLUMN IF NOT EXISTS lastglobalusersyncat bigint DEFAULT 0;

Просмотреть файл

@@ -9998,6 +9998,27 @@ func (s *RetryLayerRemoteClusterStore) Update(rc *model.RemoteCluster) (*model.R
}
func (s *RetryLayerRemoteClusterStore) UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error {
tries := 0
for {
err := s.RemoteClusterStore.UpdateLastGlobalUserSyncAt(remoteID, syncAt)
if err == nil {
return nil
}
if !isRepeatableError(err) {
return err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return err
}
timepkg.Sleep(100 * timepkg.Millisecond)
}
}
func (s *RetryLayerRemoteClusterStore) UpdateTopics(remoteClusterID string, topics string) (*model.RemoteCluster, error) {
tries := 0

Просмотреть файл

@@ -44,6 +44,7 @@ func remoteClusterFields(prefix string) []string {
prefix + "CreatorId",
prefix + "PluginID",
prefix + "Options",
prefix + "LastGlobalUserSyncAt",
}
}
@@ -99,7 +100,8 @@ func (s sqlRemoteClusterStore) Update(remoteCluster *model.RemoteCluster) (*mode
DefaultTeamId = :DefaultTeamId,
Topics = :Topics,
PluginID = :PluginID,
Options = :Options
Options = :Options,
LastGlobalUserSyncAt = :LastGlobalUserSyncAt
WHERE RemoteId = :RemoteId AND Name = :Name`
if _, err := s.GetMaster().NamedExec(query, remoteCluster); err != nil {
@@ -310,3 +312,24 @@ func (s sqlRemoteClusterStore) SetLastPingAt(remoteClusterId string) error {
}
return nil
}
func (s sqlRemoteClusterStore) UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error {
query := s.getQueryBuilder().
Update("RemoteClusters").
Set("LastGlobalUserSyncAt", syncAt).
Where(sq.Eq{"RemoteId": remoteID})
result, err := s.GetMaster().ExecBuilder(query)
if err != nil {
return errors.Wrap(err, "failed to update LastGlobalUserSyncAt for RemoteCluster")
}
count, err := result.RowsAffected()
if err != nil {
return errors.Wrap(err, "failed to determine rows affected")
}
if count == 0 {
return fmt.Errorf("remote cluster not found: %s", remoteID)
}
return nil
}

Просмотреть файл

@@ -584,8 +584,15 @@ func (us SqlUserStore) GetEtagForAllProfiles() string {
func (us SqlUserStore) GetAllProfiles(options *model.UserGetOptions) ([]*model.User, error) {
isPostgreSQL := us.DriverName() == model.DatabaseDriverPostgres
// Determine ordering based on Sort option - default to Username ASC for backwards compatibility
orderBy := "Users.Username ASC"
if options.Sort == "update_at_asc" {
orderBy = "Users.UpdateAt ASC"
}
query := us.usersQuery.
OrderBy("Users.Username ASC").
OrderBy(orderBy).
Offset(uint64(options.Page * options.PerPage)).Limit(uint64(options.PerPage))
query = applyViewRestrictionsFilter(query, options.ViewRestrictions, true)
@@ -599,6 +606,10 @@ func (us SqlUserStore) GetAllProfiles(options *model.UserGetOptions) ([]*model.U
query = query.Where("Users.DeleteAt = 0")
}
if options.UpdatedAfter > 0 {
query = query.Where(sq.Gt{"Users.UpdateAt": options.UpdatedAfter})
}
users := []*model.User{}
if err := us.GetReplica().SelectBuilder(&users, query); err != nil {
return nil, errors.Wrap(err, "failed to get User profiles")

Просмотреть файл

@@ -562,6 +562,7 @@ type RemoteClusterStore interface {
GetAll(offset, limit int, filter model.RemoteClusterQueryFilter) ([]*model.RemoteCluster, error)
UpdateTopics(remoteClusterID string, topics string) (*model.RemoteCluster, error)
SetLastPingAt(remoteClusterID string) error
UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error
}
type ComplianceStore interface {

Просмотреть файл

@@ -210,6 +210,24 @@ func (_m *RemoteClusterStore) Update(rc *model.RemoteCluster) (*model.RemoteClus
return r0, r1
}
// UpdateLastGlobalUserSyncAt provides a mock function with given fields: remoteID, syncAt
func (_m *RemoteClusterStore) UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error {
ret := _m.Called(remoteID, syncAt)
if len(ret) == 0 {
panic("no return value specified for UpdateLastGlobalUserSyncAt")
}
var r0 error
if rf, ok := ret.Get(0).(func(string, int64) error); ok {
r0 = rf(remoteID, syncAt)
} else {
r0 = ret.Error(0)
}
return r0
}
// UpdateTopics provides a mock function with given fields: remoteClusterID, topics
func (_m *RemoteClusterStore) UpdateTopics(remoteClusterID string, topics string) (*model.RemoteCluster, error) {
ret := _m.Called(remoteClusterID, topics)

Просмотреть файл

@@ -32,6 +32,7 @@ func TestSharedChannelStore(t *testing.T, rctx request.CTX, ss store.Store, s Sq
t.Run("HasRemote", func(t *testing.T) { testHasRemote(t, rctx, ss) })
t.Run("GetRemoteForUser", func(t *testing.T) { testGetRemoteForUser(t, rctx, ss) })
t.Run("UpdateSharedChannelRemoteNextSyncAt", func(t *testing.T) { testUpdateSharedChannelRemoteCursor(t, rctx, ss) })
t.Run("UpdateGlobalUserSyncCursor", func(t *testing.T) { testUpdateGlobalUserSyncCursor(t, rctx, ss) })
t.Run("DeleteSharedChannelRemote", func(t *testing.T) { testDeleteSharedChannelRemote(t, rctx, ss) })
t.Run("SaveSharedChannelUser", func(t *testing.T) { testSaveSharedChannelUser(t, rctx, ss) })
@@ -932,6 +933,38 @@ func testUpdateSharedChannelRemoteCursor(t *testing.T, rctx request.CTX, ss stor
})
}
func testUpdateGlobalUserSyncCursor(t *testing.T, rctx request.CTX, ss store.Store) {
// Create a remote cluster first
rc := &model.RemoteCluster{
RemoteId: model.NewId(),
SiteURL: "http://example.com",
CreatorId: model.NewId(),
Name: "test",
}
rcSaved, err := ss.RemoteCluster().Save(rc)
require.NoError(t, err, "couldn't save remote cluster", err)
futureTimestamp := model.GetMillis() + 3600000 // 1 hour in the future
t.Run("Update global user sync cursor for remote", func(t *testing.T) {
err := ss.RemoteCluster().UpdateLastGlobalUserSyncAt(rcSaved.RemoteId, futureTimestamp)
require.NoError(t, err, "update global user sync cursor should not error", err)
// Verify that the LastGlobalUserSyncAt field was updated in the RemoteCluster table
// Small sleep to ensure the transaction is committed
time.Sleep(10 * time.Millisecond)
updatedRC, err := ss.RemoteCluster().Get(rcSaved.RemoteId, false)
require.NoError(t, err)
require.NotZero(t, updatedRC.LastGlobalUserSyncAt, "LastGlobalUserSyncAt should not be zero")
require.Equal(t, futureTimestamp, updatedRC.LastGlobalUserSyncAt)
})
t.Run("Update global user sync cursor for non-existent remote", func(t *testing.T) {
err := ss.RemoteCluster().UpdateLastGlobalUserSyncAt(model.NewId(), futureTimestamp)
require.Error(t, err, "update non-existent remote should error", err)
})
}
func testDeleteSharedChannelRemote(t *testing.T, rctx request.CTX, ss store.Store) {
channel, err := createTestChannel(ss, rctx, "test_remote_delete")
require.NoError(t, err)

Просмотреть файл

@@ -610,6 +610,32 @@ func testUserStoreGetAllProfiles(t *testing.T, rctx request.CTX, ss store.Store)
}, actual)
})
t.Run("filter by UpdatedAfter", func(t *testing.T) {
// Update a user to ensure we have a recent update time
updateTime := model.GetMillis()
u2.FirstName = "Updated"
_, updateErr := ss.User().Update(rctx, u2, false)
require.NoError(t, updateErr)
// Query with the UpdatedAfter filter
actual, userErr := ss.User().GetAllProfiles(&model.UserGetOptions{
Page: 0,
PerPage: 10,
UpdatedAfter: updateTime - 1, // Subtract 1 to ensure we capture the update
})
require.NoError(t, userErr)
require.Contains(t, actual, sanitized(u2), "User updated after the specified time should be in the results")
// Query with a future time, should return no results
actual, userErr = ss.User().GetAllProfiles(&model.UserGetOptions{
Page: 0,
PerPage: 10,
UpdatedAfter: updateTime + 10000, // Future time
})
require.NoError(t, userErr)
require.NotContains(t, actual, sanitized(u2), "Users updated before the specified future time should not be in the results")
})
u8, err := ss.User().Save(rctx, &model.User{
Email: MakeEmail(),
Username: "u8" + model.NewId(),

Просмотреть файл

@@ -7909,6 +7909,22 @@ func (s *TimerLayerRemoteClusterStore) Update(rc *model.RemoteCluster) (*model.R
return result, err
}
func (s *TimerLayerRemoteClusterStore) UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error {
start := time.Now()
err := s.RemoteClusterStore.UpdateLastGlobalUserSyncAt(remoteID, syncAt)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("RemoteClusterStore.UpdateLastGlobalUserSyncAt", success, elapsed)
}
return err
}
func (s *TimerLayerRemoteClusterStore) UpdateTopics(remoteClusterID string, topics string) (*model.RemoteCluster, error) {
start := time.Now()

Просмотреть файл

@@ -289,3 +289,19 @@ func (rcs *Service) pause() {
rcs.server.Log().Debug("Remote Cluster Service inactive")
}
// SetActive forces the service to be active or inactive
func (rcs *Service) SetActive(active bool) {
rcs.mux.Lock()
defer rcs.mux.Unlock()
if rcs.active == active {
return
}
if active {
rcs.resume()
} else {
rcs.pause()
}
}

Просмотреть файл

@@ -24,6 +24,7 @@ const (
TopicSync = "sharedchannel_sync"
TopicChannelInvite = "sharedchannel_invite"
TopicUploadCreate = "sharedchannel_upload"
TopicGlobalUserSync = "sharedchannel_global_user_sync"
MaxRetries = 3
MaxUsersPerSync = 25
NotifyRemoteOfflineThreshold = time.Second * 10
@@ -98,6 +99,7 @@ type Service struct {
syncTopicListenerId string
inviteTopicListenerId string
uploadTopicListenerId string
globalSyncTopicListenerId string
siteURL *url.URL
}
@@ -130,6 +132,7 @@ func (scs *Service) Start() error {
scs.syncTopicListenerId = rcs.AddTopicListener(TopicSync, scs.onReceiveSyncMessage)
scs.inviteTopicListenerId = rcs.AddTopicListener(TopicChannelInvite, scs.onReceiveChannelInvite)
scs.uploadTopicListenerId = rcs.AddTopicListener(TopicUploadCreate, scs.onReceiveUploadCreate)
scs.globalSyncTopicListenerId = rcs.AddTopicListener(TopicGlobalUserSync, scs.onReceiveSyncMessage)
scs.connectionStateListenerId = rcs.AddConnectionStateListener(scs.onConnectionStateChange)
scs.mux.Unlock()
@@ -248,6 +251,9 @@ func (scs *Service) onConnectionStateChange(rc *model.RemoteCluster, online bool
// when a previously offline remote comes back online force a sync.
scs.SendPendingInvitesForRemote(rc)
scs.ForceSyncForRemote(rc)
// Schedule global user sync if feature is enabled
scs.scheduleGlobalUserSync(rc)
}
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Remote cluster connection status changed",
@@ -278,3 +284,47 @@ func (scs *Service) notifyClientsForSharedChannelUpdate(channel *model.Channel)
messageWs.Add("channel_id", channel.Id)
scs.app.Publish(messageWs)
}
// isGlobalUserSyncEnabled checks if the global user sync feature is enabled
func (scs *Service) isGlobalUserSyncEnabled() bool {
cfg := scs.server.Config()
return cfg.FeatureFlags.EnableSyncAllUsersForRemoteCluster ||
(cfg.ConnectedWorkspacesSettings.SyncUsersOnConnectionOpen != nil && *cfg.ConnectedWorkspacesSettings.SyncUsersOnConnectionOpen)
}
// scheduleGlobalUserSync schedules a task to sync all users with a remote cluster
func (scs *Service) scheduleGlobalUserSync(rc *model.RemoteCluster) {
if !scs.isGlobalUserSyncEnabled() {
return
}
// Schedule the sync task
go func() {
// Create a special sync task with empty channelID
// This empty channelID is a deliberate marker for a global user sync task
task := newSyncTask("", "", rc.RemoteId, nil, nil)
task.schedule = time.Now().Add(NotifyMinimumDelay)
scs.addTask(task)
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Scheduled global user sync task for remote",
mlog.String("remote", rc.DisplayName),
mlog.String("remoteId", rc.RemoteId),
)
}()
}
// OnReceiveSyncMessageForTesting exposes onReceiveSyncMessage for testing
func (scs *Service) OnReceiveSyncMessageForTesting(msg model.RemoteClusterMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
return scs.onReceiveSyncMessage(msg, rc, response)
}
// GetUserSyncBatchSizeForTesting returns the configured batch size for user syncing (exported for testing)
func (scs *Service) GetUserSyncBatchSizeForTesting() int {
return scs.getGlobalUserSyncBatchSize()
}
// HandleSyncAllUsersForTesting exposes syncAllUsers for testing
func (scs *Service) HandleSyncAllUsersForTesting(rc *model.RemoteCluster) error {
return scs.syncAllUsers(rc)
}

Просмотреть файл

@@ -24,8 +24,8 @@ var (
)
func (scs *Service) onReceiveSyncMessage(msg model.RemoteClusterMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
if msg.Topic != TopicSync {
return fmt.Errorf("wrong topic, expected `%s`, got `%s`", TopicSync, msg.Topic)
if msg.Topic != TopicSync && msg.Topic != TopicGlobalUserSync {
return fmt.Errorf("wrong topic, expected `%s` or `%s`, got `%s`", TopicSync, TopicGlobalUserSync, msg.Topic)
}
if len(msg.Payload) == 0 {
@@ -47,6 +47,36 @@ func (scs *Service) onReceiveSyncMessage(msg model.RemoteClusterMsg, rc *model.R
return scs.processSyncMessage(request.EmptyContext(scs.server.Log()), &sm, rc, response)
}
func (scs *Service) processGlobalUserSync(c request.CTX, syncMsg *model.SyncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
syncResp := model.SyncResponse{
UserErrors: make([]string, 0),
UsersSyncd: make([]string, 0),
}
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Processing global user sync",
mlog.String("remote", rc.Name),
mlog.Int("user_count", len(syncMsg.Users)),
)
// Process all users in the sync message
for _, user := range syncMsg.Users {
if userSaved, err := scs.upsertSyncUser(c, user, nil, rc); err != nil {
syncResp.UserErrors = append(syncResp.UserErrors, user.Id)
} else {
syncResp.UsersSyncd = append(syncResp.UsersSyncd, userSaved.Id)
if syncResp.UsersLastUpdateAt < user.UpdateAt {
syncResp.UsersLastUpdateAt = user.UpdateAt
}
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Global user upserted via sync",
mlog.String("remote", rc.Name),
mlog.String("user_id", user.Id),
)
}
}
return response.SetPayload(syncResp)
}
func (scs *Service) processSyncMessage(c request.CTX, syncMsg *model.SyncMsg, rc *model.RemoteCluster, response *remotecluster.Response) error {
var targetChannel *model.Channel
var team *model.Team
@@ -68,6 +98,25 @@ func (scs *Service) processSyncMessage(c request.CTX, syncMsg *model.SyncMsg, rc
mlog.Int("status_count", len(syncMsg.Statuses)),
)
// Check if this is a global user sync message (no channel ID and only users)
if syncMsg.ChannelId == "" {
if len(syncMsg.Posts) != 0 ||
len(syncMsg.Reactions) != 0 ||
len(syncMsg.Statuses) != 0 {
return fmt.Errorf("global user sync message should not contain posts, reactions or statuses")
}
if len(syncMsg.Users) == 0 {
return nil
}
// Check if feature flag is enabled
if !scs.isGlobalUserSyncEnabled() {
return nil
}
return scs.processGlobalUserSync(c, syncMsg, rc, response)
}
// For regular sync messages, we need a specific channel
if targetChannel, err = scs.server.GetStore().Channel().Get(syncMsg.ChannelId, true); err != nil {
// if the channel doesn't exist then none of these sync items are going to work.
return fmt.Errorf("channel not found processing sync message: %w", err)
@@ -239,7 +288,7 @@ func (scs *Service) upsertSyncUser(c request.CTX, user *model.User, channel *mod
// Instead of undoing what succeeded on any failure we simply do all steps each
// time. AddUserToChannel & AddUserToTeamByTeamId do not error if user was already
// added and exit quickly. Not needed for DMs where teamId is empty.
if channel.TeamId != "" {
if channel != nil && channel.TeamId != "" {
// add user to team
if err := scs.app.AddUserToTeamByTeamId(request.EmptyContext(scs.server.Log()), channel.TeamId, userSaved); err != nil {
return nil, fmt.Errorf("error adding sync user to Team: %w", err)

Просмотреть файл

@@ -571,14 +571,14 @@ func (scs *Service) shouldUserSync(user *model.User, channelID string, rc *model
if _, err = scs.server.GetStore().SharedChannel().SaveUser(scu); err != nil {
scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error adding user to shared channel users",
mlog.String("user_id", user.Id),
mlog.String("channel_id", user.Id),
mlog.String("channel_id", channelID),
mlog.String("remote_id", rc.RemoteId),
mlog.Err(err),
)
} else {
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "Added user to shared channel users",
mlog.String("user_id", user.Id),
mlog.String("channel_id", user.Id),
mlog.String("channel_id", channelID),
mlog.String("remote_id", rc.RemoteId),
)
}

Просмотреть файл

@@ -39,8 +39,9 @@ type syncData struct {
statuses []*model.Status
attachments []attachment
resultRepeat bool
resultNextCursor model.GetPostsSinceForSyncCursor
resultRepeat bool
resultNextCursor model.GetPostsSinceForSyncCursor
GlobalUserSyncLastTimestamp int64
}
func newSyncData(task syncTask, rc *model.RemoteCluster, scr *model.SharedChannelRemote) *syncData {
@@ -83,6 +84,12 @@ func (sd *syncData) setDataFromMsg(msg *model.SyncMsg) {
// channels are very active.
// Returning an error forces a retry on the task.
func (scs *Service) syncForRemote(task syncTask, rc *model.RemoteCluster) error {
// Empty channelID indicates a global user sync task
// Normal syncTasks always include a valid channelID
if task.channelID == "" {
return scs.syncAllUsers(rc)
}
rcs := scs.server.GetRemoteClusterService()
if rcs == nil {
return fmt.Errorf("cannot update remote cluster %s for channel id %s; Remote Cluster Service not enabled", rc.Name, task.channelID)
@@ -241,6 +248,7 @@ func (scs *Service) fetchUsersForSync(sd *syncData) error {
return err
}
// Don't sync users back to the remote cluster they originated from
for _, u := range users {
if u.GetRemoteID() != sd.rc.RemoteId {
sd.users[u.Id] = u
@@ -495,7 +503,7 @@ func (scs *Service) filterPostsForSync(sd *syncData) {
continue
}
// don't sync a post back to the remote it came from.
// don't sync a post back to the remote cluster it came from.
if p.GetRemoteID() == sd.rc.RemoteId {
continue
}
@@ -578,6 +586,11 @@ func (scs *Service) sendUserSyncData(sd *syncData) error {
msg.Users = sd.users
err := scs.sendSyncMsgToRemote(msg, sd.rc, func(syncResp model.SyncResponse, errResp error) {
// Only update cursor on successful sync
if errResp == nil && sd.GlobalUserSyncLastTimestamp > 0 {
scs.updateGlobalSyncCursor(sd.rc, sd.GlobalUserSyncLastTimestamp)
}
for _, userID := range syncResp.UsersSyncd {
if err := scs.server.GetStore().SharedChannel().UpdateUserLastSyncAt(userID, sd.task.channelID, sd.rc.RemoteId); err != nil {
scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Cannot update shared channel user LastSyncAt",
@@ -691,6 +704,204 @@ func (scs *Service) sendProfileImageSyncData(sd *syncData) {
}
}
// shouldUserSyncGlobal determines if a user needs to be synchronized globally.
// Compares user's update timestamp with the remote cluster's LastGlobalUserSyncAt.
func (scs *Service) shouldUserSyncGlobal(user *model.User, rc *model.RemoteCluster) (bool, error) {
// Don't sync users back to the remote cluster they originated from
if user.IsRemote() && user.GetRemoteID() == rc.RemoteId {
return false, nil
}
// Calculate latest update time for this user (profile or picture)
latestUserUpdateTime := user.UpdateAt
if user.LastPictureUpdate > latestUserUpdateTime {
latestUserUpdateTime = user.LastPictureUpdate
}
// For initial sync (LastGlobalUserSyncAt=0), sync all users
// For incremental sync, only sync users updated after the last sync
if rc.LastGlobalUserSyncAt == 0 {
return true, nil
}
return latestUserUpdateTime > rc.LastGlobalUserSyncAt, nil
}
// syncAllUsers synchronizes all local users to a remote cluster.
// This is called when a connection with a remote cluster is established or when handling a global user sync task.
// Uses cursor-based approach with LastGlobalUserSyncAt to resume after interruptions.
func (scs *Service) syncAllUsers(rc *model.RemoteCluster) error {
// Check if feature is enabled
if !scs.server.Config().FeatureFlags.EnableSyncAllUsersForRemoteCluster {
return nil
}
if !rc.IsOnline() {
return fmt.Errorf("remote cluster %s is not online", rc.RemoteId)
}
// Start metrics tracking
metrics := scs.server.GetMetrics()
start := time.Now()
defer func() {
if metrics != nil {
metrics.IncrementSharedChannelsSyncCounter(rc.RemoteId)
metrics.ObserveSharedChannelsSyncCollectionDuration(rc.RemoteId, time.Since(start).Seconds())
}
}()
batchSize := scs.getGlobalUserSyncBatchSize()
// Create sync data with collected users
sd := &syncData{
task: syncTask{remoteID: rc.RemoteId},
rc: rc,
scr: &model.SharedChannelRemote{RemoteId: rc.RemoteId},
users: make(map[string]*model.User),
}
// Collect users to sync
users, latestTimestamp, _, hasMore, err := scs.collectUsersForGlobalSync(rc, batchSize)
if err != nil {
return err
}
// Exit early if no users to sync
if len(users) == 0 {
scs.server.Log().Log(mlog.LvlSharedChannelServiceDebug, "No users to sync for remote cluster",
mlog.String("remote_id", rc.RemoteId))
return nil
}
// Add users to sync data
sd.users = users
sd.GlobalUserSyncLastTimestamp = latestTimestamp
// Send the collected users to remote
if err := scs.sendUserSyncData(sd); err != nil {
scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error sending user batch during sync",
mlog.String("remote_id", rc.RemoteId),
mlog.Err(err),
)
return fmt.Errorf("error sending user batch during sync: %w", err)
}
// Schedule next batch if needed
if hasMore {
scs.scheduleNextUserSyncBatch(rc, latestTimestamp, batchSize, len(users))
}
return nil
}
// getGlobalUserSyncBatchSize returns the configured batch size for user syncing
func (scs *Service) getGlobalUserSyncBatchSize() int {
batchSize := MaxUsersPerSync
if scs.server.Config().ConnectedWorkspacesSettings.GlobalUserSyncBatchSize != nil {
configValue := *scs.server.Config().ConnectedWorkspacesSettings.GlobalUserSyncBatchSize
if configValue > 0 && configValue <= 200 {
batchSize = configValue
}
}
return batchSize
}
// collectUsersForGlobalSync fetches users that need to be synced to the remote
func (scs *Service) collectUsersForGlobalSync(rc *model.RemoteCluster, batchSize int) (map[string]*model.User, int64, int, bool, error) {
options := &model.UserGetOptions{
Page: 0,
PerPage: 100, // Database fetch batch size
Active: true,
Sort: "update_at_asc", // Order by UpdateAt ASC to ensure cursor consistency
}
// Only use UpdatedAfter for incremental syncs, not the initial sync
// This ensures users with UpdateAt=0 are included in the first sync
if rc.LastGlobalUserSyncAt > 0 {
options.UpdatedAfter = rc.LastGlobalUserSyncAt
}
users := make(map[string]*model.User)
latestTimestamp := rc.LastGlobalUserSyncAt
totalCount := 0
// Page through database results
for {
batch, err := scs.server.GetStore().User().GetAllProfiles(options)
if err != nil {
scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Error fetching users for global sync",
mlog.String("remote_id", rc.RemoteId),
mlog.Err(err),
)
return nil, 0, 0, false, err
}
if len(batch) == 0 {
break // No more users to process
}
totalCount += len(batch)
// Process each user in this database page
for _, user := range batch {
// Stop if we've reached batch limit
if len(users) >= batchSize {
return users, latestTimestamp, totalCount, true, nil
}
// Skip users from remotes
if user.IsRemote() {
continue
}
// Check if user needs syncing
needsSync, _ := scs.shouldUserSyncGlobal(user, rc)
if !needsSync {
continue
}
// Add user and update cursor timestamp
users[user.Id] = user
userUpdateTime := max(user.UpdateAt, user.LastPictureUpdate)
if userUpdateTime > latestTimestamp {
latestTimestamp = userUpdateTime
}
}
// Check if we've reached the end of results
if len(batch) < options.PerPage {
break
}
// Move to next page
options.Page++
}
return users, latestTimestamp, totalCount, false, nil
}
// updateGlobalSyncCursor updates the LastGlobalUserSyncAt value for the remote cluster
func (scs *Service) updateGlobalSyncCursor(rc *model.RemoteCluster, newTimestamp int64) {
if err := scs.server.GetStore().RemoteCluster().UpdateLastGlobalUserSyncAt(rc.RemoteId, newTimestamp); err == nil {
rc.LastGlobalUserSyncAt = newTimestamp
} else {
scs.server.Log().Log(mlog.LvlSharedChannelServiceError, "Failed to update global user sync cursor",
mlog.String("remote_id", rc.RemoteId),
mlog.Err(err),
)
}
}
// scheduleNextUserSyncBatch creates a new task for the next batch of user sync
func (scs *Service) scheduleNextUserSyncBatch(rc *model.RemoteCluster, timestamp int64, batchSize, processedCount int) {
// Use timestamp as userID to make each batch task unique
// This prevents task ID collisions between different batches for the same remote
timestampStr := fmt.Sprintf("%d", timestamp)
task := newSyncTask("", timestampStr, rc.RemoteId, nil, nil)
task.schedule = time.Now().Add(NotifyMinimumDelay)
scs.addTask(task)
}
// sendSyncMsgToRemote synchronously sends the sync message to the remote cluster (or plugin).
func (scs *Service) sendSyncMsgToRemote(msg *model.SyncMsg, rc *model.RemoteCluster, f sendSyncMsgResultFunc) error {
rcs := scs.server.GetRemoteClusterService()
@@ -706,7 +917,15 @@ func (scs *Service) sendSyncMsgToRemote(msg *model.SyncMsg, rc *model.RemoteClus
if err != nil {
return err
}
rcMsg := model.NewRemoteClusterMsg(TopicSync, b)
// Use appropriate topic based on message type
topic := TopicSync
if msg.ChannelId == "" && len(msg.Users) > 0 &&
len(msg.Posts) == 0 && len(msg.Reactions) == 0 &&
len(msg.Statuses) == 0 {
topic = TopicGlobalUserSync
}
rcMsg := model.NewRemoteClusterMsg(topic, b)
ctx, cancel := context.WithTimeout(context.Background(), remotecluster.SendTimeout)
defer cancel()

Просмотреть файл

@@ -3459,6 +3459,8 @@ type ConnectedWorkspacesSettings struct {
EnableSharedChannels *bool
EnableRemoteClusterService *bool
DisableSharedChannelsStatusSync *bool
SyncUsersOnConnectionOpen *bool
GlobalUserSyncBatchSize *int
MaxPostsPerSync *int
}
@@ -3483,6 +3485,14 @@ func (c *ConnectedWorkspacesSettings) SetDefaults(isUpdate bool, e ExperimentalS
c.DisableSharedChannelsStatusSync = NewPointer(false)
}
if c.SyncUsersOnConnectionOpen == nil {
c.SyncUsersOnConnectionOpen = NewPointer(false)
}
if c.GlobalUserSyncBatchSize == nil {
c.GlobalUserSyncBatchSize = NewPointer(25) // Default to MaxUsersPerSync
}
if c.MaxPostsPerSync == nil {
c.MaxPostsPerSync = NewPointer(ConnectedWorkspacesSettingsDefaultMaxPostsPerSync)
}

Просмотреть файл

@@ -25,6 +25,9 @@ type FeatureFlags struct {
// Enable plugins in shared channels.
EnableSharedChannelsPlugins bool
// Enable syncing all users for remote clusters in shared channels
EnableSyncAllUsersForRemoteCluster bool
// AppsEnabled toggles the Apps framework functionalities both in server and client side
AppsEnabled bool
@@ -69,6 +72,7 @@ func (f *FeatureFlags) SetDefaults() {
f.TestBoolFeature = false
f.EnableRemoteClusterService = false
f.EnableSharedChannelsDMs = false
f.EnableSyncAllUsersForRemoteCluster = false
f.EnableSharedChannelsPlugins = true
f.AppsEnabled = false
f.NormalizeLdapDNs = false

Просмотреть файл

@@ -53,37 +53,39 @@ func (bm *Bitmask) UnsetBit(flag Bitmask) {
}
type RemoteCluster struct {
RemoteId string `json:"remote_id"`
RemoteTeamId string `json:"remote_team_id"` // Deprecated: this field is no longer used. It's only kept for backwards compatibility.
Name string `json:"name"`
DisplayName string `json:"display_name"`
SiteURL string `json:"site_url"`
DefaultTeamId string `json:"default_team_id"`
CreateAt int64 `json:"create_at"`
DeleteAt int64 `json:"delete_at"`
LastPingAt int64 `json:"last_ping_at"`
Token string `json:"token"`
RemoteToken string `json:"remote_token"`
Topics string `json:"topics"`
CreatorId string `json:"creator_id"`
PluginID string `json:"plugin_id"` // non-empty when sync message are to be delivered via plugin API
Options Bitmask `json:"options"` // bit-flag set of options
RemoteId string `json:"remote_id"`
RemoteTeamId string `json:"remote_team_id"` // Deprecated: this field is no longer used. It's only kept for backwards compatibility.
Name string `json:"name"`
DisplayName string `json:"display_name"`
SiteURL string `json:"site_url"`
DefaultTeamId string `json:"default_team_id"`
CreateAt int64 `json:"create_at"`
DeleteAt int64 `json:"delete_at"`
LastPingAt int64 `json:"last_ping_at"`
LastGlobalUserSyncAt int64 `json:"last_global_user_sync_at"` // Timestamp of last global user sync
Token string `json:"token"`
RemoteToken string `json:"remote_token"`
Topics string `json:"topics"`
CreatorId string `json:"creator_id"`
PluginID string `json:"plugin_id"` // non-empty when sync message are to be delivered via plugin API
Options Bitmask `json:"options"` // bit-flag set of options
}
func (rc *RemoteCluster) Auditable() map[string]any {
return map[string]any{
"remote_id": rc.RemoteId,
"remote_team_id": rc.RemoteTeamId,
"name": rc.Name,
"display_name": rc.DisplayName,
"site_url": rc.SiteURL,
"default_team_id": rc.DefaultTeamId,
"create_at": rc.CreateAt,
"delete_at": rc.DeleteAt,
"last_ping_at": rc.LastPingAt,
"creator_id": rc.CreatorId,
"plugin_id": rc.PluginID,
"options": rc.Options,
"remote_id": rc.RemoteId,
"remote_team_id": rc.RemoteTeamId,
"name": rc.Name,
"display_name": rc.DisplayName,
"site_url": rc.SiteURL,
"default_team_id": rc.DefaultTeamId,
"create_at": rc.CreateAt,
"delete_at": rc.DeleteAt,
"last_ping_at": rc.LastPingAt,
"last_global_user_sync_at": rc.LastGlobalUserSyncAt,
"creator_id": rc.CreatorId,
"plugin_id": rc.PluginID,
"options": rc.Options,
}
}

Просмотреть файл

@@ -40,6 +40,8 @@ type UserGetOptions struct {
Page int
// Page size
PerPage int
// Filters the users that have been updated after the given time
UpdatedAfter int64
}
type UserGetByIdsOptions struct {