MM-62751: [Shared Channels] Allow remote users to be discoverable in the create DM/GM modal (#30918)

server/channels/app/shared_channel_sync_self_referential_utils_test.go

@@ -0,0 +1,248 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package app
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/platform/services/remotecluster"
+	"github.com/mattermost/mattermost/server/v8/platform/services/sharedchannel"
+	"github.com/stretchr/testify/require"
+)
+
+// writeOKResponse writes a standard OK JSON response in the format expected by remotecluster
+func writeOKResponse(w http.ResponseWriter) {
+	response := &remotecluster.Response{
+		Status: "OK",
+		Err:    "",
+	}
+
+	// Set empty sync response as payload
+	syncResp := &model.SyncResponse{}
+	_ = response.SetPayload(syncResp)
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+	respBytes, _ := json.Marshal(response)
+	_, _ = w.Write(respBytes)
+}
+
+// SelfReferentialSyncHandler handles incoming sync messages for self-referential tests.
+// It processes the messages, updates cursors, and returns proper responses.
+type SelfReferentialSyncHandler struct {
+	t                *testing.T
+	service          *sharedchannel.Service
+	selfCluster      *model.RemoteCluster
+	syncMessageCount *int32
+
+	// Callbacks for capturing sync data
+	OnIndividualSync func(userId string, messageNumber int32)
+	OnBatchSync      func(userIds []string, messageNumber int32)
+	OnGlobalUserSync func(userIds []string, messageNumber int32)
+}
+
+// NewSelfReferentialSyncHandler creates a new handler for processing sync messages in tests
+func NewSelfReferentialSyncHandler(t *testing.T, service *sharedchannel.Service, selfCluster *model.RemoteCluster) *SelfReferentialSyncHandler {
+	count := int32(0)
+	return &SelfReferentialSyncHandler{
+		t:                t,
+		service:          service,
+		selfCluster:      selfCluster,
+		syncMessageCount: &count,
+	}
+}
+
+// HandleRequest processes incoming HTTP requests for the test server.
+// This handler includes common remote cluster endpoints to simulate a real remote cluster:
+// - /api/v4/remotecluster/msg: Main sync message endpoint
+// - /api/v4/remotecluster/ping: Ping endpoint to maintain online status (prevents offline after 5 minutes)
+// - /api/v4/remotecluster/confirm_invite: Invitation confirmation endpoint
+func (h *SelfReferentialSyncHandler) HandleRequest(w http.ResponseWriter, r *http.Request) {
+	switch r.URL.Path {
+	case "/api/v4/remotecluster/msg":
+		currentCall := atomic.AddInt32(h.syncMessageCount, 1)
+
+		// Read and process the sync message
+		body, _ := io.ReadAll(r.Body)
+
+		// The message is wrapped in a RemoteClusterFrame
+		var frame model.RemoteClusterFrame
+
+		err := json.Unmarshal(body, &frame)
+		if err == nil {
+			// Process the message to update cursor
+			response := &remotecluster.Response{}
+			processErr := h.service.OnReceiveSyncMessageForTesting(frame.Msg, h.selfCluster, response)
+			if processErr != nil {
+				response.Status = "ERROR"
+				response.Err = processErr.Error()
+			} else {
+				// Success - build a proper sync response
+				response.Status = "OK"
+				response.Err = ""
+
+				var syncMsg model.SyncMsg
+				if unmarshalErr := json.Unmarshal(frame.Msg.Payload, &syncMsg); unmarshalErr == nil {
+					syncResp := &model.SyncResponse{}
+
+					// Handle global user sync
+					if len(syncMsg.Users) > 0 {
+						userIds := make([]string, 0, len(syncMsg.Users))
+						for userId := range syncMsg.Users {
+							userIds = append(userIds, userId)
+							syncResp.UsersSyncd = append(syncResp.UsersSyncd, userId)
+						}
+						if h.OnGlobalUserSync != nil {
+							h.OnGlobalUserSync(userIds, currentCall)
+						}
+					}
+
+					_ = response.SetPayload(syncResp)
+				}
+			}
+
+			// Send the proper response
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			respBytes, _ := json.Marshal(response)
+			_, _ = w.Write(respBytes)
+			return
+		}
+
+		writeOKResponse(w)
+
+	case "/api/v4/remotecluster/ping":
+		writeOKResponse(w)
+
+	case "/api/v4/remotecluster/confirm_invite":
+		writeOKResponse(w)
+
+	default:
+		writeOKResponse(w)
+	}
+}
+
+// GetSyncMessageCount returns the current count of sync messages received
+func (h *SelfReferentialSyncHandler) GetSyncMessageCount() int32 {
+	return atomic.LoadInt32(h.syncMessageCount)
+}
+
+// GetUsersFromSyncMsg extracts user IDs from a sync message
+func GetUsersFromSyncMsg(msg model.SyncMsg) []string {
+	var userIds []string
+
+	// Extract from users field
+	for userId := range msg.Users {
+		userIds = append(userIds, userId)
+	}
+
+	return userIds
+}
+
+// EnsureCleanState ensures a clean test state by removing all shared channels, remote clusters,
+// and extra team/channel members. This helps prevent state pollution between tests.
+func EnsureCleanState(t *testing.T, th *TestHelper, ss store.Store) {
+	t.Helper()
+
+	// Clear all shared channels and remotes from previous tests
+	allSharedChannels, _ := ss.SharedChannel().GetAll(0, 1000, model.SharedChannelFilterOpts{})
+	for _, sc := range allSharedChannels {
+		// Delete all remotes for this channel
+		remotes, _ := ss.SharedChannel().GetRemotes(0, 100, model.SharedChannelRemoteFilterOpts{ChannelId: sc.ChannelId})
+		for _, remote := range remotes {
+			_, _ = ss.SharedChannel().DeleteRemote(remote.Id)
+		}
+		// Delete the shared channel
+		_, _ = ss.SharedChannel().Delete(sc.ChannelId)
+	}
+
+	// Delete all remote clusters
+	allRemoteClusters, _ := ss.RemoteCluster().GetAll(0, 1000, model.RemoteClusterQueryFilter{})
+	for _, rc := range allRemoteClusters {
+		_, _ = ss.RemoteCluster().Delete(rc.RemoteId)
+	}
+
+	// Remove all channel members from test channels (except the basic team/channel setup)
+	channels, _ := ss.Channel().GetAll(th.BasicTeam.Id)
+	for _, channel := range channels {
+		// Skip direct message and group channels, and skip the default channels
+		if channel.Type != model.ChannelTypeDirect && channel.Type != model.ChannelTypeGroup &&
+			channel.Id != th.BasicChannel.Id {
+			members, _ := ss.Channel().GetMembers(channel.Id, 0, 10000)
+			for _, member := range members {
+				_ = ss.Channel().RemoveMember(th.Context, channel.Id, member.UserId)
+			}
+		}
+	}
+
+	// Remove all users from teams except the basic test users
+	teams, _ := ss.Team().GetAll()
+	for _, team := range teams {
+		if team.Id == th.BasicTeam.Id {
+			members, _ := ss.Team().GetMembers(team.Id, 0, 10000, nil)
+			for _, member := range members {
+				// Keep only the basic test users
+				if member.UserId != th.BasicUser.Id && member.UserId != th.BasicUser2.Id &&
+					member.UserId != th.SystemAdminUser.Id {
+					_ = ss.Team().RemoveMember(th.Context, team.Id, member.UserId)
+				}
+			}
+		}
+	}
+
+	// Get all active users and deactivate non-basic ones
+	options := &model.UserGetOptions{
+		Page:    0,
+		PerPage: 200,
+		Active:  true,
+	}
+	users, _ := ss.User().GetAllProfiles(options)
+	for _, user := range users {
+		// Keep only the basic test users active
+		if user.Id != th.BasicUser.Id && user.Id != th.BasicUser2.Id &&
+			user.Id != th.SystemAdminUser.Id {
+			// Deactivate the user (soft delete)
+			user.DeleteAt = model.GetMillis()
+			_, _ = ss.User().Update(th.Context, user, true)
+		}
+	}
+
+	// Verify cleanup is complete
+	require.Eventually(t, func() bool {
+		sharedChannels, _ := ss.SharedChannel().GetAll(0, 1000, model.SharedChannelFilterOpts{})
+		remoteClusters, _ := ss.RemoteCluster().GetAll(0, 1000, model.RemoteClusterQueryFilter{})
+		return len(sharedChannels) == 0 && len(remoteClusters) == 0
+	}, 2*time.Second, 100*time.Millisecond, "Failed to clean up shared channels and remote clusters")
+
+	// Reset batch size to default to ensure test isolation
+	defaultBatchSize := 20
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		cfg.ConnectedWorkspacesSettings.GlobalUserSyncBatchSize = &defaultBatchSize
+	})
+
+	// Ensure services are running and ready
+	scsInterface := th.App.Srv().GetSharedChannelSyncService()
+	if scs, ok := scsInterface.(*sharedchannel.Service); ok {
+		require.Eventually(t, func() bool {
+			return scs.Active()
+		}, 2*time.Second, 100*time.Millisecond, "Shared channel service should be active")
+	}
+
+	rcService := th.App.Srv().GetRemoteClusterService()
+	if rcService != nil {
+		if rc, ok := rcService.(*remotecluster.Service); ok {
+			rc.SetActive(true)
+		}
+		require.Eventually(t, func() bool {
+			return rcService.Active()
+		}, 2*time.Second, 100*time.Millisecond, "Remote cluster service should be active")
+	}
+}

server/channels/db/migrations/migrations.list

@@ -273,6 +273,8 @@ channels/db/migrations/mysql/000137_update_attribute_view.down.sql
 channels/db/migrations/mysql/000137_update_attribute_view.up.sql
 channels/db/migrations/mysql/000138_add_default_category_name_to_channel.down.sql
 channels/db/migrations/mysql/000138_add_default_category_name_to_channel.up.sql
+channels/db/migrations/mysql/000139_remoteclusters_add_last_global_user_sync_at.down.sql
+channels/db/migrations/mysql/000139_remoteclusters_add_last_global_user_sync_at.up.sql
 channels/db/migrations/postgres/000001_create_teams.down.sql
 channels/db/migrations/postgres/000001_create_teams.up.sql
 channels/db/migrations/postgres/000002_create_team_members.down.sql
@@ -547,3 +549,5 @@ channels/db/migrations/postgres/000137_update_attribute_view.down.sql
 channels/db/migrations/postgres/000137_update_attribute_view.up.sql
 channels/db/migrations/postgres/000138_add_default_category_name_to_channel.down.sql
 channels/db/migrations/postgres/000138_add_default_category_name_to_channel.up.sql
+channels/db/migrations/postgres/000139_remoteclusters_add_last_global_user_sync_at.down.sql
+channels/db/migrations/postgres/000139_remoteclusters_add_last_global_user_sync_at.up.sql

server/channels/db/migrations/mysql/000139_remoteclusters_add_last_global_user_sync_at.down.sql

@@ -0,0 +1,14 @@
+SET @preparedStatement = (SELECT IF(
+    (
+        SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
+        WHERE TABLE_SCHEMA = DATABASE()
+        AND TABLE_NAME = 'RemoteClusters'
+        AND COLUMN_NAME = 'LastGlobalUserSyncAt'
+    ) > 0,
+    'ALTER TABLE RemoteClusters DROP COLUMN LastGlobalUserSyncAt',
+    'SELECT 1'
+));
+
+PREPARE alterIfExists FROM @preparedStatement;
+EXECUTE alterIfExists;
+DEALLOCATE PREPARE alterIfExists;

server/channels/db/migrations/mysql/000139_remoteclusters_add_last_global_user_sync_at.up.sql

@@ -0,0 +1,14 @@
+SET @preparedStatement = (SELECT IF(
+    (
+        SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS
+        WHERE TABLE_SCHEMA = DATABASE()
+        AND TABLE_NAME = 'RemoteClusters'
+        AND COLUMN_NAME = 'LastGlobalUserSyncAt'
+    ) > 0,
+    'SELECT 1',
+    'ALTER TABLE RemoteClusters ADD COLUMN LastGlobalUserSyncAt bigint DEFAULT 0'
+));
+
+PREPARE alterIfNotExists FROM @preparedStatement;
+EXECUTE alterIfNotExists;
+DEALLOCATE PREPARE alterIfNotExists;

server/channels/db/migrations/postgres/000139_remoteclusters_add_last_global_user_sync_at.down.sql

@@ -0,0 +1 @@
+ALTER TABLE remoteclusters DROP COLUMN IF EXISTS lastglobalusersyncat;

server/channels/db/migrations/postgres/000139_remoteclusters_add_last_global_user_sync_at.up.sql

@@ -0,0 +1 @@
+ALTER TABLE remoteclusters ADD COLUMN IF NOT EXISTS lastglobalusersyncat bigint DEFAULT 0;

server/channels/store/retrylayer/retrylayer.go

@@ -9998,6 +9998,27 @@ func (s *RetryLayerRemoteClusterStore) Update(rc *model.RemoteCluster) (*model.R
 
 }
 
+func (s *RetryLayerRemoteClusterStore) UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error {
+
+	tries := 0
+	for {
+		err := s.RemoteClusterStore.UpdateLastGlobalUserSyncAt(remoteID, syncAt)
+		if err == nil {
+			return nil
+		}
+		if !isRepeatableError(err) {
+			return err
+		}
+		tries++
+		if tries >= 3 {
+			err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
+			return err
+		}
+		timepkg.Sleep(100 * timepkg.Millisecond)
+	}
+
+}
+
 func (s *RetryLayerRemoteClusterStore) UpdateTopics(remoteClusterID string, topics string) (*model.RemoteCluster, error) {
 
 	tries := 0

server/channels/store/sqlstore/remote_cluster_store.go

@@ -44,6 +44,7 @@ func remoteClusterFields(prefix string) []string {
 		prefix + "CreatorId",
 		prefix + "PluginID",
 		prefix + "Options",
+		prefix + "LastGlobalUserSyncAt",
 	}
 }
 
@@ -99,7 +100,8 @@ func (s sqlRemoteClusterStore) Update(remoteCluster *model.RemoteCluster) (*mode
 			DefaultTeamId = :DefaultTeamId,
 			Topics = :Topics,
 			PluginID = :PluginID,
-			Options = :Options
+			Options = :Options,
+			LastGlobalUserSyncAt = :LastGlobalUserSyncAt
 			WHERE RemoteId = :RemoteId AND Name = :Name`
 
 	if _, err := s.GetMaster().NamedExec(query, remoteCluster); err != nil {
@@ -310,3 +312,24 @@ func (s sqlRemoteClusterStore) SetLastPingAt(remoteClusterId string) error {
 	}
 	return nil
 }
+
+func (s sqlRemoteClusterStore) UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error {
+	query := s.getQueryBuilder().
+		Update("RemoteClusters").
+		Set("LastGlobalUserSyncAt", syncAt).
+		Where(sq.Eq{"RemoteId": remoteID})
+
+	result, err := s.GetMaster().ExecBuilder(query)
+	if err != nil {
+		return errors.Wrap(err, "failed to update LastGlobalUserSyncAt for RemoteCluster")
+	}
+
+	count, err := result.RowsAffected()
+	if err != nil {
+		return errors.Wrap(err, "failed to determine rows affected")
+	}
+	if count == 0 {
+		return fmt.Errorf("remote cluster not found: %s", remoteID)
+	}
+	return nil
+}

server/channels/store/sqlstore/user_store.go

@@ -584,8 +584,15 @@ func (us SqlUserStore) GetEtagForAllProfiles() string {
 
 func (us SqlUserStore) GetAllProfiles(options *model.UserGetOptions) ([]*model.User, error) {
 	isPostgreSQL := us.DriverName() == model.DatabaseDriverPostgres
+
+	// Determine ordering based on Sort option - default to Username ASC for backwards compatibility
+	orderBy := "Users.Username ASC"
+	if options.Sort == "update_at_asc" {
+		orderBy = "Users.UpdateAt ASC"
+	}
+
 	query := us.usersQuery.
-		OrderBy("Users.Username ASC").
+		OrderBy(orderBy).
 		Offset(uint64(options.Page * options.PerPage)).Limit(uint64(options.PerPage))
 
 	query = applyViewRestrictionsFilter(query, options.ViewRestrictions, true)
@@ -599,6 +606,10 @@ func (us SqlUserStore) GetAllProfiles(options *model.UserGetOptions) ([]*model.U
 		query = query.Where("Users.DeleteAt = 0")
 	}
 
+	if options.UpdatedAfter > 0 {
+		query = query.Where(sq.Gt{"Users.UpdateAt": options.UpdatedAfter})
+	}
+
 	users := []*model.User{}
 	if err := us.GetReplica().SelectBuilder(&users, query); err != nil {
 		return nil, errors.Wrap(err, "failed to get User profiles")

server/channels/store/store.go

@@ -562,6 +562,7 @@ type RemoteClusterStore interface {
 	GetAll(offset, limit int, filter model.RemoteClusterQueryFilter) ([]*model.RemoteCluster, error)
 	UpdateTopics(remoteClusterID string, topics string) (*model.RemoteCluster, error)
 	SetLastPingAt(remoteClusterID string) error
+	UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error
 }
 
 type ComplianceStore interface {

server/channels/store/storetest/mocks/RemoteClusterStore.go

@@ -210,6 +210,24 @@ func (_m *RemoteClusterStore) Update(rc *model.RemoteCluster) (*model.RemoteClus
 	return r0, r1
 }
 
+// UpdateLastGlobalUserSyncAt provides a mock function with given fields: remoteID, syncAt
+func (_m *RemoteClusterStore) UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error {
+	ret := _m.Called(remoteID, syncAt)
+
+	if len(ret) == 0 {
+		panic("no return value specified for UpdateLastGlobalUserSyncAt")
+	}
+
+	var r0 error
+	if rf, ok := ret.Get(0).(func(string, int64) error); ok {
+		r0 = rf(remoteID, syncAt)
+	} else {
+		r0 = ret.Error(0)
+	}
+
+	return r0
+}
+
 // UpdateTopics provides a mock function with given fields: remoteClusterID, topics
 func (_m *RemoteClusterStore) UpdateTopics(remoteClusterID string, topics string) (*model.RemoteCluster, error) {
 	ret := _m.Called(remoteClusterID, topics)

server/channels/store/storetest/shared_channel_store.go

@@ -32,6 +32,7 @@ func TestSharedChannelStore(t *testing.T, rctx request.CTX, ss store.Store, s Sq
 	t.Run("HasRemote", func(t *testing.T) { testHasRemote(t, rctx, ss) })
 	t.Run("GetRemoteForUser", func(t *testing.T) { testGetRemoteForUser(t, rctx, ss) })
 	t.Run("UpdateSharedChannelRemoteNextSyncAt", func(t *testing.T) { testUpdateSharedChannelRemoteCursor(t, rctx, ss) })
+	t.Run("UpdateGlobalUserSyncCursor", func(t *testing.T) { testUpdateGlobalUserSyncCursor(t, rctx, ss) })
 	t.Run("DeleteSharedChannelRemote", func(t *testing.T) { testDeleteSharedChannelRemote(t, rctx, ss) })
 
 	t.Run("SaveSharedChannelUser", func(t *testing.T) { testSaveSharedChannelUser(t, rctx, ss) })
@@ -932,6 +933,38 @@ func testUpdateSharedChannelRemoteCursor(t *testing.T, rctx request.CTX, ss stor
 	})
 }
 
+func testUpdateGlobalUserSyncCursor(t *testing.T, rctx request.CTX, ss store.Store) {
+	// Create a remote cluster first
+	rc := &model.RemoteCluster{
+		RemoteId:  model.NewId(),
+		SiteURL:   "http://example.com",
+		CreatorId: model.NewId(),
+		Name:      "test",
+	}
+	rcSaved, err := ss.RemoteCluster().Save(rc)
+	require.NoError(t, err, "couldn't save remote cluster", err)
+
+	futureTimestamp := model.GetMillis() + 3600000 // 1 hour in the future
+
+	t.Run("Update global user sync cursor for remote", func(t *testing.T) {
+		err := ss.RemoteCluster().UpdateLastGlobalUserSyncAt(rcSaved.RemoteId, futureTimestamp)
+		require.NoError(t, err, "update global user sync cursor should not error", err)
+
+		// Verify that the LastGlobalUserSyncAt field was updated in the RemoteCluster table
+		// Small sleep to ensure the transaction is committed
+		time.Sleep(10 * time.Millisecond)
+		updatedRC, err := ss.RemoteCluster().Get(rcSaved.RemoteId, false)
+		require.NoError(t, err)
+		require.NotZero(t, updatedRC.LastGlobalUserSyncAt, "LastGlobalUserSyncAt should not be zero")
+		require.Equal(t, futureTimestamp, updatedRC.LastGlobalUserSyncAt)
+	})
+
+	t.Run("Update global user sync cursor for non-existent remote", func(t *testing.T) {
+		err := ss.RemoteCluster().UpdateLastGlobalUserSyncAt(model.NewId(), futureTimestamp)
+		require.Error(t, err, "update non-existent remote should error", err)
+	})
+}
+
 func testDeleteSharedChannelRemote(t *testing.T, rctx request.CTX, ss store.Store) {
 	channel, err := createTestChannel(ss, rctx, "test_remote_delete")
 	require.NoError(t, err)

server/channels/store/storetest/user_store.go

@@ -610,6 +610,32 @@ func testUserStoreGetAllProfiles(t *testing.T, rctx request.CTX, ss store.Store)
 		}, actual)
 	})
 
+	t.Run("filter by UpdatedAfter", func(t *testing.T) {
+		// Update a user to ensure we have a recent update time
+		updateTime := model.GetMillis()
+		u2.FirstName = "Updated"
+		_, updateErr := ss.User().Update(rctx, u2, false)
+		require.NoError(t, updateErr)
+
+		// Query with the UpdatedAfter filter
+		actual, userErr := ss.User().GetAllProfiles(&model.UserGetOptions{
+			Page:         0,
+			PerPage:      10,
+			UpdatedAfter: updateTime - 1, // Subtract 1 to ensure we capture the update
+		})
+		require.NoError(t, userErr)
+		require.Contains(t, actual, sanitized(u2), "User updated after the specified time should be in the results")
+
+		// Query with a future time, should return no results
+		actual, userErr = ss.User().GetAllProfiles(&model.UserGetOptions{
+			Page:         0,
+			PerPage:      10,
+			UpdatedAfter: updateTime + 10000, // Future time
+		})
+		require.NoError(t, userErr)
+		require.NotContains(t, actual, sanitized(u2), "Users updated before the specified future time should not be in the results")
+	})
+
 	u8, err := ss.User().Save(rctx, &model.User{
 		Email:    MakeEmail(),
 		Username: "u8" + model.NewId(),

server/channels/store/timerlayer/timerlayer.go

@@ -7909,6 +7909,22 @@ func (s *TimerLayerRemoteClusterStore) Update(rc *model.RemoteCluster) (*model.R
 	return result, err
 }
 
+func (s *TimerLayerRemoteClusterStore) UpdateLastGlobalUserSyncAt(remoteID string, syncAt int64) error {
+	start := time.Now()
+
+	err := s.RemoteClusterStore.UpdateLastGlobalUserSyncAt(remoteID, syncAt)
+
+	elapsed := float64(time.Since(start)) / float64(time.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("RemoteClusterStore.UpdateLastGlobalUserSyncAt", success, elapsed)
+	}
+	return err
+}
+
 func (s *TimerLayerRemoteClusterStore) UpdateTopics(remoteClusterID string, topics string) (*model.RemoteCluster, error) {
 	start := time.Now()