Remote Cluster Service
- provides ability for multiple Mattermost cluster instances to create a trusted connection with each other and exchange messages
- trusted connections are managed via slash commands (for now)
- facilitates features requiring inter-cluster communication, such as Shared Channels
Shared Channels Service
- provides ability to shared channels between one or more Mattermost cluster instances (using trusted connection)
- sharing/unsharing of channels is managed via slash commands (for now)
Этот коммит содержится в:
Doug Lauder
2021-04-01 13:44:56 -04:00
коммит произвёл GitHub
родитель ff980266ac
Коммит 02196e04fa
137 изменённых файлов: 15137 добавлений и 262 удалений

Просмотреть файл

@@ -125,8 +125,12 @@ type Routes struct {
Cloud *mux.Router // 'api/v4/cloud'
Imports *mux.Router // 'api/v4/imports'
Exports *mux.Router // 'api/v4/exports'
Export *mux.Router // 'api/v4/exports/{export_name:.+\\.zip}'
RemoteCluster *mux.Router // 'api/v4/remotecluster'
SharedChannels *mux.Router // 'api/v4/sharedchannels'
}
type API struct {
@@ -243,6 +247,9 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
api.BaseRoutes.Exports = api.BaseRoutes.ApiRoot.PathPrefix("/exports").Subrouter()
api.BaseRoutes.Export = api.BaseRoutes.Exports.PathPrefix("/{export_name:.+\\.zip}").Subrouter()
api.BaseRoutes.RemoteCluster = api.BaseRoutes.ApiRoot.PathPrefix("/remotecluster").Subrouter()
api.BaseRoutes.SharedChannels = api.BaseRoutes.ApiRoot.PathPrefix("/sharedchannels").Subrouter()
api.InitUser()
api.InitBot()
api.InitTeam()
@@ -280,6 +287,8 @@ func Init(configservice configservice.ConfigService, globalOptionsFunc app.AppOp
api.InitAction()
api.InitCloud()
api.InitImport()
api.InitRemoteCluster()
api.InitSharedChannels()
api.InitExport()
root.Handle("/api/v4/{anything:.*}", http.HandlerFunc(api.Handle404))

Просмотреть файл

@@ -72,6 +72,26 @@ func (api *API) CloudApiKeyRequired(h func(*Context, http.ResponseWriter, *http.
}
// RemoteClusterTokenRequired provides a handler for remote cluster requests to /remotecluster endpoints.
func (api *API) RemoteClusterTokenRequired(h func(*Context, http.ResponseWriter, *http.Request)) http.Handler {
handler := &web.Handler{
GetGlobalAppOptions: api.GetGlobalAppOptions,
HandleFunc: h,
HandlerName: web.GetHandlerName(h),
RequireSession: false,
RequireCloudKey: false,
RequireRemoteClusterToken: true,
TrustRequester: false,
RequireMfa: false,
IsStatic: false,
IsLocal: false,
}
if *api.ConfigService.Config().ServiceSettings.WebserverMode == "gzip" {
return gziphandler.GzipHandler(handler)
}
return handler
}
// ApiSessionRequiredMfa provides a handler for API endpoints which require a logged-in user session but when accessed,
// if MFA is enabled, the MFA process is not yet complete, and therefore the requirement to have completed the MFA
// authentication must be waived.

Просмотреть файл

@@ -618,7 +618,7 @@ func TestCreatePostCheckOnlineStatus(t *testing.T) {
}
case <-timeout:
// We just skip the test instead of failing because waiting for more than 5 seconds
// to get a response does not make sense, and it will unncessarily slow down
// to get a response does not make sense, and it will unnecessarily slow down
// the tests further in an already congested CI environment.
t.Skip("timed out waiting for event")
}
@@ -2035,7 +2035,7 @@ func TestDeletePostMessage(t *testing.T) {
}
case <-timeout:
// We just skip the test instead of failing because waiting for more than 5 seconds
// to get a response does not make sense, and it will unncessarily slow down
// to get a response does not make sense, and it will unnecessarily slow down
// the tests further in an already congested CI environment.
t.Skip("timed out waiting for event")
}

214
api4/remote_cluster.go Обычный файл
Просмотреть файл

@@ -0,0 +1,214 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package api4
import (
"encoding/json"
"net/http"
"time"
"github.com/mattermost/mattermost-server/v5/audit"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/services/remotecluster"
)
func (api *API) InitRemoteCluster() {
api.BaseRoutes.RemoteCluster.Handle("/ping", api.RemoteClusterTokenRequired(remoteClusterPing)).Methods("POST")
api.BaseRoutes.RemoteCluster.Handle("/msg", api.RemoteClusterTokenRequired(remoteClusterAcceptMessage)).Methods("POST")
api.BaseRoutes.RemoteCluster.Handle("/confirm_invite", api.RemoteClusterTokenRequired(remoteClusterConfirmInvite)).Methods("POST")
api.BaseRoutes.RemoteCluster.Handle("/upload/{upload_id:[A-Za-z0-9]+}", api.RemoteClusterTokenRequired(uploadRemoteData)).Methods("POST")
}
func remoteClusterPing(c *Context, w http.ResponseWriter, r *http.Request) {
// make sure remote cluster service is enabled.
if _, appErr := c.App.GetRemoteClusterService(); appErr != nil {
c.Err = appErr
return
}
frame, appErr := model.RemoteClusterFrameFromJSON(r.Body)
if appErr != nil {
c.Err = appErr
return
}
if appErr = frame.IsValid(); appErr != nil {
c.Err = appErr
return
}
remoteId := c.GetRemoteID(r)
if remoteId != frame.RemoteId {
c.SetInvalidRemoteIdError(frame.RemoteId)
return
}
rc, err := c.App.GetRemoteCluster(frame.RemoteId)
if err != nil {
c.SetInvalidRemoteIdError(frame.RemoteId)
return
}
ping, err := model.RemoteClusterPingFromRawJSON(frame.Msg.Payload)
if err != nil {
c.SetInvalidParam("msg.payload")
return
}
ping.RecvAt = model.GetMillis()
if metrics := c.App.Metrics(); metrics != nil {
metrics.IncrementRemoteClusterMsgReceivedCounter(rc.RemoteId)
}
resp, _ := json.Marshal(ping)
w.Write(resp)
}
func remoteClusterAcceptMessage(c *Context, w http.ResponseWriter, r *http.Request) {
// make sure remote cluster service is running.
service, appErr := c.App.GetRemoteClusterService()
if appErr != nil {
c.Err = appErr
return
}
frame, appErr := model.RemoteClusterFrameFromJSON(r.Body)
if appErr != nil {
c.Err = appErr
return
}
if appErr = frame.IsValid(); appErr != nil {
c.Err = appErr
return
}
auditRec := c.MakeAuditRecord("remoteClusterAcceptMessage", audit.Fail)
defer c.LogAuditRec(auditRec)
remoteId := c.GetRemoteID(r)
if remoteId != frame.RemoteId {
c.SetInvalidRemoteIdError(frame.RemoteId)
return
}
rc, err := c.App.GetRemoteCluster(frame.RemoteId)
if err != nil {
c.SetInvalidRemoteIdError(frame.RemoteId)
return
}
auditRec.AddMeta("remoteCluster", rc)
// pass message to Remote Cluster Service and write response
resp := service.ReceiveIncomingMsg(rc, frame.Msg)
b, errMarshall := json.Marshal(resp)
if errMarshall != nil {
c.Err = model.NewAppError("remoteClusterAcceptMessage", "api.marshal_error", nil, errMarshall.Error(), http.StatusInternalServerError)
return
}
w.Write(b)
}
func remoteClusterConfirmInvite(c *Context, w http.ResponseWriter, r *http.Request) {
// make sure remote cluster service is running.
if _, appErr := c.App.GetRemoteClusterService(); appErr != nil {
c.Err = appErr
return
}
frame, appErr := model.RemoteClusterFrameFromJSON(r.Body)
if appErr != nil {
c.Err = appErr
return
}
if appErr = frame.IsValid(); appErr != nil {
c.Err = appErr
return
}
auditRec := c.MakeAuditRecord("remoteClusterAcceptInvite", audit.Fail)
defer c.LogAuditRec(auditRec)
remoteId := c.GetRemoteID(r)
if remoteId != frame.RemoteId {
c.SetInvalidRemoteIdError(frame.RemoteId)
return
}
rc, err := c.App.GetRemoteCluster(frame.RemoteId)
if err != nil {
c.SetInvalidRemoteIdError(frame.RemoteId)
return
}
auditRec.AddMeta("remoteCluster", rc)
if time.Since(model.GetTimeForMillis(rc.CreateAt)) > remotecluster.InviteExpiresAfter {
c.Err = model.NewAppError("remoteClusterAcceptMessage", "api.context.invitation_expired.error", nil, "", http.StatusBadRequest)
return
}
confirm, appErr := model.RemoteClusterInviteFromRawJSON(frame.Msg.Payload)
if appErr != nil {
c.Err = appErr
return
}
rc.RemoteTeamId = confirm.RemoteTeamId
rc.SiteURL = confirm.SiteURL
rc.RemoteToken = confirm.Token
if _, err := c.App.UpdateRemoteCluster(rc); err != nil {
c.Err = err
return
}
auditRec.Success()
ReturnStatusOK(w)
}
func uploadRemoteData(c *Context, w http.ResponseWriter, r *http.Request) {
if !*c.App.Config().FileSettings.EnableFileAttachments {
c.Err = model.NewAppError("uploadRemoteData", "api.file.attachments.disabled.app_error",
nil, "", http.StatusNotImplemented)
return
}
c.RequireUploadId()
if c.Err != nil {
return
}
auditRec := c.MakeAuditRecord("uploadRemoteData", audit.Fail)
defer c.LogAuditRec(auditRec)
auditRec.AddMeta("upload_id", c.Params.UploadId)
us, err := c.App.GetUploadSession(c.Params.UploadId)
if err != nil {
c.Err = err
return
}
if us.RemoteId != c.GetRemoteID(r) {
c.Err = model.NewAppError("uploadRemoteData", "api.context.remote_id_mismatch.app_error",
nil, "", http.StatusUnauthorized)
return
}
info, err := doUploadData(c, us, r)
if err != nil {
c.Err = err
return
}
auditRec.Success()
if info == nil {
w.WriteHeader(http.StatusNoContent)
return
}
w.Write([]byte(info.ToJson()))
}

76
api4/shared_channel.go Обычный файл
Просмотреть файл

@@ -0,0 +1,76 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package api4
import (
"encoding/json"
"net/http"
"github.com/mattermost/mattermost-server/v5/model"
)
func (api *API) InitSharedChannels() {
api.BaseRoutes.SharedChannels.Handle("/{team_id:[A-Za-z0-9]+}", api.ApiSessionRequired(getSharedChannels)).Methods("GET")
api.BaseRoutes.SharedChannels.Handle("/remote_info/{remote_id:[A-Za-z0-9]+}", api.ApiSessionRequired(getRemoteClusterInfo)).Methods("GET")
}
func getSharedChannels(c *Context, w http.ResponseWriter, r *http.Request) {
c.RequireTeamId()
if c.Err != nil {
return
}
// make sure remote cluster service is enabled.
if _, appErr := c.App.GetRemoteClusterService(); appErr != nil {
c.Err = appErr
return
}
opts := model.SharedChannelFilterOpts{
TeamId: c.Params.TeamId,
}
channels, appErr := c.App.GetSharedChannels(c.Params.Page, c.Params.PerPage, opts)
if appErr != nil {
c.Err = appErr
return
}
b, err := json.Marshal(channels)
if err != nil {
c.SetJSONEncodingError()
return
}
w.Write(b)
}
func getRemoteClusterInfo(c *Context, w http.ResponseWriter, r *http.Request) {
c.RequireRemoteId()
if c.Err != nil {
return
}
// make sure remote cluster service is enabled.
if _, appErr := c.App.GetRemoteClusterService(); appErr != nil {
c.Err = appErr
return
}
// GetRemoteClusterForUser will only return a remote if the user is a member of at
// least one channel shared by the remote. All other cases return error.
rc, appErr := c.App.GetRemoteClusterForUser(c.Params.RemoteId, c.App.Session().UserId)
if appErr != nil {
c.Err = appErr
return
}
remoteInfo := rc.ToRemoteClusterInfo()
b, err := json.Marshal(remoteInfo)
if err != nil {
c.SetJSONEncodingError()
return
}
w.Write(b)
}

229
api4/shared_channel_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,229 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package api4
import (
"fmt"
"math/rand"
"sort"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost-server/v5/app"
"github.com/mattermost/mattermost-server/v5/model"
)
var (
rnd = rand.New(rand.NewSource(time.Now().UnixNano()))
)
func TestGetAllSharedChannels(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
const pages = 3
const pageSize = 7
mockService := app.NewMockRemoteClusterService(nil, app.MockOptionRemoteClusterServiceWithActive(true))
th.App.Srv().SetRemoteClusterService(mockService)
savedIds := make([]string, 0, pages*pageSize)
// make some shared channels
for i := 0; i < pages*pageSize; i++ {
channel := th.CreateChannelWithClientAndTeam(th.Client, model.CHANNEL_OPEN, th.BasicTeam.Id)
sc := &model.SharedChannel{
ChannelId: channel.Id,
TeamId: channel.TeamId,
Home: randomBool(),
ShareName: fmt.Sprintf("test_share_%d", i),
CreatorId: th.BasicChannel.CreatorId,
RemoteId: model.NewId(),
}
_, err := th.App.SaveSharedChannel(sc)
require.NoError(t, err)
savedIds = append(savedIds, channel.Id)
}
sort.Strings(savedIds)
t.Run("get shared channels paginated", func(t *testing.T) {
channelIds := make([]string, 0, 21)
for i := 0; i < pages; i++ {
channels, resp := th.Client.GetAllSharedChannels(th.BasicTeam.Id, i, pageSize)
CheckNoError(t, resp)
channelIds = append(channelIds, getIds(channels)...)
}
sort.Strings(channelIds)
// ids lists should now match
assert.Equal(t, savedIds, channelIds, "id lists should match")
})
t.Run("get shared channels for invalid team", func(t *testing.T) {
channels, resp := th.Client.GetAllSharedChannels(model.NewId(), 0, 100)
CheckNoError(t, resp)
assert.Empty(t, channels)
})
}
func getIds(channels []*model.SharedChannel) []string {
ids := make([]string, 0, len(channels))
for _, c := range channels {
ids = append(ids, c.ChannelId)
}
return ids
}
func randomBool() bool {
return rnd.Intn(2) != 0
}
func TestGetRemoteClusterById(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
mockService := app.NewMockRemoteClusterService(nil, app.MockOptionRemoteClusterServiceWithActive(true))
th.App.Srv().SetRemoteClusterService(mockService)
// for this test we need a user that belongs to a channel that
// is shared with the requested remote id.
// create a remote cluster
rc := &model.RemoteCluster{
RemoteId: model.NewId(),
DisplayName: "Test1",
RemoteTeamId: model.NewId(),
SiteURL: model.NewId(),
CreatorId: model.NewId(),
}
rc, appErr := th.App.AddRemoteCluster(rc)
require.Nil(t, appErr)
// create a shared channel
sc := &model.SharedChannel{
ChannelId: th.BasicChannel.Id,
TeamId: th.BasicChannel.TeamId,
Home: false,
ShareName: "test_share",
CreatorId: th.BasicChannel.CreatorId,
RemoteId: rc.RemoteId,
}
sc, err := th.App.SaveSharedChannel(sc)
require.NoError(t, err)
// create a shared channel remote to connect them
scr := &model.SharedChannelRemote{
Id: model.NewId(),
ChannelId: sc.ChannelId,
CreatorId: sc.CreatorId,
IsInviteAccepted: true,
IsInviteConfirmed: true,
RemoteId: sc.RemoteId,
}
_, err = th.App.SaveSharedChannelRemote(scr)
require.NoError(t, err)
t.Run("valid remote, user is member", func(t *testing.T) {
rcInfo, resp := th.Client.GetRemoteClusterInfo(rc.RemoteId)
CheckNoError(t, resp)
assert.Equal(t, rc.DisplayName, rcInfo.DisplayName)
})
t.Run("invalid remote", func(t *testing.T) {
_, resp := th.Client.GetRemoteClusterInfo(model.NewId())
CheckNotFoundStatus(t, resp)
})
}
func TestCreateDirectChannelWithRemoteUser(t *testing.T) {
t.Run("creates a local DM channel that is shared", func(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
Client := th.Client
defer Client.Logout()
localUser := th.BasicUser
remoteUser := th.CreateUser()
remoteUser.RemoteId = model.NewString(model.NewId())
remoteUser, err := th.App.UpdateUser(remoteUser, false)
require.Nil(t, err)
dm, resp := Client.CreateDirectChannel(localUser.Id, remoteUser.Id)
CheckNoError(t, resp)
channelName := model.GetDMNameFromIds(localUser.Id, remoteUser.Id)
require.Equal(t, channelName, dm.Name, "dm name didn't match")
assert.True(t, dm.IsShared())
})
t.Run("sends a shared channel invitation to the remote", func(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
Client := th.Client
defer Client.Logout()
mockService := app.NewMockSharedChannelService(nil, app.MockOptionSharedChannelServiceWithActive(true))
th.App.Srv().SetSharedChannelSyncService(mockService)
localUser := th.BasicUser
remoteUser := th.CreateUser()
rc := &model.RemoteCluster{
DisplayName: "test",
Token: model.NewId(),
CreatorId: localUser.Id,
}
rc, err := th.App.AddRemoteCluster(rc)
require.Nil(t, err)
remoteUser.RemoteId = model.NewString(rc.RemoteId)
remoteUser, err = th.App.UpdateUser(remoteUser, false)
require.Nil(t, err)
dm, resp := Client.CreateDirectChannel(localUser.Id, remoteUser.Id)
CheckNoError(t, resp)
channelName := model.GetDMNameFromIds(localUser.Id, remoteUser.Id)
require.Equal(t, channelName, dm.Name, "dm name didn't match")
require.True(t, dm.IsShared())
assert.Equal(t, 1, mockService.NumInvitations())
})
t.Run("does not send a shared channel invitation to the remote when creator is remote", func(t *testing.T) {
th := Setup(t).InitBasic()
defer th.TearDown()
Client := th.Client
defer Client.Logout()
mockService := app.NewMockSharedChannelService(nil, app.MockOptionSharedChannelServiceWithActive(true))
th.App.Srv().SetSharedChannelSyncService(mockService)
localUser := th.BasicUser
remoteUser := th.CreateUser()
rc := &model.RemoteCluster{
DisplayName: "test",
Token: model.NewId(),
CreatorId: localUser.Id,
}
rc, err := th.App.AddRemoteCluster(rc)
require.Nil(t, err)
remoteUser.RemoteId = model.NewString(rc.RemoteId)
remoteUser, err = th.App.UpdateUser(remoteUser, false)
require.Nil(t, err)
dm, resp := Client.CreateDirectChannel(remoteUser.Id, localUser.Id)
CheckNoError(t, resp)
channelName := model.GetDMNameFromIds(localUser.Id, remoteUser.Id)
require.Equal(t, channelName, dm.Name, "dm name didn't match")
require.True(t, dm.IsShared())
assert.Zero(t, mockService.NumInvitations())
})
}

Просмотреть файл

@@ -33,6 +33,10 @@ func createUpload(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
// these are not supported for client uploads; shared channels only.
us.RemoteId = ""
us.ReqFileId = ""
auditRec := c.MakeAuditRecord("createUpload", audit.Fail)
defer c.LogAuditRec(auditRec)
auditRec.AddMeta("upload", us)
@@ -119,33 +123,7 @@ func uploadData(c *Context, w http.ResponseWriter, r *http.Request) {
}
}
boundary, parseErr := parseMultipartRequestHeader(r)
if parseErr != nil && !errors.Is(parseErr, http.ErrNotMultipart) {
c.Err = model.NewAppError("uploadData", "api.upload.upload_data.invalid_content_type",
nil, parseErr.Error(), http.StatusBadRequest)
return
}
var rd io.Reader
if boundary != "" {
mr := multipart.NewReader(r.Body, boundary)
p, partErr := mr.NextPart()
if partErr != nil {
c.Err = model.NewAppError("uploadData", "api.upload.upload_data.multipart_error",
nil, partErr.Error(), http.StatusBadRequest)
return
}
rd = p
} else {
if r.ContentLength > (us.FileSize - us.FileOffset) {
c.Err = model.NewAppError("uploadData", "api.upload.upload_data.invalid_content_length",
nil, "", http.StatusBadRequest)
return
}
rd = r.Body
}
info, err := c.App.UploadData(us, rd)
info, err := doUploadData(c, us, r)
if err != nil {
c.Err = err
return
@@ -160,3 +138,30 @@ func uploadData(c *Context, w http.ResponseWriter, r *http.Request) {
w.Write([]byte(info.ToJson()))
}
func doUploadData(c *Context, us *model.UploadSession, r *http.Request) (*model.FileInfo, *model.AppError) {
boundary, parseErr := parseMultipartRequestHeader(r)
if parseErr != nil && !errors.Is(parseErr, http.ErrNotMultipart) {
return nil, model.NewAppError("uploadData", "api.upload.upload_data.invalid_content_type",
nil, parseErr.Error(), http.StatusBadRequest)
}
var rd io.Reader
if boundary != "" {
mr := multipart.NewReader(r.Body, boundary)
p, partErr := mr.NextPart()
if partErr != nil {
return nil, model.NewAppError("uploadData", "api.upload.upload_data.multipart_error",
nil, partErr.Error(), http.StatusBadRequest)
}
rd = p
} else {
if r.ContentLength > (us.FileSize - us.FileOffset) {
return nil, model.NewAppError("uploadData", "api.upload.upload_data.invalid_content_length",
nil, "", http.StatusBadRequest)
}
rd = r.Body
}
return c.App.UploadData(us, rd)
}