MM-9728: Online migration for advanced permissions phase 2 (#8744)

* MM-9728: Online migration for advanced permissions phase 2

* Add unit tests for new store functions.

* Move migration specific code to own file.

* Add migration state function test.

* Style fixes.

* Add i18n strings.

* Fix mocks.

* Add TestMain to migrations package tests.

* Fix typo.

* Fix review comments.

* Fix up the "Check if migration is done" check to actually work.

store/sqlstore/channel_store.go

@@ -1739,3 +1739,71 @@ func (s SqlChannelStore) GetChannelsByScheme(schemeId string, offset int, limit
 		}
 	})
 }
+
+// This function does the Advanced Permissions Phase 2 migration for ChannelMember objects. It performs the migration
+// in batches as a single transaction per batch to ensure consistency but to also minimise execution time to avoid
+// causing unnecessary table locks. **THIS FUNCTION SHOULD NOT BE USED FOR ANY OTHER PURPOSE.** Executing this function
+// *after* the new Schemes functionality has been used on an installation will have unintended consequences.
+func (s SqlChannelStore) MigrateChannelMembers(fromChannelId string, fromUserId string) store.StoreChannel {
+	return store.Do(func(result *store.StoreResult) {
+		var transaction *gorp.Transaction
+		var err error
+
+		if transaction, err = s.GetMaster().Begin(); err != nil {
+			result.Err = model.NewAppError("SqlChannelStore.MigrateChannelMembers", "store.sql_channel.migrate_channel_members.open_transaction.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var channelMembers []channelMember
+		if _, err := transaction.Select(&channelMembers, "SELECT * from ChannelMembers WHERE (ChannelId, UserId) > (:FromChannelId, :FromUserId) ORDER BY ChannelId, UserId LIMIT 100", map[string]interface{}{"FromChannelId": fromChannelId, "FromUserId": fromUserId}); err != nil {
+			result.Err = model.NewAppError("SqlChannelStore.MigrateChannelMembers", "store.sql_channel.migrate_channel_members.select.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		if len(channelMembers) == 0 {
+			// No more channel members in query result means that the migration has finished.
+			return
+		}
+
+		for _, member := range channelMembers {
+			roles := strings.Fields(member.Roles)
+			var newRoles []string
+			member.SchemeAdmin = sql.NullBool{Bool: false, Valid: true}
+			member.SchemeUser = sql.NullBool{Bool: false, Valid: true}
+			for _, role := range roles {
+				if role == model.CHANNEL_ADMIN_ROLE_ID {
+					member.SchemeAdmin = sql.NullBool{Bool: true, Valid: true}
+				} else if role == model.CHANNEL_USER_ROLE_ID {
+					member.SchemeUser = sql.NullBool{Bool: true, Valid: true}
+				} else {
+					newRoles = append(newRoles, role)
+				}
+			}
+			member.Roles = strings.Join(newRoles, " ")
+
+			if _, err := transaction.Update(&member); err != nil {
+				if err2 := transaction.Rollback(); err2 != nil {
+					result.Err = model.NewAppError("SqlChannelStore.MigrateChannelMembers", "store.sql_channel.migrate_channel_members.rollback_transaction.app_error", nil, err2.Error(), http.StatusInternalServerError)
+					return
+				}
+				result.Err = model.NewAppError("SqlChannelStore.MigrateChannelMembers", "store.sql_channel.migrate_channel_members.update.app_error", nil, err.Error(), http.StatusInternalServerError)
+				return
+			}
+
+		}
+
+		if err := transaction.Commit(); err != nil {
+			if err2 := transaction.Rollback(); err2 != nil {
+				result.Err = model.NewAppError("SqlChannelStore.MigrateChannelMembers", "store.sql_channel.migrate_channel_members.rollback_transaction.app_error", nil, err2.Error(), http.StatusInternalServerError)
+				return
+			}
+			result.Err = model.NewAppError("SqlChannelStore.MigrateChannelMembers", "store.sql_channel.migrate_channel_members.commit_transaction.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		data := make(map[string]string)
+		data["ChannelId"] = channelMembers[len(channelMembers)-1].ChannelId
+		data["UserId"] = channelMembers[len(channelMembers)-1].UserId
+		result.Data = data
+	})
+}

store/sqlstore/team_store.go

@@ -9,6 +9,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/mattermost/gorp"
 	"github.com/mattermost/mattermost-server/model"
 	"github.com/mattermost/mattermost-server/store"
 )
@@ -725,3 +726,71 @@ func (s SqlTeamStore) GetTeamsByScheme(schemeId string, offset int, limit int) s
 		}
 	})
 }
+
+// This function does the Advanced Permissions Phase 2 migration for TeamMember objects. It performs the migration
+// in batches as a single transaction per batch to ensure consistency but to also minimise execution time to avoid
+// causing unnecessary table locks. **THIS FUNCTION SHOULD NOT BE USED FOR ANY OTHER PURPOSE.** Executing this function
+// *after* the new Schemes functionality has been used on an installation will have unintended consequences.
+func (s SqlTeamStore) MigrateTeamMembers(fromTeamId string, fromUserId string) store.StoreChannel {
+	return store.Do(func(result *store.StoreResult) {
+		var transaction *gorp.Transaction
+		var err error
+
+		if transaction, err = s.GetMaster().Begin(); err != nil {
+			result.Err = model.NewAppError("SqlTeamStore.MigrateTeamMembers", "store.sql_team.migrate_team_members.open_transaction.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		var teamMembers []teamMember
+		if _, err := transaction.Select(&teamMembers, "SELECT * from TeamMembers WHERE (TeamId, UserId) > (:FromTeamId, :FromUserId) ORDER BY TeamId, UserId LIMIT 100", map[string]interface{}{"FromTeamId": fromTeamId, "FromUserId": fromUserId}); err != nil {
+			result.Err = model.NewAppError("SqlTeamStore.MigrateTeamMembers", "store.sql_team.migrate_team_members.select.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		if len(teamMembers) == 0 {
+			// No more team members in query result means that the migration has finished.
+			return
+		}
+
+		for _, member := range teamMembers {
+			roles := strings.Fields(member.Roles)
+			var newRoles []string
+			member.SchemeAdmin = sql.NullBool{Bool: false, Valid: true}
+			member.SchemeUser = sql.NullBool{Bool: false, Valid: true}
+			for _, role := range roles {
+				if role == model.TEAM_ADMIN_ROLE_ID {
+					member.SchemeAdmin = sql.NullBool{Bool: true, Valid: true}
+				} else if role == model.TEAM_USER_ROLE_ID {
+					member.SchemeUser = sql.NullBool{Bool: true, Valid: true}
+				} else {
+					newRoles = append(newRoles, role)
+				}
+			}
+			member.Roles = strings.Join(newRoles, " ")
+
+			if _, err := transaction.Update(&member); err != nil {
+				if err2 := transaction.Rollback(); err2 != nil {
+					result.Err = model.NewAppError("SqlTeamStore.MigrateTeamMembers", "store.sql_team.migrate_team_members.rollback_transaction.app_error", nil, err2.Error(), http.StatusInternalServerError)
+					return
+				}
+				result.Err = model.NewAppError("SqlTeamStore.MigrateTeamMembers", "store.sql_team.migrate_team_members.update.app_error", nil, err.Error(), http.StatusInternalServerError)
+				return
+			}
+
+		}
+
+		if err := transaction.Commit(); err != nil {
+			if err2 := transaction.Rollback(); err2 != nil {
+				result.Err = model.NewAppError("SqlTeamStore.MigrateTeamMembers", "store.sql_team.migrate_team_members.rollback_transaction.app_error", nil, err2.Error(), http.StatusInternalServerError)
+				return
+			}
+			result.Err = model.NewAppError("SqlTeamStore.MigrateTeamMembers", "store.sql_team.migrate_team_members.commit_transaction.app_error", nil, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		data := make(map[string]string)
+		data["TeamId"] = teamMembers[len(teamMembers)-1].TeamId
+		data["UserId"] = teamMembers[len(teamMembers)-1].UserId
+		result.Data = data
+	})
+}

store/store.go

@@ -105,6 +105,7 @@ type TeamStore interface {
 	RemoveAllMembersByUser(userId string) StoreChannel
 	UpdateLastTeamIconUpdate(teamId string, curTime int64) StoreChannel
 	GetTeamsByScheme(schemeId string, offset int, limit int) StoreChannel
+	MigrateTeamMembers(fromTeamId string, fromUserId string) StoreChannel
 }
 
 type ChannelStore interface {
@@ -163,6 +164,7 @@ type ChannelStore interface {
 	GetChannelUnread(channelId, userId string) StoreChannel
 	ClearCaches()
 	GetChannelsByScheme(schemeId string, offset int, limit int) StoreChannel
+	MigrateChannelMembers(fromChannelId string, fromUserId string) StoreChannel
 }
 
 type ChannelMemberHistoryStore interface {

store/storetest/channel_store.go

@@ -5,6 +5,7 @@ package storetest
 
 import (
 	"sort"
+	"strings"
 	"testing"
 	"time"
 
@@ -52,6 +53,7 @@ func TestChannelStore(t *testing.T, ss store.Store) {
 	t.Run("GetPinnedPosts", func(t *testing.T) { testChannelStoreGetPinnedPosts(t, ss) })
 	t.Run("MaxChannelsPerTeam", func(t *testing.T) { testChannelStoreMaxChannelsPerTeam(t, ss) })
 	t.Run("GetChannelsByScheme", func(t *testing.T) { testChannelStoreGetChannelsByScheme(t, ss) })
+	t.Run("MigrateChannelMembers", func(t *testing.T) { testChannelStoreMigrateChannelMembers(t, ss) })
 
 }
 
@@ -2254,3 +2256,76 @@ func testChannelStoreGetChannelsByScheme(t *testing.T, ss store.Store) {
 	d3 := res3.Data.(model.ChannelList)
 	assert.Len(t, d3, 0)
 }
+
+func testChannelStoreMigrateChannelMembers(t *testing.T, ss store.Store) {
+	s1 := model.NewId()
+	c1 := &model.Channel{
+		TeamId:      model.NewId(),
+		DisplayName: "Name",
+		Name:        model.NewId(),
+		Type:        model.CHANNEL_OPEN,
+		SchemeId:    &s1,
+	}
+	c1 = (<-ss.Channel().Save(c1, 100)).Data.(*model.Channel)
+
+	cm1 := &model.ChannelMember{
+		ChannelId:     c1.Id,
+		UserId:        model.NewId(),
+		ExplicitRoles: "channel_admin channel_user",
+		NotifyProps:   model.GetDefaultChannelNotifyProps(),
+	}
+	cm2 := &model.ChannelMember{
+		ChannelId:     c1.Id,
+		UserId:        model.NewId(),
+		ExplicitRoles: "channel_user",
+		NotifyProps:   model.GetDefaultChannelNotifyProps(),
+	}
+	cm3 := &model.ChannelMember{
+		ChannelId:     c1.Id,
+		UserId:        model.NewId(),
+		ExplicitRoles: "something_else",
+		NotifyProps:   model.GetDefaultChannelNotifyProps(),
+	}
+
+	cm1 = (<-ss.Channel().SaveMember(cm1)).Data.(*model.ChannelMember)
+	cm2 = (<-ss.Channel().SaveMember(cm2)).Data.(*model.ChannelMember)
+	cm3 = (<-ss.Channel().SaveMember(cm3)).Data.(*model.ChannelMember)
+
+	lastDoneChannelId := strings.Repeat("0", 26)
+	lastDoneUserId := strings.Repeat("0", 26)
+
+	for {
+		res := <-ss.Channel().MigrateChannelMembers(lastDoneChannelId, lastDoneUserId)
+		if assert.Nil(t, res.Err) {
+			if res.Data == nil {
+				break
+			}
+			data := res.Data.(map[string]string)
+			lastDoneChannelId = data["ChannelId"]
+			lastDoneUserId = data["UserId"]
+		}
+	}
+
+	ss.Channel().ClearCaches()
+
+	res1 := <-ss.Channel().GetMember(cm1.ChannelId, cm1.UserId)
+	assert.Nil(t, res1.Err)
+	cm1b := res1.Data.(*model.ChannelMember)
+	assert.Equal(t, "", cm1b.ExplicitRoles)
+	assert.True(t, cm1b.SchemeUser)
+	assert.True(t, cm1b.SchemeAdmin)
+
+	res2 := <-ss.Channel().GetMember(cm2.ChannelId, cm2.UserId)
+	assert.Nil(t, res2.Err)
+	cm2b := res2.Data.(*model.ChannelMember)
+	assert.Equal(t, "", cm2b.ExplicitRoles)
+	assert.True(t, cm2b.SchemeUser)
+	assert.False(t, cm2b.SchemeAdmin)
+
+	res3 := <-ss.Channel().GetMember(cm3.ChannelId, cm3.UserId)
+	assert.Nil(t, res3.Err)
+	cm3b := res3.Data.(*model.ChannelMember)
+	assert.Equal(t, "something_else", cm3b.ExplicitRoles)
+	assert.False(t, cm3b.SchemeUser)
+	assert.False(t, cm3b.SchemeAdmin)
+}

store/storetest/mocks/ChannelStore.go

@@ -583,6 +583,22 @@ func (_m *ChannelStore) IsUserInChannelUseCache(userId string, channelId string)
 	return r0
 }
 
+// MigrateChannelMembers provides a mock function with given fields: fromChannelId, fromUserId
+func (_m *ChannelStore) MigrateChannelMembers(fromChannelId string, fromUserId string) store.StoreChannel {
+	ret := _m.Called(fromChannelId, fromUserId)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(string, string) store.StoreChannel); ok {
+		r0 = rf(fromChannelId, fromUserId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}
+
 // PermanentDelete provides a mock function with given fields: channelId
 func (_m *ChannelStore) PermanentDelete(channelId string) store.StoreChannel {
 	ret := _m.Called(channelId)

store/storetest/mocks/TeamStore.go

@@ -301,6 +301,22 @@ func (_m *TeamStore) GetTotalMemberCount(teamId string) store.StoreChannel {
 	return r0
 }
 
+// MigrateTeamMembers provides a mock function with given fields: fromTeamId, fromUserId
+func (_m *TeamStore) MigrateTeamMembers(fromTeamId string, fromUserId string) store.StoreChannel {
+	ret := _m.Called(fromTeamId, fromUserId)
+
+	var r0 store.StoreChannel
+	if rf, ok := ret.Get(0).(func(string, string) store.StoreChannel); ok {
+		r0 = rf(fromTeamId, fromUserId)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(store.StoreChannel)
+		}
+	}
+
+	return r0
+}
+
 // PermanentDelete provides a mock function with given fields: teamId
 func (_m *TeamStore) PermanentDelete(teamId string) store.StoreChannel {
 	ret := _m.Called(teamId)

store/storetest/team_store.go

@@ -4,6 +4,7 @@
 package storetest
 
 import (
+	"strings"
 	"testing"
 	"time"
 
@@ -39,6 +40,7 @@ func TestTeamStore(t *testing.T, ss store.Store) {
 	t.Run("GetChannelUnreadsForTeam", func(t *testing.T) { testGetChannelUnreadsForTeam(t, ss) })
 	t.Run("UpdateLastTeamIconUpdate", func(t *testing.T) { testUpdateLastTeamIconUpdate(t, ss) })
 	t.Run("GetTeamsByScheme", func(t *testing.T) { testGetTeamsByScheme(t, ss) })
+	t.Run("MigrateTeamMembers", func(t *testing.T) { testTeamStoreMigrateTeamMembers(t, ss) })
 }
 
 func testTeamStoreSave(t *testing.T, ss store.Store) {
@@ -1098,3 +1100,72 @@ func testGetTeamsByScheme(t *testing.T, ss store.Store) {
 	d3 := res3.Data.([]*model.Team)
 	assert.Len(t, d3, 0)
 }
+
+func testTeamStoreMigrateTeamMembers(t *testing.T, ss store.Store) {
+	s1 := model.NewId()
+	t1 := &model.Team{
+		DisplayName: "Name",
+		Name:        "z-z-z" + model.NewId() + "b",
+		Email:       model.NewId() + "@nowhere.com",
+		Type:        model.TEAM_OPEN,
+		InviteId:    model.NewId(),
+		SchemeId:    &s1,
+	}
+	t1 = store.Must(ss.Team().Save(t1)).(*model.Team)
+
+	tm1 := &model.TeamMember{
+		TeamId:        t1.Id,
+		UserId:        model.NewId(),
+		ExplicitRoles: "team_admin team_user",
+	}
+	tm2 := &model.TeamMember{
+		TeamId:        t1.Id,
+		UserId:        model.NewId(),
+		ExplicitRoles: "team_user",
+	}
+	tm3 := &model.TeamMember{
+		TeamId:        t1.Id,
+		UserId:        model.NewId(),
+		ExplicitRoles: "something_else",
+	}
+
+	tm1 = (<-ss.Team().SaveMember(tm1, -1)).Data.(*model.TeamMember)
+	tm2 = (<-ss.Team().SaveMember(tm2, -1)).Data.(*model.TeamMember)
+	tm3 = (<-ss.Team().SaveMember(tm3, -1)).Data.(*model.TeamMember)
+
+	lastDoneTeamId := strings.Repeat("0", 26)
+	lastDoneUserId := strings.Repeat("0", 26)
+
+	for {
+		res := <-ss.Team().MigrateTeamMembers(lastDoneTeamId, lastDoneUserId)
+		if assert.Nil(t, res.Err) {
+			if res.Data == nil {
+				break
+			}
+			data := res.Data.(map[string]string)
+			lastDoneTeamId = data["TeamId"]
+			lastDoneUserId = data["UserId"]
+		}
+	}
+
+	res1 := <-ss.Team().GetMember(tm1.TeamId, tm1.UserId)
+	assert.Nil(t, res1.Err)
+	tm1b := res1.Data.(*model.TeamMember)
+	assert.Equal(t, "", tm1b.ExplicitRoles)
+	assert.True(t, tm1b.SchemeUser)
+	assert.True(t, tm1b.SchemeAdmin)
+
+	res2 := <-ss.Team().GetMember(tm2.TeamId, tm2.UserId)
+	assert.Nil(t, res2.Err)
+	tm2b := res2.Data.(*model.TeamMember)
+	assert.Equal(t, "", tm2b.ExplicitRoles)
+	assert.True(t, tm2b.SchemeUser)
+	assert.False(t, tm2b.SchemeAdmin)
+
+	res3 := <-ss.Team().GetMember(tm3.TeamId, tm3.UserId)
+	assert.Nil(t, res3.Err)
+	tm3b := res3.Data.(*model.TeamMember)
+	assert.Equal(t, "something_else", tm3b.ExplicitRoles)
+	assert.False(t, tm3b.SchemeUser)
+	assert.False(t, tm3b.SchemeAdmin)
+}