[MM-31094] Replication Lag (#16888)

* MM-31094: Adds tooling to develop and test using a MySQL instance with replication lag. Adds some lazy lookups to fallback to master if results are not found.

* MM-31094: Removes mysql-read-replica from default docker services.

* MM-31094: Switches (store..SessionStore).Get and (store.TeamStore).GetMember to using context.Context.

* MM-31094: Updates (store.UsersStore).Get to use context.

* MM-31094: Updates (store.PostStore).Get to use context.

* MM-31094: Removes feature flag and config setting.

* MM-31094: Rolls back some master reads.

* MM-31094: Rolls a non-cache read.

* MM-31094: Removes feature flag from the store.

* MM-31094: Removes unused constant and struct field.

* MM-31094: Removes some old feature flag references.

* MM-31094: Fixes some tests.

* MM-31094: App layers fix.

* MM-31094: Fixes mocks.

* MM-31094: Don't reparse flag.

* MM-31094: No reparse.

* MM-31094: Removed unused FeatureFlags field.

* MM-31094: Removes unnecessary feature flags variable declarations.

* MM-31094: Fixes copy-paste error.

* MM-31094: Fixes logical error.

* MM-30194: Removes test method from store.

* Revert "MM-30194: Removes test method from store."

This reverts commit d5a6e8529bd5f4d993824c828e239d009b05e567.

* MM-31094: Conforming to make's strange syntax.

* MM-31094: Configures helper for read replica with option.

* MM-31094: Adds some missing ctx's.

* MM-31094: WIP

* MM-31094: Updates test names.

* MM-31094: WIP

* MM-31094: Removes unnecessary master reads.

* MM-31094: ID case changes out of scope.

* MM-31094: Removes unused context.

* MM-31094: Switches to a helper. Removes some var naming changes. Fixes a merge error.

* MM-31094: Removes SQLITE db driver ref.

* MM-31094: Layer generate fix.

* MM-31094: Removes unnecessary changes.

* MM-31094: Moves test method.

* MM-31094: Re-add previous fix.

* MM-31094: Removes make command for dev.

* MM-31094: Fix for login.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

api4/apitestlib.go

@@ -1006,7 +1006,7 @@ func (th *TestHelper) MakeUserChannelAdmin(user *model.User, channel *model.Chan
 func (th *TestHelper) UpdateUserToTeamAdmin(user *model.User, team *model.Team) {
 	utils.DisableDebugLogForTest()
 
-	if tm, err := th.App.Srv().Store.Team().GetMember(team.Id, user.Id); err == nil {
+	if tm, err := th.App.Srv().Store.Team().GetMember(context.Background(), team.Id, user.Id); err == nil {
 		tm.SchemeAdmin = true
 		if _, err = th.App.Srv().Store.Team().UpdateMember(tm); err != nil {
 			utils.EnableDebugLogForTest()
@@ -1023,7 +1023,7 @@ func (th *TestHelper) UpdateUserToTeamAdmin(user *model.User, team *model.Team)
 func (th *TestHelper) UpdateUserToNonTeamAdmin(user *model.User, team *model.Team) {
 	utils.DisableDebugLogForTest()
 
-	if tm, err := th.App.Srv().Store.Team().GetMember(team.Id, user.Id); err == nil {
+	if tm, err := th.App.Srv().Store.Team().GetMember(context.Background(), team.Id, user.Id); err == nil {
 		tm.SchemeAdmin = false
 		if _, err = th.App.Srv().Store.Team().UpdateMember(tm); err != nil {
 			utils.EnableDebugLogForTest()

api4/main_test.go

@@ -4,16 +4,25 @@
 package api4
 
 import (
+	"flag"
 	"testing"
 
 	"github.com/mattermost/mattermost-server/v5/shared/mlog"
 	"github.com/mattermost/mattermost-server/v5/testlib"
 )
 
+var replicaFlag bool
+
 func TestMain(m *testing.M) {
+	if f := flag.Lookup("mysql-replica"); f == nil {
+		flag.BoolVar(&replicaFlag, "mysql-replica", false, "")
+		flag.Parse()
+	}
+
 	var options = testlib.HelperOptions{
 		EnableStore:     true,
 		EnableResources: true,
+		WithReadReplica: replicaFlag,
 	}
 
 	mlog.DisableZap()

api4/user_test.go

@@ -3348,6 +3348,45 @@ func TestLogin(t *testing.T) {
 	})
 }
 
+func TestLoginWithLag(t *testing.T) {
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+	th.Client.Logout()
+
+	t.Run("with replication lag, caches cleared", func(t *testing.T) {
+		if !replicaFlag {
+			t.Skipf("requires test flag: -mysql-replica")
+		}
+
+		if *th.App.Srv().Config().SqlSettings.DriverName != model.DATABASE_DRIVER_MYSQL {
+			t.Skipf("requires %q database driver", model.DATABASE_DRIVER_MYSQL)
+		}
+
+		mainHelper.SQLStore.UpdateLicense(model.NewTestLicense("ldap"))
+		mainHelper.ToggleReplicasOff()
+
+		err := th.App.RevokeAllSessions(th.BasicUser.Id)
+		require.Nil(t, err)
+
+		mainHelper.ToggleReplicasOn()
+		defer mainHelper.ToggleReplicasOff()
+
+		cmdErr := mainHelper.SetReplicationLagForTesting(5)
+		require.Nil(t, cmdErr)
+		defer mainHelper.SetReplicationLagForTesting(0)
+
+		_, resp := th.Client.Login(th.BasicUser.Email, th.BasicUser.Password)
+		CheckNoError(t, resp)
+
+		err = th.App.Srv().InvalidateAllCaches()
+		require.Nil(t, err)
+
+		session, err := th.App.GetSession(th.Client.AuthToken)
+		require.Nil(t, err)
+		require.NotNil(t, session)
+	})
+}
+
 func TestLoginCookies(t *testing.T) {
 	t.Run("should return cookies with X-Requested-With header", func(t *testing.T) {
 		th := Setup(t).InitBasic()