[MM-31094] Replication Lag (#16888)

* MM-31094: Adds tooling to develop and test using a MySQL instance with replication lag. Adds some lazy lookups to fallback to master if results are not found.

* MM-31094: Removes mysql-read-replica from default docker services.

* MM-31094: Switches (store..SessionStore).Get and (store.TeamStore).GetMember to using context.Context.

* MM-31094: Updates (store.UsersStore).Get to use context.

* MM-31094: Updates (store.PostStore).Get to use context.

* MM-31094: Removes feature flag and config setting.

* MM-31094: Rolls back some master reads.

* MM-31094: Rolls a non-cache read.

* MM-31094: Removes feature flag from the store.

* MM-31094: Removes unused constant and struct field.

* MM-31094: Removes some old feature flag references.

* MM-31094: Fixes some tests.

* MM-31094: App layers fix.

* MM-31094: Fixes mocks.

* MM-31094: Don't reparse flag.

* MM-31094: No reparse.

* MM-31094: Removed unused FeatureFlags field.

* MM-31094: Removes unnecessary feature flags variable declarations.

* MM-31094: Fixes copy-paste error.

* MM-31094: Fixes logical error.

* MM-30194: Removes test method from store.

* Revert "MM-30194: Removes test method from store."

This reverts commit d5a6e8529bd5f4d993824c828e239d009b05e567.

* MM-31094: Conforming to make's strange syntax.

* MM-31094: Configures helper for read replica with option.

* MM-31094: Adds some missing ctx's.

* MM-31094: WIP

* MM-31094: Updates test names.

* MM-31094: WIP

* MM-31094: Removes unnecessary master reads.

* MM-31094: ID case changes out of scope.

* MM-31094: Removes unused context.

* MM-31094: Switches to a helper. Removes some var naming changes. Fixes a merge error.

* MM-31094: Removes SQLITE db driver ref.

* MM-31094: Layer generate fix.

* MM-31094: Removes unnecessary changes.

* MM-31094: Moves test method.

* MM-31094: Re-add previous fix.

* MM-31094: Removes make command for dev.

* MM-31094: Fix for login.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

store/sqlstore/group_store.go

@@ -734,7 +734,7 @@ func (s *SqlGroupStore) TeamMembersToAdd(since int64, teamID *string) ([]*model.
 
 	var teamMembers []*model.UserTeamIDPair
 
-	_, err = s.GetReplica().Select(&teamMembers, query, params...)
+	_, err = s.GetMaster().Select(&teamMembers, query, params...)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find UserTeamIDPairs")
 	}
@@ -771,7 +771,7 @@ func (s *SqlGroupStore) ChannelMembersToAdd(since int64, channelID *string) ([]*
 
 	var channelMembers []*model.UserChannelIDPair
 
-	_, err = s.GetReplica().Select(&channelMembers, query, params...)
+	_, err = s.GetMaster().Select(&channelMembers, query, params...)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find UserChannelIDPairs")
 	}
@@ -1447,7 +1447,7 @@ func (s *SqlGroupStore) PermittedSyncableAdmins(syncableID string, syncableType
 	}
 
 	var userIDs []string
-	if _, err = s.GetReplica().Select(&userIDs, query, args...); err != nil {
+	if _, err = s.GetMaster().Select(&userIDs, query, args...); err != nil {
 		return nil, errors.Wrapf(err, "failed to find User ids")
 	}
 

store/sqlstore/post_store.go

@@ -455,7 +455,7 @@ func (s *SqlPostStore) getPostWithCollapsedThreads(id string, extended bool) (*m
 	return s.prepareThreadedResponse([]*postWithExtra{&post}, extended, false)
 }
 
-func (s *SqlPostStore) Get(id string, skipFetchThreads, collapsedThreads, collapsedThreadsExtended bool) (*model.PostList, error) {
+func (s *SqlPostStore) Get(ctx context.Context, id string, skipFetchThreads, collapsedThreads, collapsedThreadsExtended bool) (*model.PostList, error) {
 	if collapsedThreads {
 		return s.getPostWithCollapsedThreads(id, collapsedThreadsExtended)
 	}
@@ -467,7 +467,7 @@ func (s *SqlPostStore) Get(id string, skipFetchThreads, collapsedThreads, collap
 
 	var post model.Post
 	postFetchQuery := "SELECT p.*, (SELECT count(Posts.Id) FROM Posts WHERE Posts.RootId = (CASE WHEN p.RootId = '' THEN p.Id ELSE p.RootId END) AND Posts.DeleteAt = 0) as ReplyCount FROM Posts p WHERE p.Id = :Id AND p.DeleteAt = 0"
-	err := s.GetReplica().SelectOne(&post, postFetchQuery, map[string]interface{}{"Id": id})
+	err := s.DBFromContext(ctx).SelectOne(&post, postFetchQuery, map[string]interface{}{"Id": id})
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, store.NewErrNotFound("Post", id)

store/sqlstore/session_store.go

@@ -73,10 +73,10 @@ func (me SqlSessionStore) Save(session *model.Session) (*model.Session, error) {
 	return session, nil
 }
 
-func (me SqlSessionStore) Get(sessionIdOrToken string) (*model.Session, error) {
+func (me SqlSessionStore) Get(ctx context.Context, sessionIdOrToken string) (*model.Session, error) {
 	var sessions []*model.Session
 
-	if _, err := me.GetReplica().Select(&sessions, "SELECT * FROM Sessions WHERE Token = :Token OR Id = :Id LIMIT 1", map[string]interface{}{"Token": sessionIdOrToken, "Id": sessionIdOrToken}); err != nil {
+	if _, err := me.DBFromContext(ctx).Select(&sessions, "SELECT * FROM Sessions WHERE Token = :Token OR Id = :Id LIMIT 1", map[string]interface{}{"Token": sessionIdOrToken, "Id": sessionIdOrToken}); err != nil {
 		return nil, errors.Wrapf(err, "failed to find Sessions with sessionIdOrToken=%s", sessionIdOrToken)
 	} else if len(sessions) == 0 {
 		return nil, store.NewErrNotFound("Session", fmt.Sprintf("sessionIdOrToken=%s", sessionIdOrToken))
@@ -249,7 +249,7 @@ func (me SqlSessionStore) UpdateDeviceId(id string, deviceId string, expiresAt i
 }
 
 func (me SqlSessionStore) UpdateProps(session *model.Session) error {
-	oldSession, err := me.Get(session.Id)
+	oldSession, err := me.Get(context.Background(), session.Id)
 	if err != nil {
 		return err
 	}

store/sqlstore/store.go

@@ -132,7 +132,7 @@ type SqlStore struct {
 	rrCounter      int64
 	srCounter      int64
 	master         *gorp.DbMap
-	replicas       []*gorp.DbMap
+	Replicas       []*gorp.DbMap
 	searchReplicas []*gorp.DbMap
 	stores         SqlStoreStores
 	settings       *model.SqlSettings
@@ -318,9 +318,9 @@ func (ss *SqlStore) initConnection() {
 	ss.master = setupConnection("master", *ss.settings.DataSource, ss.settings)
 
 	if len(ss.settings.DataSourceReplicas) > 0 {
-		ss.replicas = make([]*gorp.DbMap, len(ss.settings.DataSourceReplicas))
+		ss.Replicas = make([]*gorp.DbMap, len(ss.settings.DataSourceReplicas))
 		for i, replica := range ss.settings.DataSourceReplicas {
-			ss.replicas[i] = setupConnection(fmt.Sprintf("replica-%v", i), replica, ss.settings)
+			ss.Replicas[i] = setupConnection(fmt.Sprintf("replica-%v", i), replica, ss.settings)
 		}
 	}
 
@@ -395,8 +395,8 @@ func (ss *SqlStore) GetReplica() *gorp.DbMap {
 		return ss.GetMaster()
 	}
 
-	rrNum := atomic.AddInt64(&ss.rrCounter, 1) % int64(len(ss.replicas))
-	return ss.replicas[rrNum]
+	rrNum := atomic.AddInt64(&ss.rrCounter, 1) % int64(len(ss.Replicas))
+	return ss.Replicas[rrNum]
 }
 
 func (ss *SqlStore) TotalMasterDbConnections() int {
@@ -409,7 +409,7 @@ func (ss *SqlStore) TotalReadDbConnections() int {
 	}
 
 	count := 0
-	for _, db := range ss.replicas {
+	for _, db := range ss.Replicas {
 		count = count + db.Db.Stats().OpenConnections
 	}
 
@@ -1008,9 +1008,9 @@ func IsUniqueConstraintError(err error, indexName []string) bool {
 }
 
 func (ss *SqlStore) GetAllConns() []*gorp.DbMap {
-	all := make([]*gorp.DbMap, len(ss.replicas)+1)
-	copy(all, ss.replicas)
-	all[len(ss.replicas)] = ss.master
+	all := make([]*gorp.DbMap, len(ss.Replicas)+1)
+	copy(all, ss.Replicas)
+	all[len(ss.Replicas)] = ss.master
 	return all
 }
 
@@ -1033,7 +1033,7 @@ func (ss *SqlStore) RecycleDBConnections(d time.Duration) {
 
 func (ss *SqlStore) Close() {
 	ss.master.Db.Close()
-	for _, replica := range ss.replicas {
+	for _, replica := range ss.Replicas {
 		replica.Db.Close()
 	}
 
@@ -1210,6 +1210,10 @@ func (ss *SqlStore) UpdateLicense(license *model.License) {
 	ss.license = license
 }
 
+func (ss *SqlStore) GetLicense() *model.License {
+	return ss.license
+}
+
 func (ss *SqlStore) migrate(direction migrationDirection) error {
 	var driver database.Driver
 	var err error

store/sqlstore/store_test.go

@@ -36,7 +36,7 @@ var storeTypes []*storeType
 func newStoreType(name, driver string) *storeType {
 	return &storeType{
 		Name:        name,
-		SqlSettings: storetest.MakeSqlSettings(driver),
+		SqlSettings: storetest.MakeSqlSettings(driver, false),
 	}
 }
 
@@ -560,9 +560,9 @@ func TestVersionString(t *testing.T) {
 func makeSqlSettings(driver string) *model.SqlSettings {
 	switch driver {
 	case model.DATABASE_DRIVER_POSTGRES:
-		return storetest.MakeSqlSettings(driver)
+		return storetest.MakeSqlSettings(driver, false)
 	case model.DATABASE_DRIVER_MYSQL:
-		return storetest.MakeSqlSettings(driver)
+		return storetest.MakeSqlSettings(driver, false)
 	}
 
 	return nil

store/sqlstore/team_store.go

@@ -988,7 +988,7 @@ func (s SqlTeamStore) UpdateMember(member *model.TeamMember) (*model.TeamMember,
 }
 
 // GetMember returns a single member of the team that matches the teamId and userId provided as parameters.
-func (s SqlTeamStore) GetMember(teamId string, userId string) (*model.TeamMember, error) {
+func (s SqlTeamStore) GetMember(ctx context.Context, teamId string, userId string) (*model.TeamMember, error) {
 	query := s.getTeamMembersWithSchemeSelectQuery().
 		Where(sq.Eq{"TeamMembers.TeamId": teamId}).
 		Where(sq.Eq{"TeamMembers.UserId": userId})
@@ -999,7 +999,7 @@ func (s SqlTeamStore) GetMember(teamId string, userId string) (*model.TeamMember
 	}
 
 	var dbMember teamMemberWithSchemeRoles
-	err = s.GetReplica().SelectOne(&dbMember, queryString, args...)
+	err = s.DBFromContext(ctx).SelectOne(&dbMember, queryString, args...)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, store.NewErrNotFound("TeamMember", fmt.Sprintf("teamId=%s, userId=%s", teamId, userId))