MM-50427: Make MM survive DB replica outage (#22888)

We monitor the health of DB replicas, and on a fatal error,
take them out of the pool.

On a separate goroutine, we keep pinging the unhealthy replicas,
and on getting a good response back, we add them back to the pool.

https://mattermost.atlassian.net/browse/MM-50427

```release-note
Mattermost is now resilient against DB replica outages and will
dynamically choose a replica if it's alive.

Also added a config parameter ReplicaMonitorIntervalSeconds
whose default value is 5. This controls how frequently unhealthy
replicas will be monitored for liveness check.
```

Co-authored-by: Mattermost Build <build@mattermost.com>

server/channels/store/storetest/mocks/Store.go

@@ -346,22 +346,6 @@ func (_m *Store) GetInternalReplicaDB() *sql.DB {
 	return r0
 }
 
-// GetInternalReplicaDBs provides a mock function with given fields:
-func (_m *Store) GetInternalReplicaDBs() []*sql.DB {
-	ret := _m.Called()
-
-	var r0 []*sql.DB
-	if rf, ok := ret.Get(0).(func() []*sql.DB); ok {
-		r0 = rf()
-	} else {
-		if ret.Get(0) != nil {
-			r0 = ret.Get(0).([]*sql.DB)
-		}
-	}
-
-	return r0
-}
-
 // Group provides a mock function with given fields:
 func (_m *Store) Group() store.GroupStore {
 	ret := _m.Called()

server/channels/store/storetest/settings.go

@@ -261,6 +261,7 @@ func MakeSqlSettings(driver string, withReplica bool) *model.SqlSettings {
 	}
 
 	log("Created temporary " + driver + " database " + dbName)
+	settings.ReplicaMonitorIntervalSeconds = model.NewInt(5)
 
 	return settings
 }