MM-24310: Recycle DB connections properly (#14378)

Automatic Merge

store/sqlstore/supplier.go

@@ -426,7 +426,7 @@ func (ss *SqlSupplier) DoesTableExist(tableName string) bool {
 
 	} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
 		count, err := ss.GetMaster().SelectInt(
-			`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
+			`SELECT count(name) FROM sqlite_master WHERE type='table' AND name=?`,
 			tableName,
 		)
 
@@ -1004,6 +1004,23 @@ func (ss *SqlSupplier) GetAllConns() []*gorp.DbMap {
 	return all
 }
 
+// RecycleDBConnections closes active connections by setting the max conn lifetime
+// to d, and then resets them back to their original duration.
+func (ss *SqlSupplier) RecycleDBConnections(d time.Duration) {
+	// Get old time.
+	originalDuration := time.Duration(*ss.settings.ConnMaxLifetimeMilliseconds) * time.Millisecond
+	// Set the max lifetimes for all connections.
+	for _, conn := range ss.GetAllConns() {
+		conn.Db.SetConnMaxLifetime(d)
+	}
+	// Wait for that period with an additional 2 seconds of scheduling delay.
+	time.Sleep(d + 2*time.Second)
+	// Reset max lifetime back to original value.
+	for _, conn := range ss.GetAllConns() {
+		conn.Db.SetConnMaxLifetime(originalDuration)
+	}
+}
+
 func (ss *SqlSupplier) Close() {
 	ss.master.Db.Close()
 	for _, replica := range ss.replicas {

store/sqlstore/supplier_test.go

@@ -5,7 +5,9 @@ package sqlstore_test
 
 import (
 	"regexp"
+	"sync"
 	"testing"
+	"time"
 
 	"github.com/mattermost/gorp"
 	_ "github.com/mattn/go-sqlite3"
@@ -151,6 +153,46 @@ func TestGetDbVersion(t *testing.T) {
 	}
 }
 
+func TestRecycleDBConns(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping recycle DBConns test")
+	}
+	testDrivers := []string{
+		model.DATABASE_DRIVER_POSTGRES,
+		model.DATABASE_DRIVER_MYSQL,
+		model.DATABASE_DRIVER_SQLITE,
+	}
+
+	for _, driver := range testDrivers {
+		t.Run(driver, func(t *testing.T) {
+			settings := makeSqlSettings(driver)
+			supplier := sqlstore.NewSqlSupplier(*settings, nil)
+
+			var wg sync.WaitGroup
+			tables := []string{"Posts", "Channels", "Users"}
+			for _, table := range tables {
+				wg.Add(1)
+				go func(table string) {
+					defer wg.Done()
+					query := `SELECT count(*) FROM ` + table
+					_, err := supplier.GetMaster().SelectInt(query)
+					assert.NoError(t, err)
+				}(table)
+			}
+			wg.Wait()
+
+			stats := supplier.GetMaster().Db.Stats()
+			assert.Equal(t, 0, int(stats.MaxLifetimeClosed), "unexpected number of connections closed due to maxlifetime")
+
+			supplier.RecycleDBConnections(2 * time.Second)
+			// We cannot reliably control exactly how many open connections are there. So we
+			// just do a basic check and confirm that atleast one has been closed.
+			stats = supplier.GetMaster().Db.Stats()
+			assert.Greater(t, int(stats.MaxLifetimeClosed), 0, "unexpected number of connections closed due to maxlifetime")
+		})
+	}
+}
+
 func TestGetAllConns(t *testing.T) {
 	t.Parallel()
 	testCases := []struct {