jobs: add config cleanup job (#19987)

config/database.go

@@ -416,3 +416,12 @@ func (ds *DatabaseStore) String() string {
 func (ds *DatabaseStore) Close() error {
 	return ds.db.Close()
 }
+
+// removes configurations from database if they are older than threshold.
+func (ds *DatabaseStore) cleanUp(thresholdCreatAt int) error {
+	if _, err := ds.db.NamedExec("DELETE FROM Configurations Where CreateAt < :timestamp", map[string]interface{}{"timestamp": thresholdCreatAt}); err != nil {
+		return errors.Wrap(err, "unable to clean Configurations table")
+	}
+
+	return nil
+}

config/database_test.go

@@ -1113,3 +1113,50 @@ func TestDatabaseStoreString(t *testing.T) {
 		assert.False(t, strings.Contains(maskedDSN, "mostest"))
 	}
 }
+
+func TestCleanUp(t *testing.T) {
+	_, tearDown := setupConfigDatabase(t, emptyConfig, nil)
+	defer tearDown()
+
+	ds, err := newTestDatabaseStore(nil)
+	require.NoError(t, err)
+	require.NotNil(t, ds)
+	defer ds.Close()
+
+	dbs, ok := ds.backingStore.(*DatabaseStore)
+	require.True(t, ok, "should be a DatabaseStore instance")
+
+	b, err := marshalConfig(ds.config)
+	require.NoError(t, err)
+
+	ds.config.JobSettings.CleanupConfigThresholdDays = model.NewInt(30) // we set 30 days as threshold
+
+	now := time.Now()
+	for i := 0; i < 5; i++ {
+		// 20 days, we expect to remove at least 3 configuration values from the store
+		// first 2 (0 and 1) will be within a month constraint, others will be older than
+		// a month hence we expect 3 configurations to be removed from the database.
+		m := -1 * i * 24 * 20
+		params := map[string]interface{}{
+			"id":        model.NewId(),
+			"value":     string(b),
+			"create_at": model.GetMillisForTime(now.Add(time.Duration(m) * time.Hour)),
+		}
+
+		_, err = dbs.db.NamedExec("INSERT INTO Configurations (Id, Value, CreateAt) VALUES (:id, :value, :create_at)", params)
+		require.NoError(t, err)
+	}
+	var initialCount int
+	row := dbs.db.QueryRow("SELECT COUNT(*) FROM Configurations")
+	err = row.Scan(&initialCount)
+	require.NoError(t, err)
+
+	err = ds.CleanUp()
+	require.NoError(t, err)
+
+	var count int
+	row = dbs.db.QueryRow("SELECT COUNT(*) FROM Configurations")
+	err = row.Scan(&count)
+	require.NoError(t, err)
+	require.True(t, count+3 == initialCount)
+}

config/store.go

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"reflect"
 	"sync"
+	"time"
 
 	"github.com/pkg/errors"
 
@@ -396,3 +397,16 @@ func (s *Store) IsReadOnly() bool {
 	defer s.configLock.RUnlock()
 	return s.readOnly
 }
+
+// Cleanup removes outdated configurations from the database.
+// this is a no-op function for FileStore type backing store.
+func (s *Store) CleanUp() error {
+	switch bs := s.backingStore.(type) {
+	case *DatabaseStore:
+		dur := time.Duration(*s.config.JobSettings.CleanupConfigThresholdDays) * time.Hour * 24
+		expiry := model.GetMillisForTime(time.Now().Add(-dur))
+		return bs.cleanUp(int(expiry))
+	default:
+		return nil
+	}
+}