Add functionality to cleanup old jobs (#18646)

* Add functionality to cleanup old jobs

Historically, we never cleaned up old jobs from the DB
leading to them being accumulated forever.

This PR introduces functionality to cleanup old jobs
older than a defined threshold.

The functionality is set to false by default and has
to be enabled for it to work.

```release-note
2 new config settings were added.

JobSettings.CleanupOldJobs: This indicates whether to clean up old jobs
from the DB or not. Default is false.

JobSettings.CleanupJobsThresholdHours: This defines the time gap in hours beyond
which older jobs will be removed. This has no effect if the above config
setting is set to false. Default is -1
```

* fix copy pasta

```release-note
NONE
```

* address review comments

```release-note
NONE
```

* Fix lint

```release-note
NONE
```

* Use single config option for everything

```release-note
NONE
```

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

store/sqlstore/job_store.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+	"time"
 
 	sq "github.com/Masterminds/squirrel"
 	"github.com/mattermost/gorp"
@@ -17,6 +18,10 @@ import (
 	"github.com/mattermost/mattermost-server/v6/store"
 )
 
+const (
+	jobsCleanupDelay = 100 * time.Millisecond
+)
+
 type SqlJobStore struct {
 	*SqlStore
 }
@@ -286,3 +291,31 @@ func (jss SqlJobStore) Delete(id string) (string, error) {
 	}
 	return id, nil
 }
+
+func (jss SqlJobStore) Cleanup(expiryTime int64, batchSize int) error {
+	var query string
+	if jss.DriverName() == model.DatabaseDriverPostgres {
+		query = "DELETE FROM Jobs WHERE Id IN (SELECT Id FROM Jobs WHERE CreateAt < ? AND (Status != ? AND Status != ?) ORDER BY CreateAt ASC LIMIT ?)"
+	} else {
+		query = "DELETE FROM Jobs WHERE CreateAt < ? AND (Status != ? AND Status != ?) ORDER BY CreateAt ASC LIMIT ?"
+	}
+
+	var rowsAffected int64 = 1
+
+	for rowsAffected > 0 {
+		sqlResult, err := jss.GetMasterX().Exec(query,
+			expiryTime, model.JobStatusInProgress, model.JobStatusPending, batchSize)
+		if err != nil {
+			return errors.Wrap(err, "unable to delete jobs")
+		}
+		var rowErr error
+		rowsAffected, rowErr = sqlResult.RowsAffected()
+		if rowErr != nil {
+			return errors.Wrap(err, "unable to delete jobs")
+		}
+
+		time.Sleep(jobsCleanupDelay)
+	}
+
+	return nil
+}