MM-53747: Do not start if job is in-progress as well. (#24115)

We missed this out last time. It's possible in an HA
scenario for a second pod to start later while the other
job is in-progress. In that case, it would schedule
two jobs.

https://mattermost.atlassian.net/browse/MM-53747

```release-note
NONE
```

server/channels/store/sqlstore/job_store.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/go-sql-driver/mysql"
+	"github.com/lib/pq"
 	sq "github.com/mattermost/squirrel"
 	"github.com/pkg/errors"
 
@@ -48,7 +50,70 @@ func (jss SqlJobStore) Save(job *model.Job) (*model.Job, error) {
 	}
 
 	if _, err = jss.GetMasterX().Exec(queryString, args...); err != nil {
-		return nil, errors.Wrap(err, "failed to save Preference")
+		return nil, errors.Wrap(err, "failed to save Job")
+	}
+
+	return job, nil
+}
+
+func (jss SqlJobStore) SaveOnce(job *model.Job) (*model.Job, error) {
+	jsonData, err := json.Marshal(job.Data)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed marshalling job data")
+	}
+	if jss.IsBinaryParamEnabled() {
+		jsonData = AppendBinaryFlag(jsonData)
+	}
+
+	tx, err := jss.GetMasterX().BeginXWithIsolation(&sql.TxOptions{
+		Isolation: sql.LevelSerializable,
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "begin_transaction")
+	}
+	defer finalizeTransactionX(tx, &err)
+
+	query, args, err := jss.getQueryBuilder().
+		Select("COUNT(*)").
+		From("Jobs").
+		Where(sq.Eq{
+			"Status": []string{model.JobStatusPending, model.JobStatusInProgress},
+			"Type":   job.Type,
+		}).ToSql()
+	if err != nil {
+		return nil, errors.Wrap(err, "job_tosql")
+	}
+
+	var count int64
+	err = tx.Get(&count, query, args...)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to count pending and in-progress jobs with type=%s", job.Type)
+	}
+
+	if count > 0 {
+		return nil, nil
+	}
+
+	query, args, err = jss.getQueryBuilder().
+		Insert("Jobs").
+		Columns("Id", "Type", "Priority", "CreateAt", "StartAt", "LastActivityAt", "Status", "Progress", "Data").
+		Values(job.Id, job.Type, job.Priority, job.CreateAt, job.StartAt, job.LastActivityAt, job.Status, job.Progress, jsonData).ToSql()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to generate sqlquery")
+	}
+
+	if _, err = tx.Exec(query, args...); err != nil {
+		if isRepeatableError(err) {
+			return nil, nil
+		}
+		return nil, errors.Wrap(err, "failed to save Job")
+	}
+
+	if err = tx.Commit(); err != nil {
+		if isRepeatableError(err) {
+			return nil, nil
+		}
+		return nil, errors.Wrap(err, "commit_transaction")
 	}
 
 	return job, nil
@@ -342,3 +407,24 @@ func (jss SqlJobStore) Cleanup(expiryTime int64, batchSize int) error {
 
 	return nil
 }
+
+const mySQLDeadlockCode = uint16(1213)
+
+// isRepeatableError is a bit of copied code from retrylayer.go.
+// A little copying is fine because we don't want to import another package
+// in the store layer
+func isRepeatableError(err error) bool {
+	var pqErr *pq.Error
+	var mysqlErr *mysql.MySQLError
+	switch {
+	case errors.As(err, &pqErr):
+		if pqErr.Code == "40001" || pqErr.Code == "40P01" {
+			return true
+		}
+	case errors.As(err, &mysqlErr):
+		if mysqlErr.Number == mySQLDeadlockCode {
+			return true
+		}
+	}
+	return false
+}