MM-53747: Do not start if job is in-progress as well. (#24115)

We missed this out last time. It's possible in an HA
scenario for a second pod to start later while the other
job is in-progress. In that case, it would schedule
two jobs.

https://mattermost.atlassian.net/browse/MM-53747

```release-note
NONE
```

server/channels/store/storetest/job_store.go

@@ -5,10 +5,12 @@ package storetest
 
 import (
 	"errors"
+	"sync"
 	"testing"
 
 	"time"
 
+	"github.com/lib/pq"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -18,6 +20,7 @@ import (
 
 func TestJobStore(t *testing.T, ss store.Store) {
 	t.Run("JobSaveGet", func(t *testing.T) { testJobSaveGet(t, ss) })
+	t.Run("JobSaveOnce", func(t *testing.T) { testJobSaveOnce(t, ss) })
 	t.Run("JobGetAllByType", func(t *testing.T) { testJobGetAllByType(t, ss) })
 	t.Run("JobGetAllByTypeAndStatus", func(t *testing.T) { testJobGetAllByTypeAndStatus(t, ss) })
 	t.Run("JobGetAllByTypePage", func(t *testing.T) { testJobGetAllByTypePage(t, ss) })
@@ -56,6 +59,51 @@ func testJobSaveGet(t *testing.T, ss store.Store) {
 	require.Equal(t, "12345", received.Data["Total"])
 }
 
+func testJobSaveOnce(t *testing.T, ss store.Store) {
+	var wg sync.WaitGroup
+
+	ids := make([]string, 2)
+	for i := 0; i < 2; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			job := &model.Job{
+				Id:     model.NewId(),
+				Type:   model.JobTypeS3PathMigration,
+				Status: model.JobStatusPending,
+				Data: map[string]string{
+					"Processed":     "0",
+					"Total":         "12345",
+					"LastProcessed": "abcd",
+				},
+			}
+
+			job, err := ss.Job().SaveOnce(job)
+			if err != nil {
+				var pqErr *pq.Error
+				if errors.As(err, &pqErr) {
+					t.Logf("%#v\n", pqErr)
+				}
+			}
+			require.NoError(t, err)
+
+			if job != nil {
+				ids[i] = job.Id
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	cnt, err := ss.Job().GetCountByStatusAndType(model.JobStatusPending, model.JobTypeS3PathMigration)
+	require.NoError(t, err)
+	assert.Equal(t, 1, int(cnt))
+
+	for _, id := range ids {
+		ss.Job().Delete(id)
+	}
+}
+
 func testJobGetAllByType(t *testing.T, ss store.Store) {
 	jobType := model.NewId()
 

server/channels/store/storetest/mocks/JobStore.go

@@ -336,6 +336,32 @@ func (_m *JobStore) Save(job *model.Job) (*model.Job, error) {
 	return r0, r1
 }
 
+// SaveOnce provides a mock function with given fields: job
+func (_m *JobStore) SaveOnce(job *model.Job) (*model.Job, error) {
+	ret := _m.Called(job)
+
+	var r0 *model.Job
+	var r1 error
+	if rf, ok := ret.Get(0).(func(*model.Job) (*model.Job, error)); ok {
+		return rf(job)
+	}
+	if rf, ok := ret.Get(0).(func(*model.Job) *model.Job); ok {
+		r0 = rf(job)
+	} else {
+		if ret.Get(0) != nil {
+			r0 = ret.Get(0).(*model.Job)
+		}
+	}
+
+	if rf, ok := ret.Get(1).(func(*model.Job) error); ok {
+		r1 = rf(job)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
+}
+
 // UpdateOptimistically provides a mock function with given fields: job, currentStatus
 func (_m *JobStore) UpdateOptimistically(job *model.Job, currentStatus string) (bool, error) {
 	ret := _m.Called(job, currentStatus)