MM-42412 - guard watcher stop/stopped channels with mutex to prevent race condition (#19759)

* MM-42412 - Guard watcher stop/stopped channels with mutex to prevent race condition

* reinitialize channels on Stop fn call

Co-authored-by: Pablo Velez Vidal <pablo.velez@mattermost.com>
Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

jobs/jobs_watcher.go

@@ -26,6 +26,8 @@ type Watcher struct {
 
 func (srv *JobServer) MakeWatcher(workers *Workers, pollingInterval int) *Watcher {
 	return &Watcher{
+		stop:            make(chan struct{}),
+		stopped:         make(chan struct{}),
 		pollingInterval: pollingInterval,
 		workers:         workers,
 		srv:             srv,
@@ -34,8 +36,6 @@ func (srv *JobServer) MakeWatcher(workers *Workers, pollingInterval int) *Watche
 
 func (watcher *Watcher) Start() {
 	mlog.Debug("Watcher Started")
-	watcher.stop = make(chan struct{})
-	watcher.stopped = make(chan struct{})
 	// Delay for some random number of milliseconds before starting to ensure that multiple
 	// instances of the jobserver  don't poll at a time too close to each other.
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -61,6 +61,9 @@ func (watcher *Watcher) Stop() {
 	mlog.Debug("Watcher Stopping")
 	close(watcher.stop)
 	<-watcher.stopped
+
+	watcher.stop = make(chan struct{})
+	watcher.stopped = make(chan struct{})
 }
 
 func (watcher *Watcher) PollAndNotify() {

jobs/server_test.go

@@ -11,7 +11,6 @@ import (
 )
 
 func TestStartWorkers(t *testing.T) {
-	t.Skip("MM-42412")
 	t.Run("uninitialized", func(t *testing.T) {
 		jobServer, _, _ := makeJobServer(t)
 		err := jobServer.StartWorkers()
@@ -44,7 +43,6 @@ func TestStartWorkers(t *testing.T) {
 }
 
 func TestStopWorkers(t *testing.T) {
-	t.Skip("MM-42412")
 	t.Run("uninitialized", func(t *testing.T) {
 		jobServer, _, _ := makeJobServer(t)
 		err := jobServer.StopWorkers()