app/platform: fix an issue about accessing to a closed db on shutdown (#21549)

app/platform/goroutines.go

@@ -21,8 +21,8 @@ func (ps *PlatformService) Go(f func()) {
 	}()
 }
 
-// WaitForGoroutines blocks until all goroutines created by App.Go exit.
-func (ps *PlatformService) WaitForGoroutines() {
+// waitForGoroutines blocks until all goroutines created by PlatformService.Go() exit.
+func (ps *PlatformService) waitForGoroutines() {
 	for atomic.LoadInt32(&ps.goroutineCount) != 0 {
 		<-ps.goroutineExitSignal
 	}

app/platform/service.go

@@ -390,6 +390,12 @@ func (ps *PlatformService) Shutdown() error {
 
 	ps.RemoveLicenseListener(ps.licenseListenerId)
 
+	// we need to wait the goroutines to finish before closing the store
+	// and this needs to be called after hub stop because hub generates goroutines
+	// when it is active. If we wait first we have no mechanism to prevent adding
+	// more go routines hence they still going to be invoked.
+	ps.waitForGoroutines()
+
 	if ps.Store != nil {
 		ps.Store.Close()
 	}

app/platform/service_test.go

@@ -4,10 +4,13 @@
 package platform
 
 import (
+	"math/rand"
 	"net/http"
 	"os"
 	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/mattermost/mattermost-server/v6/config"
 	"github.com/mattermost/mattermost-server/v6/einterfaces/mocks"
@@ -154,3 +157,24 @@ func TestMetrics(t *testing.T) {
 		mockMetricsImpl.AssertExpectations(t)
 	})
 }
+
+func TestShutdown(t *testing.T) {
+	t.Run("should shutdown gracefully", func(t *testing.T) {
+		th := Setup(t)
+		rand.Seed(time.Now().UnixNano())
+
+		// we create plenty of go routines to make sure we wait for all of them
+		// to finish before shutting down
+		for i := 0; i < 1000; i++ {
+			th.Service.Go(func() {
+				time.Sleep(time.Millisecond * time.Duration(rand.Intn(20)))
+			})
+		}
+
+		err := th.Service.Shutdown()
+		require.NoError(t, err)
+
+		// assert that there are no more go routines running
+		require.Zero(t, atomic.LoadInt32(&th.Service.goroutineCount))
+	})
+}

app/server.go

@@ -706,8 +706,6 @@ func (s *Server) Shutdown() {
 	s.StopPushNotificationsHubWorkers()
 	s.htmlTemplateWatcher.Close()
 
-	s.WaitForGoroutines()
-
 	s.platform.StopSearchEngine()
 
 	s.Audit.Shutdown()
@@ -814,11 +812,6 @@ func (s *Server) GoBuffered(f func()) {
 	s.platform.GoBuffered(f)
 }
 
-// WaitForGoroutines blocks until all goroutines created by App.Go exit.
-func (s *Server) WaitForGoroutines() {
-	s.platform.WaitForGoroutines()
-}
-
 var corsAllowedMethods = []string{
 	"POST",
 	"GET",