improved OnDeactivate handling (#11988)

* Deactivate plugins in parallel to improve shutdown time
* Give plugins at most 10s to handle OnDeactivate before forcefully terminating

app/plugin_shutdown_test.go

@@ -0,0 +1,69 @@
+package app
+
+import (
+	"testing"
+	"time"
+)
+
+func TestPluginShutdownTest(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping test to verify forced shutdown of slow plugin")
+	}
+
+	th := Setup(t).InitBasic()
+	defer th.TearDown()
+
+	tearDown, _, _ := SetAppEnvironmentWithPlugins(t,
+		[]string{
+			`
+			package main
+
+			import (
+				"github.com/mattermost/mattermost-server/plugin"
+			)
+
+			type MyPlugin struct {
+				plugin.MattermostPlugin
+			}
+
+			func main() {
+				plugin.ClientMain(&MyPlugin{})
+			}
+			`,
+			`
+			package main
+
+			import (
+				"github.com/mattermost/mattermost-server/plugin"
+			)
+
+			type MyPlugin struct {
+				plugin.MattermostPlugin
+			}
+
+			func (p *MyPlugin) OnDeactivate() error {
+				c := make(chan bool)
+				<-c
+
+				return nil
+			}
+
+			func main() {
+				plugin.ClientMain(&MyPlugin{})
+			}
+		`,
+		}, th.App, th.App.NewPluginAPI)
+	defer tearDown()
+
+	done := make(chan bool)
+	go func() {
+		defer close(done)
+		th.App.ShutDownPlugins()
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(15 * time.Second):
+		t.Fatal("failed to force plugin shutdown after 10 seconds")
+	}
+}

plugin/environment.go

@@ -288,16 +288,42 @@ func (env *Environment) Shutdown() {
 		env.pluginHealthCheckJob.Cancel()
 	}
 
+	var wg sync.WaitGroup
 	env.registeredPlugins.Range(func(key, value interface{}) bool {
 		rp := value.(*registeredPlugin)
 
-		if rp.supervisor != nil {
+		if rp.supervisor == nil {
+			return true
+		}
+
+		wg.Add(1)
+
+		done := make(chan bool)
+		go func() {
+			defer close(done)
 			if err := rp.supervisor.Hooks().OnDeactivate(); err != nil {
 				env.logger.Error("Plugin OnDeactivate() error", mlog.String("plugin_id", rp.BundleInfo.Manifest.Id), mlog.Err(err))
 			}
-			rp.supervisor.Shutdown()
-		}
+		}()
 
+		go func() {
+			defer wg.Done()
+
+			select {
+			case <-time.After(10 * time.Second):
+				env.logger.Warn("Plugin OnDeactivate() failed to complete in 10 seconds", mlog.String("plugin_id", rp.BundleInfo.Manifest.Id))
+			case <-done:
+			}
+
+			rp.supervisor.Shutdown()
+		}()
+
+		return true
+	})
+
+	wg.Wait()
+
+	env.registeredPlugins.Range(func(key, value interface{}) bool {
 		env.registeredPlugins.Delete(key)
 
 		return true