MM-49485: Properly disable plugin in product mode (round 2) (#22195)

There were multiple issues at play here.

1. If the plugin was disabled in ch.syncPlugins,
then the prepackaged plugin would not be processed.
Therefore, if there was an earlier version of the plugin
that remained in S3, that would show up instead.

Therefore, we have to let the full plugin extraction
procedure go ahead, but not enable focalboard.

2. We also need to send the disablePlugin signal
to the other node. But the signal won't be sent
until the cluster leader is elected and inter-node
communication is set up. Therefore, we need to listen
to the ClusterLeaderChanged event and send out
a disable plugin event as well.

When both these issues are taken care of,
then we would correctly see the version of the plugin
from the prepackaged_plugins directory in the disabled
state.

https://mattermost.atlassian.net/browse/MM-49485

```release-note
NONE
```

app/channels.go

@@ -284,6 +284,11 @@ func (ch *Channels) Start() error {
 
 	})
 
+	// This needs to be done after initPlugins has completed,
+	// because we want the full plugin processing to be complete before disabling it.
+	ch.disableBoardsIfNeeded()
+	ch.srv.AddClusterLeaderChangedListener(ch.disableBoardsIfNeeded)
+
 	// TODO: This should be moved to the platform service.
 	if err := ch.srv.platform.EnsureAsymmetricSigningKey(); err != nil {
 		return errors.Wrapf(err, "unable to ensure asymmetric signing key")
@@ -367,3 +372,16 @@ func (ch *Channels) HooksForPluginOrProduct(id string) (plugin.Hooks, error) {
 
 	return nil, fmt.Errorf("could not find hooks for id %s", id)
 }
+
+func (ch *Channels) disableBoardsIfNeeded() {
+	// Disable focalboard in product mode.
+	if ch.srv.Config().FeatureFlags.BoardsProduct {
+		// disablePlugin automatically checks if the plugin is running or not,
+		// and if it isn't, it returns an error. Therefore we ignore those errors.
+		// We don't want to check here again if the plugin is enabled or not.
+		appErr := ch.disablePlugin(model.PluginIdFocalboard)
+		if appErr != nil && appErr.Id != "app.plugin.not_installed.app_error" && appErr.Id != "app.plugin.disabled.app_error" {
+			ch.srv.Log().Error("Error disabling plugin in product mode", mlog.Err(appErr))
+		}
+	}
+}

app/plugin.go

@@ -161,6 +161,11 @@ func (ch *Channels) syncPluginsActiveState() {
 				defer wg.Done()
 
 				pluginID := plugin.Manifest.Id
+				// We skip it from activating here. It is disabled later, at a higher level
+				// from *Channels.Start.
+				if ch.srv.Config().FeatureFlags.BoardsProduct && pluginID == model.PluginIdFocalboard {
+					return
+				}
 				updatedManifest, activated, err := pluginsEnvironment.Activate(pluginID)
 				if err != nil {
 					plugin.WrapLogger(ch.srv.Log()).Error("Unable to activate plugin", mlog.Err(err))
@@ -305,16 +310,6 @@ func (ch *Channels) syncPlugins() *model.AppError {
 
 	var wg sync.WaitGroup
 	for _, plugin := range availablePlugins {
-		// Disable focalboard in product mode.
-		if plugin.Manifest.Id == model.PluginIdFocalboard && ch.cfgSvc.Config().FeatureFlags.BoardsProduct {
-			mlog.Info("Plugin cannot run in product mode, disabling.", mlog.String("plugin_id", model.PluginIdFocalboard))
-			appErr := ch.disablePlugin(model.PluginIdFocalboard)
-			if appErr != nil {
-				mlog.Error("Error disabling plugin", mlog.Err(err))
-			}
-			continue
-		}
-
 		wg.Add(1)
 		go func(pluginID string) {
 			defer wg.Done()

app/plugin_install.go

@@ -393,6 +393,11 @@ func (ch *Channels) installExtractedPlugin(manifest *model.Manifest, fromPluginD
 			return manifest, nil
 		}
 
+		// We skip it from activating here. It is disabled later, at a higher level
+		// from *Channels.Start.
+		if ch.srv.Config().FeatureFlags.BoardsProduct && manifest.Id == model.PluginIdFocalboard {
+			return manifest, nil
+		}
 		updatedManifest, _, err := pluginsEnvironment.Activate(manifest.Id)
 		if err != nil {
 			return nil, model.NewAppError("installExtractedPlugin", "app.plugin.restart.app_error", nil, "", http.StatusInternalServerError).Wrap(err)