MM-57018: support reattaching plugins (#26421)

* ProfileImageBytes for EnsureBotOptions

* leverage plugintest.NewAPI

* fix linting

* add UpdateUserRoles to plugin api

* MM-57018: support reattaching plugins

Expose a local-only API for reattaching plugins: instead of the server starting and managing the process itself, allow the plugin to be launched externally (eg within a unit test) and reattach to an existing server instance to provide the unit test with a fully functional RPC API, sidestepping the need for mocking the plugin API in most cases.

In the future, this may become the basis for running plugins in a sidecar container.

Fixes: https://mattermost.atlassian.net/browse/MM-57018

* drop unused supervisor.pid

* factor out checkMinServerVersion

* factor out startPluginServer

* restore missing setPluginState on successful reattach

* avoid passing around a stale registeredPlugin

* inline initializePluginImplementation

* have IsValid return an error

* explicitly close rpcClient

In the case of reattached plugins, the Unix socket won't necessarily disappear leaving the muxBrokers blocked indefinitely. And `Kill()` doesn't do anything if there's no process being managed.

* explicitly detachPlugin

* emphasize gRPC not being supported

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

server/public/plugin/supervisor.go

@@ -23,15 +23,58 @@ import (
 )
 
 type supervisor struct {
-	lock        sync.RWMutex
-	client      *plugin.Client
-	hooks       Hooks
-	implemented [TotalHooksID]bool
-	pid         int
-	hooksClient *hooksRPCClient
+	lock         sync.RWMutex
+	client       *plugin.Client
+	hooks        Hooks
+	implemented  [TotalHooksID]bool
+	hooksClient  *hooksRPCClient
+	isReattached bool
 }
 
-func newSupervisor(pluginInfo *model.BundleInfo, apiImpl API, driver Driver, parentLogger *mlog.Logger, metrics metricsInterface) (retSupervisor *supervisor, retErr error) {
+func WithExecutableFromManifest(pluginInfo *model.BundleInfo) func(*supervisor, *plugin.ClientConfig) error {
+	return func(_ *supervisor, clientConfig *plugin.ClientConfig) error {
+		executable := pluginInfo.Manifest.GetExecutableForRuntime(runtime.GOOS, runtime.GOARCH)
+		if executable == "" {
+			return fmt.Errorf("backend executable not found for environment: %s/%s", runtime.GOOS, runtime.GOARCH)
+		}
+
+		executable = filepath.Clean(filepath.Join(".", executable))
+		if strings.HasPrefix(executable, "..") {
+			return fmt.Errorf("invalid backend executable: %s", executable)
+		}
+
+		executable = filepath.Join(pluginInfo.Path, executable)
+
+		cmd := exec.Command(executable)
+
+		// This doesn't add more security than before
+		// but removes the SecureConfig is nil warning.
+		// https://mattermost.atlassian.net/browse/MM-49167
+		pluginChecksum, err := getPluginExecutableChecksum(executable)
+		if err != nil {
+			return errors.Wrapf(err, "unable to generate plugin checksum")
+		}
+
+		clientConfig.Cmd = cmd
+		clientConfig.SecureConfig = &plugin.SecureConfig{
+			Checksum: pluginChecksum,
+			Hash:     sha256.New(),
+		}
+
+		return nil
+	}
+}
+
+func WithReattachConfig(pluginReattachConfig *model.PluginReattachConfig) func(*supervisor, *plugin.ClientConfig) error {
+	return func(sup *supervisor, clientConfig *plugin.ClientConfig) error {
+		clientConfig.Reattach = pluginReattachConfig.ToHashicorpPluginReattachmentConfig()
+		sup.isReattached = true
+
+		return nil
+	}
+}
+
+func newSupervisor(pluginInfo *model.BundleInfo, apiImpl API, driver Driver, parentLogger *mlog.Logger, metrics metricsInterface, opts ...func(*supervisor, *plugin.ClientConfig) error) (retSupervisor *supervisor, retErr error) {
 	sup := supervisor{}
 	defer func() {
 		if retErr != nil {
@@ -54,49 +97,28 @@ func newSupervisor(pluginInfo *model.BundleInfo, apiImpl API, driver Driver, par
 		},
 	}
 
-	executable := pluginInfo.Manifest.GetExecutableForRuntime(runtime.GOOS, runtime.GOARCH)
-	if executable == "" {
-		return nil, fmt.Errorf("backend executable not found for environment: %s/%s", runtime.GOOS, runtime.GOARCH)
-	}
-
-	executable = filepath.Clean(filepath.Join(".", executable))
-	if strings.HasPrefix(executable, "..") {
-		return nil, fmt.Errorf("invalid backend executable: %s", executable)
-	}
-
-	executable = filepath.Join(pluginInfo.Path, executable)
-
-	cmd := exec.Command(executable)
-
-	// This doesn't add more security than before
-	// but removes the SecureConfig is nil warning.
-	// https://mattermost.atlassian.net/browse/MM-49167
-	pluginChecksum, err := getPluginExecutableChecksum(executable)
-	if err != nil {
-		return nil, errors.Wrapf(err, "unable to generate plugin checksum")
-	}
-
-	sup.client = plugin.NewClient(&plugin.ClientConfig{
+	clientConfig := &plugin.ClientConfig{
 		HandshakeConfig: handshake,
 		Plugins:         pluginMap,
-		Cmd:             cmd,
 		SyncStdout:      wrappedLogger.With(mlog.String("source", "plugin_stdout")).StdLogWriter(),
 		SyncStderr:      wrappedLogger.With(mlog.String("source", "plugin_stderr")).StdLogWriter(),
 		Logger:          hclogAdaptedLogger,
 		StartTimeout:    time.Second * 3,
-		SecureConfig: &plugin.SecureConfig{
-			Checksum: pluginChecksum,
-			Hash:     sha256.New(),
-		},
-	})
+	}
+	for _, opt := range opts {
+		err := opt(&sup, clientConfig)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to apply option")
+		}
+	}
+
+	sup.client = plugin.NewClient(clientConfig)
 
 	rpcClient, err := sup.client.Client()
 	if err != nil {
 		return nil, err
 	}
 
-	sup.pid = cmd.Process.Pid
-
 	raw, err := rpcClient.Dispense("hooks")
 	if err != nil {
 		return nil, err
@@ -126,6 +148,20 @@ func (sup *supervisor) Shutdown() {
 	sup.lock.RLock()
 	defer sup.lock.RUnlock()
 	if sup.client != nil {
+		// For reattached plugins, Kill() is mostly a no-op, so manually clean up the
+		// underlying rpcClient. This might be something to upstream unless we're doing
+		// something else wrong.
+		if sup.isReattached {
+			rpcClient, err := sup.client.Client()
+			if err != nil {
+				mlog.Warn("Failed to obtain rpcClient on Shutdown")
+			} else {
+				if err = rpcClient.Close(); err != nil {
+					mlog.Warn("Failed to close rpcClient on Shutdown")
+				}
+			}
+		}
+
 		sup.client.Kill()
 	}