PLT-7407: Back-end plugin mechanism (#7177)

* begin backend plugin wip

* flesh out rpcplugin. everything done except for minor supervisor stubs

* done with basic plugin infrastructure

* simplify tests

* remove unused test lines

plugin/pluginenv/environment.go

@@ -0,0 +1,123 @@
+// Package pluginenv provides high level functionality for discovering and launching plugins.
+package pluginenv
+
+import (
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/mattermost/platform/plugin"
+)
+
+type APIProviderFunc func(*plugin.Manifest) (plugin.API, error)
+type SupervisorProviderFunc func(*plugin.BundleInfo) (plugin.Supervisor, error)
+
+// Environment represents an environment that plugins are discovered and launched in.
+type Environment struct {
+	searchPath         string
+	apiProvider        APIProviderFunc
+	supervisorProvider SupervisorProviderFunc
+	activePlugins      map[string]plugin.Supervisor
+}
+
+type Option func(*Environment)
+
+// Creates a new environment. At a minimum, the APIProvider and SearchPath options are required.
+func New(options ...Option) (*Environment, error) {
+	env := &Environment{
+		activePlugins: make(map[string]plugin.Supervisor),
+	}
+	for _, opt := range options {
+		opt(env)
+	}
+	if env.supervisorProvider == nil {
+		env.supervisorProvider = DefaultSupervisorProvider
+	}
+	if env.searchPath == "" {
+		return nil, fmt.Errorf("a search path must be provided")
+	} else if env.apiProvider == nil {
+		return nil, fmt.Errorf("an api provider must be provided")
+	}
+	return env, nil
+}
+
+// Returns a list of all plugins found within the environment.
+func (env *Environment) Plugins() ([]*plugin.BundleInfo, error) {
+	return ScanSearchPath(env.searchPath)
+}
+
+// Returns the ids of the currently active plugins.
+func (env *Environment) ActivePluginIds() (ids []string) {
+	for id := range env.activePlugins {
+		ids = append(ids, id)
+	}
+	return
+}
+
+// Activates the plugin with the given id.
+func (env *Environment) ActivatePlugin(id string) error {
+	if _, ok := env.activePlugins[id]; ok {
+		return fmt.Errorf("plugin already active: %v", id)
+	}
+	plugins, err := ScanSearchPath(env.searchPath)
+	if err != nil {
+		return err
+	}
+	var plugin *plugin.BundleInfo
+	for _, p := range plugins {
+		if p.Manifest != nil && p.Manifest.Id == id {
+			if plugin != nil {
+				return fmt.Errorf("multiple plugins found: %v", id)
+			}
+			plugin = p
+		}
+	}
+	if plugin == nil {
+		return fmt.Errorf("plugin not found: %v", id)
+	}
+	supervisor, err := env.supervisorProvider(plugin)
+	if err != nil {
+		return errors.Wrapf(err, "unable to create supervisor for plugin: %v", id)
+	}
+	api, err := env.apiProvider(plugin.Manifest)
+	if err != nil {
+		return errors.Wrapf(err, "unable to get api for plugin: %v", id)
+	}
+	if err := supervisor.Start(); err != nil {
+		return errors.Wrapf(err, "unable to start plugin: %v", id)
+	}
+	if err := supervisor.Hooks().OnActivate(api); err != nil {
+		supervisor.Stop()
+		return errors.Wrapf(err, "unable to activate plugin: %v", id)
+	}
+	env.activePlugins[id] = supervisor
+	return nil
+}
+
+// Deactivates the plugin with the given id.
+func (env *Environment) DeactivatePlugin(id string) error {
+	if supervisor, ok := env.activePlugins[id]; !ok {
+		return fmt.Errorf("plugin not active: %v", id)
+	} else {
+		delete(env.activePlugins, id)
+		err := supervisor.Hooks().OnDeactivate()
+		if serr := supervisor.Stop(); err == nil {
+			err = serr
+		}
+		return err
+	}
+}
+
+// Deactivates all plugins and gracefully shuts down the environment.
+func (env *Environment) Shutdown() (errs []error) {
+	for _, supervisor := range env.activePlugins {
+		if err := supervisor.Hooks().OnDeactivate(); err != nil {
+			errs = append(errs, err)
+		}
+		if err := supervisor.Stop(); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	env.activePlugins = make(map[string]plugin.Supervisor)
+	return
+}

plugin/pluginenv/environment_test.go

@@ -0,0 +1,291 @@
+package pluginenv
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/platform/plugin"
+	"github.com/mattermost/platform/plugin/plugintest"
+)
+
+type MockProvider struct {
+	mock.Mock
+}
+
+func (m *MockProvider) API(manifest *plugin.Manifest) (plugin.API, error) {
+	ret := m.Called()
+	if ret.Get(0) == nil {
+		return nil, ret.Error(1)
+	}
+	return ret.Get(0).(plugin.API), ret.Error(1)
+}
+
+func (m *MockProvider) Supervisor(bundle *plugin.BundleInfo) (plugin.Supervisor, error) {
+	ret := m.Called()
+	if ret.Get(0) == nil {
+		return nil, ret.Error(1)
+	}
+	return ret.Get(0).(plugin.Supervisor), ret.Error(1)
+}
+
+type MockSupervisor struct {
+	mock.Mock
+}
+
+func (m *MockSupervisor) Start() error {
+	return m.Called().Error(0)
+}
+
+func (m *MockSupervisor) Stop() error {
+	return m.Called().Error(0)
+}
+
+func (m *MockSupervisor) Hooks() plugin.Hooks {
+	return m.Called().Get(0).(plugin.Hooks)
+}
+
+func initTmpDir(t *testing.T, files map[string]string) string {
+	success := false
+	dir, err := ioutil.TempDir("", "mm-plugin-test")
+	require.NoError(t, err)
+	defer func() {
+		if !success {
+			os.RemoveAll(dir)
+		}
+	}()
+
+	for name, contents := range files {
+		path := filepath.Join(dir, name)
+		parent := filepath.Dir(path)
+		require.NoError(t, os.MkdirAll(parent, 0700))
+		f, err := os.Create(path)
+		require.NoError(t, err)
+		_, err = f.WriteString(contents)
+		f.Close()
+		require.NoError(t, err)
+	}
+
+	success = true
+	return dir
+}
+
+func TestNew_MissingOptions(t *testing.T) {
+	dir := initTmpDir(t, map[string]string{
+		"foo/plugin.json": `{"id": "foo"}`,
+	})
+	defer os.RemoveAll(dir)
+
+	var provider MockProvider
+	defer provider.AssertExpectations(t)
+
+	env, err := New(
+		APIProvider(provider.API),
+	)
+	assert.Nil(t, env)
+	assert.Error(t, err)
+
+	env, err = New(
+		SearchPath(dir),
+	)
+	assert.Nil(t, env)
+	assert.Error(t, err)
+}
+
+func TestEnvironment(t *testing.T) {
+	dir := initTmpDir(t, map[string]string{
+		".foo/plugin.json": `{"id": "foo"}`,
+		"foo/bar":          "asdf",
+		"foo/plugin.json":  `{"id": "foo"}`,
+		"bar/zxc":          "qwer",
+		"baz/plugin.yaml":  "id: baz",
+		"bad/plugin.json":  "asd",
+		"qwe":              "asd",
+	})
+	defer os.RemoveAll(dir)
+
+	var provider MockProvider
+	defer provider.AssertExpectations(t)
+
+	env, err := New(
+		SearchPath(dir),
+		APIProvider(provider.API),
+		SupervisorProvider(provider.Supervisor),
+	)
+	require.NoError(t, err)
+	defer env.Shutdown()
+
+	plugins, err := env.Plugins()
+	assert.NoError(t, err)
+	assert.Len(t, plugins, 3)
+
+	assert.Error(t, env.ActivatePlugin("x"))
+
+	var api struct{ plugin.API }
+	var supervisor MockSupervisor
+	defer supervisor.AssertExpectations(t)
+	var hooks plugintest.Hooks
+	defer hooks.AssertExpectations(t)
+
+	provider.On("API").Return(&api, nil)
+	provider.On("Supervisor").Return(&supervisor, nil)
+
+	supervisor.On("Start").Return(nil)
+	supervisor.On("Stop").Return(nil)
+	supervisor.On("Hooks").Return(&hooks)
+
+	hooks.On("OnActivate", &api).Return(nil)
+
+	assert.NoError(t, env.ActivatePlugin("foo"))
+	assert.Equal(t, env.ActivePluginIds(), []string{"foo"})
+	assert.Error(t, env.ActivatePlugin("foo"))
+
+	hooks.On("OnDeactivate").Return(nil)
+	assert.NoError(t, env.DeactivatePlugin("foo"))
+	assert.Error(t, env.DeactivatePlugin("foo"))
+
+	assert.NoError(t, env.ActivatePlugin("foo"))
+	assert.Equal(t, env.ActivePluginIds(), []string{"foo"})
+	assert.Empty(t, env.Shutdown())
+}
+
+func TestEnvironment_DuplicatePluginError(t *testing.T) {
+	dir := initTmpDir(t, map[string]string{
+		"foo/plugin.json":  `{"id": "foo"}`,
+		"foo2/plugin.json": `{"id": "foo"}`,
+	})
+	defer os.RemoveAll(dir)
+
+	var provider MockProvider
+	defer provider.AssertExpectations(t)
+
+	env, err := New(
+		SearchPath(dir),
+		APIProvider(provider.API),
+		SupervisorProvider(provider.Supervisor),
+	)
+	require.NoError(t, err)
+	defer env.Shutdown()
+
+	assert.Error(t, env.ActivatePlugin("foo"))
+	assert.Empty(t, env.ActivePluginIds())
+}
+
+func TestEnvironment_BadSearchPathError(t *testing.T) {
+	var provider MockProvider
+	defer provider.AssertExpectations(t)
+
+	env, err := New(
+		SearchPath("thissearchpathshouldnotexist!"),
+		APIProvider(provider.API),
+		SupervisorProvider(provider.Supervisor),
+	)
+	require.NoError(t, err)
+	defer env.Shutdown()
+
+	assert.Error(t, env.ActivatePlugin("foo"))
+	assert.Empty(t, env.ActivePluginIds())
+}
+
+func TestEnvironment_ActivatePluginErrors(t *testing.T) {
+	dir := initTmpDir(t, map[string]string{
+		"foo/plugin.json": `{"id": "foo"}`,
+	})
+	defer os.RemoveAll(dir)
+
+	var provider MockProvider
+
+	env, err := New(
+		SearchPath(dir),
+		APIProvider(provider.API),
+		SupervisorProvider(provider.Supervisor),
+	)
+	require.NoError(t, err)
+	defer env.Shutdown()
+
+	var api struct{ plugin.API }
+	var supervisor MockSupervisor
+	var hooks plugintest.Hooks
+
+	for name, setup := range map[string]func(){
+		"SupervisorProviderError": func() {
+			provider.On("Supervisor").Return(nil, fmt.Errorf("test error"))
+		},
+		"APIProviderError": func() {
+			provider.On("API").Return(plugin.API(nil), fmt.Errorf("test error"))
+			provider.On("Supervisor").Return(&supervisor, nil)
+		},
+		"SupervisorError": func() {
+			provider.On("API").Return(&api, nil)
+			provider.On("Supervisor").Return(&supervisor, nil)
+
+			supervisor.On("Start").Return(fmt.Errorf("test error"))
+		},
+		"HooksError": func() {
+			provider.On("API").Return(&api, nil)
+			provider.On("Supervisor").Return(&supervisor, nil)
+
+			supervisor.On("Start").Return(nil)
+			supervisor.On("Stop").Return(nil)
+			supervisor.On("Hooks").Return(&hooks)
+
+			hooks.On("OnActivate", &api).Return(fmt.Errorf("test error"))
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			supervisor.Mock = mock.Mock{}
+			hooks.Mock = mock.Mock{}
+			provider.Mock = mock.Mock{}
+			setup()
+			assert.Error(t, env.ActivatePlugin("foo"))
+			assert.Empty(t, env.ActivePluginIds())
+			supervisor.AssertExpectations(t)
+			hooks.AssertExpectations(t)
+			provider.AssertExpectations(t)
+		})
+	}
+}
+
+func TestEnvironment_ShutdownError(t *testing.T) {
+	dir := initTmpDir(t, map[string]string{
+		"foo/plugin.json": `{"id": "foo"}`,
+	})
+	defer os.RemoveAll(dir)
+
+	var provider MockProvider
+	defer provider.AssertExpectations(t)
+
+	env, err := New(
+		SearchPath(dir),
+		APIProvider(provider.API),
+		SupervisorProvider(provider.Supervisor),
+	)
+	require.NoError(t, err)
+	defer env.Shutdown()
+
+	var api struct{ plugin.API }
+	var supervisor MockSupervisor
+	defer supervisor.AssertExpectations(t)
+	var hooks plugintest.Hooks
+	defer hooks.AssertExpectations(t)
+
+	provider.On("API").Return(&api, nil)
+	provider.On("Supervisor").Return(&supervisor, nil)
+
+	supervisor.On("Start").Return(nil)
+	supervisor.On("Stop").Return(fmt.Errorf("test error"))
+	supervisor.On("Hooks").Return(&hooks)
+
+	hooks.On("OnActivate", &api).Return(nil)
+	hooks.On("OnDeactivate").Return(fmt.Errorf("test error"))
+
+	assert.NoError(t, env.ActivatePlugin("foo"))
+	assert.Equal(t, env.ActivePluginIds(), []string{"foo"})
+	assert.Len(t, env.Shutdown(), 2)
+}

plugin/pluginenv/options.go

@@ -0,0 +1,42 @@
+package pluginenv
+
+import (
+	"fmt"
+
+	"github.com/mattermost/platform/plugin"
+	"github.com/mattermost/platform/plugin/rpcplugin"
+)
+
+// APIProvider specifies a function that provides an API implementation to each plugin.
+func APIProvider(provider APIProviderFunc) Option {
+	return func(env *Environment) {
+		env.apiProvider = provider
+	}
+}
+
+// SupervisorProvider specifies a function that provides a Supervisor implementation to each plugin.
+// If unspecified, DefaultSupervisorProvider is used.
+func SupervisorProvider(provider SupervisorProviderFunc) Option {
+	return func(env *Environment) {
+		env.supervisorProvider = provider
+	}
+}
+
+// SearchPath specifies a directory that contains the plugins to launch.
+func SearchPath(path string) Option {
+	return func(env *Environment) {
+		env.searchPath = path
+	}
+}
+
+// DefaultSupervisorProvider chooses a supervisor based on the plugin's manifest contents. E.g. if
+// the manifest specifies a backend executable, it will be given an rpcplugin.Supervisor.
+func DefaultSupervisorProvider(bundle *plugin.BundleInfo) (plugin.Supervisor, error) {
+	if bundle.Manifest == nil {
+		return nil, fmt.Errorf("a manifest is required")
+	}
+	if bundle.Manifest.Backend == nil {
+		return nil, fmt.Errorf("invalid manifest: at this time, only backend plugins are supported")
+	}
+	return rpcplugin.SupervisorProvider(bundle)
+}

plugin/pluginenv/options_test.go

@@ -0,0 +1,32 @@
+package pluginenv
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/platform/plugin"
+	"github.com/mattermost/platform/plugin/rpcplugin"
+)
+
+func TestDefaultSupervisorProvider(t *testing.T) {
+	_, err := DefaultSupervisorProvider(&plugin.BundleInfo{})
+	assert.Error(t, err)
+
+	_, err = DefaultSupervisorProvider(&plugin.BundleInfo{
+		Manifest: &plugin.Manifest{},
+	})
+	assert.Error(t, err)
+
+	supervisor, err := DefaultSupervisorProvider(&plugin.BundleInfo{
+		Manifest: &plugin.Manifest{
+			Backend: &plugin.ManifestBackend{
+				Executable: "foo",
+			},
+		},
+	})
+	require.NoError(t, err)
+	_, ok := supervisor.(*rpcplugin.Supervisor)
+	assert.True(t, ok)
+}

plugin/pluginenv/search_path.go

@@ -0,0 +1,32 @@
+package pluginenv
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/mattermost/platform/plugin"
+)
+
+// Performs a full scan of the given path.
+//
+// This function will return info for all subdirectories that appear to be plugins (i.e. all
+// subdirectories containing plugin manifest files, regardless of whether they could actually be
+// parsed).
+//
+// Plugins are found non-recursively and paths beginning with a dot are always ignored.
+func ScanSearchPath(path string) ([]*plugin.BundleInfo, error) {
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+	var ret []*plugin.BundleInfo
+	for _, file := range files {
+		if !file.IsDir() || file.Name()[0] == '.' {
+			continue
+		}
+		if info := plugin.BundleInfoForPath(filepath.Join(path, file.Name())); info.ManifestPath != "" {
+			ret = append(ret, info)
+		}
+	}
+	return ret, nil
+}

plugin/pluginenv/search_path_test.go

@@ -0,0 +1,62 @@
+package pluginenv
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/platform/plugin"
+)
+
+func TestScanSearchPath(t *testing.T) {
+	dir := initTmpDir(t, map[string]string{
+		".foo/plugin.json": `{"id": "foo"}`,
+		"foo/bar":          "asdf",
+		"foo/plugin.json":  `{"id": "foo"}`,
+		"bar/zxc":          "qwer",
+		"baz/plugin.yaml":  "id: baz",
+		"bad/plugin.json":  "asd",
+		"qwe":              "asd",
+	})
+	defer os.RemoveAll(dir)
+
+	plugins, err := ScanSearchPath(dir)
+	require.NoError(t, err)
+	assert.Len(t, plugins, 3)
+	assert.Contains(t, plugins, &plugin.BundleInfo{
+		Path:         filepath.Join(dir, "foo"),
+		ManifestPath: filepath.Join(dir, "foo", "plugin.json"),
+		Manifest: &plugin.Manifest{
+			Id: "foo",
+		},
+	})
+	assert.Contains(t, plugins, &plugin.BundleInfo{
+		Path:         filepath.Join(dir, "baz"),
+		ManifestPath: filepath.Join(dir, "baz", "plugin.yaml"),
+		Manifest: &plugin.Manifest{
+			Id: "baz",
+		},
+	})
+	foundError := false
+	for _, x := range plugins {
+		if x.ManifestError != nil {
+			assert.Equal(t, x.Path, filepath.Join(dir, "bad"))
+			assert.Equal(t, x.ManifestPath, filepath.Join(dir, "bad", "plugin.json"))
+			syntexError, ok := x.ManifestError.(*json.SyntaxError)
+			assert.True(t, ok)
+			assert.EqualValues(t, 1, syntexError.Offset)
+			foundError = true
+		}
+	}
+	assert.True(t, foundError)
+}
+
+func TestScanSearchPath_Error(t *testing.T) {
+	plugins, err := ScanSearchPath("not a valid path!")
+	assert.Nil(t, plugins)
+	assert.Error(t, err)
+}