[MM-31790] Support Packet Generation BACKEND (#16667)

* init commit

* clean up the code

* make mocks

* fix translations

* mocks and lint fixes

* add tests

* little fixes

* Update i18n/en.json

Co-authored-by: Scott Bishel <scott.bishel@mattermost.com>

* Update i18n/en.json

Co-authored-by: Scott Bishel <scott.bishel@mattermost.com>

* Update i18n/en.json

Co-authored-by: Scott Bishel <scott.bishel@mattermost.com>

* Update i18n/en.json

Co-authored-by: Scott Bishel <scott.bishel@mattermost.com>

* Update i18n/en.json

Co-authored-by: Scott Bishel <scott.bishel@mattermost.com>

* Address Comments

* fix i18n

* update api endpoint

* add enable file and file level for conditional show of banner

* Address Comments

* Make it more clear about returns

* Create zip file utility function

* update en.json

* address comments

* write tests

* check for data in test

* remove warning string

* Correct expected and actual

* set database through environment variables

* reset environment variable at end of test

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Co-authored-by: Scott Bishel <scott.bishel@mattermost.com>

app/app_iface.go

@@ -308,6 +308,9 @@ type AppIface interface {
 	TeamMembersMinusGroupMembers(teamID string, groupIDs []string, page, perPage int) ([]*model.UserWithGroups, int64, *model.AppError)
 	// This function migrates the default built in roles from code/config to the database.
 	DoAdvancedPermissionsMigration()
+	// This function zip's up all the files in fileDatas array and then saves it to the directory specified with the specified zip file name
+	// Ensure the zip file name ends with a .zip
+	CreateZipFileAndAddFiles(fileBackend filesstore.FileBackend, fileDatas []model.FileData, zipFileName, directory string) error
 	// This to be used for places we check the users password when they are already logged in
 	DoubleCheckPassword(user *model.User, password string) *model.AppError
 	// UpdateBotActive marks a bot as active or inactive, along with its corresponding user.
@@ -506,6 +509,7 @@ type AppIface interface {
 	FindTeamByName(name string) bool
 	GenerateMfaSecret(userId string) (*model.MfaSecret, *model.AppError)
 	GeneratePublicLink(siteURL string, info *model.FileInfo) string
+	GenerateSupportPacket() []model.FileData
 	GetActivePluginManifests() ([]*model.Manifest, *model.AppError)
 	GetAllChannels(page, perPage int, opts model.ChannelSearchOpts) (*model.ChannelListWithTeamData, *model.AppError)
 	GetAllChannelsCount(opts model.ChannelSearchOpts) (int64, *model.AppError)

app/file.go

@@ -4,6 +4,7 @@
 package app
 
 import (
+	"archive/zip"
 	"bytes"
 	"crypto/sha256"
 	"encoding/base64"
@@ -18,6 +19,8 @@ import (
 	"mime/multipart"
 	"net/http"
 	"net/url"
+	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -1258,3 +1261,49 @@ func (a *App) CopyFileInfos(userId string, fileIds []string) ([]string, *model.A
 
 	return newFileIds, nil
 }
+
+// This function zip's up all the files in fileDatas array and then saves it to the directory specified with the specified zip file name
+// Ensure the zip file name ends with a .zip
+func (a *App) CreateZipFileAndAddFiles(fileBackend filesstore.FileBackend, fileDatas []model.FileData, zipFileName, directory string) error {
+	// Create Zip File (temporarily stored on disk)
+	conglomerateZipFile, err := os.Create(zipFileName)
+	if err != nil {
+		return err
+	}
+	defer os.Remove(zipFileName)
+
+	// Create a new zip archive.
+	zipFileWriter := zip.NewWriter(conglomerateZipFile)
+
+	// Populate Zip file with File Datas array
+	err = populateZipfile(zipFileWriter, fileDatas)
+	if err != nil {
+		return err
+	}
+
+	conglomerateZipFile.Seek(0, 0)
+	_, err = fileBackend.WriteFile(conglomerateZipFile, path.Join(directory, zipFileName))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// This is a implementation of Go's example of writing files to zip (with slight modification)
+// https://golang.org/src/archive/zip/example_test.go
+func populateZipfile(w *zip.Writer, fileDatas []model.FileData) error {
+	defer w.Close()
+	for _, fd := range fileDatas {
+		f, err := w.Create(fd.Filename)
+		if err != nil {
+			return err
+		}
+
+		_, err = f.Write(fd.Body)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

app/file_test.go

@@ -4,6 +4,7 @@
 package app
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -14,6 +15,8 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v5/model"
+	"github.com/mattermost/mattermost-server/v5/plugin/plugintest/mock"
+	"github.com/mattermost/mattermost-server/v5/services/filesstore/mocks"
 	"github.com/mattermost/mattermost-server/v5/utils/fileutils"
 )
 
@@ -269,6 +272,23 @@ func TestMigrateFilenamesToFileInfos(t *testing.T) {
 	assert.Equal(t, 0, len(infos))
 }
 
+func TestCreateZipFileAndAddFiles(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	mockBackend := mocks.FileBackend{}
+	mockBackend.On("WriteFile", mock.Anything, "directory-to-heaven/zip-file-name-to-heaven.zip").Return(int64(666), errors.New("Only those who dare to fail greatly can ever achieve greatly"))
+
+	err := th.App.CreateZipFileAndAddFiles(&mockBackend, []model.FileData{}, "zip-file-name-to-heaven.zip", "directory-to-heaven")
+	require.NotNil(t, err)
+	require.Equal(t, err.Error(), "Only those who dare to fail greatly can ever achieve greatly")
+
+	mockBackend = mocks.FileBackend{}
+	mockBackend.On("WriteFile", mock.Anything, "directory-to-heaven/zip-file-name-to-heaven.zip").Return(int64(666), nil)
+	err = th.App.CreateZipFileAndAddFiles(&mockBackend, []model.FileData{}, "zip-file-name-to-heaven.zip", "directory-to-heaven")
+	require.Nil(t, err)
+}
+
 func TestCopyFileInfos(t *testing.T) {
 	th := Setup(t)
 	defer th.TearDown()

app/opentracing/opentracing_layer.go

@@ -2416,6 +2416,28 @@ func (a *OpenTracingAppLayer) CreateWebhookPost(userId string, channel *model.Ch
 	return resultVar0, resultVar1
 }
 
+func (a *OpenTracingAppLayer) CreateZipFileAndAddFiles(fileBackend filesstore.FileBackend, fileDatas []model.FileData, zipFileName string, directory string) error {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.CreateZipFileAndAddFiles")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0 := a.app.CreateZipFileAndAddFiles(fileBackend, fileDatas, zipFileName, directory)
+
+	if resultVar0 != nil {
+		span.LogFields(spanlog.Error(resultVar0))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0
+}
+
 func (a *OpenTracingAppLayer) DBHealthCheckDelete() error {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DBHealthCheckDelete")
@@ -3943,6 +3965,23 @@ func (a *OpenTracingAppLayer) GeneratePublicLink(siteURL string, info *model.Fil
 	return resultVar0
 }
 
+func (a *OpenTracingAppLayer) GenerateSupportPacket() []model.FileData {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GenerateSupportPacket")
+
+	a.ctx = newCtx
+	a.app.Srv().Store.SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store.SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0 := a.app.GenerateSupportPacket()
+
+	return resultVar0
+}
+
 func (a *OpenTracingAppLayer) GetActivePluginManifests() ([]*model.Manifest, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetActivePluginManifests")

app/server.go

@@ -6,8 +6,10 @@ package app
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"hash/maphash"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
@@ -23,6 +25,8 @@ import (
 	"syscall"
 	"time"
 
+	"gopkg.in/yaml.v2"
+
 	"github.com/getsentry/sentry-go"
 	sentryhttp "github.com/getsentry/sentry-go/http"
 	"github.com/gorilla/mux"
@@ -636,6 +640,12 @@ func (s *Server) AppOptions() []AppOption {
 	}
 }
 
+// Return Database type (postgres or mysql) and current version of Mattermost
+func (s *Server) DatabaseTypeAndMattermostVersion() (string, string) {
+	mattermostVersion, _ := s.Store.System().GetByName("Version")
+	return *s.Config().SqlSettings.DriverName, mattermostVersion.Value
+}
+
 // initLogging initializes and configures the logger. This may be called more than once.
 func (s *Server) initLogging() error {
 	if s.Log == nil {
@@ -1611,3 +1621,178 @@ func (s *Server) HttpService() httpservice.HTTPService {
 func (s *Server) SetLog(l *mlog.Logger) {
 	s.Log = l
 }
+
+func (a *App) GenerateSupportPacket() []model.FileData {
+	// If any errors we come across within this function, we will log it in a warning.txt file so that we know why certain files did not get produced if any
+	var warnings []string
+
+	// Creating an array of files that we are going to be adding to our zip file
+	fileDatas := []model.FileData{}
+
+	// A array of the functions that we can iterate through since they all have the same return value
+	functions := []func() (*model.FileData, string){
+		a.generateSupportPacketYaml,
+		a.createPluginsFile,
+		a.createSanitizedConfigFile,
+		a.getMattermostLog,
+		a.getNotificationsLog,
+	}
+
+	for _, fn := range functions {
+		fileData, warning := fn()
+
+		if fileData != nil {
+			fileDatas = append(fileDatas, *fileData)
+		} else {
+			warnings = append(warnings, warning)
+		}
+	}
+
+	// Adding a warning.txt file to the fileDatas if any warning
+	if len(warnings) > 0 {
+		finalWarning := strings.Join(warnings, "\n")
+		fileDatas = append(fileDatas, model.FileData{
+			Filename: "warning.txt",
+			Body:     []byte(finalWarning),
+		})
+	}
+
+	return fileDatas
+}
+
+func (a *App) getNotificationsLog() (*model.FileData, string) {
+	var warning string
+
+	// Getting notifications.log
+	if *a.Srv().Config().NotificationLogSettings.EnableFile {
+		// notifications.log
+		notificationsLog := utils.GetNotificationsLogFileLocation(*a.Srv().Config().LogSettings.FileLocation)
+
+		notificationsLogFileData, notificationsLogFileDataErr := ioutil.ReadFile(notificationsLog)
+
+		if notificationsLogFileDataErr == nil {
+			fileData := model.FileData{
+				Filename: "notifications.log",
+				Body:     notificationsLogFileData,
+			}
+			return &fileData, ""
+		}
+
+		warning = fmt.Sprintf("ioutil.ReadFile(notificationsLog) Error: %s", notificationsLogFileDataErr.Error())
+
+	} else {
+		warning = "Unable to retrieve notifications.log because LogSettings: EnableFile is false in config.json"
+	}
+
+	return nil, warning
+}
+
+func (a *App) getMattermostLog() (*model.FileData, string) {
+	var warning string
+
+	// Getting mattermost.log
+	if *a.Srv().Config().LogSettings.EnableFile {
+		// mattermost.log
+		mattermostLog := utils.GetLogFileLocation(*a.Srv().Config().LogSettings.FileLocation)
+
+		mattermostLogFileData, mattermostLogFileDataErr := ioutil.ReadFile(mattermostLog)
+
+		if mattermostLogFileDataErr == nil {
+			fileData := model.FileData{
+				Filename: "mattermost.log",
+				Body:     mattermostLogFileData,
+			}
+			return &fileData, ""
+		}
+		warning = fmt.Sprintf("ioutil.ReadFile(mattermostLog) Error: %s", mattermostLogFileDataErr.Error())
+
+	} else {
+		warning = "Unable to retrieve mattermost.log because LogSettings: EnableFile is false in config.json"
+	}
+
+	return nil, warning
+}
+
+func (a *App) createSanitizedConfigFile() (*model.FileData, string) {
+	// Getting sanitized config, prettifying it, and then adding it to our file data array
+	sanitizedConfigPrettyJSON, err := json.MarshalIndent(a.GetSanitizedConfig(), "", "    ")
+	if err == nil {
+		fileData := model.FileData{
+			Filename: "sanitized_config.json",
+			Body:     sanitizedConfigPrettyJSON,
+		}
+		return &fileData, ""
+	}
+
+	warning := fmt.Sprintf("json.MarshalIndent(c.App.GetSanitizedConfig()) Error: %s", err.Error())
+	return nil, warning
+}
+
+func (a *App) createPluginsFile() (*model.FileData, string) {
+	var warning string
+
+	// Getting the plugins installed on the server, prettify it, and then add them to the file data array
+	pluginsResponse, appErr := a.GetPlugins()
+	if appErr == nil {
+		pluginsPrettyJSON, err := json.MarshalIndent(pluginsResponse, "", "    ")
+		if err == nil {
+			fileData := model.FileData{
+				Filename: "plugins.json",
+				Body:     pluginsPrettyJSON,
+			}
+
+			return &fileData, ""
+		}
+
+		warning = fmt.Sprintf("json.MarshalIndent(pluginsResponse) Error: %s", err.Error())
+	} else {
+		warning = fmt.Sprintf("c.App.GetPlugins() Error: %s", appErr.Error())
+	}
+
+	return nil, warning
+}
+
+func (a *App) generateSupportPacketYaml() (*model.FileData, string) {
+	// Here we are getting information regarding Elastic Search
+	var elasticServerVersion string
+	var elasticServerPlugins []string
+	if a.Srv().SearchEngine.ElasticsearchEngine != nil {
+		elasticServerVersion = a.Srv().SearchEngine.ElasticsearchEngine.GetFullVersion()
+		elasticServerPlugins = a.Srv().SearchEngine.ElasticsearchEngine.GetPlugins()
+	}
+
+	// Here we are getting information regarding LDAP
+	ldapInterface := a.Srv().Ldap
+	var vendorName, vendorVersion string
+	if ldapInterface != nil {
+		vendorName, vendorVersion = ldapInterface.GetVendorNameAndVendorVersion()
+	}
+
+	// Here we are getting information regarding the database (mysql/postgres + current Mattermost version)
+	databaseType, databaseVersion := a.Srv().DatabaseTypeAndMattermostVersion()
+
+	// Creating the struct for support packet yaml file
+	supportPacket := model.SupportPacket{
+		ServerOS:             runtime.GOOS,
+		ServerArchitecture:   runtime.GOARCH,
+		DatabaseType:         databaseType,
+		DatabaseVersion:      databaseVersion,
+		LdapVendorName:       vendorName,
+		LdapVendorVersion:    vendorVersion,
+		ElasticServerVersion: elasticServerVersion,
+		ElasticServerPlugins: elasticServerPlugins,
+	}
+
+	// Marshal to a Yaml File
+	supportPacketYaml, err := yaml.Marshal(&supportPacket)
+	if err == nil {
+		fileData := model.FileData{
+			Filename: "support_packet.yaml",
+			Body:     supportPacketYaml,
+		}
+		return &fileData, ""
+	}
+
+	warning := fmt.Sprintf("yaml.Marshal(&supportPacket) Error: %s", err.Error())
+	return nil, warning
+}

app/server_test.go

@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/getsentry/sentry-go"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/mattermost/mattermost-server/v5/config"
@@ -161,6 +162,184 @@ func TestStartServerTLSSuccess(t *testing.T) {
 	require.NoError(t, serverErr)
 }
 
+func TestDatabaseTypeAndMattermostVersion(t *testing.T) {
+	sqlDrivernameEnvironment := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
+	defer os.Setenv("MM_SQLSETTINGS_DRIVERNAME", sqlDrivernameEnvironment)
+
+	os.Setenv("MM_SQLSETTINGS_DRIVERNAME", "postgres")
+
+	th := Setup(t)
+	defer th.TearDown()
+
+	databaseType, mattermostVersion := th.Server.DatabaseTypeAndMattermostVersion()
+	assert.Equal(t, "postgres", databaseType)
+	assert.Equal(t, "5.31.0", mattermostVersion)
+
+	os.Setenv("MM_SQLSETTINGS_DRIVERNAME", "mysql")
+
+	th2 := Setup(t)
+	defer th2.TearDown()
+
+	databaseType, mattermostVersion = th2.Server.DatabaseTypeAndMattermostVersion()
+	assert.Equal(t, "mysql", databaseType)
+	assert.Equal(t, "5.31.0", mattermostVersion)
+}
+
+func TestGenerateSupportPacket(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	d1 := []byte("hello\ngo\n")
+	err := ioutil.WriteFile("mattermost.log", d1, 0777)
+	require.Nil(t, err)
+	err = ioutil.WriteFile("notifications.log", d1, 0777)
+	require.Nil(t, err)
+
+	fileDatas := th.App.GenerateSupportPacket()
+	testFiles := []string{"support_packet.yaml", "plugins.json", "sanitized_config.json", "mattermost.log", "notifications.log"}
+	for i, fileData := range fileDatas {
+		require.NotNil(t, fileData)
+		assert.Equal(t, testFiles[i], fileData.Filename)
+		assert.Positive(t, len(fileData.Body))
+	}
+
+	// Remove these two files and ensure that warning.txt file is generated
+	err = os.Remove("notifications.log")
+	require.Nil(t, err)
+	err = os.Remove("mattermost.log")
+	require.Nil(t, err)
+	fileDatas = th.App.GenerateSupportPacket()
+	testFiles = []string{"support_packet.yaml", "plugins.json", "sanitized_config.json", "warning.txt"}
+	for i, fileData := range fileDatas {
+		require.NotNil(t, fileData)
+		assert.Equal(t, testFiles[i], fileData.Filename)
+		assert.Positive(t, len(fileData.Body))
+	}
+}
+
+func TestGetNotificationsLog(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	// Disable notifications file to get an error
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.NotificationLogSettings.EnableFile = false
+	})
+
+	fileData, warning := th.App.getNotificationsLog()
+	assert.Nil(t, fileData)
+	assert.Equal(t, warning, "Unable to retrieve notifications.log because LogSettings: EnableFile is false in config.json")
+
+	// Enable notifications file but delete any notifications file to get an error trying to read the file
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.NotificationLogSettings.EnableFile = true
+	})
+
+	// If any previous notifications.log file, lets delete it
+	os.Remove("notifications.log")
+
+	fileData, warning = th.App.getNotificationsLog()
+	assert.Nil(t, fileData)
+	assert.Contains(t, warning, "ioutil.ReadFile(notificationsLog) Error:")
+
+	// Happy path where we have file and no warning
+	d1 := []byte("hello\ngo\n")
+	err := ioutil.WriteFile("notifications.log", d1, 0777)
+	defer os.Remove("notifications.log")
+	require.Nil(t, err)
+
+	fileData, warning = th.App.getNotificationsLog()
+	require.NotNil(t, fileData)
+	assert.Equal(t, "notifications.log", fileData.Filename)
+	assert.Positive(t, len(fileData.Body))
+	assert.Empty(t, warning)
+}
+
+func TestGetMattermostLog(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	// disable mattermost log file setting in config so we should get an warning
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.LogSettings.EnableFile = false
+	})
+
+	fileData, warning := th.App.getMattermostLog()
+	assert.Nil(t, fileData)
+	assert.Equal(t, "Unable to retrieve mattermost.log because LogSettings: EnableFile is false in config.json", warning)
+
+	// We enable the setting but delete any mattermost log file
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.LogSettings.EnableFile = true
+	})
+
+	// If any previous mattermost.log file, lets delete it
+	os.Remove("mattermost.log")
+
+	fileData, warning = th.App.getMattermostLog()
+	assert.Nil(t, fileData)
+	assert.Contains(t, warning, "ioutil.ReadFile(mattermostLog) Error:")
+
+	// Happy path where we get a log file and no warning
+	d1 := []byte("hello\ngo\n")
+	err := ioutil.WriteFile("mattermost.log", d1, 0777)
+	defer os.Remove("mattermost.log")
+	require.Nil(t, err)
+
+	fileData, warning = th.App.getMattermostLog()
+	require.NotNil(t, fileData)
+	assert.Equal(t, "mattermost.log", fileData.Filename)
+	assert.Positive(t, len(fileData.Body))
+	assert.Empty(t, warning)
+}
+
+func TestCreateSanitizedConfigFile(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	// Happy path where we have a sanitized config file with no warning
+	fileData, warning := th.App.createSanitizedConfigFile()
+	require.NotNil(t, fileData)
+	assert.Equal(t, "sanitized_config.json", fileData.Filename)
+	assert.Positive(t, len(fileData.Body))
+	assert.Empty(t, warning)
+}
+
+func TestCreatePluginsFile(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	// Happy path where we have a plugins file with no warning
+	fileData, warning := th.App.createPluginsFile()
+	require.NotNil(t, fileData)
+	assert.Equal(t, "plugins.json", fileData.Filename)
+	assert.Positive(t, len(fileData.Body))
+	assert.Empty(t, warning)
+
+	// Turn off plugins so we can get an error
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.PluginSettings.Enable = false
+	})
+
+	// Plugins off in settings so no fileData and we get a warning instead
+	fileData, warning = th.App.createPluginsFile()
+	assert.Nil(t, fileData)
+	assert.Contains(t, warning, "c.App.GetPlugins() Error:")
+}
+
+func TestGenerateSupportPacketYaml(t *testing.T) {
+	th := Setup(t)
+	defer th.TearDown()
+
+	// Happy path where we have a support packet yaml file without any warnings
+	fileData, warning := th.App.generateSupportPacketYaml()
+	require.NotNil(t, fileData)
+	assert.Equal(t, "support_packet.yaml", fileData.Filename)
+	assert.Positive(t, len(fileData.Body))
+	assert.Empty(t, warning)
+
+}
+
 func TestStartServerTLSVersion(t *testing.T) {
 	s, err := NewServer()
 	require.NoError(t, err)