server/public/ -- pre-requisite changes (#23278)

* invert depdendency: filestore -> model

* markdown: nolint:misspell

* inline jsonutils within model

* push model.GetInfoForBytes -> channels/app

* push channel/utils.CompileGo* -> plugin/utils

* push plugin/scheduler -> channels/jobs/plugins

* push utils.Copy(File|Dir) -> model

* oauthproiders/gitlab -> channels/app/oauthproviders/gitlab

* decouple plugin from einterfaces.MetricsInterface

* fix TestGetInfoForFile

* Revert "Run golangci in server CI (#23240)"

This reverts commit 349e5d4573.

* add model/utils

---------

Co-authored-by: Agniva De Sarker <agnivade@yahoo.co.in>

server/channels/api4/user_test.go

@@ -26,7 +26,7 @@ import (
 	"github.com/mattermost/mattermost-server/server/v8/model"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mail"
 
-	_ "github.com/mattermost/mattermost-server/server/v8/model/oauthproviders/gitlab"
+	_ "github.com/mattermost/mattermost-server/server/v8/channels/app/oauthproviders/gitlab"
 )
 
 func TestCreateUser(t *testing.T) {

server/channels/app/emoji.go

@@ -131,7 +131,7 @@ func (a *App) UploadEmojiImage(c request.CTX, id string, imageData *multipart.Fi
 	if config.Width > MaxEmojiWidth || config.Height > MaxEmojiHeight {
 		data := buf.Bytes()
 		newbuf := bytes.NewBuffer(nil)
-		info, err := model.GetInfoForBytes(imageData.Filename, bytes.NewReader(data), len(data))
+		info, err := getInfoForBytes(imageData.Filename, bytes.NewReader(data), len(data))
 		if err != nil {
 			return err
 		}

server/channels/app/file.go

@@ -66,7 +66,7 @@ func (a *App) FileBackend() filestore.FileBackend {
 }
 
 func (a *App) CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError {
-	fileBackendSettings := settings.ToFileBackendSettings(false, false)
+	fileBackendSettings := filestore.NewFileBackendSettingsFromConfig(settings, false, false)
 	err := fileBackendSettings.CheckMandatoryS3Fields()
 	if err != nil {
 		return model.NewAppError("CheckMandatoryS3Fields", "api.admin.test_s3.missing_s3_bucket", nil, "", http.StatusBadRequest).Wrap(err)
@@ -96,7 +96,7 @@ func (a *App) TestFileStoreConnection() *model.AppError {
 func (a *App) TestFileStoreConnectionWithConfig(cfg *model.FileSettings) *model.AppError {
 	license := a.Srv().License()
 	insecure := a.Config().ServiceSettings.EnableInsecureOutgoingConnections
-	backend, err := filestore.NewFileBackend(cfg.ToFileBackendSettings(license != nil && *license.Features.Compliance, insecure != nil && *insecure))
+	backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(cfg, license != nil && *license.Features.Compliance, insecure != nil && *insecure))
 	if err != nil {
 		return model.NewAppError("FileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
 	}
@@ -260,7 +260,7 @@ func (a *App) getInfoForFilename(post *model.Post, teamID, channelID, userID, ol
 		return nil
 	}
 
-	info, err := model.GetInfoForBytes(name, bytes.NewReader(data), len(data))
+	info, err := getInfoForBytes(name, bytes.NewReader(data), len(data))
 	if err != nil {
 		mlog.Warn(
 			"Unable to fully decode file info when migrating post to use FileInfos",
@@ -879,7 +879,7 @@ func (a *App) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTe
 	channelID := filepath.Base(rawChannelId)
 	userID := filepath.Base(rawUserId)
 
-	info, err := model.GetInfoForBytes(filename, bytes.NewReader(data), len(data))
+	info, err := getInfoForBytes(filename, bytes.NewReader(data), len(data))
 	if err != nil {
 		err.StatusCode = http.StatusBadRequest
 		return nil, data, err

server/channels/app/file_info.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package app
+
+import (
+	"image"
+	"io"
+	"mime"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/server/v8/channels/utils/imgutils"
+	"github.com/mattermost/mattermost-server/server/v8/model"
+)
+
+func getInfoForBytes(name string, data io.ReadSeeker, size int) (*model.FileInfo, *model.AppError) {
+	info := &model.FileInfo{
+		Name: name,
+		Size: int64(size),
+	}
+	var err *model.AppError
+
+	extension := strings.ToLower(filepath.Ext(name))
+	info.MimeType = mime.TypeByExtension(extension)
+
+	if extension != "" {
+		// The client expects a file extension without the leading period
+		info.Extension = extension[1:]
+	} else {
+		info.Extension = extension
+	}
+
+	if info.IsImage() {
+		// Only set the width and height if it's actually an image that we can understand
+		if config, _, err := image.DecodeConfig(data); err == nil {
+			info.Width = config.Width
+			info.Height = config.Height
+
+			if info.MimeType == "image/gif" {
+				// Just show the gif itself instead of a preview image for animated gifs
+				data.Seek(0, io.SeekStart)
+				frameCount, err := imgutils.CountGIFFrames(data)
+				if err != nil {
+					// Still return the rest of the info even though it doesn't appear to be an actual gif
+					info.HasPreviewImage = true
+					return info, model.NewAppError("getInfoForBytes", "app.file_info.get.gif.app_error", nil, "", http.StatusBadRequest).Wrap(err)
+				}
+				info.HasPreviewImage = frameCount == 1
+			} else {
+				info.HasPreviewImage = true
+			}
+		}
+	}
+
+	return info, err
+}

server/channels/app/file_info_test.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package app
+
+import (
+	"bytes"
+	"encoding/base64"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetInfoForFile(t *testing.T) {
+	fakeFile := make([]byte, 1000)
+
+	pngFile, err := os.ReadFile("tests/test.png")
+	require.NoError(t, err, "Failed to load test.png")
+
+	// base 64 encoded version of handtinywhite.gif from http://probablyprogramming.com/2009/03/15/the-tiniest-gif-ever
+	gifFile, _ := base64.StdEncoding.DecodeString("R0lGODlhAQABAIABAP///wAAACwAAAAAAQABAAACAkQBADs=")
+
+	animatedGifFile, err := os.ReadFile("tests/testgif.gif")
+	require.NoError(t, err, "Failed to load testgif.gif")
+
+	var ttc = []struct {
+		testName                string
+		filename                string
+		file                    []byte
+		usePrefixForMime        bool
+		expectedExtension       string
+		expectedSize            int
+		expectedMime            string
+		expectedWidth           int
+		expectedHeight          int
+		expectedHasPreviewImage bool
+	}{
+		{
+			testName:                "Text File",
+			filename:                "file.txt",
+			file:                    fakeFile,
+			usePrefixForMime:        true,
+			expectedExtension:       "txt",
+			expectedSize:            1000,
+			expectedMime:            "text/plain",
+			expectedWidth:           0,
+			expectedHeight:          0,
+			expectedHasPreviewImage: false,
+		},
+		{
+			testName:                "PNG file",
+			filename:                "test.png",
+			file:                    pngFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "png",
+			expectedSize:            279591,
+			expectedMime:            "image/png",
+			expectedWidth:           408,
+			expectedHeight:          336,
+			expectedHasPreviewImage: true,
+		},
+		{
+			testName:                "Static Gif File",
+			filename:                "handtinywhite.gif",
+			file:                    gifFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "gif",
+			expectedSize:            35,
+			expectedMime:            "image/gif",
+			expectedWidth:           1,
+			expectedHeight:          1,
+			expectedHasPreviewImage: true,
+		},
+		{
+			testName:                "Animated Gif File",
+			filename:                "testgif.gif",
+			file:                    animatedGifFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "gif",
+			expectedSize:            38689,
+			expectedMime:            "image/gif",
+			expectedWidth:           118,
+			expectedHeight:          118,
+			expectedHasPreviewImage: false,
+		},
+		{
+			testName:                "No extension File",
+			filename:                "filewithoutextension",
+			file:                    fakeFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "",
+			expectedSize:            1000,
+			expectedMime:            "",
+			expectedWidth:           0,
+			expectedHeight:          0,
+			expectedHasPreviewImage: false,
+		},
+		{
+			// Always make the extension lower case to make it easier to use in other places
+			testName:                "Uppercase extension File",
+			filename:                "file.TXT",
+			file:                    fakeFile,
+			usePrefixForMime:        true,
+			expectedExtension:       "txt",
+			expectedSize:            1000,
+			expectedMime:            "text/plain",
+			expectedWidth:           0,
+			expectedHeight:          0,
+			expectedHasPreviewImage: false,
+		},
+		{
+			// Don't error out for image formats we don't support
+			testName:                "Not supported File",
+			filename:                "file.tif",
+			file:                    fakeFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "tif",
+			expectedSize:            1000,
+			expectedMime:            "image/tiff",
+			expectedWidth:           0,
+			expectedHeight:          0,
+			expectedHasPreviewImage: false,
+		},
+	}
+
+	for _, tc := range ttc {
+		t.Run(tc.testName, func(t *testing.T) {
+			info, appErr := getInfoForBytes(tc.filename, bytes.NewReader(tc.file), len(tc.file))
+			require.Nil(t, appErr)
+
+			assert.Equalf(t, tc.filename, info.Name, "Got incorrect filename: %v", info.Name)
+			assert.Equalf(t, tc.expectedExtension, info.Extension, "Got incorrect extension: %v", info.Extension)
+			assert.EqualValuesf(t, tc.expectedSize, info.Size, "Got incorrect size: %v", info.Size)
+			assert.Equalf(t, tc.expectedWidth, info.Width, "Got incorrect width: %v", info.Width)
+			assert.Equalf(t, tc.expectedHeight, info.Height, "Got incorrect height: %v", info.Height)
+			assert.Equalf(t, tc.expectedHasPreviewImage, info.HasPreviewImage, "Got incorrect has preview image: %v", info.HasPreviewImage)
+
+			if tc.usePrefixForMime {
+				assert.Truef(t, strings.HasPrefix(info.MimeType, tc.expectedMime), "Got incorrect mime type: %v", info.MimeType)
+			} else {
+				assert.Equalf(t, tc.expectedMime, info.MimeType, "Got incorrect mime type: %v", info.MimeType)
+			}
+		})
+	}
+}

server/channels/app/oauthproviders/gitlab/gitlab.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package oauthgitlab
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/server/v8/channels/einterfaces"
+	"github.com/mattermost/mattermost-server/server/v8/model"
+)
+
+type GitLabProvider struct {
+}
+
+type GitLabUser struct {
+	Id       int64  `json:"id"`
+	Username string `json:"username"`
+	Login    string `json:"login"`
+	Email    string `json:"email"`
+	Name     string `json:"name"`
+}
+
+func init() {
+	provider := &GitLabProvider{}
+	einterfaces.RegisterOAuthProvider(model.UserAuthServiceGitlab, provider)
+}
+
+func userFromGitLabUser(glu *GitLabUser) *model.User {
+	user := &model.User{}
+	username := glu.Username
+	if username == "" {
+		username = glu.Login
+	}
+	user.Username = model.CleanUsername(username)
+	splitName := strings.Split(glu.Name, " ")
+	if len(splitName) == 2 {
+		user.FirstName = splitName[0]
+		user.LastName = splitName[1]
+	} else if len(splitName) >= 2 {
+		user.FirstName = splitName[0]
+		user.LastName = strings.Join(splitName[1:], " ")
+	} else {
+		user.FirstName = glu.Name
+	}
+	user.Email = glu.Email
+	user.Email = strings.ToLower(user.Email)
+	userId := glu.getAuthData()
+	user.AuthData = &userId
+	user.AuthService = model.UserAuthServiceGitlab
+
+	return user
+}
+
+func gitLabUserFromJSON(data io.Reader) (*GitLabUser, error) {
+	decoder := json.NewDecoder(data)
+	var glu GitLabUser
+	err := decoder.Decode(&glu)
+	if err != nil {
+		return nil, err
+	}
+	return &glu, nil
+}
+
+func (glu *GitLabUser) IsValid() error {
+	if glu.Id == 0 {
+		return errors.New("user id can't be 0")
+	}
+
+	if glu.Email == "" {
+		return errors.New("user e-mail should not be empty")
+	}
+
+	return nil
+}
+
+func (glu *GitLabUser) getAuthData() string {
+	return strconv.FormatInt(glu.Id, 10)
+}
+
+func (m *GitLabProvider) GetUserFromJSON(data io.Reader, tokenUser *model.User) (*model.User, error) {
+	glu, err := gitLabUserFromJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	if err = glu.IsValid(); err != nil {
+		return nil, err
+	}
+
+	return userFromGitLabUser(glu), nil
+}
+
+func (m *GitLabProvider) GetSSOSettings(config *model.Config, service string) (*model.SSOSettings, error) {
+	return &config.GitLabSettings, nil
+}
+
+func (m *GitLabProvider) GetUserFromIdToken(idToken string) (*model.User, error) {
+	return nil, nil
+}
+
+func (m *GitLabProvider) IsSameUser(dbUser, oauthUser *model.User) bool {
+	return dbUser.AuthData == oauthUser.AuthData
+}

server/channels/app/platform/service.go

@@ -225,7 +225,7 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 	// Step 3: Initialize filestore
 	if ps.filestore == nil {
 		insecure := ps.Config().ServiceSettings.EnableInsecureOutgoingConnections
-		backend, err2 := filestore.NewFileBackend(ps.Config().FileSettings.ToFileBackendSettings(license != nil && *license.Features.Compliance, insecure != nil && *insecure))
+		backend, err2 := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&ps.Config().FileSettings, license != nil && *license.Features.Compliance, insecure != nil && *insecure))
 		if err2 != nil {
 			return nil, fmt.Errorf("failed to initialize filebackend: %w", err2)
 		}

server/channels/app/plugin_api_test.go

@@ -27,11 +27,11 @@ import (
 
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/request"
 	"github.com/mattermost/mattermost-server/server/v8/channels/einterfaces/mocks"
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils"
 	"github.com/mattermost/mattermost-server/server/v8/channels/utils/fileutils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/i18n"
 	"github.com/mattermost/mattermost-server/server/v8/plugin"
+	"github.com/mattermost/mattermost-server/server/v8/plugin/utils"
 )
 
 func getDefaultPluginSettingsSchema() string {

server/channels/app/plugin_hooks_test.go

@@ -21,10 +21,10 @@ import (
 
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/request"
 	"github.com/mattermost/mattermost-server/server/v8/channels/einterfaces/mocks"
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
 	"github.com/mattermost/mattermost-server/server/v8/plugin"
 	"github.com/mattermost/mattermost-server/server/v8/plugin/plugintest"
+	"github.com/mattermost/mattermost-server/server/v8/plugin/utils"
 )
 
 func SetAppEnvironmentWithPlugins(t *testing.T, pluginCode []string, app *App, apiFunc func(*model.Manifest) plugin.API) (func(), []string, []error) {

server/channels/app/plugin_install.go

@@ -45,8 +45,8 @@ import (
 
 	"github.com/blang/semver"
 
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
+	"github.com/mattermost/mattermost-server/server/v8/model/utils"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/filestore"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mlog"
 	"github.com/mattermost/mattermost-server/server/v8/plugin"

server/channels/app/server.go

@@ -47,6 +47,7 @@ import (
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/last_accessible_post"
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/migrations"
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/notify_admin"
+	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/plugins"
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/product_notices"
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/resend_invitation_email"
 	"github.com/mattermost/mattermost-server/server/v8/channels/product"
@@ -70,7 +71,6 @@ import (
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mail"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mlog"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/templates"
-	"github.com/mattermost/mattermost-server/server/v8/plugin/scheduler"
 )
 
 // declaring this as var to allow overriding in tests
@@ -1500,8 +1500,8 @@ func (s *Server) initJobs() {
 
 	s.Jobs.RegisterJobType(
 		model.JobTypePlugins,
-		scheduler.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
-		scheduler.MakeScheduler(s.Jobs),
+		plugins.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
+		plugins.MakeScheduler(s.Jobs),
 	)
 
 	s.Jobs.RegisterJobType(

server/channels/app/user_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
+	oauthgitlab "github.com/mattermost/mattermost-server/server/v8/channels/app/oauthproviders/gitlab"
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/request"
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/users"
 	"github.com/mattermost/mattermost-server/server/v8/channels/einterfaces"
@@ -26,7 +27,6 @@ import (
 	storemocks "github.com/mattermost/mattermost-server/server/v8/channels/store/storetest/mocks"
 	"github.com/mattermost/mattermost-server/server/v8/channels/utils/testutils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
-	oauthgitlab "github.com/mattermost/mattermost-server/server/v8/model/oauthproviders/gitlab"
 )
 
 func TestCreateOAuthUser(t *testing.T) {

server/channels/app/users/profile_picture.go

@@ -59,7 +59,7 @@ func (us *UserService) GetProfileImage(user *model.User) ([]byte, bool, error) {
 func (us *UserService) FileBackend() (filestore.FileBackend, error) {
 	license := us.license()
 	insecure := us.config().ServiceSettings.EnableInsecureOutgoingConnections
-	backend, err := filestore.NewFileBackend(us.config().FileSettings.ToFileBackendSettings(license != nil && *license.Features.Compliance, insecure != nil && *insecure))
+	backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&us.config().FileSettings, license != nil && *license.Features.Compliance, insecure != nil && *insecure))
 	if err != nil {
 		return nil, err
 	}

server/channels/jobs/plugins/scheduler.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package plugins
+
+import (
+	"time"
+
+	"github.com/mattermost/mattermost-server/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost-server/server/v8/model"
+)
+
+const schedFreq = 24 * time.Hour
+
+func MakeScheduler(jobServer *jobs.JobServer) model.Scheduler {
+	isEnabled := func(cfg *model.Config) bool {
+		return true
+	}
+	return jobs.NewPeriodicScheduler(jobServer, model.JobTypePlugins, schedFreq, isEnabled)
+}

server/channels/jobs/plugins/worker.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package plugins
+
+import (
+	"github.com/mattermost/mattermost-server/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost-server/server/v8/model"
+	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mlog"
+)
+
+type AppIface interface {
+	DeleteAllExpiredPluginKeys() *model.AppError
+}
+
+type Worker struct {
+	name      string
+	stop      chan bool
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	app       AppIface
+}
+
+func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
+	worker := Worker{
+		name:      "Plugins",
+		stop:      make(chan bool, 1),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		app:       app,
+	}
+
+	return &worker
+}
+
+func (worker *Worker) Run() {
+	mlog.Debug("Worker started", mlog.String("worker", worker.name))
+
+	defer func() {
+		mlog.Debug("Worker finished", mlog.String("worker", worker.name))
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			mlog.Debug("Worker received stop signal", mlog.String("worker", worker.name))
+			return
+		case job := <-worker.jobs:
+			mlog.Debug("Worker received a new candidate job.", mlog.String("worker", worker.name))
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *Worker) Stop() {
+	mlog.Debug("Worker stopping", mlog.String("worker", worker.name))
+	worker.stop <- true
+	<-worker.stopped
+}
+
+func (worker *Worker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *Worker) IsEnabled(cfg *model.Config) bool {
+	return true
+}
+
+func (worker *Worker) DoJob(job *model.Job) {
+	if claimed, err := worker.jobServer.ClaimJob(job); err != nil {
+		mlog.Info("Worker experienced an error while trying to claim job",
+			mlog.String("worker", worker.name),
+			mlog.String("job_id", job.Id),
+			mlog.String("error", err.Error()))
+		return
+	} else if !claimed {
+		return
+	}
+
+	if err := worker.app.DeleteAllExpiredPluginKeys(); err != nil {
+		mlog.Error("Worker: Failed to delete expired keys", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+		worker.setJobError(job, err)
+		return
+	}
+
+	mlog.Info("Worker: Job is complete", mlog.String("worker", worker.name), mlog.String("job_id", job.Id))
+	worker.setJobSuccess(job)
+}
+
+func (worker *Worker) setJobSuccess(job *model.Job) {
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		mlog.Error("Worker: Failed to set success for job", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+		worker.setJobError(job, err)
+	}
+}
+
+func (worker *Worker) setJobError(job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		mlog.Error("Worker: Failed to set job error", mlog.String("worker", worker.name), mlog.String("job_id", job.Id), mlog.String("error", err.Error()))
+	}
+}

server/channels/testlib/resources.go

@@ -12,9 +12,9 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils"
 	"github.com/mattermost/mattermost-server/server/v8/channels/utils/fileutils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
+	"github.com/mattermost/mattermost-server/server/v8/model/utils"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/filestore"
 )
 

server/channels/utils/file.go

@@ -5,119 +5,9 @@ package utils
 
 import (
 	"errors"
-	"fmt"
 	"io"
-	"os"
-	"path/filepath"
 )
 
-// CopyFile will copy a file from src path to dst path.
-// Overwrites any existing files at dst.
-// Permissions are copied from file at src to the new file at dst.
-func CopyFile(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return
-	}
-	defer in.Close()
-
-	if err = os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil {
-		return
-	}
-	out, err := os.Create(dst)
-	if err != nil {
-		return
-	}
-	defer func() {
-		if e := out.Close(); e != nil {
-			err = e
-		}
-	}()
-
-	_, err = io.Copy(out, in)
-	if err != nil {
-		return
-	}
-
-	err = out.Sync()
-	if err != nil {
-		return
-	}
-
-	stat, err := os.Stat(src)
-	if err != nil {
-		return
-	}
-	err = os.Chmod(dst, stat.Mode())
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-// CopyDir will copy a directory and all contained files and directories.
-// src must exist and dst must not exist.
-// Permissions are preserved when possible. Symlinks are skipped.
-func CopyDir(src string, dst string) (err error) {
-	src = filepath.Clean(src)
-	dst = filepath.Clean(dst)
-
-	stat, err := os.Stat(src)
-	if err != nil {
-		return
-	}
-	if !stat.IsDir() {
-		return fmt.Errorf("source must be a directory")
-	}
-
-	_, err = os.Stat(dst)
-	if err != nil && !os.IsNotExist(err) {
-		return
-	}
-	if err == nil {
-		return fmt.Errorf("destination already exists")
-	}
-
-	err = os.MkdirAll(dst, stat.Mode())
-	if err != nil {
-		return
-	}
-
-	items, err := os.ReadDir(src)
-	if err != nil {
-		return
-	}
-
-	for _, item := range items {
-		srcPath := filepath.Join(src, item.Name())
-		dstPath := filepath.Join(dst, item.Name())
-
-		if item.IsDir() {
-			err = CopyDir(srcPath, dstPath)
-			if err != nil {
-				return
-			}
-		} else {
-			info, ierr := item.Info()
-			if ierr != nil {
-				continue
-			}
-
-			if info.Mode()&os.ModeSymlink != 0 {
-				continue
-			}
-
-			err = CopyFile(srcPath, dstPath)
-			if err != nil {
-				return
-			}
-		}
-	}
-
-	return
-}
-
 var SizeLimitExceeded = errors.New("Size limit exceeded")
 
 type LimitedReaderWithError struct {

server/channels/utils/file_test.go

@@ -7,63 +7,11 @@ import (
 	"bytes"
 	"crypto/rand"
 	"io"
-	"os"
-	"path/filepath"
 	"testing"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestCopyDir(t *testing.T) {
-	srcDir, err := os.MkdirTemp("", "src")
-	require.NoError(t, err)
-	defer os.RemoveAll(srcDir)
-
-	dstParentDir, err := os.MkdirTemp("", "dstparent")
-	require.NoError(t, err)
-	defer os.RemoveAll(dstParentDir)
-
-	dstDir := filepath.Join(dstParentDir, "dst")
-
-	tempFile := "temp.txt"
-	err = os.WriteFile(filepath.Join(srcDir, tempFile), []byte("test file"), 0655)
-	require.NoError(t, err)
-
-	childDir := "child"
-	err = os.Mkdir(filepath.Join(srcDir, childDir), 0777)
-	require.NoError(t, err)
-
-	childTempFile := "childtemp.txt"
-	err = os.WriteFile(filepath.Join(srcDir, childDir, childTempFile), []byte("test file"), 0755)
-	require.NoError(t, err)
-
-	err = CopyDir(srcDir, dstDir)
-	assert.NoError(t, err)
-
-	stat, err := os.Stat(filepath.Join(dstDir, tempFile))
-	assert.NoError(t, err)
-	assert.Equal(t, uint32(0655), uint32(stat.Mode()))
-	assert.False(t, stat.IsDir())
-	data, err := os.ReadFile(filepath.Join(dstDir, tempFile))
-	assert.NoError(t, err)
-	assert.Equal(t, "test file", string(data))
-
-	stat, err = os.Stat(filepath.Join(dstDir, childDir))
-	assert.NoError(t, err)
-	assert.True(t, stat.IsDir())
-
-	stat, err = os.Stat(filepath.Join(dstDir, childDir, childTempFile))
-	assert.NoError(t, err)
-	assert.Equal(t, uint32(0755), uint32(stat.Mode()))
-	assert.False(t, stat.IsDir())
-	data, err = os.ReadFile(filepath.Join(dstDir, childDir, childTempFile))
-	assert.NoError(t, err)
-	assert.Equal(t, "test file", string(data))
-
-	err = CopyDir(srcDir, dstDir)
-	assert.Error(t, err)
-}
 func TestLimitedReaderWithError(t *testing.T) {
 	t.Run("read less than max size", func(t *testing.T) {
 		maxBytes := 10

server/channels/utils/jsonutils/json.go

@@ -1,56 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.txt for license information.
-
-package jsonutils
-
-import (
-	"bytes"
-	"encoding/json"
-
-	"github.com/pkg/errors"
-)
-
-type HumanizedJSONError struct {
-	Err       error
-	Line      int
-	Character int
-}
-
-func (e *HumanizedJSONError) Error() string {
-	return e.Err.Error()
-}
-
-// HumanizeJSONError extracts error offsets and annotates the error with useful context
-func HumanizeJSONError(err error, data []byte) error {
-	if syntaxError, ok := err.(*json.SyntaxError); ok {
-		return NewHumanizedJSONError(syntaxError, data, syntaxError.Offset)
-	} else if unmarshalError, ok := err.(*json.UnmarshalTypeError); ok {
-		return NewHumanizedJSONError(unmarshalError, data, unmarshalError.Offset)
-	} else {
-		return err
-	}
-}
-
-func NewHumanizedJSONError(err error, data []byte, offset int64) *HumanizedJSONError {
-	if err == nil {
-		return nil
-	}
-
-	if offset < 0 || offset > int64(len(data)) {
-		return &HumanizedJSONError{
-			Err: errors.Wrapf(err, "invalid offset %d", offset),
-		}
-	}
-
-	lineSep := []byte{'\n'}
-
-	line := bytes.Count(data[:offset], lineSep) + 1
-	lastLineOffset := bytes.LastIndex(data[:offset], lineSep)
-	character := int(offset) - (lastLineOffset + 1) + 1
-
-	return &HumanizedJSONError{
-		Line:      line,
-		Character: character,
-		Err:       errors.Wrapf(err, "parsing error at line %d, character %d", line, character),
-	}
-}

server/channels/utils/jsonutils/json_test.go

@@ -1,235 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.txt for license information.
-
-package jsonutils_test
-
-import (
-	"encoding/json"
-	"reflect"
-	"testing"
-
-	"github.com/pkg/errors"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils/jsonutils"
-)
-
-func TestHumanizeJsonError(t *testing.T) {
-	t.Parallel()
-
-	type testType struct{}
-
-	testCases := []struct {
-		Description string
-		Data        []byte
-		Err         error
-		ExpectedErr string
-	}{
-		{
-			"nil error",
-			[]byte{},
-			nil,
-			"",
-		},
-		{
-			"non-special error",
-			[]byte{},
-			errors.New("test"),
-			"test",
-		},
-		{
-			"syntax error, offset 17, middle of line 3",
-			[]byte("line 1\nline 2\nline 3"),
-			&json.SyntaxError{
-				// msg can't be set
-				Offset: 17,
-			},
-			"parsing error at line 3, character 4: ",
-		},
-		{
-			"unmarshal type error, offset 17, middle of line 3",
-			[]byte("line 1\nline 2\nline 3"),
-			&json.UnmarshalTypeError{
-				Value:  "bool",
-				Type:   reflect.TypeOf(testType{}),
-				Offset: 17,
-				Struct: "struct",
-				Field:  "field",
-			},
-			"parsing error at line 3, character 4: json: cannot unmarshal bool into Go struct field struct.field of type jsonutils_test.testType",
-		},
-	}
-
-	for _, testCase := range testCases {
-		testCase := testCase
-		t.Run(testCase.Description, func(t *testing.T) {
-			actual := jsonutils.HumanizeJSONError(testCase.Err, testCase.Data)
-			if testCase.ExpectedErr == "" {
-				assert.NoError(t, actual)
-			} else {
-				assert.EqualError(t, actual, testCase.ExpectedErr)
-			}
-		})
-	}
-}
-
-func TestNewHumanizedJSONError(t *testing.T) {
-	t.Parallel()
-
-	testCases := []struct {
-		Description string
-		Data        []byte
-		Offset      int64
-		Err         error
-		Expected    *jsonutils.HumanizedJSONError
-	}{
-		{
-			"nil error",
-			[]byte{},
-			0,
-			nil,
-			nil,
-		},
-		{
-			"offset -1, before start of string",
-			[]byte("line 1\nline 2\nline 3"),
-			-1,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err: errors.Wrap(errors.New("message"), "invalid offset -1"),
-			},
-		},
-		{
-			"offset 0, start of string",
-			[]byte("line 1\nline 2\nline 3"),
-			0,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 1, character 1"),
-				Line:      1,
-				Character: 1,
-			},
-		},
-		{
-			"offset 5, end of line 1",
-			[]byte("line 1\nline 2\nline 3"),
-			5,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 1, character 6"),
-				Line:      1,
-				Character: 6,
-			},
-		},
-		{
-			"offset 6, new line at end end of line 1",
-			[]byte("line 1\nline 2\nline 3"),
-			6,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 1, character 7"),
-				Line:      1,
-				Character: 7,
-			},
-		},
-		{
-			"offset 7, start of line 2",
-			[]byte("line 1\nline 2\nline 3"),
-			7,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 2, character 1"),
-				Line:      2,
-				Character: 1,
-			},
-		},
-		{
-			"offset 12, end of line 2",
-			[]byte("line 1\nline 2\nline 3"),
-			12,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 2, character 6"),
-				Line:      2,
-				Character: 6,
-			},
-		},
-		{
-			"offset 13, newline at end of line 2",
-			[]byte("line 1\nline 2\nline 3"),
-			13,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 2, character 7"),
-				Line:      2,
-				Character: 7,
-			},
-		},
-		{
-			"offset 17, middle of line 3",
-			[]byte("line 1\nline 2\nline 3"),
-			17,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 3, character 4"),
-				Line:      3,
-				Character: 4,
-			},
-		},
-		{
-			"offset 19, end of string",
-			[]byte("line 1\nline 2\nline 3"),
-			19,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 3, character 6"),
-				Line:      3,
-				Character: 6,
-			},
-		},
-		{
-			"offset 20, offset = length of string",
-			[]byte("line 1\nline 2\nline 3"),
-			20,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 3, character 7"),
-				Line:      3,
-				Character: 7,
-			},
-		},
-		{
-			"offset 21, offset = length of string, after newline",
-			[]byte("line 1\nline 2\nline 3\n"),
-			21,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err:       errors.Wrap(errors.New("message"), "parsing error at line 4, character 1"),
-				Line:      4,
-				Character: 1,
-			},
-		},
-		{
-			"offset 21, offset > length of string",
-			[]byte("line 1\nline 2\nline 3"),
-			21,
-			errors.New("message"),
-			&jsonutils.HumanizedJSONError{
-				Err: errors.Wrap(errors.New("message"), "invalid offset 21"),
-			},
-		},
-	}
-
-	for _, testCase := range testCases {
-		testCase := testCase
-		t.Run(testCase.Description, func(t *testing.T) {
-			actual := jsonutils.NewHumanizedJSONError(testCase.Err, testCase.Data, testCase.Offset)
-			if testCase.Expected != nil && actual.Err != nil {
-				if assert.EqualValues(t, testCase.Expected.Err.Error(), actual.Err.Error()) {
-					actual.Err = testCase.Expected.Err
-				}
-			}
-			assert.Equal(t, testCase.Expected, actual)
-		})
-	}
-}

server/channels/utils/test_files_compiler.go

@@ -1,73 +0,0 @@
-// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
-// See LICENSE.txt for license information.
-
-package utils
-
-import (
-	"bytes"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func CompileGo(t *testing.T, sourceCode, outputPath string) {
-	dir, err := os.MkdirTemp(".", "")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	dir, err = filepath.Abs(dir)
-	require.NoError(t, err)
-
-	// Write out main.go given the source code.
-	main := filepath.Join(dir, "main.go")
-	err = os.WriteFile(main, []byte(sourceCode), 0600)
-	require.NoError(t, err)
-
-	_, sourceFile, _, ok := runtime.Caller(0)
-	require.True(t, ok)
-	serverPath := filepath.Dir(filepath.Dir(sourceFile))
-
-	out := &bytes.Buffer{}
-	cmd := exec.Command("go", "build", "-o", outputPath, main)
-	cmd.Dir = serverPath
-	cmd.Stdout = out
-	cmd.Stderr = out
-	err = cmd.Run()
-	if err != nil {
-		t.Log("Go compile errors:\n", out.String())
-	}
-	require.NoError(t, err, "failed to compile go")
-}
-
-func CompileGoTest(t *testing.T, sourceCode, outputPath string) {
-	dir, err := os.MkdirTemp(".", "")
-	require.NoError(t, err)
-	defer os.RemoveAll(dir)
-
-	dir, err = filepath.Abs(dir)
-	require.NoError(t, err)
-
-	// Write out main.go given the source code.
-	main := filepath.Join(dir, "main_test.go")
-	err = os.WriteFile(main, []byte(sourceCode), 0600)
-	require.NoError(t, err)
-
-	_, sourceFile, _, ok := runtime.Caller(0)
-	require.True(t, ok)
-	serverPath := filepath.Dir(filepath.Dir(sourceFile))
-
-	out := &bytes.Buffer{}
-	cmd := exec.Command("go", "test", "-c", "-o", outputPath, main)
-	cmd.Dir = serverPath
-	cmd.Stdout = out
-	cmd.Stderr = out
-	err = cmd.Run()
-	if err != nil {
-		t.Log("Go compile errors:\n", out.String())
-	}
-	require.NoError(t, err, "failed to compile go")
-}

server/channels/web/web_test.go

@@ -21,11 +21,11 @@ import (
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/request"
 	"github.com/mattermost/mattermost-server/server/v8/channels/store/localcachelayer"
 	"github.com/mattermost/mattermost-server/server/v8/channels/store/storetest/mocks"
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils"
 	"github.com/mattermost/mattermost-server/server/v8/config"
 	"github.com/mattermost/mattermost-server/server/v8/model"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mlog"
 	"github.com/mattermost/mattermost-server/server/v8/plugin"
+	"github.com/mattermost/mattermost-server/server/v8/plugin/utils"
 )
 
 var apiClient *model.Client4