server/public/ -- pre-requisite changes (#23278)

* invert depdendency: filestore -> model

* markdown: nolint:misspell

* inline jsonutils within model

* push model.GetInfoForBytes -> channels/app

* push channel/utils.CompileGo* -> plugin/utils

* push plugin/scheduler -> channels/jobs/plugins

* push utils.Copy(File|Dir) -> model

* oauthproiders/gitlab -> channels/app/oauthproviders/gitlab

* decouple plugin from einterfaces.MetricsInterface

* fix TestGetInfoForFile

* Revert "Run golangci in server CI (#23240)"

This reverts commit 349e5d4573.

* add model/utils

---------

Co-authored-by: Agniva De Sarker <agnivade@yahoo.co.in>

server/channels/app/emoji.go

@@ -131,7 +131,7 @@ func (a *App) UploadEmojiImage(c request.CTX, id string, imageData *multipart.Fi
 	if config.Width > MaxEmojiWidth || config.Height > MaxEmojiHeight {
 		data := buf.Bytes()
 		newbuf := bytes.NewBuffer(nil)
-		info, err := model.GetInfoForBytes(imageData.Filename, bytes.NewReader(data), len(data))
+		info, err := getInfoForBytes(imageData.Filename, bytes.NewReader(data), len(data))
 		if err != nil {
 			return err
 		}

server/channels/app/file.go

@@ -66,7 +66,7 @@ func (a *App) FileBackend() filestore.FileBackend {
 }
 
 func (a *App) CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError {
-	fileBackendSettings := settings.ToFileBackendSettings(false, false)
+	fileBackendSettings := filestore.NewFileBackendSettingsFromConfig(settings, false, false)
 	err := fileBackendSettings.CheckMandatoryS3Fields()
 	if err != nil {
 		return model.NewAppError("CheckMandatoryS3Fields", "api.admin.test_s3.missing_s3_bucket", nil, "", http.StatusBadRequest).Wrap(err)
@@ -96,7 +96,7 @@ func (a *App) TestFileStoreConnection() *model.AppError {
 func (a *App) TestFileStoreConnectionWithConfig(cfg *model.FileSettings) *model.AppError {
 	license := a.Srv().License()
 	insecure := a.Config().ServiceSettings.EnableInsecureOutgoingConnections
-	backend, err := filestore.NewFileBackend(cfg.ToFileBackendSettings(license != nil && *license.Features.Compliance, insecure != nil && *insecure))
+	backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(cfg, license != nil && *license.Features.Compliance, insecure != nil && *insecure))
 	if err != nil {
 		return model.NewAppError("FileBackend", "api.file.no_driver.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
 	}
@@ -260,7 +260,7 @@ func (a *App) getInfoForFilename(post *model.Post, teamID, channelID, userID, ol
 		return nil
 	}
 
-	info, err := model.GetInfoForBytes(name, bytes.NewReader(data), len(data))
+	info, err := getInfoForBytes(name, bytes.NewReader(data), len(data))
 	if err != nil {
 		mlog.Warn(
 			"Unable to fully decode file info when migrating post to use FileInfos",
@@ -879,7 +879,7 @@ func (a *App) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTe
 	channelID := filepath.Base(rawChannelId)
 	userID := filepath.Base(rawUserId)
 
-	info, err := model.GetInfoForBytes(filename, bytes.NewReader(data), len(data))
+	info, err := getInfoForBytes(filename, bytes.NewReader(data), len(data))
 	if err != nil {
 		err.StatusCode = http.StatusBadRequest
 		return nil, data, err

server/channels/app/file_info.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package app
+
+import (
+	"image"
+	"io"
+	"mime"
+	"net/http"
+	"path/filepath"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/server/v8/channels/utils/imgutils"
+	"github.com/mattermost/mattermost-server/server/v8/model"
+)
+
+func getInfoForBytes(name string, data io.ReadSeeker, size int) (*model.FileInfo, *model.AppError) {
+	info := &model.FileInfo{
+		Name: name,
+		Size: int64(size),
+	}
+	var err *model.AppError
+
+	extension := strings.ToLower(filepath.Ext(name))
+	info.MimeType = mime.TypeByExtension(extension)
+
+	if extension != "" {
+		// The client expects a file extension without the leading period
+		info.Extension = extension[1:]
+	} else {
+		info.Extension = extension
+	}
+
+	if info.IsImage() {
+		// Only set the width and height if it's actually an image that we can understand
+		if config, _, err := image.DecodeConfig(data); err == nil {
+			info.Width = config.Width
+			info.Height = config.Height
+
+			if info.MimeType == "image/gif" {
+				// Just show the gif itself instead of a preview image for animated gifs
+				data.Seek(0, io.SeekStart)
+				frameCount, err := imgutils.CountGIFFrames(data)
+				if err != nil {
+					// Still return the rest of the info even though it doesn't appear to be an actual gif
+					info.HasPreviewImage = true
+					return info, model.NewAppError("getInfoForBytes", "app.file_info.get.gif.app_error", nil, "", http.StatusBadRequest).Wrap(err)
+				}
+				info.HasPreviewImage = frameCount == 1
+			} else {
+				info.HasPreviewImage = true
+			}
+		}
+	}
+
+	return info, err
+}

server/channels/app/file_info_test.go

@@ -0,0 +1,148 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package app
+
+import (
+	"bytes"
+	"encoding/base64"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetInfoForFile(t *testing.T) {
+	fakeFile := make([]byte, 1000)
+
+	pngFile, err := os.ReadFile("tests/test.png")
+	require.NoError(t, err, "Failed to load test.png")
+
+	// base 64 encoded version of handtinywhite.gif from http://probablyprogramming.com/2009/03/15/the-tiniest-gif-ever
+	gifFile, _ := base64.StdEncoding.DecodeString("R0lGODlhAQABAIABAP///wAAACwAAAAAAQABAAACAkQBADs=")
+
+	animatedGifFile, err := os.ReadFile("tests/testgif.gif")
+	require.NoError(t, err, "Failed to load testgif.gif")
+
+	var ttc = []struct {
+		testName                string
+		filename                string
+		file                    []byte
+		usePrefixForMime        bool
+		expectedExtension       string
+		expectedSize            int
+		expectedMime            string
+		expectedWidth           int
+		expectedHeight          int
+		expectedHasPreviewImage bool
+	}{
+		{
+			testName:                "Text File",
+			filename:                "file.txt",
+			file:                    fakeFile,
+			usePrefixForMime:        true,
+			expectedExtension:       "txt",
+			expectedSize:            1000,
+			expectedMime:            "text/plain",
+			expectedWidth:           0,
+			expectedHeight:          0,
+			expectedHasPreviewImage: false,
+		},
+		{
+			testName:                "PNG file",
+			filename:                "test.png",
+			file:                    pngFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "png",
+			expectedSize:            279591,
+			expectedMime:            "image/png",
+			expectedWidth:           408,
+			expectedHeight:          336,
+			expectedHasPreviewImage: true,
+		},
+		{
+			testName:                "Static Gif File",
+			filename:                "handtinywhite.gif",
+			file:                    gifFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "gif",
+			expectedSize:            35,
+			expectedMime:            "image/gif",
+			expectedWidth:           1,
+			expectedHeight:          1,
+			expectedHasPreviewImage: true,
+		},
+		{
+			testName:                "Animated Gif File",
+			filename:                "testgif.gif",
+			file:                    animatedGifFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "gif",
+			expectedSize:            38689,
+			expectedMime:            "image/gif",
+			expectedWidth:           118,
+			expectedHeight:          118,
+			expectedHasPreviewImage: false,
+		},
+		{
+			testName:                "No extension File",
+			filename:                "filewithoutextension",
+			file:                    fakeFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "",
+			expectedSize:            1000,
+			expectedMime:            "",
+			expectedWidth:           0,
+			expectedHeight:          0,
+			expectedHasPreviewImage: false,
+		},
+		{
+			// Always make the extension lower case to make it easier to use in other places
+			testName:                "Uppercase extension File",
+			filename:                "file.TXT",
+			file:                    fakeFile,
+			usePrefixForMime:        true,
+			expectedExtension:       "txt",
+			expectedSize:            1000,
+			expectedMime:            "text/plain",
+			expectedWidth:           0,
+			expectedHeight:          0,
+			expectedHasPreviewImage: false,
+		},
+		{
+			// Don't error out for image formats we don't support
+			testName:                "Not supported File",
+			filename:                "file.tif",
+			file:                    fakeFile,
+			usePrefixForMime:        false,
+			expectedExtension:       "tif",
+			expectedSize:            1000,
+			expectedMime:            "image/tiff",
+			expectedWidth:           0,
+			expectedHeight:          0,
+			expectedHasPreviewImage: false,
+		},
+	}
+
+	for _, tc := range ttc {
+		t.Run(tc.testName, func(t *testing.T) {
+			info, appErr := getInfoForBytes(tc.filename, bytes.NewReader(tc.file), len(tc.file))
+			require.Nil(t, appErr)
+
+			assert.Equalf(t, tc.filename, info.Name, "Got incorrect filename: %v", info.Name)
+			assert.Equalf(t, tc.expectedExtension, info.Extension, "Got incorrect extension: %v", info.Extension)
+			assert.EqualValuesf(t, tc.expectedSize, info.Size, "Got incorrect size: %v", info.Size)
+			assert.Equalf(t, tc.expectedWidth, info.Width, "Got incorrect width: %v", info.Width)
+			assert.Equalf(t, tc.expectedHeight, info.Height, "Got incorrect height: %v", info.Height)
+			assert.Equalf(t, tc.expectedHasPreviewImage, info.HasPreviewImage, "Got incorrect has preview image: %v", info.HasPreviewImage)
+
+			if tc.usePrefixForMime {
+				assert.Truef(t, strings.HasPrefix(info.MimeType, tc.expectedMime), "Got incorrect mime type: %v", info.MimeType)
+			} else {
+				assert.Equalf(t, tc.expectedMime, info.MimeType, "Got incorrect mime type: %v", info.MimeType)
+			}
+		})
+	}
+}

server/channels/app/oauthproviders/gitlab/gitlab.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package oauthgitlab
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/server/v8/channels/einterfaces"
+	"github.com/mattermost/mattermost-server/server/v8/model"
+)
+
+type GitLabProvider struct {
+}
+
+type GitLabUser struct {
+	Id       int64  `json:"id"`
+	Username string `json:"username"`
+	Login    string `json:"login"`
+	Email    string `json:"email"`
+	Name     string `json:"name"`
+}
+
+func init() {
+	provider := &GitLabProvider{}
+	einterfaces.RegisterOAuthProvider(model.UserAuthServiceGitlab, provider)
+}
+
+func userFromGitLabUser(glu *GitLabUser) *model.User {
+	user := &model.User{}
+	username := glu.Username
+	if username == "" {
+		username = glu.Login
+	}
+	user.Username = model.CleanUsername(username)
+	splitName := strings.Split(glu.Name, " ")
+	if len(splitName) == 2 {
+		user.FirstName = splitName[0]
+		user.LastName = splitName[1]
+	} else if len(splitName) >= 2 {
+		user.FirstName = splitName[0]
+		user.LastName = strings.Join(splitName[1:], " ")
+	} else {
+		user.FirstName = glu.Name
+	}
+	user.Email = glu.Email
+	user.Email = strings.ToLower(user.Email)
+	userId := glu.getAuthData()
+	user.AuthData = &userId
+	user.AuthService = model.UserAuthServiceGitlab
+
+	return user
+}
+
+func gitLabUserFromJSON(data io.Reader) (*GitLabUser, error) {
+	decoder := json.NewDecoder(data)
+	var glu GitLabUser
+	err := decoder.Decode(&glu)
+	if err != nil {
+		return nil, err
+	}
+	return &glu, nil
+}
+
+func (glu *GitLabUser) IsValid() error {
+	if glu.Id == 0 {
+		return errors.New("user id can't be 0")
+	}
+
+	if glu.Email == "" {
+		return errors.New("user e-mail should not be empty")
+	}
+
+	return nil
+}
+
+func (glu *GitLabUser) getAuthData() string {
+	return strconv.FormatInt(glu.Id, 10)
+}
+
+func (m *GitLabProvider) GetUserFromJSON(data io.Reader, tokenUser *model.User) (*model.User, error) {
+	glu, err := gitLabUserFromJSON(data)
+	if err != nil {
+		return nil, err
+	}
+	if err = glu.IsValid(); err != nil {
+		return nil, err
+	}
+
+	return userFromGitLabUser(glu), nil
+}
+
+func (m *GitLabProvider) GetSSOSettings(config *model.Config, service string) (*model.SSOSettings, error) {
+	return &config.GitLabSettings, nil
+}
+
+func (m *GitLabProvider) GetUserFromIdToken(idToken string) (*model.User, error) {
+	return nil, nil
+}
+
+func (m *GitLabProvider) IsSameUser(dbUser, oauthUser *model.User) bool {
+	return dbUser.AuthData == oauthUser.AuthData
+}

server/channels/app/platform/service.go

@@ -225,7 +225,7 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
 	// Step 3: Initialize filestore
 	if ps.filestore == nil {
 		insecure := ps.Config().ServiceSettings.EnableInsecureOutgoingConnections
-		backend, err2 := filestore.NewFileBackend(ps.Config().FileSettings.ToFileBackendSettings(license != nil && *license.Features.Compliance, insecure != nil && *insecure))
+		backend, err2 := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&ps.Config().FileSettings, license != nil && *license.Features.Compliance, insecure != nil && *insecure))
 		if err2 != nil {
 			return nil, fmt.Errorf("failed to initialize filebackend: %w", err2)
 		}

server/channels/app/plugin_api_test.go

@@ -27,11 +27,11 @@ import (
 
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/request"
 	"github.com/mattermost/mattermost-server/server/v8/channels/einterfaces/mocks"
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils"
 	"github.com/mattermost/mattermost-server/server/v8/channels/utils/fileutils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/i18n"
 	"github.com/mattermost/mattermost-server/server/v8/plugin"
+	"github.com/mattermost/mattermost-server/server/v8/plugin/utils"
 )
 
 func getDefaultPluginSettingsSchema() string {

server/channels/app/plugin_hooks_test.go

@@ -21,10 +21,10 @@ import (
 
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/request"
 	"github.com/mattermost/mattermost-server/server/v8/channels/einterfaces/mocks"
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
 	"github.com/mattermost/mattermost-server/server/v8/plugin"
 	"github.com/mattermost/mattermost-server/server/v8/plugin/plugintest"
+	"github.com/mattermost/mattermost-server/server/v8/plugin/utils"
 )
 
 func SetAppEnvironmentWithPlugins(t *testing.T, pluginCode []string, app *App, apiFunc func(*model.Manifest) plugin.API) (func(), []string, []error) {

server/channels/app/plugin_install.go

@@ -45,8 +45,8 @@ import (
 
 	"github.com/blang/semver"
 
-	"github.com/mattermost/mattermost-server/server/v8/channels/utils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
+	"github.com/mattermost/mattermost-server/server/v8/model/utils"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/filestore"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mlog"
 	"github.com/mattermost/mattermost-server/server/v8/plugin"

server/channels/app/server.go

@@ -47,6 +47,7 @@ import (
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/last_accessible_post"
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/migrations"
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/notify_admin"
+	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/plugins"
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/product_notices"
 	"github.com/mattermost/mattermost-server/server/v8/channels/jobs/resend_invitation_email"
 	"github.com/mattermost/mattermost-server/server/v8/channels/product"
@@ -70,7 +71,6 @@ import (
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mail"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/mlog"
 	"github.com/mattermost/mattermost-server/server/v8/platform/shared/templates"
-	"github.com/mattermost/mattermost-server/server/v8/plugin/scheduler"
 )
 
 // declaring this as var to allow overriding in tests
@@ -1500,8 +1500,8 @@ func (s *Server) initJobs() {
 
 	s.Jobs.RegisterJobType(
 		model.JobTypePlugins,
-		scheduler.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
-		scheduler.MakeScheduler(s.Jobs),
+		plugins.MakeWorker(s.Jobs, New(ServerConnector(s.Channels()))),
+		plugins.MakeScheduler(s.Jobs),
 	)
 
 	s.Jobs.RegisterJobType(

server/channels/app/user_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
+	oauthgitlab "github.com/mattermost/mattermost-server/server/v8/channels/app/oauthproviders/gitlab"
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/request"
 	"github.com/mattermost/mattermost-server/server/v8/channels/app/users"
 	"github.com/mattermost/mattermost-server/server/v8/channels/einterfaces"
@@ -26,7 +27,6 @@ import (
 	storemocks "github.com/mattermost/mattermost-server/server/v8/channels/store/storetest/mocks"
 	"github.com/mattermost/mattermost-server/server/v8/channels/utils/testutils"
 	"github.com/mattermost/mattermost-server/server/v8/model"
-	oauthgitlab "github.com/mattermost/mattermost-server/server/v8/model/oauthproviders/gitlab"
 )
 
 func TestCreateOAuthUser(t *testing.T) {

server/channels/app/users/profile_picture.go

@@ -59,7 +59,7 @@ func (us *UserService) GetProfileImage(user *model.User) ([]byte, bool, error) {
 func (us *UserService) FileBackend() (filestore.FileBackend, error) {
 	license := us.license()
 	insecure := us.config().ServiceSettings.EnableInsecureOutgoingConnections
-	backend, err := filestore.NewFileBackend(us.config().FileSettings.ToFileBackendSettings(license != nil && *license.Features.Compliance, insecure != nil && *insecure))
+	backend, err := filestore.NewFileBackend(filestore.NewFileBackendSettingsFromConfig(&us.config().FileSettings, license != nil && *license.Features.Compliance, insecure != nil && *insecure))
 	if err != nil {
 		return nil, err
 	}