[MM-53454] Add export file settings + slash command for public link (#23915)

Co-authored-by: Mattermost Build <build@mattermost.com>
Этот коммит содержится в:
Julien Tant
2023-07-19 13:01:39 -07:00
коммит произвёл GitHub
родитель f3b1f33dff
Коммит 077c16ef61
29 изменённых файлов: 1024 добавлений и 135 удалений

Просмотреть файл

@@ -17,6 +17,7 @@ func (api *API) InitExport() {
api.BaseRoutes.Exports.Handle("", api.APISessionRequired(listExports)).Methods("GET")
api.BaseRoutes.Export.Handle("", api.APISessionRequired(deleteExport)).Methods("DELETE")
api.BaseRoutes.Export.Handle("", api.APISessionRequired(downloadExport)).Methods("GET")
api.BaseRoutes.Export.Handle("/presign-url", api.APISessionRequired(generatePresignURLExport)).Methods("POST")
}
func listExports(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -66,7 +67,7 @@ func downloadExport(c *Context, w http.ResponseWriter, r *http.Request) {
}
filePath := filepath.Join(*c.App.Config().ExportSettings.Directory, c.Params.ExportName)
if ok, err := c.App.FileExists(filePath); err != nil {
if ok, err := c.App.ExportFileExists(filePath); err != nil {
c.Err = err
return
} else if !ok {
@@ -74,7 +75,7 @@ func downloadExport(c *Context, w http.ResponseWriter, r *http.Request) {
return
}
file, err := c.App.FileReader(filePath)
file, err := c.App.ExportFileReader(filePath)
if err != nil {
c.Err = err
return
@@ -84,3 +85,30 @@ func downloadExport(c *Context, w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/zip")
http.ServeContent(w, r, c.Params.ExportName, time.Time{}, file)
}
func generatePresignURLExport(c *Context, w http.ResponseWriter, r *http.Request) {
auditRec := c.MakeAuditRecord("generatePresignURLExport", audit.Fail)
defer c.LogAuditRec(auditRec)
audit.AddEventParameter(auditRec, "export_name", c.Params.ExportName)
if !c.IsSystemAdmin() {
c.SetPermissionError(model.PermissionManageSystem)
return
}
res, appErr := c.App.GeneratePresignURLForExport(c.Params.ExportName)
if appErr != nil {
c.Err = appErr
return
}
data, err := json.Marshal(res)
if err != nil {
c.Err = model.NewAppError("generatePresignURLExport", "app.export.marshal.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
return
}
w.Write(data)
auditRec.Success()
}

Просмотреть файл

@@ -7,4 +7,5 @@ func (api *API) InitExportLocal() {
api.BaseRoutes.Exports.Handle("", api.APILocal(listExports)).Methods("GET")
api.BaseRoutes.Export.Handle("", api.APILocal(deleteExport)).Methods("DELETE")
api.BaseRoutes.Export.Handle("", api.APILocal(downloadExport)).Methods("GET")
api.BaseRoutes.Export.Handle("/presign-url", api.APILocal(generatePresignURLExport)).Methods("POST")
}

Просмотреть файл

@@ -56,6 +56,8 @@ type AppIface interface {
// AddUserToChannel adds a user to a given channel.
AddUserToChannel(c request.CTX, user *model.User, channel *model.Channel, skipTeamMemberIntegrityCheck bool) (*model.ChannelMember, *model.AppError)
// Caller must close the first return value
ExportFileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
// Caller must close the first return value
FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError)
// ChannelMembersMinusGroupMembers returns the set of users in the given channel minus the set of users in the given
// groups.
@@ -565,6 +567,9 @@ type AppIface interface {
DownloadFromURL(downloadURL string) ([]byte, error)
EnableUserAccessToken(token *model.UserAccessToken) *model.AppError
EnvironmentConfig(filter func(reflect.StructField) bool) map[string]any
ExportFileBackend() filestore.FileBackend
ExportFileExists(path string) (bool, *model.AppError)
ExportFileModTime(path string) (time.Time, *model.AppError)
ExportPermissions(w io.Writer) error
ExtractContentFromFileInfo(fileInfo *model.FileInfo) error
FetchSamlMetadataFromIdp(url string) ([]byte, *model.AppError)
@@ -578,6 +583,7 @@ type AppIface interface {
FindTeamByName(name string) bool
FinishSendAdminNotifyPost(trial bool, now int64, pluginBasedData map[string][]*model.NotifyAdminData)
GenerateMfaSecret(userID string) (*model.MfaSecret, *model.AppError)
GeneratePresignURLForExport(name string) (*model.PresignURLResponse, *model.AppError)
GeneratePublicLink(siteURL string, info *model.FileInfo) string
GenerateSupportPacket() []model.FileData
GetAcknowledgementsForPost(postID string) ([]*model.PostAcknowledgement, *model.AppError)
@@ -914,6 +920,7 @@ type AppIface interface {
ListAllCommands(teamID string, T i18n.TranslateFunc) ([]*model.Command, *model.AppError)
ListDirectory(path string) ([]string, *model.AppError)
ListDirectoryRecursively(path string) ([]string, *model.AppError)
ListExportDirectory(path string) ([]string, *model.AppError)
ListExports() ([]string, *model.AppError)
ListImports() ([]string, *model.AppError)
ListPluginKeys(pluginID string, page, perPage int) ([]string, *model.AppError)
@@ -978,6 +985,7 @@ type AppIface interface {
RemoveChannelsFromRetentionPolicy(policyID string, channelIDs []string) *model.AppError
RemoveCustomStatus(c request.CTX, userID string) *model.AppError
RemoveDirectory(path string) *model.AppError
RemoveExportFile(path string) *model.AppError
RemoveFile(path string) *model.AppError
RemoveLdapPrivateCertificate() *model.AppError
RemoveLdapPublicCertificate() *model.AppError
@@ -1178,6 +1186,8 @@ type AppIface interface {
VerifyEmailFromToken(c request.CTX, userSuppliedTokenString string) *model.AppError
VerifyUserEmail(userID, email string) *model.AppError
ViewChannel(c request.CTX, view *model.ChannelView, userID string, currentSessionId string, collapsedThreadsSupported bool) (map[string]int64, *model.AppError)
WriteExportFile(fr io.Reader, path string) (int64, *model.AppError)
WriteExportFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError)
WriteFile(fr io.Reader, path string) (int64, *model.AppError)
WriteFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError)
}

Просмотреть файл

@@ -35,11 +35,12 @@ type licenseSvc interface {
// Channels contains all channels related state.
type Channels struct {
srv *Server
cfgSvc product.ConfigService
filestore filestore.FileBackend
licenseSvc licenseSvc
routerSvc *routerService
srv *Server
cfgSvc product.ConfigService
filestore filestore.FileBackend
exportFilestore filestore.FileBackend
licenseSvc licenseSvc
routerSvc *routerService
postActionCookieSecret []byte
@@ -91,10 +92,11 @@ func init() {
return NewChannels(services)
},
Dependencies: map[product.ServiceKey]struct{}{
ServerKey: {},
product.ConfigKey: {},
product.LicenseKey: {},
product.FilestoreKey: {},
ServerKey: {},
product.ConfigKey: {},
product.LicenseKey: {},
product.FilestoreKey: {},
product.ExportFilestoreKey: {},
},
})
}
@@ -119,6 +121,7 @@ func NewChannels(services map[product.ServiceKey]any) (*Channels, error) {
product.ConfigKey,
product.LicenseKey,
product.FilestoreKey,
product.ExportFilestoreKey,
}
for _, svcKey := range requiredServices {
svc, ok := services[svcKey]
@@ -139,6 +142,12 @@ func NewChannels(services map[product.ServiceKey]any) (*Channels, error) {
return nil, errors.New("Filestore service did not satisfy FileBackend interface")
}
ch.filestore = filestore
case product.ExportFilestoreKey:
exportFilestore, ok := svc.(filestore.FileBackend)
if !ok {
return nil, errors.New("Export filestore service did not satisfy FileBackend interface")
}
ch.exportFilestore = exportFilestore
case product.LicenseKey:
svc, ok := svc.(licenseSvc)
if !ok {

Просмотреть файл

@@ -11,6 +11,7 @@ import (
"io"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"strings"
@@ -23,6 +24,7 @@ import (
"github.com/mattermost/mattermost/server/v8/channels/app/imports"
"github.com/mattermost/mattermost/server/v8/channels/app/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
)
// We use this map to identify the exportable preferences.
@@ -817,7 +819,7 @@ func (a *App) exportFile(outPath, filePath string, zipWr *zip.Writer) *model.App
}
func (a *App) ListExports() ([]string, *model.AppError) {
exports, appErr := a.ListDirectory(*a.Config().ExportSettings.Directory)
exports, appErr := a.ListExportDirectory(*a.Config().ExportSettings.Directory)
if appErr != nil {
return nil, appErr
}
@@ -830,16 +832,51 @@ func (a *App) ListExports() ([]string, *model.AppError) {
return results, nil
}
func (a *App) GeneratePresignURLForExport(name string) (*model.PresignURLResponse, *model.AppError) {
if !a.Config().FeatureFlags.EnableExportDirectDownload {
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.featureflag.app_error", nil, "", http.StatusInternalServerError)
}
if !*a.Config().FileSettings.DedicatedExportStore {
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.config.app_error", nil, "", http.StatusInternalServerError)
}
b := a.ExportFileBackend()
backend, ok := b.(filestore.FileBackendWithLinkGenerator)
if !ok {
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.driver.app_error", nil, "", http.StatusInternalServerError)
}
p := path.Join(*a.Config().ExportSettings.Directory, filepath.Base(name))
found, err := b.FileExists(p)
if err != nil {
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.fileexist.app_error", nil, "", http.StatusInternalServerError)
}
if !found {
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.notfound.app_error", nil, "", http.StatusInternalServerError)
}
link, exp, err := backend.GeneratePublicLink(p)
if err != nil {
return nil, model.NewAppError("GeneratePresignURLForExport", "app.eport.generate_presigned_url.link.app_error", nil, "", http.StatusInternalServerError)
}
return &model.PresignURLResponse{
URL: link,
Expiration: exp,
}, nil
}
func (a *App) DeleteExport(name string) *model.AppError {
filePath := filepath.Join(*a.Config().ExportSettings.Directory, name)
if ok, err := a.FileExists(filePath); err != nil {
if ok, err := a.ExportFileExists(filePath); err != nil {
return err
} else if !ok {
return nil
}
return a.RemoveFile(filePath)
return a.RemoveExportFile(filePath)
}
func updateJobProgress(logger mlog.LoggerIFace, store store.Store, job *model.Job, key string, value int) {

Просмотреть файл

@@ -65,6 +65,10 @@ func (a *App) FileBackend() filestore.FileBackend {
return a.ch.filestore
}
func (a *App) ExportFileBackend() filestore.FileBackend {
return a.ch.exportFilestore
}
func (a *App) CheckMandatoryS3Fields(settings *model.FileSettings) *model.AppError {
fileBackendSettings := filestore.NewFileBackendSettingsFromConfig(settings, false, false)
err := fileBackendSettings.CheckMandatoryS3Fields()
@@ -111,31 +115,56 @@ func (a *App) ReadFile(path string) ([]byte, *model.AppError) {
return a.ch.srv.ReadFile(path)
}
func (s *Server) fileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
result, nErr := s.FileBackend().Reader(path)
func fileReader(backend filestore.FileBackend, path string) (filestore.ReadCloseSeeker, *model.AppError) {
result, nErr := backend.Reader(path)
if nErr != nil {
return nil, model.NewAppError("FileReader", "api.file.file_reader.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return result, nil
}
func (s *Server) fileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
return fileReader(s.FileBackend(), path)
}
func (s *Server) exportFileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
return fileReader(s.ExportFileBackend(), path)
}
// Caller must close the first return value
func (a *App) FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
return a.Srv().fileReader(path)
}
// Caller must close the first return value
func (a *App) ExportFileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
return a.Srv().exportFileReader(path)
}
func (a *App) FileExists(path string) (bool, *model.AppError) {
return a.Srv().fileExists(path)
}
func (a *App) ExportFileExists(path string) (bool, *model.AppError) {
return a.Srv().exportFileExists(path)
}
func (s *Server) fileExists(path string) (bool, *model.AppError) {
result, nErr := s.FileBackend().FileExists(path)
return fileExists(s.FileBackend(), path)
}
func fileExists(backend filestore.FileBackend, path string) (bool, *model.AppError) {
result, nErr := backend.FileExists(path)
if nErr != nil {
return false, model.NewAppError("FileExists", "api.file.file_exists.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return result, nil
}
func (s *Server) exportFileExists(path string) (bool, *model.AppError) {
return fileExists(s.ExportFileBackend(), path)
}
func (a *App) FileSize(path string) (int64, *model.AppError) {
size, nErr := a.FileBackend().FileSize(path)
if nErr != nil {
@@ -144,8 +173,8 @@ func (a *App) FileSize(path string) (int64, *model.AppError) {
return size, nil
}
func (a *App) FileModTime(path string) (time.Time, *model.AppError) {
modTime, nErr := a.FileBackend().FileModTime(path)
func fileModTime(backend filestore.FileBackend, path string) (time.Time, *model.AppError) {
modTime, nErr := backend.FileModTime(path)
if nErr != nil {
return time.Time{}, model.NewAppError("FileModTime", "api.file.file_mod_time.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
@@ -153,6 +182,14 @@ func (a *App) FileModTime(path string) (time.Time, *model.AppError) {
return modTime, nil
}
func (a *App) FileModTime(path string) (time.Time, *model.AppError) {
return fileModTime(a.FileBackend(), path)
}
func (a *App) ExportFileModTime(path string) (time.Time, *model.AppError) {
return fileModTime(a.ExportFileBackend(), path)
}
func (a *App) MoveFile(oldPath, newPath string) *model.AppError {
nErr := a.FileBackend().MoveFile(oldPath, newPath)
if nErr != nil {
@@ -169,17 +206,33 @@ func (a *App) WriteFile(fr io.Reader, path string) (int64, *model.AppError) {
return a.Srv().writeFile(fr, path)
}
func (s *Server) writeFile(fr io.Reader, path string) (int64, *model.AppError) {
result, nErr := s.FileBackend().WriteFile(fr, path)
func writeFile(backend filestore.FileBackend, fr io.Reader, path string) (int64, *model.AppError) {
result, nErr := backend.WriteFile(fr, path)
if nErr != nil {
return result, model.NewAppError("WriteFile", "api.file.write_file.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return result, nil
}
func (s *Server) writeFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError) {
func (s *Server) writeFile(fr io.Reader, path string) (int64, *model.AppError) {
return writeFile(s.FileBackend(), fr, path)
}
func (s *Server) writeExportFile(fr io.Reader, path string) (int64, *model.AppError) {
return writeFile(s.ExportFileBackend(), fr, path)
}
func (a *App) WriteExportFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError) {
return a.Srv().writeExportFileContext(ctx, fr, path)
}
func (a *App) WriteExportFile(fr io.Reader, path string) (int64, *model.AppError) {
return a.Srv().writeExportFile(fr, path)
}
func writeFileContext(ctx context.Context, backend filestore.FileBackend, fr io.Reader, path string) (int64, *model.AppError) {
// Check if we can provide a custom context, otherwise just use the default method.
written, err := filestore.TryWriteFileContext(s.FileBackend(), ctx, fr, path)
written, err := filestore.TryWriteFileContext(backend, ctx, fr, path)
if err != nil {
return written, model.NewAppError("WriteFile", "api.file.write_file.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
}
@@ -187,6 +240,14 @@ func (s *Server) writeFileContext(ctx context.Context, fr io.Reader, path string
return written, nil
}
func (s *Server) writeFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError) {
return writeFileContext(ctx, s.FileBackend(), fr, path)
}
func (s *Server) writeExportFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError) {
return writeFileContext(ctx, s.ExportFileBackend(), fr, path)
}
func (a *App) AppendFile(fr io.Reader, path string) (int64, *model.AppError) {
result, nErr := a.FileBackend().AppendFile(fr, path)
if nErr != nil {
@@ -199,24 +260,43 @@ func (a *App) RemoveFile(path string) *model.AppError {
return a.Srv().removeFile(path)
}
func (s *Server) removeFile(path string) *model.AppError {
nErr := s.FileBackend().RemoveFile(path)
func (a *App) RemoveExportFile(path string) *model.AppError {
return a.Srv().removeExportFile(path)
}
func removeFile(backend filestore.FileBackend, path string) *model.AppError {
nErr := backend.RemoveFile(path)
if nErr != nil {
return model.NewAppError("RemoveFile", "api.file.remove_file.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return nil
}
func (s *Server) removeFile(path string) *model.AppError {
return removeFile(s.FileBackend(), path)
}
func (s *Server) removeExportFile(path string) *model.AppError {
return removeFile(s.ExportFileBackend(), path)
}
func (a *App) ListDirectory(path string) ([]string, *model.AppError) {
return a.Srv().listDirectory(path, false)
}
func (a *App) ListExportDirectory(path string) ([]string, *model.AppError) {
return a.Srv().listExportDirectory(path, false)
}
func (a *App) ListDirectoryRecursively(path string) ([]string, *model.AppError) {
return a.Srv().listDirectory(path, true)
}
func (s *Server) listDirectory(path string, recursion bool) ([]string, *model.AppError) {
backend := s.FileBackend()
return listDirectory(s.FileBackend(), path, recursion)
}
func listDirectory(backend filestore.FileBackend, path string, recursion bool) ([]string, *model.AppError) {
var paths []string
var nErr error
@@ -227,12 +307,16 @@ func (s *Server) listDirectory(path string, recursion bool) ([]string, *model.Ap
}
if nErr != nil {
return nil, model.NewAppError("ListDirectory", "api.file.list_directory.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
return nil, model.NewAppError("ListExportDirectory", "api.file.list_directory.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
}
return paths, nil
}
func (s *Server) listExportDirectory(path string, recursion bool) ([]string, *model.AppError) {
return listDirectory(s.ExportFileBackend(), path, recursion)
}
func (a *App) RemoveDirectory(path string) *model.AppError {
nErr := a.FileBackend().RemoveDirectory(path)
if nErr != nil {

Просмотреть файл

@@ -1454,13 +1454,15 @@ func TestPushNotificationRace(t *testing.T) {
ConfigStore: memoryStore,
},
platform.SetFileStore(&fmocks.FileBackend{}),
platform.SetExportFileStore(&fmocks.FileBackend{}),
platform.StoreOverride(mockStore))
require.NoError(t, err)
serviceMap := map[product.ServiceKey]any{
ServerKey: s,
product.ConfigKey: s.platform,
product.LicenseKey: &licenseWrapper{s},
product.FilestoreKey: s.FileBackend(),
ServerKey: s,
product.ConfigKey: s.platform,
product.LicenseKey: &licenseWrapper{s},
product.FilestoreKey: s.FileBackend(),
product.ExportFilestoreKey: s.ExportFileBackend(),
}
ch, err := NewChannels(serviceMap)
require.NoError(t, err)

Просмотреть файл

@@ -4104,6 +4104,89 @@ func (a *OpenTracingAppLayer) ExecuteCommand(c request.CTX, args *model.CommandA
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) ExportFileBackend() filestore.FileBackend {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ExportFileBackend")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0 := a.app.ExportFileBackend()
return resultVar0
}
func (a *OpenTracingAppLayer) ExportFileExists(path string) (bool, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ExportFileExists")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.ExportFileExists(path)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) ExportFileModTime(path string) (time.Time, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ExportFileModTime")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.ExportFileModTime(path)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) ExportFileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ExportFileReader")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.ExportFileReader(path)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) ExportPermissions(w io.Writer) error {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ExportPermissions")
@@ -4478,6 +4561,28 @@ func (a *OpenTracingAppLayer) GenerateMfaSecret(userID string) (*model.MfaSecret
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) GeneratePresignURLForExport(name string) (*model.PresignURLResponse, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GeneratePresignURLForExport")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.GeneratePresignURLForExport(name)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) GeneratePublicLink(siteURL string, info *model.FileInfo) string {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GeneratePublicLink")
@@ -12393,6 +12498,28 @@ func (a *OpenTracingAppLayer) ListDirectoryRecursively(path string) ([]string, *
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) ListExportDirectory(path string) ([]string, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ListExportDirectory")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.ListExportDirectory(path)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) ListExports() ([]string, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ListExports")
@@ -13951,6 +14078,28 @@ func (a *OpenTracingAppLayer) RemoveDirectory(path string) *model.AppError {
return resultVar0
}
func (a *OpenTracingAppLayer) RemoveExportFile(path string) *model.AppError {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.RemoveExportFile")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0 := a.app.RemoveExportFile(path)
if resultVar0 != nil {
span.LogFields(spanlog.Error(resultVar0))
ext.Error.Set(span, true)
}
return resultVar0
}
func (a *OpenTracingAppLayer) RemoveFile(path string) *model.AppError {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.RemoveFile")
@@ -18759,6 +18908,50 @@ func (a *OpenTracingAppLayer) ViewChannel(c request.CTX, view *model.ChannelView
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) WriteExportFile(fr io.Reader, path string) (int64, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.WriteExportFile")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.WriteExportFile(fr, path)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) WriteExportFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.WriteExportFileContext")
a.ctx = newCtx
a.app.Srv().Store().SetContext(newCtx)
defer func() {
a.app.Srv().Store().SetContext(origCtx)
a.ctx = origCtx
}()
defer span.Finish()
resultVar0, resultVar1 := a.app.WriteExportFileContext(ctx, fr, path)
if resultVar1 != nil {
span.LogFields(spanlog.Error(resultVar1))
ext.Error.Set(span, true)
}
return resultVar0, resultVar1
}
func (a *OpenTracingAppLayer) WriteFile(fr io.Reader, path string) (int64, *model.AppError) {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.WriteFile")

Просмотреть файл

@@ -82,6 +82,13 @@ func SetFileStore(filestore filestore.FileBackend) Option {
}
}
func SetExportFileStore(filestore filestore.FileBackend) Option {
return func(ps *PlatformService) error {
ps.exportFilestore = filestore
return nil
}
}
// ConfigStore applies the given config store, typically to replace the traditional sources with a memory store for testing.
func ConfigStore(configStore *config.Store) Option {
return func(ps *PlatformService) error {

Просмотреть файл

@@ -43,7 +43,8 @@ type PlatformService struct {
configStore *config.Store
filestore filestore.FileBackend
filestore filestore.FileBackend
exportFilestore filestore.FileBackend
cacheProvider cache.Provider
statusCache cache.Cache
@@ -247,6 +248,18 @@ func New(sc ServiceConfig, options ...Option) (*PlatformService, error) {
ps.filestore = backend
}
if ps.exportFilestore == nil {
ps.exportFilestore = ps.filestore
if *ps.Config().FileSettings.DedicatedExportStore {
backend, errFileBack := filestore.NewFileBackend(filestore.NewExportFileBackendSettingsFromConfig(&ps.Config().FileSettings, license != nil && *license.Features.Compliance, false))
if errFileBack != nil {
return nil, fmt.Errorf("failed to initialize export filebackend: %w", errFileBack)
}
ps.exportFilestore = backend
}
}
var err error
ps.Store, err = ps.newStore()
if err != nil {
@@ -490,3 +503,7 @@ func (ps *PlatformService) GetPluginStatuses() (model.PluginStatuses, *model.App
func (ps *PlatformService) FileBackend() filestore.FileBackend {
return ps.filestore
}
func (ps *PlatformService) ExportFileBackend() filestore.FileBackend {
return ps.exportFilestore
}

Просмотреть файл

@@ -50,29 +50,32 @@ func TestInitializeProducts(t *testing.T) {
t.Run("2 products and no circular dependency", func(t *testing.T) {
serviceMap := map[product.ServiceKey]any{
product.ConfigKey: nil,
product.LicenseKey: nil,
product.FilestoreKey: nil,
product.ClusterKey: nil,
product.ConfigKey: nil,
product.LicenseKey: nil,
product.FilestoreKey: nil,
product.ExportFilestoreKey: nil,
product.ClusterKey: nil,
}
products := map[string]product.Manifest{
"productA": {
Initializer: newProductA,
Dependencies: map[product.ServiceKey]struct{}{
product.ConfigKey: {},
product.LicenseKey: {},
product.FilestoreKey: {},
product.ClusterKey: {},
product.ConfigKey: {},
product.LicenseKey: {},
product.FilestoreKey: {},
product.ExportFilestoreKey: {},
product.ClusterKey: {},
},
},
"productB": {
Initializer: newProductB,
Dependencies: map[product.ServiceKey]struct{}{
product.ConfigKey: {},
testSrvKey1: {},
product.FilestoreKey: {},
product.ClusterKey: {},
product.ConfigKey: {},
testSrvKey1: {},
product.FilestoreKey: {},
product.ExportFilestoreKey: {},
product.ClusterKey: {},
},
},
}
@@ -89,30 +92,33 @@ func TestInitializeProducts(t *testing.T) {
t.Run("2 products and circular dependency", func(t *testing.T) {
serviceMap := map[product.ServiceKey]any{
product.ConfigKey: nil,
product.LicenseKey: nil,
product.FilestoreKey: nil,
product.ClusterKey: nil,
product.ConfigKey: nil,
product.LicenseKey: nil,
product.FilestoreKey: nil,
product.ExportFilestoreKey: nil,
product.ClusterKey: nil,
}
products := map[string]product.Manifest{
"productA": {
Initializer: newProductA,
Dependencies: map[product.ServiceKey]struct{}{
product.ConfigKey: {},
product.LicenseKey: {},
product.FilestoreKey: {},
product.ClusterKey: {},
testSrvKey2: {},
product.ConfigKey: {},
product.LicenseKey: {},
product.FilestoreKey: {},
product.ExportFilestoreKey: {},
product.ClusterKey: {},
testSrvKey2: {},
},
},
"productB": {
Initializer: newProductB,
Dependencies: map[product.ServiceKey]struct{}{
product.ConfigKey: {},
testSrvKey1: {},
product.FilestoreKey: {},
product.ClusterKey: {},
product.ConfigKey: {},
testSrvKey1: {},
product.FilestoreKey: {},
product.ExportFilestoreKey: {},
product.ClusterKey: {},
},
},
}
@@ -127,10 +133,11 @@ func TestInitializeProducts(t *testing.T) {
t.Run("2 products and one w/o any dependency", func(t *testing.T) {
serviceMap := map[product.ServiceKey]any{
product.ConfigKey: nil,
product.LicenseKey: nil,
product.FilestoreKey: nil,
product.ClusterKey: nil,
product.ConfigKey: nil,
product.LicenseKey: nil,
product.FilestoreKey: nil,
product.ExportFilestoreKey: nil,
product.ClusterKey: nil,
}
products := map[string]product.Manifest{

Просмотреть файл

@@ -250,21 +250,22 @@ func NewServer(options ...Option) (*Server, error) {
app := New(ServerConnector(s.Channels()))
serviceMap := map[product.ServiceKey]any{
ServerKey: s,
product.ConfigKey: s.platform,
product.LicenseKey: s.licenseWrapper,
product.FilestoreKey: s.platform.FileBackend(),
product.FileInfoStoreKey: &fileInfoWrapper{srv: s},
product.ClusterKey: s.platform,
product.UserKey: app,
product.LogKey: s.platform.Log(),
product.CloudKey: &cloudWrapper{cloud: s.Cloud},
product.KVStoreKey: s.platform,
product.StoreKey: store.NewStoreServiceAdapter(s.Store()),
product.SystemKey: &systemServiceAdapter{server: s},
product.SessionKey: app,
product.FrontendKey: app,
product.CommandKey: app,
ServerKey: s,
product.ConfigKey: s.platform,
product.LicenseKey: s.licenseWrapper,
product.FilestoreKey: s.platform.FileBackend(),
product.ExportFilestoreKey: s.platform.ExportFileBackend(),
product.FileInfoStoreKey: &fileInfoWrapper{srv: s},
product.ClusterKey: s.platform,
product.UserKey: app,
product.LogKey: s.platform.Log(),
product.CloudKey: &cloudWrapper{cloud: s.Cloud},
product.KVStoreKey: s.platform,
product.StoreKey: store.NewStoreServiceAdapter(s.Store()),
product.SystemKey: &systemServiceAdapter{server: s},
product.SessionKey: app,
product.FrontendKey: app,
product.CommandKey: app,
}
// It is important to initialize the hub only after the global logger is set
@@ -1476,6 +1477,10 @@ func (s *Server) FileBackend() filestore.FileBackend {
return s.platform.FileBackend()
}
func (s *Server) ExportFileBackend() filestore.FileBackend {
return s.platform.ExportFileBackend()
}
func (s *Server) TotalWebsocketConnections() int {
return s.Platform().TotalWebsocketConnections()
}

Просмотреть файл

@@ -0,0 +1,120 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package slashcommands
import (
"path"
"strings"
"time"
"github.com/mattermost/logr/v2"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/i18n"
"github.com/mattermost/mattermost/server/v8/channels/app"
"github.com/mattermost/mattermost/server/v8/channels/app/request"
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
)
type ExportLinkProvider struct {
}
const (
CmdExportLink = "exportlink"
LatestExportMessage = "latest"
)
func init() {
app.RegisterCommandProvider(&ExportLinkProvider{})
}
func (*ExportLinkProvider) GetTrigger() string {
return CmdExportLink
}
func (*ExportLinkProvider) GetCommand(a *app.App, T i18n.TranslateFunc) *model.Command {
if !a.Config().FeatureFlags.EnableExportDirectDownload {
return nil
}
if !*a.Config().FileSettings.DedicatedExportStore {
return nil
}
b := a.ExportFileBackend()
_, ok := b.(filestore.FileBackendWithLinkGenerator)
if !ok {
return nil
}
return &model.Command{
Trigger: CmdExportLink,
AutoComplete: true,
AutoCompleteDesc: T("api.command_exportlink.desc"),
AutoCompleteHint: T("api.command_exportlink.hint", map[string]any{
"LatestMsg": LatestExportMessage,
}),
DisplayName: T("api.command_exportlink.name"),
}
}
func (*ExportLinkProvider) DoCommand(a *app.App, c request.CTX, args *model.CommandArgs, message string) *model.CommandResponse {
if !a.SessionHasPermissionTo(*c.Session(), model.PermissionManageSystem) {
return &model.CommandResponse{ResponseType: model.CommandResponseTypeEphemeral, Text: args.T("api.command_exportlink.permission.app_error")}
}
b := a.ExportFileBackend()
_, ok := b.(filestore.FileBackendWithLinkGenerator)
if !ok {
return &model.CommandResponse{ResponseType: model.CommandResponseTypeEphemeral, Text: args.T("api.command_exportlink.driver.app_error")}
}
file := ""
if message == LatestExportMessage {
files, err := b.ListDirectory(*a.Config().ExportSettings.Directory)
if err != nil {
return &model.CommandResponse{ResponseType: model.CommandResponseTypeEphemeral, Text: args.T("api.command_exportlink.list.app_error")}
}
if len(files) == 0 {
return &model.CommandResponse{ResponseType: model.CommandResponseTypeEphemeral, Text: args.T("api.command_exportlink.empty.app_error")}
}
latestFound := time.Time{}
for _, f := range files {
// find the latest file
if !strings.HasSuffix(f, "_export.zip") {
continue
}
t, err := b.FileModTime(f)
if err != nil {
a.Log().Warn("Failed to get file mod time", logr.String("file", f), logr.Err(err))
continue
}
if t.After(latestFound) {
file = path.Base(f)
latestFound = t
}
}
}
if model.IsValidId(message) {
file = message + "_export.zip"
}
if file == "" {
file = message
}
if !strings.HasSuffix(file, "_export.zip") {
return &model.CommandResponse{ResponseType: model.CommandResponseTypeEphemeral, Text: args.T("api.command_exportlink.invalid.app_error")}
}
res, err := a.GeneratePresignURLForExport(file)
if err != nil {
return &model.CommandResponse{ResponseType: model.CommandResponseTypeEphemeral, Text: args.T("api.command_exportlink.presign.app_error")}
}
// return link
return &model.CommandResponse{ResponseType: model.CommandResponseTypeEphemeral, Text: args.T("api.command_exportlink.link.text", map[string]interface{}{
"Link": res.URL,
"Expiration": res.Expiration.String(),
})}
}

Просмотреть файл

@@ -19,9 +19,9 @@ const jobName = "ExportDelete"
type AppIface interface {
configservice.ConfigService
ListDirectory(path string) ([]string, *model.AppError)
FileModTime(path string) (time.Time, *model.AppError)
RemoveFile(path string) *model.AppError
ListExportDirectory(path string) ([]string, *model.AppError)
ExportFileModTime(path string) (time.Time, *model.AppError)
RemoveExportFile(path string) *model.AppError
}
func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
@@ -33,7 +33,7 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
exportPath := *app.Config().ExportSettings.Directory
retentionTime := time.Duration(*app.Config().ExportSettings.RetentionDays) * 24 * time.Hour
exports, appErr := app.ListDirectory(exportPath)
exports, appErr := app.ListExportDirectory(exportPath)
if appErr != nil {
return appErr
}
@@ -41,7 +41,7 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
errors := merror.New()
for i := range exports {
filename := filepath.Base(exports[i])
modTime, appErr := app.FileModTime(filepath.Join(exportPath, filename))
modTime, appErr := app.ExportFileModTime(filepath.Join(exportPath, filename))
if appErr != nil {
mlog.Debug("Worker: Failed to get file modification time",
mlog.Err(appErr), mlog.String("export", exports[i]))
@@ -51,7 +51,7 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
if time.Now().After(modTime.Add(retentionTime)) {
// remove file data from storage.
if appErr := app.RemoveFile(exports[i]); appErr != nil {
if appErr := app.RemoveExportFile(exports[i]); appErr != nil {
mlog.Debug("Worker: Failed to remove file",
mlog.Err(appErr), mlog.String("export", exports[i]))
errors.Append(appErr)

Просмотреть файл

@@ -19,8 +19,7 @@ const jobName = "ExportProcess"
type AppIface interface {
configservice.ConfigService
WriteFile(fr io.Reader, path string) (int64, *model.AppError)
WriteFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError)
WriteExportFileContext(ctx context.Context, fr io.Reader, path string) (int64, *model.AppError)
BulkExport(ctx request.CTX, writer io.Writer, outPath string, job *model.Job, opts model.BulkExportOpts) *model.AppError
Log() *mlog.Logger
}
@@ -45,7 +44,7 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface) model.Worker {
rd, wr := io.Pipe()
go func() {
_, appErr := app.WriteFileContext(context.Background(), rd, filepath.Join(outPath, exportFilename))
_, appErr := app.WriteExportFileContext(context.Background(), rd, filepath.Join(outPath, exportFilename))
if appErr != nil {
// we close the reader here to prevent a deadlock when the bulk exporter tries to
// write into the pipe while app.WriteFile has already returned. The error will be

Просмотреть файл

@@ -6,27 +6,28 @@ package product
type ServiceKey string
const (
ChannelKey ServiceKey = "channel"
ConfigKey ServiceKey = "config"
LicenseKey ServiceKey = "license"
FilestoreKey ServiceKey = "filestore"
FileInfoStoreKey ServiceKey = "fileinfostore"
ClusterKey ServiceKey = "cluster"
CloudKey ServiceKey = "cloud"
PostKey ServiceKey = "post"
TeamKey ServiceKey = "team"
UserKey ServiceKey = "user"
PermissionsKey ServiceKey = "permissions"
RouterKey ServiceKey = "router"
BotKey ServiceKey = "bot"
LogKey ServiceKey = "log"
HooksKey ServiceKey = "hooks"
KVStoreKey ServiceKey = "kvstore"
StoreKey ServiceKey = "storekey"
SystemKey ServiceKey = "systemkey"
PreferencesKey ServiceKey = "preferenceskey"
SessionKey ServiceKey = "sessionkey"
FrontendKey ServiceKey = "frontendkey"
CommandKey ServiceKey = "commandkey"
ThreadsKey ServiceKey = "threadskey"
ChannelKey ServiceKey = "channel"
ConfigKey ServiceKey = "config"
LicenseKey ServiceKey = "license"
FilestoreKey ServiceKey = "filestore"
ExportFilestoreKey ServiceKey = "exportfilestore"
FileInfoStoreKey ServiceKey = "fileinfostore"
ClusterKey ServiceKey = "cluster"
CloudKey ServiceKey = "cloud"
PostKey ServiceKey = "post"
TeamKey ServiceKey = "team"
UserKey ServiceKey = "user"
PermissionsKey ServiceKey = "permissions"
RouterKey ServiceKey = "router"
BotKey ServiceKey = "bot"
LogKey ServiceKey = "log"
HooksKey ServiceKey = "hooks"
KVStoreKey ServiceKey = "kvstore"
StoreKey ServiceKey = "storekey"
SystemKey ServiceKey = "systemkey"
PreferencesKey ServiceKey = "preferenceskey"
SessionKey ServiceKey = "sessionkey"
FrontendKey ServiceKey = "frontendkey"
CommandKey ServiceKey = "commandkey"
ThreadsKey ServiceKey = "threadskey"
)