mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[MM-55052] Improve file extraction logging (#25036)

Просмотр исходного кода
Этот коммит содержится в:
Ben Schumacher
2023-10-23 20:25:40 +02:00
коммит произвёл GitHub
родитель aa82597e2d
Коммит e864f9cfee
15 изменённых файлов: 74 добавлений и 26 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
server/channels/app/app_iface.go
Просмотреть файл

@@ -566,7 +566,7 @@ type AppIface interface {
ExportFileExists(path string) (bool, *model.AppError)
ExportFileModTime(path string) (time.Time, *model.AppError)
ExportPermissions(w io.Writer) error
ExtractContentFromFileInfo(fileInfo *model.FileInfo) error
ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileInfo) error
FetchSamlMetadataFromIdp(url string) ([]byte, *model.AppError)
FileBackend() filestore.FileBackend
FileExists(path string) (bool, *model.AppError)

16
server/channels/app/file.go
Просмотреть файл

@@ -757,7 +757,7 @@ func (a *App) UploadFileX(c *request.Context, channelID, name string, input io.R
if written > t.maxFileSize {
if fileErr := a.RemoveFile(t.fileinfo.Path); fileErr != nil {
mlog.Error("Failed to remove file", mlog.Err(fileErr))
c.Logger().Error("Failed to remove file", mlog.Err(fileErr))
}
return nil, t.newAppError("api.file.upload_file.too_large_detailed.app_error", http.StatusRequestEntityTooLarge, "Length", t.ContentLength, "Limit", t.maxFileSize)
}
@@ -796,10 +796,11 @@ func (a *App) UploadFileX(c *request.Context, channelID, name string, input io.R
if *a.Config().FileSettings.ExtractContent {
infoCopy := *t.fileinfo
crctx := c.Clone()
a.Srv().GoBuffered(func() {
err := a.ExtractContentFromFileInfo(&infoCopy)
err := a.ExtractContentFromFileInfo(crctx, &infoCopy)
if err != nil {
mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
crctx.Logger().Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
}
})
}
@@ -1047,10 +1048,11 @@ func (a *App) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTe
if *a.Config().FileSettings.ExtractContent {
infoCopy := *info
crctx := c.Clone()
a.Srv().GoBuffered(func() {
err := a.ExtractContentFromFileInfo(&infoCopy)
err := a.ExtractContentFromFileInfo(crctx, &infoCopy)
if err != nil {
mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
crctx.Logger().Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
}
})
}
@@ -1442,7 +1444,7 @@ func (a *App) SearchFilesInTeamForUser(c *request.Context, terms string, userId
return fileInfoSearchResults, a.filterInaccessibleFiles(fileInfoSearchResults, filterFileOptions{assumeSortedCreatedAt: true})
}
func (a *App) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error {
func (a *App) ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileInfo) error {
// We don't process images.
if fileInfo.IsImage() {
return nil
@@ -1453,7 +1455,7 @@ func (a *App) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error {
return errors.Wrap(aerr, "failed to open file for extract file content")
}
defer file.Close()
text, err := docextractor.Extract(fileInfo.Name, file, docextractor.ExtractSettings{
text, err := docextractor.Extract(rctx.Logger(), fileInfo.Name, file, docextractor.ExtractSettings{
ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion,
})
if err != nil {

3
server/channels/app/file_test.go
Просмотреть файл

@@ -17,6 +17,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
storemocks "github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
@@ -555,7 +556,7 @@ func TestExtractContentFromFileInfo(t *testing.T) {
}
// Test that we don't process images.
require.NoError(t, app.ExtractContentFromFileInfo(fi))
require.NoError(t, app.ExtractContentFromFileInfo(request.TestContext(t), fi))
}
func TestGetLastAccessibleFileTime(t *testing.T) {

4
server/channels/app/opentracing/opentracing_layer.go
Просмотреть файл

@@ -4248,7 +4248,7 @@ func (a *OpenTracingAppLayer) ExtendSessionExpiryIfNeeded(session *model.Session
return resultVar0
}
func (a *OpenTracingAppLayer) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error {
func (a *OpenTracingAppLayer) ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileInfo) error {
origCtx := a.ctx
span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ExtractContentFromFileInfo")
@@ -4260,7 +4260,7 @@ func (a *OpenTracingAppLayer) ExtractContentFromFileInfo(fileInfo *model.FileInf
}()
defer span.Finish()
resultVar0 := a.app.ExtractContentFromFileInfo(fileInfo)
resultVar0 := a.app.ExtractContentFromFileInfo(rctx, fileInfo)
if resultVar0 != nil {
span.LogFields(spanlog.Error(resultVar0))

7
server/channels/app/upload.go
Просмотреть файл

@@ -318,17 +318,18 @@ func (a *App) UploadData(c request.CTX, us *model.UploadSession, rd io.Reader) (
if *a.Config().FileSettings.ExtractContent {
infoCopy := *info
crctx := c.Clone()
a.Srv().Go(func() {
err := a.ExtractContentFromFileInfo(&infoCopy)
err := a.ExtractContentFromFileInfo(crctx, &infoCopy)
if err != nil {
mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
crctx.Logger().Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id))
}
})
}
// delete upload session
if storeErr := a.Srv().Store().UploadSession().Delete(us.Id); storeErr != nil {
mlog.Warn("Failed to delete UploadSession", mlog.Err(storeErr))
c.Logger().Warn("Failed to delete UploadSession", mlog.Err(storeErr))
}
return info, nil

6
server/channels/jobs/extract_content/worker.go
Просмотреть файл

@@ -8,6 +8,7 @@ import (
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/jobs"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
@@ -20,7 +21,7 @@ var ignoredFiles = map[string]bool{
}
type AppIface interface {
ExtractContentFromFileInfo(fileInfo *model.FileInfo) error
ExtractContentFromFileInfo(rctx request.CTX, fileInfo *model.FileInfo) error
}
func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store) *jobs.SimpleWorker {
@@ -66,7 +67,8 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface, store store.Store) *job
for _, fileInfo := range fileInfos {
if !ignoredFiles[fileInfo.Extension] {
logger.Debug("Extracting file", mlog.String("filename", fileInfo.Name), mlog.String("filepath", fileInfo.Path))
err = app.ExtractContentFromFileInfo(fileInfo)
err = app.ExtractContentFromFileInfo(request.EmptyContext(logger), fileInfo)
if err != nil {
logger.Warn("Failed to extract file content", mlog.Err(err), mlog.String("file_info_id", fileInfo.Id))
nErrs++