diff --git a/app/app_iface.go b/app/app_iface.go index 1106609ffc..28840b315b 100644 --- a/app/app_iface.go +++ b/app/app_iface.go @@ -515,6 +515,7 @@ type AppIface interface { EnableUserAccessToken(token *model.UserAccessToken) *model.AppError EnvironmentConfig(filter func(reflect.StructField) bool) map[string]interface{} ExportPermissions(w io.Writer) error + ExtractContentFromFileInfo(fileInfo *model.FileInfo) error FetchSamlMetadataFromIdp(url string) ([]byte, *model.AppError) FileBackend() (filestore.FileBackend, *model.AppError) FileExists(path string) (bool, *model.AppError) diff --git a/app/file.go b/app/file.go index 237f108696..721a9eb7b4 100644 --- a/app/file.go +++ b/app/file.go @@ -9,7 +9,6 @@ import ( "context" "crypto/sha256" "encoding/base64" - "errors" "fmt" "image" "image/color" @@ -30,6 +29,7 @@ import ( "github.com/disintegration/imaging" _ "github.com/oov/psd" + "github.com/pkg/errors" "github.com/rwcarlsen/goexif/exif" _ "golang.org/x/image/bmp" _ "golang.org/x/image/tiff" @@ -75,6 +75,7 @@ const ( ImageThumbnailPixelWidth = 120 ImageThumbnailPixelHeight = 100 ImagePreviewPixelWidth = 1920 + MaxContentExtractionSize = 1024 * 1024 // 1Mb ) func (a *App) FileBackend() (filestore.FileBackend, *model.AppError) { @@ -773,21 +774,9 @@ func (a *App) UploadFileX(channelID, name string, input io.Reader, if *a.Config().FileSettings.ExtractContent && a.Config().FeatureFlags.FilesSearch { infoCopy := *t.fileinfo a.Srv().Go(func() { - file, aerr := a.FileReader(t.fileinfo.Path) - if aerr != nil { - mlog.Error("Failed to open file for extract file content", mlog.Err(aerr)) - return - } - defer file.Close() - text, err := docextractor.Extract(infoCopy.Name, file, docextractor.ExtractSettings{ - ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion, - }) + err := a.ExtractContentFromFileInfo(&infoCopy) if err != nil { - mlog.Error("Failed to extract file content", mlog.Err(err)) - return - } - if storeErr := a.Srv().Store.FileInfo().SetContent(infoCopy.Id, text); storeErr != nil { - mlog.Error("Failed to save the extracted file content", mlog.Err(storeErr)) + mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id)) } }) } @@ -1043,21 +1032,9 @@ func (a *App) DoUploadFileExpectModification(now time.Time, rawTeamId string, ra if *a.Config().FileSettings.ExtractContent && a.Config().FeatureFlags.FilesSearch { infoCopy := *info a.Srv().Go(func() { - file, aerr := a.FileReader(infoCopy.Path) - if aerr != nil { - mlog.Error("Failed to open file for extract file content", mlog.Err(aerr)) - return - } - defer file.Close() - text, err := docextractor.Extract(infoCopy.Name, file, docextractor.ExtractSettings{ - ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion, - }) + err := a.ExtractContentFromFileInfo(&infoCopy) if err != nil { - mlog.Error("Failed to extract file content", mlog.Err(err)) - return - } - if storeErr := a.Srv().Store.FileInfo().SetContent(infoCopy.Id, text); storeErr != nil { - mlog.Error("Failed to save the extracted file content", mlog.Err(storeErr)) + mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id)) } }) } @@ -1396,3 +1373,26 @@ func (a *App) SearchFilesInTeamForUser(terms string, userId string, teamId strin return fileInfoSearchResults, nil } + +func (a *App) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error { + file, aerr := a.FileReader(fileInfo.Path) + if aerr != nil { + return errors.Wrap(aerr, "failed to open file for extract file content") + } + defer file.Close() + text, err := docextractor.Extract(fileInfo.Name, file, docextractor.ExtractSettings{ + ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion, + }) + if err != nil { + return errors.Wrap(err, "failed to extract file content") + } + if text != "" { + if len(text) > MaxContentExtractionSize { + text = text[0:MaxContentExtractionSize] + } + if storeErr := a.Srv().Store.FileInfo().SetContent(fileInfo.Id, text); storeErr != nil { + return errors.Wrap(storeErr, "failed to save the extracted file content") + } + } + return nil +} diff --git a/app/migrations.go b/app/migrations.go index 33bc7bde01..39d5a220e0 100644 --- a/app/migrations.go +++ b/app/migrations.go @@ -16,7 +16,6 @@ const EmojisPermissionsMigrationKey = "EmojisPermissionsMigrationComplete" const GuestRolesCreationMigrationKey = "GuestRolesCreationMigrationComplete" const SystemConsoleRolesCreationMigrationKey = "SystemConsoleRolesCreationMigrationComplete" const ContentExtractionConfigMigrationKey = "ContentExtractionConfigMigrationComplete" -const usersLimitToAutoEnableContentExtraction = 500 // This function migrates the default built in roles from code/config to the database. func (a *App) DoAdvancedPermissionsMigration() { @@ -285,35 +284,6 @@ func (a *App) DoSystemConsoleRolesCreationMigration() { } } -func (a *App) doContentExtractionConfigMigration() { - if !a.Config().FeatureFlags.FilesSearch { - return - } - // If the migration is already marked as completed, don't do it again. - if _, err := a.Srv().Store.System().GetByName(ContentExtractionConfigMigrationKey); err == nil { - return - } - - if usersCount, err := a.Srv().Store.User().Count(model.UserCountOptions{}); err != nil { - mlog.Critical("Failed to get the users count for migrating the content extraction, using default value", mlog.Err(err)) - } else { - if usersCount < usersLimitToAutoEnableContentExtraction { - a.UpdateConfig(func(config *model.Config) { - config.FileSettings.ExtractContent = model.NewBool(true) - }) - } - } - - system := model.System{ - Name: ContentExtractionConfigMigrationKey, - Value: "true", - } - - if err := a.Srv().Store.System().Save(&system); err != nil { - mlog.Critical("Failed to mark content extraction config migration as completed.", mlog.Err(err)) - } -} - func (a *App) DoAppMigrations() { a.DoAdvancedPermissionsMigration() a.DoEmojisPermissionsMigration() @@ -325,5 +295,4 @@ func (a *App) DoAppMigrations() { if err != nil { mlog.Critical("(app.App).DoPermissionsMigrations failed", mlog.Err(err)) } - a.doContentExtractionConfigMigration() } diff --git a/app/opentracing/opentracing_layer.go b/app/opentracing/opentracing_layer.go index ff3b3c6ac7..6a1fe19ddb 100644 --- a/app/opentracing/opentracing_layer.go +++ b/app/opentracing/opentracing_layer.go @@ -3795,6 +3795,28 @@ func (a *OpenTracingAppLayer) ExtendSessionExpiryIfNeeded(session *model.Session return resultVar0 } +func (a *OpenTracingAppLayer) ExtractContentFromFileInfo(fileInfo *model.FileInfo) error { + origCtx := a.ctx + span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ExtractContentFromFileInfo") + + a.ctx = newCtx + a.app.Srv().Store.SetContext(newCtx) + defer func() { + a.app.Srv().Store.SetContext(origCtx) + a.ctx = origCtx + }() + + defer span.Finish() + resultVar0 := a.app.ExtractContentFromFileInfo(fileInfo) + + if resultVar0 != nil { + span.LogFields(spanlog.Error(resultVar0)) + ext.Error.Set(span, true) + } + + return resultVar0 +} + func (a *OpenTracingAppLayer) FetchSamlMetadataFromIdp(url string) ([]byte, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.FetchSamlMetadataFromIdp") diff --git a/app/upload.go b/app/upload.go index d50397d1a0..f239f02b68 100644 --- a/app/upload.go +++ b/app/upload.go @@ -14,7 +14,6 @@ import ( "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/plugin" - "github.com/mattermost/mattermost-server/v5/services/docextractor" "github.com/mattermost/mattermost-server/v5/shared/mlog" "github.com/mattermost/mattermost-server/v5/store" ) @@ -303,15 +302,9 @@ func (a *App) UploadData(us *model.UploadSession, rd io.Reader) (*model.FileInfo if *a.Config().FileSettings.ExtractContent && a.Config().FeatureFlags.FilesSearch { infoCopy := *info a.Srv().Go(func() { - text, err := docextractor.Extract(infoCopy.Name, file, docextractor.ExtractSettings{ - ArchiveRecursion: *a.Config().FileSettings.ArchiveRecursion, - }) + err := a.ExtractContentFromFileInfo(&infoCopy) if err != nil { - mlog.Error("Failed to extract file content", mlog.Err(err)) - return - } - if storeErr := a.Srv().Store.FileInfo().SetContent(infoCopy.Id, text); storeErr != nil { - mlog.Error("Failed to save the extracted file content", mlog.Err(storeErr)) + mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", infoCopy.Id)) } }) } diff --git a/cmd/mattermost/commands/extract_content.go b/cmd/mattermost/commands/extract_content.go new file mode 100644 index 0000000000..bce1efc36b --- /dev/null +++ b/cmd/mattermost/commands/extract_content.go @@ -0,0 +1,86 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package commands + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/shared/mlog" +) + +var ExtractContentCmd = &cobra.Command{ + Use: "extract-documents-content", + Short: "Extracts the documents content", + Long: "Extracts the documents content and stores it in the database for document searche", + Example: "extract-documents-content --from=12345", + RunE: extractContentCmdF, +} + +func init() { + ExtractContentCmd.Flags().Int64("from", 0, "The timestamp of the earliest file to extract, expressed in seconds since the unix epoch.") + ExtractContentCmd.Flags().Int64("to", model.GetMillis(), "The timestamp of the latest file to extract, expressed in seconds since the unix epoch.") + RootCmd.AddCommand(ExtractContentCmd) +} + +func extractContentCmdF(command *cobra.Command, args []string) error { + a, err := InitDBCommandContextCobra(command) + if err != nil { + return err + } + defer a.Srv().Shutdown() + + if !*a.Config().FileSettings.ExtractContent || !a.Config().FeatureFlags.FilesSearch { + return errors.New("ERROR: Document extraction is not enabled") + } + + startTime, err := command.Flags().GetInt64("from") + if err != nil { + return errors.New("\"from\" flag error") + } + if startTime < 0 { + return errors.New("\"from\" must be a positive integer") + } + + endTime, err := command.Flags().GetInt64("to") + if err != nil { + return errors.New("\"to\" flag error") + } + if endTime < startTime { + return errors.New("\"to\" must be greater than from") + } + + since := startTime + for { + opts := model.GetFileInfosOptions{ + Since: since, + SortBy: model.FILEINFO_SORT_BY_CREATED, + IncludeDeleted: false, + } + fileInfos, err := a.Srv().Store.FileInfo().GetWithOptions(0, 1000, &opts) + if err != nil { + return fmt.Errorf("ERROR: Document extraction failed %v", err.Error()) + } + if len(fileInfos) == 0 { + break + } + for _, fileInfo := range fileInfos { + fmt.Println("extracting file", fileInfo.Name, fileInfo.Path) + err = a.ExtractContentFromFileInfo(fileInfo) + if err != nil { + mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", fileInfo.Id)) + } + } + lastFileInfo := fileInfos[len(fileInfos)-1] + if lastFileInfo.CreateAt > endTime { + break + } + since = lastFileInfo.CreateAt + 1 + } + + return nil +} diff --git a/model/config.go b/model/config.go index 9c605944bc..ea1d5a59dc 100644 --- a/model/config.go +++ b/model/config.go @@ -1430,7 +1430,7 @@ func (s *FileSettings) SetDefaults(isUpdate bool) { } if s.ExtractContent == nil { - s.ExtractContent = NewBool(false) + s.ExtractContent = NewBool(true) } if s.ArchiveRecursion == nil { diff --git a/services/docextractor/archive.go b/services/docextractor/archive.go index f3dc68ffb2..96194252cb 100644 --- a/services/docextractor/archive.go +++ b/services/docextractor/archive.go @@ -4,6 +4,7 @@ package docextractor import ( + "bytes" "fmt" "io" "io/ioutil" @@ -23,7 +24,7 @@ func (ae *archiveExtractor) Match(filename string) bool { return err == nil } -func (ae *archiveExtractor) Extract(name string, r io.Reader) (string, error) { +func (ae *archiveExtractor) Extract(name string, r io.ReadSeeker) (string, error) { dir, err := ioutil.TempDir(os.TempDir(), "archiver") if err != nil { return "", fmt.Errorf("error creating temporary file: %v", err) @@ -45,7 +46,14 @@ func (ae *archiveExtractor) Extract(name string, r io.Reader) (string, error) { text.WriteString(file.Name() + " ") if ae.SubExtractor != nil { filename := filepath.Base(file.Name()) - subtext, extractErr := ae.SubExtractor.Extract(filename, file) + filename = strings.ReplaceAll(filename, "-", " ") + filename = strings.ReplaceAll(filename, ".", " ") + filename = strings.ReplaceAll(filename, ",", " ") + data, err2 := ioutil.ReadAll(file) + if err2 != nil { + return err2 + } + subtext, extractErr := ae.SubExtractor.Extract(filename, bytes.NewReader(data)) if extractErr == nil { text.WriteString(subtext + " ") } diff --git a/services/docextractor/combine.go b/services/docextractor/combine.go index 4c230f2cac..447e749335 100644 --- a/services/docextractor/combine.go +++ b/services/docextractor/combine.go @@ -26,9 +26,10 @@ func (ce *combineExtractor) Match(filename string) bool { return false } -func (ce *combineExtractor) Extract(filename string, r io.Reader) (string, error) { +func (ce *combineExtractor) Extract(filename string, r io.ReadSeeker) (string, error) { for _, extractor := range ce.SubExtractors { if extractor.Match(filename) { + r.Seek(0, io.SeekStart) text, err := extractor.Extract(filename, r) if err != nil { mlog.Warn("unable to extract file content", mlog.Err(err)) diff --git a/services/docextractor/docextractor.go b/services/docextractor/docextractor.go index a82593d095..5c05af7a22 100644 --- a/services/docextractor/docextractor.go +++ b/services/docextractor/docextractor.go @@ -15,18 +15,18 @@ type ExtractSettings struct { } // Extract extract the text from a document using the system default extractors -func Extract(filename string, r io.Reader, settings ExtractSettings) (string, error) { +func Extract(filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) { return ExtractWithExtraExtractors(filename, r, settings, []Extractor{}) } // ExtractWithExtraExtractors extract the text from a document using the provided extractors beside the system default extractors. -func ExtractWithExtraExtractors(filename string, r io.Reader, settings ExtractSettings, extraExtractors []Extractor) (string, error) { +func ExtractWithExtraExtractors(filename string, r io.ReadSeeker, settings ExtractSettings, extraExtractors []Extractor) (string, error) { enabledExtractors := &combineExtractor{} for _, extraExtractor := range extraExtractors { enabledExtractors.Add(extraExtractor) } - enabledExtractors.Add(&pdfExtractor{}) enabledExtractors.Add(&documentExtractor{}) + enabledExtractors.Add(&pdfExtractor{}) if settings.ArchiveRecursion { enabledExtractors.Add(&archiveExtractor{SubExtractor: enabledExtractors}) diff --git a/services/docextractor/docextractor_test.go b/services/docextractor/docextractor_test.go index 78533decaf..e4e60b04c4 100644 --- a/services/docextractor/docextractor_test.go +++ b/services/docextractor/docextractor_test.go @@ -149,7 +149,7 @@ func (te *customTestPdfExtractor) Match(filename string) bool { return strings.HasSuffix(filename, ".pdf") } -func (te *customTestPdfExtractor) Extract(filename string, r io.Reader) (string, error) { +func (te *customTestPdfExtractor) Extract(filename string, r io.ReadSeeker) (string, error) { return "this is a text generated content", nil } @@ -159,7 +159,7 @@ func (te *failingExtractor) Match(filename string) bool { return true } -func (te *failingExtractor) Extract(filename string, r io.Reader) (string, error) { +func (te *failingExtractor) Extract(filename string, r io.ReadSeeker) (string, error) { return "", errors.New("this always fail") } diff --git a/services/docextractor/documents.go b/services/docextractor/documents.go index 38a47881d7..91ad5c428b 100644 --- a/services/docextractor/documents.go +++ b/services/docextractor/documents.go @@ -25,6 +25,7 @@ var doconvConverterByExtensions = map[string]func(io.Reader) (string, map[string "html": func(r io.Reader) (string, map[string]string, error) { return docconv.ConvertHTML(r, true) }, "pages": docconv.ConvertPages, "rtf": docconv.ConvertRTF, + "pdf": docconv.ConvertPDF, } func (de *documentExtractor) Match(filename string) bool { @@ -33,7 +34,7 @@ func (de *documentExtractor) Match(filename string) bool { return ok } -func (de *documentExtractor) Extract(filename string, r io.Reader) (string, error) { +func (de *documentExtractor) Extract(filename string, r io.ReadSeeker) (string, error) { extension := strings.TrimPrefix(path.Ext(filename), ".") converter, ok := doconvConverterByExtensions[extension] if !ok { diff --git a/services/docextractor/interface.go b/services/docextractor/interface.go index 390223da50..6ea417914d 100644 --- a/services/docextractor/interface.go +++ b/services/docextractor/interface.go @@ -10,5 +10,5 @@ import ( // Extractors define the interface needed to extract file content type Extractor interface { Match(filename string) bool - Extract(filename string, file io.Reader) (string, error) + Extract(filename string, file io.ReadSeeker) (string, error) } diff --git a/services/docextractor/mmpreview.go b/services/docextractor/mmpreview.go index 0839c834f4..a46cc3f956 100644 --- a/services/docextractor/mmpreview.go +++ b/services/docextractor/mmpreview.go @@ -10,6 +10,7 @@ package docextractor import ( "bytes" "io" + "io/ioutil" "mime/multipart" "net/http" "path" @@ -41,7 +42,7 @@ func (mpe *mmPreviewExtractor) Match(filename string) bool { return mmpreviewSupportedExtensions[extension] } -func (mpe *mmPreviewExtractor) Extract(filename string, file io.Reader) (string, error) { +func (mpe *mmPreviewExtractor) Extract(filename string, file io.ReadSeeker) (string, error) { b, w, err := createMultipartFormData("file", filename, file) if err != nil { return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.") @@ -62,10 +63,14 @@ func (mpe *mmPreviewExtractor) Extract(filename string, file io.Reader) (string, if resp.StatusCode != 200 { return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)") } - return mpe.pdfExtractor.Extract(filename, resp.Body) + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, "unable to read the response from mmpreview") + } + return mpe.pdfExtractor.Extract(filename, bytes.NewReader(data)) } -func createMultipartFormData(fieldName, fileName string, fileData io.Reader) (bytes.Buffer, *multipart.Writer, error) { +func createMultipartFormData(fieldName, fileName string, fileData io.ReadSeeker) (bytes.Buffer, *multipart.Writer, error) { var b bytes.Buffer var err error w := multipart.NewWriter(&b) diff --git a/services/docextractor/pdf.go b/services/docextractor/pdf.go index bfe3964ae3..9f9f7dce45 100644 --- a/services/docextractor/pdf.go +++ b/services/docextractor/pdf.go @@ -25,7 +25,7 @@ func (pe *pdfExtractor) Match(filename string) bool { return supportedExtensions[extension] } -func (pe *pdfExtractor) Extract(filename string, r io.Reader) (string, error) { +func (pe *pdfExtractor) Extract(filename string, r io.ReadSeeker) (string, error) { f, err := ioutil.TempFile(os.TempDir(), "pdflib") if err != nil { return "", fmt.Errorf("error creating temporary file: %v", err) diff --git a/services/docextractor/plain.go b/services/docextractor/plain.go index 3889d01981..cac6d06e1a 100644 --- a/services/docextractor/plain.go +++ b/services/docextractor/plain.go @@ -16,7 +16,7 @@ func (pe *plainExtractor) Match(filename string) bool { return true } -func (pe *plainExtractor) Extract(filename string, r io.Reader) (string, error) { +func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, error) { // This detects any visible character plus any whitespace validRanges := append(unicode.GraphicRanges, unicode.White_Space) diff --git a/store/sqlstore/file_info_store.go b/store/sqlstore/file_info_store.go index 14c66427aa..2f2adab979 100644 --- a/store/sqlstore/file_info_store.go +++ b/store/sqlstore/file_info_store.go @@ -350,19 +350,11 @@ func (fs SqlFileInfoStore) SetContent(fileId, content string) error { return errors.Wrap(err, "file_info_tosql") } - sqlResult, err := fs.GetMaster().Exec(queryString, args...) + _, err = fs.GetMaster().Exec(queryString, args...) if err != nil { return errors.Wrapf(err, "failed to update FileInfo content with id=%s", fileId) } - count, err := sqlResult.RowsAffected() - if err != nil { - // RowsAffected should never fail with the MySQL or Postgres drivers - return errors.Wrap(err, "unable to retrieve rows affected") - } else if count == 0 { - // Could not attach the file to the post - return store.NewErrInvalidInput("FileInfo", "", fmt.Sprintf("<%s>", fileId)) - } return nil }