Improve document extraction and including a document extraction command (#17183)

* Add extract documents content command

* Adding the extraction command and making the pure go pdf library as secondary option

* Improving the memory usage and docextractor interface

* Enable content extraction by default in all the instances

* Tiny improvement on archive indexing

* Adding App interface generation and the opentracing layer

* Fixing linter errors

* Addressing PR review comments

* Addressing PR review comments

services/docextractor/archive.go

@@ -4,6 +4,7 @@
 package docextractor
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -23,7 +24,7 @@ func (ae *archiveExtractor) Match(filename string) bool {
 	return err == nil
 }
 
-func (ae *archiveExtractor) Extract(name string, r io.Reader) (string, error) {
+func (ae *archiveExtractor) Extract(name string, r io.ReadSeeker) (string, error) {
 	dir, err := ioutil.TempDir(os.TempDir(), "archiver")
 	if err != nil {
 		return "", fmt.Errorf("error creating temporary file: %v", err)
@@ -45,7 +46,14 @@ func (ae *archiveExtractor) Extract(name string, r io.Reader) (string, error) {
 		text.WriteString(file.Name() + " ")
 		if ae.SubExtractor != nil {
 			filename := filepath.Base(file.Name())
-			subtext, extractErr := ae.SubExtractor.Extract(filename, file)
+			filename = strings.ReplaceAll(filename, "-", " ")
+			filename = strings.ReplaceAll(filename, ".", " ")
+			filename = strings.ReplaceAll(filename, ",", " ")
+			data, err2 := ioutil.ReadAll(file)
+			if err2 != nil {
+				return err2
+			}
+			subtext, extractErr := ae.SubExtractor.Extract(filename, bytes.NewReader(data))
 			if extractErr == nil {
 				text.WriteString(subtext + " ")
 			}

services/docextractor/combine.go

@@ -26,9 +26,10 @@ func (ce *combineExtractor) Match(filename string) bool {
 	return false
 }
 
-func (ce *combineExtractor) Extract(filename string, r io.Reader) (string, error) {
+func (ce *combineExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
 	for _, extractor := range ce.SubExtractors {
 		if extractor.Match(filename) {
+			r.Seek(0, io.SeekStart)
 			text, err := extractor.Extract(filename, r)
 			if err != nil {
 				mlog.Warn("unable to extract file content", mlog.Err(err))

services/docextractor/docextractor.go

@@ -15,18 +15,18 @@ type ExtractSettings struct {
 }
 
 // Extract extract the text from a document using the system default extractors
-func Extract(filename string, r io.Reader, settings ExtractSettings) (string, error) {
+func Extract(filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) {
 	return ExtractWithExtraExtractors(filename, r, settings, []Extractor{})
 }
 
 // ExtractWithExtraExtractors extract the text from a document using the provided extractors beside the system default extractors.
-func ExtractWithExtraExtractors(filename string, r io.Reader, settings ExtractSettings, extraExtractors []Extractor) (string, error) {
+func ExtractWithExtraExtractors(filename string, r io.ReadSeeker, settings ExtractSettings, extraExtractors []Extractor) (string, error) {
 	enabledExtractors := &combineExtractor{}
 	for _, extraExtractor := range extraExtractors {
 		enabledExtractors.Add(extraExtractor)
 	}
-	enabledExtractors.Add(&pdfExtractor{})
 	enabledExtractors.Add(&documentExtractor{})
+	enabledExtractors.Add(&pdfExtractor{})
 
 	if settings.ArchiveRecursion {
 		enabledExtractors.Add(&archiveExtractor{SubExtractor: enabledExtractors})

services/docextractor/docextractor_test.go

@@ -149,7 +149,7 @@ func (te *customTestPdfExtractor) Match(filename string) bool {
 	return strings.HasSuffix(filename, ".pdf")
 }
 
-func (te *customTestPdfExtractor) Extract(filename string, r io.Reader) (string, error) {
+func (te *customTestPdfExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
 	return "this is a text generated content", nil
 }
 
@@ -159,7 +159,7 @@ func (te *failingExtractor) Match(filename string) bool {
 	return true
 }
 
-func (te *failingExtractor) Extract(filename string, r io.Reader) (string, error) {
+func (te *failingExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
 	return "", errors.New("this always fail")
 }
 

services/docextractor/documents.go

@@ -25,6 +25,7 @@ var doconvConverterByExtensions = map[string]func(io.Reader) (string, map[string
 	"html":  func(r io.Reader) (string, map[string]string, error) { return docconv.ConvertHTML(r, true) },
 	"pages": docconv.ConvertPages,
 	"rtf":   docconv.ConvertRTF,
+	"pdf":   docconv.ConvertPDF,
 }
 
 func (de *documentExtractor) Match(filename string) bool {
@@ -33,7 +34,7 @@ func (de *documentExtractor) Match(filename string) bool {
 	return ok
 }
 
-func (de *documentExtractor) Extract(filename string, r io.Reader) (string, error) {
+func (de *documentExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
 	extension := strings.TrimPrefix(path.Ext(filename), ".")
 	converter, ok := doconvConverterByExtensions[extension]
 	if !ok {

services/docextractor/interface.go

@@ -10,5 +10,5 @@ import (
 // Extractors define the interface needed to extract file content
 type Extractor interface {
 	Match(filename string) bool
-	Extract(filename string, file io.Reader) (string, error)
+	Extract(filename string, file io.ReadSeeker) (string, error)
 }

services/docextractor/mmpreview.go

@@ -10,6 +10,7 @@ package docextractor
 import (
 	"bytes"
 	"io"
+	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"path"
@@ -41,7 +42,7 @@ func (mpe *mmPreviewExtractor) Match(filename string) bool {
 	return mmpreviewSupportedExtensions[extension]
 }
 
-func (mpe *mmPreviewExtractor) Extract(filename string, file io.Reader) (string, error) {
+func (mpe *mmPreviewExtractor) Extract(filename string, file io.ReadSeeker) (string, error) {
 	b, w, err := createMultipartFormData("file", filename, file)
 	if err != nil {
 		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
@@ -62,10 +63,14 @@ func (mpe *mmPreviewExtractor) Extract(filename string, file io.Reader) (string,
 	if resp.StatusCode != 200 {
 		return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)")
 	}
-	return mpe.pdfExtractor.Extract(filename, resp.Body)
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "unable to read the response from mmpreview")
+	}
+	return mpe.pdfExtractor.Extract(filename, bytes.NewReader(data))
 }
 
-func createMultipartFormData(fieldName, fileName string, fileData io.Reader) (bytes.Buffer, *multipart.Writer, error) {
+func createMultipartFormData(fieldName, fileName string, fileData io.ReadSeeker) (bytes.Buffer, *multipart.Writer, error) {
 	var b bytes.Buffer
 	var err error
 	w := multipart.NewWriter(&b)

services/docextractor/pdf.go

@@ -25,7 +25,7 @@ func (pe *pdfExtractor) Match(filename string) bool {
 	return supportedExtensions[extension]
 }
 
-func (pe *pdfExtractor) Extract(filename string, r io.Reader) (string, error) {
+func (pe *pdfExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
 	f, err := ioutil.TempFile(os.TempDir(), "pdflib")
 	if err != nil {
 		return "", fmt.Errorf("error creating temporary file: %v", err)

services/docextractor/plain.go

@@ -16,7 +16,7 @@ func (pe *plainExtractor) Match(filename string) bool {
 	return true
 }
 
-func (pe *plainExtractor) Extract(filename string, r io.Reader) (string, error) {
+func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
 	// This detects any visible character plus any whitespace
 	validRanges := append(unicode.GraphicRanges, unicode.White_Space)