Document extractor service (#15665)

* Document extractor service

* Fixing vendor modules

* Addressing PR Review comments

* Some small simplifications

* Fixing a linter complain

* simplifying a bit the code using package variables

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

services/docextractor/archive.go

@@ -0,0 +1,60 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mholt/archiver/v3"
+)
+
+type archiveExtractor struct {
+	SubExtractor Extractor
+}
+
+func (ae *archiveExtractor) Match(filename string) bool {
+	_, err := archiver.ByExtension(filename)
+	return err == nil
+}
+
+func (ae *archiveExtractor) Extract(name string, r io.Reader) (string, error) {
+	dir, err := ioutil.TempDir(os.TempDir(), "archiver")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	f, err := os.Create(filepath.Join(dir, name))
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+	_, err = io.Copy(f, r)
+	f.Close()
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	var text strings.Builder
+	err = archiver.Walk(f.Name(), func(file archiver.File) error {
+		text.WriteString(file.Name() + " ")
+		if ae.SubExtractor != nil {
+			filename := filepath.Base(file.Name())
+			subtext, extractErr := ae.SubExtractor.Extract(filename, file)
+			if extractErr == nil {
+				text.WriteString(subtext + " ")
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return text.String(), nil
+}

services/docextractor/combine.go

@@ -0,0 +1,41 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+
+	"github.com/mattermost/mattermost-server/v5/mlog"
+)
+
+type combineExtractor struct {
+	SubExtractors []Extractor
+}
+
+func (ce *combineExtractor) Add(extractor Extractor) {
+	ce.SubExtractors = append(ce.SubExtractors, extractor)
+}
+
+func (ce *combineExtractor) Match(filename string) bool {
+	for _, extractor := range ce.SubExtractors {
+		if extractor.Match(filename) {
+			return true
+		}
+	}
+	return false
+}
+
+func (ce *combineExtractor) Extract(filename string, r io.Reader) (string, error) {
+	for _, extractor := range ce.SubExtractors {
+		if extractor.Match(filename) {
+			text, err := extractor.Extract(filename, r)
+			if err != nil {
+				mlog.Warn("unable to extract file content", mlog.Err(err))
+				continue
+			}
+			return text, nil
+		}
+	}
+	return "", nil
+}

services/docextractor/docextractor.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+)
+
+// ExtractSettings defines the features enabled/disable during the document text extraction.
+type ExtractSettings struct {
+	ArchiveRecursion bool
+	MMPreviewURL     string
+	MMPreviewSecret  string
+}
+
+// Extract extract the text from a document using the system default extractors
+func Extract(filename string, r io.Reader, settings ExtractSettings) (string, error) {
+	return ExtractWithExtraExtractors(filename, r, settings, []Extractor{})
+}
+
+// ExtractWithExtraExtractors extract the text from a document using the provided extractors beside the system default extractors.
+func ExtractWithExtraExtractors(filename string, r io.Reader, settings ExtractSettings, extraExtractors []Extractor) (string, error) {
+	enabledExtractors := &combineExtractor{}
+	for _, extraExtractor := range extraExtractors {
+		enabledExtractors.Add(extraExtractor)
+	}
+	enabledExtractors.Add(&pdfExtractor{})
+	enabledExtractors.Add(&documentExtractor{})
+
+	if settings.ArchiveRecursion {
+		enabledExtractors.Add(&archiveExtractor{SubExtractor: enabledExtractors})
+	} else {
+		enabledExtractors.Add(&archiveExtractor{})
+	}
+
+	if settings.MMPreviewURL != "" {
+		enabledExtractors.Add(newMMPreviewExtractor(settings.MMPreviewURL, settings.MMPreviewSecret, pdfExtractor{}))
+	}
+	enabledExtractors.Add(&plainExtractor{})
+
+	if enabledExtractors.Match(filename) {
+		return enabledExtractors.Extract(filename, r)
+	}
+	return "", nil
+}

services/docextractor/docextractor_test.go

@@ -0,0 +1,186 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v5/utils/testutils"
+)
+
+func TestExtract(t *testing.T) {
+	testCases := []struct {
+		Name         string
+		TestFileName string
+		Settings     ExtractSettings
+		Contains     []string
+		NotContains  []string
+		ExpectError  bool
+	}{
+		{
+			"Plain text file",
+			"test-markdown-basics.md",
+			ExtractSettings{},
+			[]string{"followed", "separated"},
+			[]string{},
+			false,
+		},
+		{
+			"Zip file without recursion",
+			"Fake_Team_Import.zip",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Zip file with recursion",
+			"Fake_Team_Import.zip",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Rar file without recursion",
+			"Fake_Team_Import.rar",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Rar file with recursion",
+			"Fake_Team_Import.rar",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Tar.gz file without recursion",
+			"Fake_Team_Import.tar.gz",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Tar.gz file with recursion",
+			"Fake_Team_Import.tar.gz",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Pdf file",
+			"sample-doc.pdf",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Docx file",
+			"sample-doc.docx",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Pptx file",
+			"sample-doc.pptx",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			data, err := testutils.ReadTestFile(tc.TestFileName)
+			require.Nil(t, err)
+			text, err := Extract(tc.TestFileName, bytes.NewReader(data), tc.Settings)
+			if tc.ExpectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				for _, expectedString := range tc.Contains {
+					assert.Contains(t, text, expectedString)
+				}
+				for _, notExpectedString := range tc.NotContains {
+					assert.NotContains(t, text, notExpectedString)
+				}
+			}
+		})
+	}
+
+	t.Run("Unsupported binary file", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("testjpg.jpg")
+		require.Nil(t, err)
+		text, err := Extract("testjpg.jpg", bytes.NewReader(data), ExtractSettings{})
+		require.NoError(t, err)
+		require.Equal(t, "", text)
+	})
+
+	t.Run("Wrong extension", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.Nil(t, err)
+		text, err := Extract("sample-doc.docx", bytes.NewReader(data), ExtractSettings{})
+		require.NoError(t, err)
+		require.Equal(t, "", text)
+	})
+}
+
+type customTestPdfExtractor struct{}
+
+func (te *customTestPdfExtractor) Match(filename string) bool {
+	return strings.HasSuffix(filename, ".pdf")
+}
+
+func (te *customTestPdfExtractor) Extract(filename string, r io.Reader) (string, error) {
+	return "this is a text generated content", nil
+}
+
+type failingExtractor struct{}
+
+func (te *failingExtractor) Match(filename string) bool {
+	return true
+}
+
+func (te *failingExtractor) Extract(filename string, r io.Reader) (string, error) {
+	return "", errors.New("this always fail")
+}
+
+func TestExtractWithExtraExtractors(t *testing.T) {
+	t.Run("overrite existing extractor", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.Nil(t, err)
+
+		text, err := ExtractWithExtraExtractors("sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&customTestPdfExtractor{}})
+		require.NoError(t, err)
+		require.Equal(t, text, "this is a text generated content")
+	})
+
+	t.Run("failing extractor", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.Nil(t, err)
+
+		text, err := ExtractWithExtraExtractors("sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&failingExtractor{}})
+		require.NoError(t, err)
+		assert.Contains(t, text, "simple")
+		assert.Contains(t, text, "document")
+		assert.Contains(t, text, "contains")
+	})
+}

services/docextractor/documents.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"code.sajari.com/docconv"
+)
+
+type documentExtractor struct{}
+
+var doconvConverterByExtensions = map[string]func(io.Reader) (string, map[string]string, error){
+	"doc":   docconv.ConvertDoc,
+	"docx":  docconv.ConvertDocx,
+	"pptx":  docconv.ConvertPptx,
+	"odt":   docconv.ConvertODT,
+	"html":  func(r io.Reader) (string, map[string]string, error) { return docconv.ConvertHTML(r, true) },
+	"pages": docconv.ConvertPages,
+	"rtf":   docconv.ConvertRTF,
+}
+
+func (de *documentExtractor) Match(filename string) bool {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	_, ok := doconvConverterByExtensions[extension]
+	return ok
+}
+
+func (de *documentExtractor) Extract(filename string, r io.Reader) (string, error) {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	converter, ok := doconvConverterByExtensions[extension]
+	if !ok {
+		return "", errors.New("Unknown converter")
+	}
+
+	f, err := ioutil.TempFile(os.TempDir(), "docconv")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+
+	_, err = io.Copy(f, r)
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	text, _, err := converter(f)
+	if err != nil {
+		return "", err
+	}
+
+	return text, nil
+}

services/docextractor/interface.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+)
+
+// Extractors define the interface needed to extract file content
+type Extractor interface {
+	Match(filename string) bool
+	Extract(filename string, file io.Reader) (string, error)
+}

services/docextractor/mmpreview.go

@@ -0,0 +1,81 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+// MMPreview is a micro-service to convert from any libreoffice supported
+// format into a PDF file, and then we use the regular pdf extractor to convert
+// it into plain text.
+
+import (
+	"bytes"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type mmPreviewExtractor struct {
+	url          string
+	secret       string
+	pdfExtractor pdfExtractor
+}
+
+var mmpreviewSupportedExtensions = map[string]bool{
+	"ppt":  true,
+	"odp":  true,
+	"xls":  true,
+	"xlsx": true,
+	"ods":  true,
+}
+
+func newMMPreviewExtractor(url string, secret string, pdfExtractor pdfExtractor) *mmPreviewExtractor {
+	return &mmPreviewExtractor{url: url, secret: secret, pdfExtractor: pdfExtractor}
+}
+
+func (mpe *mmPreviewExtractor) Match(filename string) bool {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	return mmpreviewSupportedExtensions[extension]
+}
+
+func (mpe *mmPreviewExtractor) Extract(filename string, file io.Reader) (string, error) {
+	b, w, err := createMultipartFormData("file", filename, file)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	req, err := http.NewRequest("POST", mpe.url+"/toPDF", &b)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	if mpe.secret != "" {
+		req.Header.Add("Authentication", mpe.secret)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)")
+	}
+	return mpe.pdfExtractor.Extract(filename, resp.Body)
+}
+
+func createMultipartFormData(fieldName, fileName string, fileData io.Reader) (bytes.Buffer, *multipart.Writer, error) {
+	var b bytes.Buffer
+	var err error
+	w := multipart.NewWriter(&b)
+	var fw io.Writer
+	if fw, err = w.CreateFormFile(fieldName, fileName); err != nil {
+		return b, nil, err
+	}
+	if _, err = io.Copy(fw, fileData); err != nil {
+		return b, nil, err
+	}
+	w.Close()
+	return b, w, nil
+}

services/docextractor/pdf.go

@@ -0,0 +1,52 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/ledongthuc/pdf"
+)
+
+type pdfExtractor struct{}
+
+func (pe *pdfExtractor) Match(filename string) bool {
+	supportedExtensions := map[string]bool{
+		"pdf": true,
+	}
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	return supportedExtensions[extension]
+}
+
+func (pe *pdfExtractor) Extract(filename string, r io.Reader) (string, error) {
+	f, err := ioutil.TempFile(os.TempDir(), "pdflib")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+	size, err := io.Copy(f, r)
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	reader, err := pdf.NewReader(f, size)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	b, err := reader.GetPlainText()
+	if err != nil {
+		return "", err
+	}
+	buf.ReadFrom(b)
+	return buf.String(), nil
+}

services/docextractor/plain.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+	"io/ioutil"
+	"unicode"
+	"unicode/utf8"
+)
+
+type plainExtractor struct{}
+
+func (pe *plainExtractor) Match(filename string) bool {
+	return true
+}
+
+func (pe *plainExtractor) Extract(filename string, r io.Reader) (string, error) {
+	// This detects any visible character plus any whitespace
+	validRanges := append(unicode.GraphicRanges, unicode.White_Space)
+
+	text, _ := ioutil.ReadAll(r)
+	count := 0
+	for {
+		c, size := utf8.DecodeRune(text[count:])
+		if !unicode.In(c, validRanges...) {
+			return "", nil
+		}
+		if size == 0 {
+			break
+		}
+		count += size
+		if count > 1024 {
+			break
+		}
+	}
+
+	return string(text), nil
+}