Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

server/platform/services/docextractor/archive.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/mholt/archiver/v3"
+)
+
+type archiveExtractor struct {
+	SubExtractor Extractor
+}
+
+func (ae *archiveExtractor) Match(filename string) bool {
+	_, err := archiver.ByExtension(filename)
+	return err == nil
+}
+
+func (ae *archiveExtractor) Extract(name string, r io.ReadSeeker) (string, error) {
+	dir, err := os.MkdirTemp(os.TempDir(), "archiver")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	f, err := os.Create(filepath.Join(dir, name))
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+	_, err = io.Copy(f, r)
+	f.Close()
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	var text strings.Builder
+	err = archiver.Walk(f.Name(), func(file archiver.File) error {
+		text.WriteString(file.Name() + " ")
+		if ae.SubExtractor != nil {
+			filename := filepath.Base(file.Name())
+			filename = strings.ReplaceAll(filename, "-", " ")
+			filename = strings.ReplaceAll(filename, ".", " ")
+			filename = strings.ReplaceAll(filename, ",", " ")
+			data, err2 := io.ReadAll(file)
+			if err2 != nil {
+				return err2
+			}
+			subtext, extractErr := ae.SubExtractor.Extract(filename, bytes.NewReader(data))
+			if extractErr == nil {
+				text.WriteString(subtext + " ")
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return text.String(), nil
+}

server/platform/services/docextractor/combine.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+type combineExtractor struct {
+	SubExtractors []Extractor
+}
+
+func (ce *combineExtractor) Add(extractor Extractor) {
+	ce.SubExtractors = append(ce.SubExtractors, extractor)
+}
+
+func (ce *combineExtractor) Match(filename string) bool {
+	for _, extractor := range ce.SubExtractors {
+		if extractor.Match(filename) {
+			return true
+		}
+	}
+	return false
+}
+
+func (ce *combineExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
+	for _, extractor := range ce.SubExtractors {
+		if extractor.Match(filename) {
+			r.Seek(0, io.SeekStart)
+			text, err := extractor.Extract(filename, r)
+			if err != nil {
+				mlog.Warn("unable to extract file content", mlog.Err(err))
+				continue
+			}
+			return text, nil
+		}
+	}
+	return "", nil
+}

server/platform/services/docextractor/docextractor.go

@@ -0,0 +1,46 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+)
+
+// ExtractSettings defines the features enabled/disable during the document text extraction.
+type ExtractSettings struct {
+	ArchiveRecursion bool
+	MMPreviewURL     string
+	MMPreviewSecret  string
+}
+
+// Extract extract the text from a document using the system default extractors
+func Extract(filename string, r io.ReadSeeker, settings ExtractSettings) (string, error) {
+	return ExtractWithExtraExtractors(filename, r, settings, []Extractor{})
+}
+
+// ExtractWithExtraExtractors extract the text from a document using the provided extractors beside the system default extractors.
+func ExtractWithExtraExtractors(filename string, r io.ReadSeeker, settings ExtractSettings, extraExtractors []Extractor) (string, error) {
+	enabledExtractors := &combineExtractor{}
+	for _, extraExtractor := range extraExtractors {
+		enabledExtractors.Add(extraExtractor)
+	}
+	enabledExtractors.Add(&documentExtractor{})
+	enabledExtractors.Add(&pdfExtractor{})
+
+	if settings.ArchiveRecursion {
+		enabledExtractors.Add(&archiveExtractor{SubExtractor: enabledExtractors})
+	} else {
+		enabledExtractors.Add(&archiveExtractor{})
+	}
+
+	if settings.MMPreviewURL != "" {
+		enabledExtractors.Add(newMMPreviewExtractor(settings.MMPreviewURL, settings.MMPreviewSecret, pdfExtractor{}))
+	}
+	enabledExtractors.Add(&plainExtractor{})
+
+	if enabledExtractors.Match(filename) {
+		return enabledExtractors.Extract(filename, r)
+	}
+	return "", nil
+}

server/platform/services/docextractor/docextractor_test.go

@@ -0,0 +1,202 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+)
+
+func TestExtract(t *testing.T) {
+	testCases := []struct {
+		Name         string
+		TestFileName string
+		Settings     ExtractSettings
+		Contains     []string
+		NotContains  []string
+		ExpectError  bool
+	}{
+		{
+			"Plain text file",
+			"test-markdown-basics.md",
+			ExtractSettings{},
+			[]string{"followed", "separated", "Basic"},
+			[]string{},
+			false,
+		},
+		{
+			"Plain small text file",
+			"test-hashtags.md",
+			ExtractSettings{},
+			[]string{"should", "render", "strings"},
+			[]string{},
+			false,
+		},
+		{
+			"Zip file without recursion",
+			"Fake_Team_Import.zip",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Zip file with recursion",
+			"Fake_Team_Import.zip",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Rar file without recursion",
+			"Fake_Team_Import.rar",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Rar file with recursion",
+			"Fake_Team_Import.rar",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Tar.gz file without recursion",
+			"Fake_Team_Import.tar.gz",
+			ExtractSettings{},
+			[]string{"users", "channels", "general"},
+			[]string{"purpose", "announcements"},
+			false,
+		},
+		{
+			"Tar.gz file with recursion",
+			"Fake_Team_Import.tar.gz",
+			ExtractSettings{ArchiveRecursion: true},
+			[]string{"users", "channels", "general", "purpose", "announcements"},
+			[]string{},
+			false,
+		},
+		{
+			"Pdf file",
+			"sample-doc.pdf",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Docx file",
+			"sample-doc.docx",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Odt file",
+			"sample-doc.odt",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+		{
+			"Pptx file",
+			"sample-doc.pptx",
+			ExtractSettings{},
+			[]string{"simple", "document", "contains"},
+			[]string{},
+			false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			data, err := testutils.ReadTestFile(tc.TestFileName)
+			require.NoError(t, err)
+			text, err := Extract(tc.TestFileName, bytes.NewReader(data), tc.Settings)
+			if tc.ExpectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				for _, expectedString := range tc.Contains {
+					assert.Contains(t, text, expectedString)
+				}
+				for _, notExpectedString := range tc.NotContains {
+					assert.NotContains(t, text, notExpectedString)
+				}
+			}
+		})
+	}
+
+	t.Run("Unsupported binary file", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("testjpg.jpg")
+		require.NoError(t, err)
+		text, err := Extract("testjpg.jpg", bytes.NewReader(data), ExtractSettings{})
+		require.NoError(t, err)
+		require.Equal(t, "", text)
+	})
+
+	t.Run("Wrong docx extension", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.NoError(t, err)
+		text, err := Extract("sample-doc.docx", bytes.NewReader(data), ExtractSettings{})
+		require.NoError(t, err)
+		require.Equal(t, "", text)
+	})
+}
+
+type customTestPdfExtractor struct{}
+
+func (te *customTestPdfExtractor) Match(filename string) bool {
+	return strings.HasSuffix(filename, ".pdf")
+}
+
+func (te *customTestPdfExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
+	return "this is a text generated content", nil
+}
+
+type failingExtractor struct{}
+
+func (te *failingExtractor) Match(filename string) bool {
+	return true
+}
+
+func (te *failingExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
+	return "", errors.New("this always fail")
+}
+
+func TestExtractWithExtraExtractors(t *testing.T) {
+	t.Run("override existing extractor", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.NoError(t, err)
+
+		text, err := ExtractWithExtraExtractors("sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&customTestPdfExtractor{}})
+		require.NoError(t, err)
+		require.Equal(t, text, "this is a text generated content")
+	})
+
+	t.Run("failing extractor", func(t *testing.T) {
+		data, err := testutils.ReadTestFile("sample-doc.pdf")
+		require.NoError(t, err)
+
+		text, err := ExtractWithExtraExtractors("sample-doc.pdf", bytes.NewReader(data), ExtractSettings{}, []Extractor{&failingExtractor{}})
+		require.NoError(t, err)
+		assert.Contains(t, text, "simple")
+		assert.Contains(t, text, "document")
+		assert.Contains(t, text, "contains")
+	})
+}

server/platform/services/docextractor/documents.go

@@ -0,0 +1,55 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"errors"
+	"io"
+	"path"
+	"strings"
+
+	"code.sajari.com/docconv"
+)
+
+type documentExtractor struct{}
+
+var doconvConverterByExtensions = map[string]func(io.Reader) (string, map[string]string, error){
+	"doc":  docconv.ConvertDoc,
+	"docx": docconv.ConvertDocx,
+	"pptx": docconv.ConvertPptx,
+	"odt":  docconv.ConvertODT,
+	"html": func(r io.Reader) (string, map[string]string, error) { return docconv.ConvertHTML(r, true) },
+	// Temporarily disabled to avoid crashes on malicious .pages files
+	// "pages": docconv.ConvertPages,
+	"rtf": docconv.ConvertRTF,
+	"pdf": docconv.ConvertPDF,
+}
+
+func (de *documentExtractor) Match(filename string) bool {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	_, ok := doconvConverterByExtensions[extension]
+	return ok
+}
+
+func (de *documentExtractor) Extract(filename string, r io.ReadSeeker) (out string, outErr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			out = ""
+			outErr = errors.New("error extracting document text")
+		}
+	}()
+
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	converter, ok := doconvConverterByExtensions[extension]
+	if !ok {
+		return "", errors.New("unknown converter")
+	}
+
+	text, _, err := converter(r)
+	if err != nil {
+		return "", err
+	}
+
+	return text, nil
+}

server/platform/services/docextractor/interface.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+)
+
+// Extractors define the interface needed to extract file content
+type Extractor interface {
+	Match(filename string) bool
+	Extract(filename string, file io.ReadSeeker) (string, error)
+}

server/platform/services/docextractor/mmpreview.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+// MMPreview is a micro-service to convert from any libreoffice supported
+// format into a PDF file, and then we use the regular pdf extractor to convert
+// it into plain text.
+
+import (
+	"bytes"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"path"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type mmPreviewExtractor struct {
+	url          string
+	secret       string
+	pdfExtractor pdfExtractor
+}
+
+var mmpreviewSupportedExtensions = map[string]bool{
+	"ppt":  true,
+	"odp":  true,
+	"xls":  true,
+	"xlsx": true,
+	"ods":  true,
+}
+
+func newMMPreviewExtractor(url string, secret string, pdfExtractor pdfExtractor) *mmPreviewExtractor {
+	return &mmPreviewExtractor{url: url, secret: secret, pdfExtractor: pdfExtractor}
+}
+
+func (mpe *mmPreviewExtractor) Match(filename string) bool {
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	return mmpreviewSupportedExtensions[extension]
+}
+
+func (mpe *mmPreviewExtractor) Extract(filename string, file io.ReadSeeker) (string, error) {
+	b, w, err := createMultipartFormData("file", filename, file)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	req, err := http.NewRequest("POST", mpe.url+"/toPDF", &b)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	req.Header.Set("Content-Type", w.FormDataContentType())
+	if mpe.secret != "" {
+		req.Header.Add("Authentication", mpe.secret)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, "Unable to generate file preview using mmpreview.")
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return "", errors.New("Unable to generate file preview using mmpreview (The server has replied with an error)")
+	}
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "unable to read the response from mmpreview")
+	}
+	return mpe.pdfExtractor.Extract(filename, bytes.NewReader(data))
+}
+
+func createMultipartFormData(fieldName, fileName string, fileData io.ReadSeeker) (bytes.Buffer, *multipart.Writer, error) {
+	var b bytes.Buffer
+	var err error
+	w := multipart.NewWriter(&b)
+	var fw io.Writer
+	if fw, err = w.CreateFormFile(fieldName, fileName); err != nil {
+		return b, nil, err
+	}
+	if _, err = io.Copy(fw, fileData); err != nil {
+		return b, nil, err
+	}
+	w.Close()
+	return b, w, nil
+}

server/platform/services/docextractor/pdf.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/ledongthuc/pdf"
+)
+
+type pdfExtractor struct{}
+
+func (pe *pdfExtractor) Match(filename string) bool {
+	supportedExtensions := map[string]bool{
+		"pdf": true,
+	}
+	extension := strings.TrimPrefix(path.Ext(filename), ".")
+	return supportedExtensions[extension]
+}
+
+func (pe *pdfExtractor) Extract(filename string, r io.ReadSeeker) (out string, outErr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			out = ""
+			outErr = errors.New("error extracting pdf text")
+		}
+	}()
+	f, err := os.CreateTemp(os.TempDir(), "pdflib")
+	if err != nil {
+		return "", fmt.Errorf("error creating temporary file: %v", err)
+	}
+	defer f.Close()
+	defer os.Remove(f.Name())
+	size, err := io.Copy(f, r)
+	if err != nil {
+		return "", fmt.Errorf("error copying data into temporary file: %v", err)
+	}
+
+	reader, err := pdf.NewReader(f, size)
+	if err != nil {
+		return "", err
+	}
+
+	var buf bytes.Buffer
+	b, err := reader.GetPlainText()
+	if err != nil {
+		return "", err
+	}
+	buf.ReadFrom(b)
+	return buf.String(), nil
+}

server/platform/services/docextractor/pdf_test.go

@@ -0,0 +1,37 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+)
+
+func TestPdfEmptyFile(t *testing.T) {
+	extractor := pdfExtractor{}
+	_, err := extractor.Extract("test.pdf", bytes.NewReader([]byte{}))
+	require.Error(t, err)
+}
+
+func TestPdfFile(t *testing.T) {
+	extractor := pdfExtractor{}
+	contentText := "This is a simple document that contains some text."
+	content, err := testutils.ReadTestFile("sample-doc.pdf")
+	require.NoError(t, err)
+	extractedText, err := extractor.Extract("sample-doc.pdf", bytes.NewReader(content))
+	require.NoError(t, err)
+	require.Equal(t, contentText, extractedText)
+}
+
+func TestWrongPdfFile(t *testing.T) {
+	extractor := pdfExtractor{}
+	content, err := testutils.ReadTestFile("sample-doc.docx")
+	require.NoError(t, err)
+	_, err = extractor.Extract("sample-doc.pdf", bytes.NewReader(content))
+	require.Error(t, err)
+}

server/platform/services/docextractor/plain.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"io"
+	"unicode"
+	"unicode/utf8"
+)
+
+type plainExtractor struct{}
+
+func (pe *plainExtractor) Match(filename string) bool {
+	return true
+}
+
+func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, error) {
+	// This detects any visible character plus any whitespace
+	validRanges := append(unicode.GraphicRanges, unicode.White_Space)
+
+	runes := make([]byte, 1024)
+	total, err := r.Read(runes)
+	if err != nil && err != io.EOF {
+		return "", err
+	}
+
+	if total == 0 {
+		return "", nil
+	}
+
+	count := 0
+	for {
+		c, size := utf8.DecodeRune(runes[count:])
+		if !unicode.In(c, validRanges...) {
+			return "", nil
+		}
+		if size == 0 {
+			break
+		}
+		count += size
+
+		// subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes
+		if count > total-utf8.UTFMax {
+			break
+		}
+	}
+
+	text, _ := io.ReadAll(r)
+	return string(runes[0:total]) + string(text), nil
+}

server/platform/services/docextractor/plain_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPlainEmptyFile(t *testing.T) {
+	extractor := plainExtractor{}
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte{}))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}
+
+func TestPlainTextSmallFile(t *testing.T) {
+	extractor := plainExtractor{}
+	content := strings.Repeat("test \n", 5)
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte(content)))
+	require.NoError(t, err)
+	require.Equal(t, content, extractedText)
+}
+
+func TestPlainBigFile(t *testing.T) {
+	extractor := plainExtractor{}
+	content := strings.Repeat("test \n", 1000)
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte(content)))
+	require.NoError(t, err)
+	require.Equal(t, content, extractedText)
+}
+
+func TestSmallBinaryFile(t *testing.T) {
+	extractor := plainExtractor{}
+	notUTF8Char := byte(0x7)
+	content := bytes.Repeat([]byte{notUTF8Char}, 1000)
+	extractedText, err := extractor.Extract("test.bin", bytes.NewReader(content))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}
+
+func TestBigBinaryFile(t *testing.T) {
+	extractor := plainExtractor{}
+	notUTF8Char := byte(0x7)
+	content := bytes.Repeat([]byte{notUTF8Char}, 10000)
+	extractedText, err := extractor.Extract("test.bin", bytes.NewReader(content))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}