Removing FilesSearch feature flag (#17548)

* Removing FilesSearch feature flag

* Fixing tests

* Adding an improvement on plain text extraction

* Adding tests for plain text extraction

* Removed unneeded conversion

* Adding missed license

* Remove the feature flag from the migration

* Fixing some tests

* Updating i18n/en.json file

services/docextractor/plain.go

@@ -42,11 +42,11 @@ func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, err
 		count += size
 
 		// subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes
-		if count > total-utf8.UTFMax || count > len(runes)-utf8.UTFMax {
+		if count > total-utf8.UTFMax {
 			break
 		}
 	}
 
 	text, _ := ioutil.ReadAll(r)
-	return string(runes) + string(text), nil
+	return string(runes[0:total]) + string(text), nil
 }

services/docextractor/plain_test.go

@@ -0,0 +1,53 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package docextractor
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestPlainEmptyFile(t *testing.T) {
+	extractor := plainExtractor{}
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte{}))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}
+
+func TestPlainTextSmallFile(t *testing.T) {
+	extractor := plainExtractor{}
+	content := strings.Repeat("test \n", 5)
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte(content)))
+	require.NoError(t, err)
+	require.Equal(t, content, extractedText)
+}
+
+func TestPlainBigFile(t *testing.T) {
+	extractor := plainExtractor{}
+	content := strings.Repeat("test \n", 1000)
+	extractedText, err := extractor.Extract("test.txt", bytes.NewReader([]byte(content)))
+	require.NoError(t, err)
+	require.Equal(t, content, extractedText)
+}
+
+func TestSmallBinaryFile(t *testing.T) {
+	extractor := plainExtractor{}
+	notUTF8Char := byte(0x7)
+	content := bytes.Repeat([]byte{notUTF8Char}, 1000)
+	extractedText, err := extractor.Extract("test.bin", bytes.NewReader(content))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}
+
+func TestBigBinaryFile(t *testing.T) {
+	extractor := plainExtractor{}
+	notUTF8Char := byte(0x7)
+	content := bytes.Repeat([]byte{notUTF8Char}, 10000)
+	extractedText, err := extractor.Extract("test.bin", bytes.NewReader(content))
+	require.NoError(t, err)
+	require.Equal(t, "", extractedText)
+}

services/searchengine/bleveengine/indexer/indexing_job.go

@@ -148,10 +148,6 @@ func (worker *BleveIndexerWorker) DoJob(job *model.Job) {
 		EndAtTime:    model.GetMillis(),
 	}
 
-	if !worker.jobServer.Config().FeatureFlags.FilesSearch {
-		progress.DoneFiles = true
-	}
-
 	// Extract the start and end times, if they are set.
 	if startString, ok := job.Data["start_time"]; ok {
 		startInt, err := strconv.ParseInt(startString, 10, 64)