Improving doc extraction command (#17482)

Automatic Merge

services/docextractor/docextractor_test.go

@@ -29,7 +29,15 @@ func TestExtract(t *testing.T) {
 			"Plain text file",
 			"test-markdown-basics.md",
 			ExtractSettings{},
-			[]string{"followed", "separated"},
+			[]string{"followed", "separated", "Basic"},
+			[]string{},
+			false,
+		},
+		{
+			"Plain small text file",
+			"test-hashtags.md",
+			ExtractSettings{},
+			[]string{"should", "render", "strings"},
 			[]string{},
 			false,
 		},

services/docextractor/plain.go

@@ -20,12 +20,16 @@ func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, err
 	// This detects any visible character plus any whitespace
 	validRanges := append(unicode.GraphicRanges, unicode.White_Space)
 
-	runes := make([]byte, 1028)
-	_, err := r.Read(runes)
-	if err != nil {
+	runes := make([]byte, 1024)
+	total, err := r.Read(runes)
+	if err != nil && err != io.EOF {
 		return "", err
 	}
 
+	if total == 0 {
+		return "", nil
+	}
+
 	count := 0
 	for {
 		c, size := utf8.DecodeRune(runes[count:])
@@ -36,11 +40,13 @@ func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, err
 			break
 		}
 		count += size
-		if count > 1024 {
+
+		// subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes
+		if count > total-utf8.UTFMax || count > len(runes)-utf8.UTFMax {
 			break
 		}
 	}
 
 	text, _ := ioutil.ReadAll(r)
-	return string(text), nil
+	return string(runes) + string(text), nil
 }