Improving doc extraction command (#17482)

Automatic Merge

cmd/mattermost/commands/extract_content.go

@@ -21,9 +21,17 @@ var ExtractContentCmd = &cobra.Command{
 	RunE:    extractContentCmdF,
 }
 
+var ignoredFiles map[string]bool
+
 func init() {
+	ignoredFiles = map[string]bool{
+		"png": true, "jpg": true, "jpeg": true, "gif": true, "wmv": true,
+		"mpg": true, "mpeg": true, "mp3": true, "mp4": true, "ogg": true,
+		"ogv": true, "mov": true, "apk": true, "svg": true, "webm": true,
+		"mkv": true,
+	}
 	ExtractContentCmd.Flags().Int64("from", 0, "The timestamp of the earliest file to extract, expressed in seconds since the unix epoch.")
-	ExtractContentCmd.Flags().Int64("to", model.GetMillis(), "The timestamp of the latest file to extract, expressed in seconds since the unix epoch.")
+	ExtractContentCmd.Flags().Int64("to", model.GetMillis()/1000, "The timestamp of the latest file to extract, expressed in seconds since the unix epoch.")
 	RootCmd.AddCommand(ExtractContentCmd)
 }
 
@@ -54,7 +62,7 @@ func extractContentCmdF(command *cobra.Command, args []string) error {
 		return errors.New("\"to\" must be greater than from")
 	}
 
-	since := startTime
+	since := startTime * 1000
 	for {
 		opts := model.GetFileInfosOptions{
 			Since:          since,
@@ -69,14 +77,16 @@ func extractContentCmdF(command *cobra.Command, args []string) error {
 			break
 		}
 		for _, fileInfo := range fileInfos {
-			fmt.Println("extracting file", fileInfo.Name, fileInfo.Path)
-			err = a.ExtractContentFromFileInfo(fileInfo)
-			if err != nil {
-				mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", fileInfo.Id))
+			if !ignoredFiles[fileInfo.Extension] {
+				fmt.Println("extracting file", fileInfo.Name, fileInfo.Path)
+				err = a.ExtractContentFromFileInfo(fileInfo)
+				if err != nil {
+					mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", fileInfo.Id))
+				}
 			}
 		}
 		lastFileInfo := fileInfos[len(fileInfos)-1]
-		if lastFileInfo.CreateAt > endTime {
+		if lastFileInfo.CreateAt > endTime*1000 {
 			break
 		}
 		since = lastFileInfo.CreateAt + 1

services/docextractor/docextractor_test.go

@@ -29,7 +29,15 @@ func TestExtract(t *testing.T) {
 			"Plain text file",
 			"test-markdown-basics.md",
 			ExtractSettings{},
-			[]string{"followed", "separated"},
+			[]string{"followed", "separated", "Basic"},
+			[]string{},
+			false,
+		},
+		{
+			"Plain small text file",
+			"test-hashtags.md",
+			ExtractSettings{},
+			[]string{"should", "render", "strings"},
 			[]string{},
 			false,
 		},

services/docextractor/plain.go

@@ -20,12 +20,16 @@ func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, err
 	// This detects any visible character plus any whitespace
 	validRanges := append(unicode.GraphicRanges, unicode.White_Space)
 
-	runes := make([]byte, 1028)
-	_, err := r.Read(runes)
-	if err != nil {
+	runes := make([]byte, 1024)
+	total, err := r.Read(runes)
+	if err != nil && err != io.EOF {
 		return "", err
 	}
 
+	if total == 0 {
+		return "", nil
+	}
+
 	count := 0
 	for {
 		c, size := utf8.DecodeRune(runes[count:])
@@ -36,11 +40,13 @@ func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, err
 			break
 		}
 		count += size
-		if count > 1024 {
+
+		// subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes
+		if count > total-utf8.UTFMax || count > len(runes)-utf8.UTFMax {
 			break
 		}
 	}
 
 	text, _ := ioutil.ReadAll(r)
-	return string(text), nil
+	return string(runes) + string(text), nil
 }