diff --git a/cmd/mattermost/commands/extract_content.go b/cmd/mattermost/commands/extract_content.go index bce1efc36b..87915dbcbb 100644 --- a/cmd/mattermost/commands/extract_content.go +++ b/cmd/mattermost/commands/extract_content.go @@ -21,9 +21,17 @@ var ExtractContentCmd = &cobra.Command{ RunE: extractContentCmdF, } +var ignoredFiles map[string]bool + func init() { + ignoredFiles = map[string]bool{ + "png": true, "jpg": true, "jpeg": true, "gif": true, "wmv": true, + "mpg": true, "mpeg": true, "mp3": true, "mp4": true, "ogg": true, + "ogv": true, "mov": true, "apk": true, "svg": true, "webm": true, + "mkv": true, + } ExtractContentCmd.Flags().Int64("from", 0, "The timestamp of the earliest file to extract, expressed in seconds since the unix epoch.") - ExtractContentCmd.Flags().Int64("to", model.GetMillis(), "The timestamp of the latest file to extract, expressed in seconds since the unix epoch.") + ExtractContentCmd.Flags().Int64("to", model.GetMillis()/1000, "The timestamp of the latest file to extract, expressed in seconds since the unix epoch.") RootCmd.AddCommand(ExtractContentCmd) } @@ -54,7 +62,7 @@ func extractContentCmdF(command *cobra.Command, args []string) error { return errors.New("\"to\" must be greater than from") } - since := startTime + since := startTime * 1000 for { opts := model.GetFileInfosOptions{ Since: since, @@ -69,14 +77,16 @@ func extractContentCmdF(command *cobra.Command, args []string) error { break } for _, fileInfo := range fileInfos { - fmt.Println("extracting file", fileInfo.Name, fileInfo.Path) - err = a.ExtractContentFromFileInfo(fileInfo) - if err != nil { - mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", fileInfo.Id)) + if !ignoredFiles[fileInfo.Extension] { + fmt.Println("extracting file", fileInfo.Name, fileInfo.Path) + err = a.ExtractContentFromFileInfo(fileInfo) + if err != nil { + mlog.Error("Failed to extract file content", mlog.Err(err), mlog.String("fileInfoId", fileInfo.Id)) + } } } lastFileInfo := fileInfos[len(fileInfos)-1] - if lastFileInfo.CreateAt > endTime { + if lastFileInfo.CreateAt > endTime*1000 { break } since = lastFileInfo.CreateAt + 1 diff --git a/services/docextractor/docextractor_test.go b/services/docextractor/docextractor_test.go index e4e60b04c4..864ebaf2c1 100644 --- a/services/docextractor/docextractor_test.go +++ b/services/docextractor/docextractor_test.go @@ -29,7 +29,15 @@ func TestExtract(t *testing.T) { "Plain text file", "test-markdown-basics.md", ExtractSettings{}, - []string{"followed", "separated"}, + []string{"followed", "separated", "Basic"}, + []string{}, + false, + }, + { + "Plain small text file", + "test-hashtags.md", + ExtractSettings{}, + []string{"should", "render", "strings"}, []string{}, false, }, diff --git a/services/docextractor/plain.go b/services/docextractor/plain.go index 12393e2767..d2dc22d77a 100644 --- a/services/docextractor/plain.go +++ b/services/docextractor/plain.go @@ -20,12 +20,16 @@ func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, err // This detects any visible character plus any whitespace validRanges := append(unicode.GraphicRanges, unicode.White_Space) - runes := make([]byte, 1028) - _, err := r.Read(runes) - if err != nil { + runes := make([]byte, 1024) + total, err := r.Read(runes) + if err != nil && err != io.EOF { return "", err } + if total == 0 { + return "", nil + } + count := 0 for { c, size := utf8.DecodeRune(runes[count:]) @@ -36,11 +40,13 @@ func (pe *plainExtractor) Extract(filename string, r io.ReadSeeker) (string, err break } count += size - if count > 1024 { + + // subtract the max rune size to prevent accidentally splitted runes at the end of first 1024 bytes + if count > total-utf8.UTFMax || count > len(runes)-utf8.UTFMax { break } } text, _ := ioutil.ReadAll(r) - return string(text), nil + return string(runes) + string(text), nil }