[MM-13033] Continue bulk import on large image error (#10780)

* [MM-13033] Continue bulk import on large image error

* add tests, review comments

* Update app/import_functions.go

Co-Authored-By: liusy182 <liusy182@hotmail.com>
Этот коммит содержится в:
Siyuan Liu
2019-05-08 13:29:42 -07:00
коммит произвёл Jesús Espino
родитель 66cb36f5dc
Коммит 27566f6a06
3 изменённых файлов: 43 добавлений и 9 удалений

Просмотреть файл

@@ -6,14 +6,25 @@ package app
import (
"bufio"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"sync"
"github.com/mattermost/mattermost-server/mlog"
"github.com/mattermost/mattermost-server/model"
)
func stopOnError(err LineImportWorkerError) bool {
if err.Error.Id == "api.file.upload_file.large_image.app_error" {
mlog.Warn(fmt.Sprintf("Large image import error: %s", err.Error.Error()))
return false
}
return true
}
func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan LineImportWorkerData, errors chan<- LineImportWorkerError) {
for line := range lines {
if err := a.ImportLine(line.LineImportData, dryRun); err != nil {
@@ -65,7 +76,9 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
// Check no errors occurred while waiting for the queue to empty.
if len(errorsChan) != 0 {
err := <-errorsChan
return err.Error, err.LineNumber
if stopOnError(err) {
return err.Error, err.LineNumber
}
}
}
@@ -81,9 +94,11 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
select {
case linesChan <- LineImportWorkerData{line, lineNumber}:
case err := <-errorsChan:
close(linesChan)
wg.Wait()
return err.Error, err.LineNumber
if stopOnError(err) {
close(linesChan)
wg.Wait()
return err.Error, err.LineNumber
}
}
}
@@ -94,7 +109,9 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
// Check no errors occurred while waiting for the queue to empty.
if len(errorsChan) != 0 {
err := <-errorsChan
return err.Error, err.LineNumber
if stopOnError(err) {
return err.Error, err.LineNumber
}
}
if err := scanner.Err(); err != nil {