From d5df6a3fc473ed0f809216ce494fb359d6a87da6 Mon Sep 17 00:00:00 2001
From: Ibrahim Serdar Acikgoz <serdaracikgoz86@gmail.com>
Date: Fri, 6 Mar 2020 23:14:35 +0300
Subject: [PATCH] [MM-22314] Remove read limit at app/import which cause bulk
 import to fail (#13863)

* app/import: increase buffer size to avoid failing in large inputs

* remove commented out lock

* app/import: make maxTokenSize constant

Co-authored-by: mattermod <mattermod@users.noreply.github.com>
---
 app/import.go      | 20 ++++++++------------
 app/import_test.go | 13 +++++++++++++
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/app/import.go b/app/import.go
index 26de668985..39e379252f 100644
--- a/app/import.go
+++ b/app/import.go
@@ -16,6 +16,8 @@ import (
 	"github.com/mattermost/mattermost-server/v5/model"
 )
 
+const maxScanTokenSize = 16 * 1024 * 1024 // Need to set a higher limit than default because some customers cross the limit. See MM-22314
+
 func stopOnError(err LineImportWorkerError) bool {
 	if err.Error.Id == "api.file.upload_file.large_image.app_error" {
 		mlog.Warn("Large image import error", mlog.Err(err.Error))
@@ -35,6 +37,9 @@ func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan Lin
 
 func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model.AppError, int) {
 	scanner := bufio.NewScanner(fileReader)
+	buf := make([]byte, 0, 64*1024)
+	scanner.Buffer(buf, maxScanTokenSize)
+
 	lineNumber := 0
 
 	a.Srv().Store.LockToMaster()
@@ -104,7 +109,9 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
 	}
 
 	// No more lines. Clear out the worker queue before continuing.
-	close(linesChan)
+	if linesChan != nil {
+		close(linesChan)
+	}
 	wg.Wait()
 
 	// Check no errors occurred while waiting for the queue to empty.
@@ -119,10 +126,6 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
 		return model.NewAppError("BulkImport", "app.import.bulk_import.file_scan.error", nil, err.Error(), http.StatusInternalServerError), 0
 	}
 
-	if err := a.finalizeImport(dryRun); err != nil {
-		return err, 0
-	}
-
 	return nil, 0
 }
 
@@ -180,10 +183,3 @@ func (a *App) importLine(line LineImportData, dryRun bool) *model.AppError {
 		return model.NewAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "", http.StatusBadRequest)
 	}
 }
-
-func (a *App) finalizeImport(dryRun bool) *model.AppError {
-	if dryRun {
-		return nil
-	}
-	return nil
-}
diff --git a/app/import_test.go b/app/import_test.go
index e3c2c6d7bd..517d455b77 100644
--- a/app/import_test.go
+++ b/app/import_test.go
@@ -190,6 +190,19 @@ func TestImportBulkImport(t *testing.T) {
 	require.NotNil(t, err, "Should have failed due to missing version line on line 1.")
 	require.Equal(t, 1, line, "Should have failed due to missing version line on line 1.")
 
+	// Run bulk import using a valid and large input and a \r\n line break.
+	t.Run("", func(t *testing.T) {
+		posts := `{"type": "post"` + strings.Repeat(`, "post": {"team": "`+teamName+`", "channel": "`+channelName+`", "user": "`+username+`", "message": "Repeat after me", "create_at": 193456789012}`, 1E4) + "}"
+		data4 := `{"type": "version", "version": 1}
+{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}}
+{"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}}
+{"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `","theme": "` + teamTheme1 + `", "channels": [{"name": "` + channelName + `"}]}]}}
+{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012}}`
+		err, line = th.App.BulkImport(strings.NewReader(data4+"\r\n"+posts), false, 2)
+		require.Nil(t, err, "BulkImport should have succeeded")
+		require.Equal(t, 0, line, "BulkImport line should be 0")
+	})
+
 	t.Run("First item after version without type", func(t *testing.T) {
 		data := `{"type": "version", "version": 1}
 {"name": "custom-emoji-troll", "image": "bulkdata/emoji/trollolol.png"}`