PLT-5366, PLT-5364, PLT-5363: Bulk Import Part 1. (#5204)

This commit provides the first part of the bulk import system.

The CLI command is provided, complete with validation & apply modes. All
the basic properties of Teams and Channels can be imported. Users &
Posts will follow separately in a future commit.

app/import.go

@@ -4,9 +4,12 @@
 package app
 
 import (
+	"bufio"
 	"bytes"
+	"encoding/json"
 	"io"
 	"regexp"
+	"strings"
 	"unicode/utf8"
 
 	l4g "github.com/alecthomas/log4go"
@@ -14,7 +17,242 @@ import (
 	"github.com/mattermost/platform/utils"
 )
 
+// Import Data Models
+
+type LineImportData struct {
+	Type    string             `json:"type"`
+	Team    *TeamImportData    `json:"team"`
+	Channel *ChannelImportData `json:"channel"`
+}
+
+type TeamImportData struct {
+	Name            *string `json:"name"`
+	DisplayName     *string `json:"display_name"`
+	Type            *string `json:"type"`
+	Description     *string `json:"description"`
+	AllowOpenInvite *bool   `json:"allow_open_invite"`
+}
+
+type ChannelImportData struct {
+	Team        *string `json:"team"`
+	Name        *string `json:"name"`
+	DisplayName *string `json:"display_name"`
+	Type        *string `json:"type"`
+	Header      *string `json:"header"`
+	Purpose     *string `json:"purpose"`
+}
+
 //
+// -- Bulk Import Functions --
+// These functions import data directly into the database. Security and permission checks are bypassed but validity is
+// still enforced.
+//
+
+func BulkImport(fileReader io.Reader, dryRun bool) (*model.AppError, int) {
+	scanner := bufio.NewScanner(fileReader)
+	lineNumber := 0
+	for scanner.Scan() {
+		decoder := json.NewDecoder(strings.NewReader(scanner.Text()))
+		lineNumber++
+
+		var line LineImportData
+		if err := decoder.Decode(&line); err != nil {
+			return model.NewLocAppError("BulkImport", "app.import.bulk_import.json_decode.error", nil, err.Error()), lineNumber
+		} else {
+			if err := ImportLine(line, dryRun); err != nil {
+				return err, lineNumber
+			}
+		}
+	}
+
+	if err := scanner.Err(); err != nil {
+		return model.NewLocAppError("BulkImport", "app.import.bulk_import.file_scan.error", nil, err.Error()), 0
+	}
+
+	return nil, 0
+}
+
+func ImportLine(line LineImportData, dryRun bool) *model.AppError {
+	switch {
+	case line.Type == "team":
+		if line.Team == nil {
+			return model.NewLocAppError("BulkImport", "app.import.import_line.null_team.error", nil, "")
+		} else {
+			return ImportTeam(line.Team, dryRun)
+		}
+	case line.Type == "channel":
+		if line.Channel == nil {
+			return model.NewLocAppError("BulkImport", "app.import.import_line.null_channel.error", nil, "")
+		} else {
+			return ImportChannel(line.Channel, dryRun)
+		}
+	default:
+		return model.NewLocAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "")
+	}
+}
+
+func ImportTeam(data *TeamImportData, dryRun bool) *model.AppError {
+	if err := validateTeamImportData(data); err != nil {
+		return err
+	}
+
+	// If this is a Dry Run, do not continue any further.
+	if dryRun {
+		return nil
+	}
+
+	var team *model.Team
+	if result := <-Srv.Store.Team().GetByName(*data.Name); result.Err == nil {
+		team = result.Data.(*model.Team)
+	} else {
+		team = &model.Team{}
+	}
+
+	team.Name = *data.Name
+	team.DisplayName = *data.DisplayName
+	team.Type = *data.Type
+
+	if data.Description != nil {
+		team.Description = *data.Description
+	}
+
+	if data.AllowOpenInvite != nil {
+		team.AllowOpenInvite = *data.AllowOpenInvite
+	}
+
+	if team.Id == "" {
+		if _, err := CreateTeam(team); err != nil {
+			return err
+		}
+	} else {
+		if _, err := UpdateTeam(team); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validateTeamImportData(data *TeamImportData) *model.AppError {
+
+	if data.Name == nil {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_missing.error", nil, "")
+	} else if len(*data.Name) > model.TEAM_NAME_MAX_LENGTH {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_length.error", nil, "")
+	} else if model.IsReservedTeamName(*data.Name) {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_reserved.error", nil, "")
+	} else if !model.IsValidTeamName(*data.Name) {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.name_characters.error", nil, "")
+	}
+
+	if data.DisplayName == nil {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.display_name_missing.error", nil, "")
+	} else if utf8.RuneCountInString(*data.DisplayName) == 0 || utf8.RuneCountInString(*data.DisplayName) > model.TEAM_DISPLAY_NAME_MAX_RUNES {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.display_name_length.error", nil, "")
+	}
+
+	if data.Type == nil {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.type_missing.error", nil, "")
+	} else if *data.Type != model.TEAM_OPEN && *data.Type != model.TEAM_INVITE {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.type_invalid.error", nil, "")
+	}
+
+	if data.Description != nil && len(*data.Description) > model.TEAM_DESCRIPTION_MAX_LENGTH {
+		return model.NewLocAppError("BulkImport", "app.import.validate_team_import_data.description_length.error", nil, "")
+	}
+
+	return nil
+}
+
+func ImportChannel(data *ChannelImportData, dryRun bool) *model.AppError {
+	if err := validateChannelImportData(data); err != nil {
+		return err
+	}
+
+	// If this is a Dry Run, do not continue any further.
+	if dryRun {
+		return nil
+	}
+
+	var team *model.Team
+	if result := <-Srv.Store.Team().GetByName(*data.Team); result.Err != nil {
+		return model.NewLocAppError("BulkImport", "app.import.import_channel.team_not_found.error", map[string]interface{}{"TeamName": *data.Team}, "")
+	} else {
+		team = result.Data.(*model.Team)
+	}
+
+	var channel *model.Channel
+	if result := <-Srv.Store.Channel().GetByNameIncludeDeleted(team.Id, *data.Name); result.Err == nil {
+		channel = result.Data.(*model.Channel)
+	} else {
+		channel = &model.Channel{}
+	}
+
+	channel.TeamId = team.Id
+	channel.Name = *data.Name
+	channel.DisplayName = *data.DisplayName
+	channel.Type = *data.Type
+
+	if data.Header != nil {
+		channel.Header = *data.Header
+	}
+
+	if data.Purpose != nil {
+		channel.Purpose = *data.Purpose
+	}
+
+	if channel.Id == "" {
+		if _, err := CreateChannel(channel, false); err != nil {
+			return err
+		}
+	} else {
+		if _, err := UpdateChannel(channel); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func validateChannelImportData(data *ChannelImportData) *model.AppError {
+
+	if data.Team == nil {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.team_missing.error", nil, "")
+	}
+
+	if data.Name == nil {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_missing.error", nil, "")
+	} else if len(*data.Name) > model.CHANNEL_NAME_MAX_LENGTH {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_length.error", nil, "")
+	} else if !model.IsValidChannelIdentifier(*data.Name) {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.name_characters.error", nil, "")
+	}
+
+	if data.DisplayName == nil {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.display_name_missing.error", nil, "")
+	} else if utf8.RuneCountInString(*data.DisplayName) == 0 || utf8.RuneCountInString(*data.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.display_name_length.error", nil, "")
+	}
+
+	if data.Type == nil {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.type_missing.error", nil, "")
+	} else if *data.Type != model.CHANNEL_OPEN && *data.Type != model.CHANNEL_PRIVATE {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.type_invalid.error", nil, "")
+	}
+
+	if data.Header != nil && utf8.RuneCountInString(*data.Header) > model.CHANNEL_HEADER_MAX_RUNES {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.header_length.error", nil, "")
+	}
+
+	if data.Purpose != nil && utf8.RuneCountInString(*data.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES {
+		return model.NewLocAppError("BulkImport", "app.import.validate_channel_import_data.purpose_length.error", nil, "")
+	}
+
+	return nil
+}
+
+//
+// -- Old SlackImport Functions --
 // Import functions are sutible for entering posts and users into the database without
 // some of the usual checks. (IsValid is still run)
 //
@@ -73,7 +311,7 @@ func ImportUser(team *model.Team, user *model.User) *model.User {
 	}
 }
 
-func ImportChannel(channel *model.Channel) *model.Channel {
+func OldImportChannel(channel *model.Channel) *model.Channel {
 	if result := <-Srv.Store.Channel().Save(channel); result.Err != nil {
 		return nil
 	} else {

app/import_test.go

@@ -0,0 +1,552 @@
+// Copyright (c) 2017 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"github.com/mattermost/platform/model"
+	"strings"
+	"testing"
+)
+
+func ptrStr(s string) *string {
+	return &s
+}
+
+func ptrInt64(i int64) *int64 {
+	return &i
+}
+
+func ptrBool(b bool) *bool {
+	return &b
+}
+
+func TestImportValidateTeamImportData(t *testing.T) {
+
+	// Test with minimum required valid properties.
+	data := TeamImportData{
+		Name:        ptrStr("teamname"),
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+	}
+	if err := validateTeamImportData(&data); err != nil {
+		t.Fatal("Validation failed but should have been valid.")
+	}
+
+	// Test with various invalid names.
+	data = TeamImportData{
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+	}
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to missing name.")
+	}
+
+	data.Name = ptrStr(strings.Repeat("abcdefghij", 7))
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to too long name.")
+	}
+
+	data.Name = ptrStr("login")
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to reserved word in name.")
+	}
+
+	data.Name = ptrStr("Test::''ASD")
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to non alphanum characters in name.")
+	}
+
+	data.Name = ptrStr("A")
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to short name.")
+	}
+
+	// Test team various invalid display names.
+	data = TeamImportData{
+		Name: ptrStr("teamname"),
+		Type: ptrStr("O"),
+	}
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to missing display_name.")
+	}
+
+	data.DisplayName = ptrStr("")
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to empty display_name.")
+	}
+
+	data.DisplayName = ptrStr(strings.Repeat("abcdefghij", 7))
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to too long display_name.")
+	}
+
+	// Test with various valid and invalid types.
+	data = TeamImportData{
+		Name:        ptrStr("teamname"),
+		DisplayName: ptrStr("Display Name"),
+	}
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to missing type.")
+	}
+
+	data.Type = ptrStr("A")
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to invalid type.")
+	}
+
+	data.Type = ptrStr("I")
+	if err := validateTeamImportData(&data); err != nil {
+		t.Fatal("Should have succeeded with valid type.")
+	}
+
+	// Test with all the combinations of optional parameters.
+	data = TeamImportData{
+		Name:            ptrStr("teamname"),
+		DisplayName:     ptrStr("Display Name"),
+		Type:            ptrStr("O"),
+		Description:     ptrStr("The team description."),
+		AllowOpenInvite: ptrBool(true),
+	}
+	if err := validateTeamImportData(&data); err != nil {
+		t.Fatal("Should have succeeded with valid optional properties.")
+	}
+
+	data.AllowOpenInvite = ptrBool(false)
+	if err := validateTeamImportData(&data); err != nil {
+		t.Fatal("Should have succeeded with allow open invites false.")
+	}
+
+	data.Description = ptrStr(strings.Repeat("abcdefghij ", 26))
+	if err := validateTeamImportData(&data); err == nil {
+		t.Fatal("Should have failed due to too long description.")
+	}
+}
+
+func TestImportValidateChannelImportData(t *testing.T) {
+
+	// Test with minimum required valid properties.
+	data := ChannelImportData{
+		Team:        ptrStr("teamname"),
+		Name:        ptrStr("channelname"),
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+	}
+	if err := validateChannelImportData(&data); err != nil {
+		t.Fatal("Validation failed but should have been valid.")
+	}
+
+	// Test with missing team.
+	data = ChannelImportData{
+		Name:        ptrStr("channelname"),
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+	}
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to missing team.")
+	}
+
+	// Test with various invalid names.
+	data = ChannelImportData{
+		Team:        ptrStr("teamname"),
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+	}
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to missing name.")
+	}
+
+	data.Name = ptrStr(strings.Repeat("abcdefghij", 7))
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to too long name.")
+	}
+
+	data.Name = ptrStr("Test::''ASD")
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to non alphanum characters in name.")
+	}
+
+	data.Name = ptrStr("A")
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to short name.")
+	}
+
+	// Test team various invalid display names.
+	data = ChannelImportData{
+		Team: ptrStr("teamname"),
+		Name: ptrStr("channelname"),
+		Type: ptrStr("O"),
+	}
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to missing display_name.")
+	}
+
+	data.DisplayName = ptrStr("")
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to empty display_name.")
+	}
+
+	data.DisplayName = ptrStr(strings.Repeat("abcdefghij", 7))
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to too long display_name.")
+	}
+
+	// Test with various valid and invalid types.
+	data = ChannelImportData{
+		Team:        ptrStr("teamname"),
+		Name:        ptrStr("channelname"),
+		DisplayName: ptrStr("Display Name"),
+	}
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to missing type.")
+	}
+
+	data.Type = ptrStr("A")
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to invalid type.")
+	}
+
+	data.Type = ptrStr("P")
+	if err := validateChannelImportData(&data); err != nil {
+		t.Fatal("Should have succeeded with valid type.")
+	}
+
+	// Test with all the combinations of optional parameters.
+	data = ChannelImportData{
+		Team:        ptrStr("teamname"),
+		Name:        ptrStr("channelname"),
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+		Header:      ptrStr("Channel Header Here"),
+		Purpose:     ptrStr("Channel Purpose Here"),
+	}
+	if err := validateChannelImportData(&data); err != nil {
+		t.Fatal("Should have succeeded with valid optional properties.")
+	}
+
+	data.Header = ptrStr(strings.Repeat("abcdefghij ", 103))
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to too long header.")
+	}
+
+	data.Header = ptrStr("Channel Header Here")
+	data.Purpose = ptrStr(strings.Repeat("abcdefghij ", 26))
+	if err := validateChannelImportData(&data); err == nil {
+		t.Fatal("Should have failed due to too long purpose.")
+	}
+}
+
+func TestImportImportTeam(t *testing.T) {
+	_ = Setup()
+
+	// Check how many teams are in the database.
+	var teamsCount int64
+	if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil {
+		teamsCount = r.Data.(int64)
+	} else {
+		t.Fatalf("Failed to get team count.")
+	}
+
+	data := TeamImportData{
+		Name:            ptrStr(model.NewId()),
+		DisplayName:     ptrStr("Display Name"),
+		Type:            ptrStr("XXX"),
+		Description:     ptrStr("The team description."),
+		AllowOpenInvite: ptrBool(true),
+	}
+
+	// Try importing an invalid team in dryRun mode.
+	if err := ImportTeam(&data, true); err == nil {
+		t.Fatalf("Should have received an error importing an invalid team.")
+	}
+
+	// Do a valid team in dry-run mode.
+	data.Type = ptrStr("O")
+	if err := ImportTeam(&data, true); err != nil {
+		t.Fatalf("Received an error validating valid team.")
+	}
+
+	// Check that no more teams are in the DB.
+	if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil {
+		if r.Data.(int64) != teamsCount {
+			t.Fatalf("Teams got persisted in dry run mode.")
+		}
+	} else {
+		t.Fatalf("Failed to get team count.")
+	}
+
+	// Do an invalid team in apply mode, check db changes.
+	data.Type = ptrStr("XXX")
+	if err := ImportTeam(&data, false); err == nil {
+		t.Fatalf("Import should have failed on invalid team.")
+	}
+
+	// Check that no more teams are in the DB.
+	if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil {
+		if r.Data.(int64) != teamsCount {
+			t.Fatalf("Invalid team got persisted.")
+		}
+	} else {
+		t.Fatalf("Failed to get team count.")
+	}
+
+	// Do a valid team in apply mode, check db changes.
+	data.Type = ptrStr("O")
+	if err := ImportTeam(&data, false); err != nil {
+		t.Fatalf("Received an error importing valid team.")
+	}
+
+	// Check that one more team is in the DB.
+	if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil {
+		if r.Data.(int64)-1 != teamsCount {
+			t.Fatalf("Team did not get saved in apply run mode.", r.Data.(int64), teamsCount)
+		}
+	} else {
+		t.Fatalf("Failed to get team count.")
+	}
+
+	// Get the team and check that all the fields are correct.
+	if team, err := GetTeamByName(*data.Name); err != nil {
+		t.Fatalf("Failed to get team from database.")
+	} else {
+		if team.DisplayName != *data.DisplayName || team.Type != *data.Type || team.Description != *data.Description || team.AllowOpenInvite != *data.AllowOpenInvite {
+			t.Fatalf("Imported team properties do not match import data.")
+		}
+	}
+
+	// Alter all the fields of that team (apart from unique identifier) and import again.
+	data.DisplayName = ptrStr("Display Name 2")
+	data.Type = ptrStr("P")
+	data.Description = ptrStr("The new description")
+	data.AllowOpenInvite = ptrBool(false)
+
+	// Check that the original number of teams are again in the DB (because this query doesn't include deleted).
+	data.Type = ptrStr("O")
+	if err := ImportTeam(&data, false); err != nil {
+		t.Fatalf("Received an error importing updated valid team.")
+	}
+
+	if r := <-Srv.Store.Team().AnalyticsTeamCount(); r.Err == nil {
+		if r.Data.(int64)-1 != teamsCount {
+			t.Fatalf("Team alterations did not get saved in apply run mode.", r.Data.(int64), teamsCount)
+		}
+	} else {
+		t.Fatalf("Failed to get team count.")
+	}
+
+	// Get the team and check that all fields are correct.
+	if team, err := GetTeamByName(*data.Name); err != nil {
+		t.Fatalf("Failed to get team from database.")
+	} else {
+		if team.DisplayName != *data.DisplayName || team.Type != *data.Type || team.Description != *data.Description || team.AllowOpenInvite != *data.AllowOpenInvite {
+			t.Fatalf("Updated team properties do not match import data.")
+		}
+	}
+}
+
+func TestImportImportChannel(t *testing.T) {
+	_ = Setup()
+
+	// Import a Team.
+	teamName := model.NewId()
+	ImportTeam(&TeamImportData{
+		Name:        &teamName,
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+	}, false)
+	var team *model.Team
+	if te, err := GetTeamByName(teamName); err != nil {
+		t.Fatalf("Failed to get team from database.")
+	} else {
+		team = te
+	}
+
+	// Check how many channels are in the database.
+	var channelCount int64
+	if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil {
+		channelCount = r.Data.(int64)
+	} else {
+		t.Fatalf("Failed to get team count.")
+	}
+
+	// Do an invalid channel in dry-run mode.
+	data := ChannelImportData{
+		Team:        &teamName,
+		DisplayName: ptrStr("Display Name"),
+		Type:        ptrStr("O"),
+		Header:      ptrStr("Channe Header"),
+		Purpose:     ptrStr("Channel Purpose"),
+	}
+	if err := ImportChannel(&data, true); err == nil {
+		t.Fatalf("Expected error due to invalid name.")
+	}
+
+	// Check that no more channels are in the DB.
+	if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil {
+		if r.Data.(int64) != channelCount {
+			t.Fatalf("Channels got persisted in dry run mode.")
+		}
+	} else {
+		t.Fatalf("Failed to get channel count.")
+	}
+
+	// Do a valid channel with a nonexistent team in dry-run mode.
+	data.Name = ptrStr("channelname")
+	data.Team = ptrStr(model.NewId())
+	if err := ImportChannel(&data, true); err != nil {
+		t.Fatalf("Expected success as cannot validate channel name in dry run mode.")
+	}
+
+	// Check that no more channels are in the DB.
+	if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil {
+		if r.Data.(int64) != channelCount {
+			t.Fatalf("Channels got persisted in dry run mode.")
+		}
+	} else {
+		t.Fatalf("Failed to get channel count.")
+	}
+
+	// Do a valid channel in dry-run mode.
+	data.Team = &teamName
+	if err := ImportChannel(&data, true); err != nil {
+		t.Fatalf("Expected success as valid team.")
+	}
+
+	// Check that no more channels are in the DB.
+	if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil {
+		if r.Data.(int64) != channelCount {
+			t.Fatalf("Channels got persisted in dry run mode.")
+		}
+	} else {
+		t.Fatalf("Failed to get channel count.")
+	}
+
+	// Do an invalid channel in apply mode.
+	data.Name = nil
+	if err := ImportChannel(&data, false); err == nil {
+		t.Fatalf("Expected error due to invalid name (apply mode).")
+	}
+
+	// Check that no more channels are in the DB.
+	if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil {
+		if r.Data.(int64) != channelCount {
+			t.Fatalf("Invalid channel got persisted in apply mode.")
+		}
+	} else {
+		t.Fatalf("Failed to get channel count.")
+	}
+
+	// Do a valid channel in apply mode with a nonexistant team.
+	data.Name = ptrStr("channelname")
+	data.Team = ptrStr(model.NewId())
+	if err := ImportChannel(&data, false); err == nil {
+		t.Fatalf("Expected error due to non-existant team (apply mode).")
+	}
+
+	// Check that no more channels are in the DB.
+	if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil {
+		if r.Data.(int64) != channelCount {
+			t.Fatalf("Invalid team channel got persisted in apply mode.")
+		}
+	} else {
+		t.Fatalf("Failed to get channel count.")
+	}
+
+	// Do a valid channel in apply mode.
+	data.Team = &teamName
+	if err := ImportChannel(&data, false); err != nil {
+		t.Fatalf("Expected success in apply mode: %v", err.Error())
+	}
+
+	// Check that no more channels are in the DB.
+	if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil {
+		if r.Data.(int64) != channelCount+1 {
+			t.Fatalf("Channels did not get persisted in apply mode: found %v expected %v + 1", r.Data.(int64), channelCount)
+		}
+	} else {
+		t.Fatalf("Failed to get channel count.")
+	}
+
+	// Get the Channel and check all the fields are correct.
+	if channel, err := GetChannelByName(*data.Name, team.Id); err != nil {
+		t.Fatalf("Failed to get channel from database.")
+	} else {
+		if channel.Name != *data.Name || channel.DisplayName != *data.DisplayName || channel.Type != *data.Type || channel.Header != *data.Header || channel.Purpose != *data.Purpose {
+			t.Fatalf("Imported team properties do not match Import Data.")
+		}
+	}
+
+	// Alter all the fields of that channel.
+	data.DisplayName = ptrStr("Chaned Disp Name")
+	data.Type = ptrStr(model.CHANNEL_PRIVATE)
+	data.Header = ptrStr("New Header")
+	data.Purpose = ptrStr("New Purpose")
+	if err := ImportChannel(&data, false); err != nil {
+		t.Fatalf("Expected success in apply mode: %v", err.Error())
+	}
+
+	// Check channel count the same.
+	if r := <-Srv.Store.Channel().AnalyticsTypeCount("", model.CHANNEL_OPEN); r.Err == nil {
+		if r.Data.(int64) != channelCount {
+			t.Fatalf("Updated channel did not get correctly persisted in apply mode.")
+		}
+	} else {
+		t.Fatalf("Failed to get channel count.")
+	}
+
+	// Get the Channel and check all the fields are correct.
+	if channel, err := GetChannelByName(*data.Name, team.Id); err != nil {
+		t.Fatalf("Failed to get channel from database.")
+	} else {
+		if channel.Name != *data.Name || channel.DisplayName != *data.DisplayName || channel.Type != *data.Type || channel.Header != *data.Header || channel.Purpose != *data.Purpose {
+			t.Fatalf("Updated team properties do not match Import Data.")
+		}
+	}
+
+}
+
+func TestImportImportLine(t *testing.T) {
+	_ = Setup()
+
+	// Try import line with an invalid type.
+	line := LineImportData{
+		Type: "gibberish",
+	}
+
+	if err := ImportLine(line, false); err == nil {
+		t.Fatalf("Expected an error when importing a line with invalid type.")
+	}
+
+	// Try import line with team type but nil team.
+	line.Type = "team"
+	if err := ImportLine(line, false); err == nil {
+		t.Fatalf("Expected an error when importing a line of type team with a nil team.")
+	}
+
+	// Try import line with channel type but nil channel.
+	line.Type = "channel"
+	if err := ImportLine(line, false); err == nil {
+		t.Fatalf("Expected an error when importing a line with type channel with a nil channel.")
+	}
+}
+
+func TestImportBulkImport(t *testing.T) {
+	_ = Setup()
+
+	teamName := model.NewId()
+	channelName := model.NewId()
+
+	// Run bulk import with a valid 1 of everything.
+	data1 := `{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}}
+{"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}}`
+
+	if err, line := BulkImport(strings.NewReader(data1), false); err != nil || line != 0 {
+		t.Fatalf("BulkImport should have succeeded: %v, %v", err.Error(), line)
+	}
+
+	// Run bulk import using a string that contains a line with invalid json.
+	data2 := `{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "vinewy665jam3n6oxzhsdgajly"}`
+	if err, line := BulkImport(strings.NewReader(data2), false); err == nil || line != 1 {
+		t.Fatalf("Should have failed due to invalid JSON on line 1.")
+	}
+}

app/slackimport.go

@@ -484,7 +484,7 @@ func SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[str
 
 		if mChannel == nil {
 			// Haven't found an existing channel to merge with. Try importing it as a new one.
-			mChannel = ImportChannel(&newChannel)
+			mChannel = OldImportChannel(&newChannel)
 			if mChannel == nil {
 				l4g.Warn(utils.T("api.slackimport.slack_add_channels.import_failed.warn"), newChannel.DisplayName)
 				log.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))

cmd/platform/import.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"os"
 
+	"fmt"
 	"github.com/mattermost/platform/app"
 	"github.com/spf13/cobra"
 )
@@ -23,8 +24,19 @@ var slackImportCmd = &cobra.Command{
 	RunE:    slackImportCmdF,
 }
 
+var bulkImportCmd = &cobra.Command{
+	Use:     "bulk [file]",
+	Short:   "Import bulk data.",
+	Long:    "Import data from a Mattermost Bulk Import File.",
+	Example: "  import bulk bulk_data.json",
+	RunE:    bulkImportCmdF,
+}
+
 func init() {
+	bulkImportCmd.Flags().Bool("apply", false, "Save the import data to the database. Use with caution - this cannot be reverted.")
+
 	importCmd.AddCommand(
+		bulkImportCmd,
 		slackImportCmd,
 	)
 }
@@ -60,3 +72,47 @@ func slackImportCmdF(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func bulkImportCmdF(cmd *cobra.Command, args []string) error {
+	initDBCommandContextCobra(cmd)
+
+	apply, err := cmd.Flags().GetBool("apply")
+	if err != nil {
+		return errors.New("Apply flag error")
+	}
+
+	if len(args) != 1 {
+		return errors.New("Incorrect number of arguments.")
+	}
+
+	fileReader, err := os.Open(args[0])
+	if err != nil {
+		return err
+	}
+	defer fileReader.Close()
+
+	if apply {
+		CommandPrettyPrintln("Running Bulk Import. This may take a long time.")
+	} else {
+		CommandPrettyPrintln("Running Bulk Import Data Validation.")
+		CommandPrettyPrintln("** This checks the validity of the entities in the data file, but does not persist any changes **")
+		CommandPrettyPrintln("Use the --apply flag to perform the actual data import.")
+	}
+
+	CommandPrettyPrintln("")
+
+	if err, lineNumber := app.BulkImport(fileReader, !apply); err != nil {
+		CommandPrettyPrintln(err.Error())
+		if lineNumber != 0 {
+			CommandPrettyPrintln(fmt.Sprintf("Error occurred on data file line %v", lineNumber))
+		}
+	}
+
+	if apply {
+		CommandPrettyPrintln("Finished Bulk Import.")
+	} else {
+		CommandPrettyPrintln("Validation complete. You can now perform the import by rerunning this command with the --apply flag.")
+	}
+
+	return nil
+}

i18n/en.json

@@ -2779,6 +2779,118 @@
     "id": "app.channel.post_update_channel_purpose_message.updated_to",
     "translation": "%s updated the channel purpose to: %s"
   },
+  {
+    "id": "app.import.bulk_import.json_decode.error",
+    "translation": "JSON decode of line failed."
+  },
+  {
+    "id": "app.import.bulk_import.file_scan.error",
+    "translation": "Error reading import data file."
+  },
+  {
+    "id": "app.import.import_line.null_team.error",
+    "translation": "Import data line has type \"team\" but the team object is null."
+  },
+  {
+    "id": "app.import.import_line.null_channel.error",
+    "translation": "Import data line has type \"channel\" but the channel object is null."
+  },
+  {
+    "id": "app.import.import_line.unknown_line_type.error",
+    "translation": "Import data line has unknown type \"{{.Type}}\"."
+  },
+  {
+    "id": "app.import.validate_team_import_data.name_missing.error",
+    "translation": "Missing required team property: name."
+  },
+  {
+    "id": "app.import.validate_team_import_data.name_length.error",
+    "translation": "Team name is too long."
+  },
+  {
+    "id": "app.import.validate_team_import_data.name_reserved.error",
+    "translation": "Team name contains reserved word."
+  },
+  {
+    "id": "app.import.validate_team_import_data.name_characters.error",
+    "translation": "Team name contains invalid characters."
+  },
+  {
+    "id": "app.import.validate_team_import_data.display_name_missing.error",
+    "translation": "Missing required team property: display_name"
+  },
+  {
+    "id": "app.import.validate_team_import_data.display_name_length.error",
+    "translation": "Team display_name is not within permitted length constraints."
+  },
+  {
+    "id": "app.import.validate_team_import_data.type_missing.error",
+    "translation": "Missing required team property: type"
+  },
+  {
+    "id": "app.import.validate_team_import_data.type_invalid.error",
+    "translation": "Team type is not valid."
+  },
+  {
+    "id": "app.import.validate_team_import_data.create_at_zero.error",
+    "translation": "Team create_at must not be 0 if provided."
+  },
+  {
+    "id": "app.import.validate_team_import_data.description_length.error",
+    "translation": "Team description is too long."
+  },
+  {
+    "id": "app.import.validate_team_import_data.allowed_domains_length.error",
+    "translation": "Team allowed_domains is too long."
+  },
+  {
+    "id": "app.import.import_channel.team_not_found.error",
+    "translation": "Error importing channel. Team with name \"{{.TeamName}}\" could not be found."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.team_missing.error",
+    "translation": "Missing required channel property: team"
+  },
+  {
+    "id": "app.import.validate_channel_import_data.name_missing.error",
+    "translation": "Missing required channel property: name"
+  },
+  {
+    "id": "app.import.validate_channel_import_data.name_length.error",
+    "translation": "Channel name is too long."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.name_characters.error",
+    "translation": "Channel name contains invalid characters."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.display_name_missing.error",
+    "translation": "Missing required channel property: display_name."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.display_name_length.error",
+    "translation": "Channel display_name is not within permitted length constraints."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.type_missing.error",
+    "translation": "Missing required channel property: type."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.type_invalid.error",
+    "translation": "Channel type is invalid."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.create_at_zero.error",
+    "translation": "Channel create_at must not be 0 if provided."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.header_length.error",
+    "translation": "Channel header is too long."
+  },
+  {
+    "id": "app.import.validate_channel_import_data.purpose_length.error",
+    "translation": "Channel purpose is too long."
+  },
   {
     "id": "authentication.permissions.create_team_roles.description",
     "translation": "Ability to create new teams"

model/team.go

@@ -13,8 +13,14 @@ import (
 )
 
 const (
-	TEAM_OPEN   = "O"
-	TEAM_INVITE = "I"
+	TEAM_OPEN                       = "O"
+	TEAM_INVITE                     = "I"
+	TEAM_ALLOWED_DOMAINS_MAX_LENGTH = 500
+	TEAM_COMPANY_NAME_MAX_LENGTH    = 64
+	TEAM_DESCRIPTION_MAX_LENGTH     = 255
+	TEAM_DISPLAY_NAME_MAX_RUNES     = 64
+	TEAM_EMAIL_MAX_LENGTH           = 128
+	TEAM_NAME_MAX_LENGTH            = 64
 )
 
 type Team struct {
@@ -123,7 +129,7 @@ func (o *Team) IsValid() *AppError {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.update_at.app_error", nil, "id="+o.Id)
 	}
 
-	if len(o.Email) > 128 {
+	if len(o.Email) > TEAM_EMAIL_MAX_LENGTH {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.email.app_error", nil, "id="+o.Id)
 	}
 
@@ -131,15 +137,15 @@ func (o *Team) IsValid() *AppError {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.email.app_error", nil, "id="+o.Id)
 	}
 
-	if utf8.RuneCountInString(o.DisplayName) == 0 || utf8.RuneCountInString(o.DisplayName) > 64 {
+	if utf8.RuneCountInString(o.DisplayName) == 0 || utf8.RuneCountInString(o.DisplayName) > TEAM_DISPLAY_NAME_MAX_RUNES {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.name.app_error", nil, "id="+o.Id)
 	}
 
-	if len(o.Name) > 64 {
+	if len(o.Name) > TEAM_NAME_MAX_LENGTH {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.url.app_error", nil, "id="+o.Id)
 	}
 
-	if len(o.Description) > 255 {
+	if len(o.Description) > TEAM_DESCRIPTION_MAX_LENGTH {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.description.app_error", nil, "id="+o.Id)
 	}
 
@@ -155,11 +161,11 @@ func (o *Team) IsValid() *AppError {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.type.app_error", nil, "id="+o.Id)
 	}
 
-	if len(o.CompanyName) > 64 {
+	if len(o.CompanyName) > TEAM_COMPANY_NAME_MAX_LENGTH {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.company.app_error", nil, "id="+o.Id)
 	}
 
-	if len(o.AllowedDomains) > 500 {
+	if len(o.AllowedDomains) > TEAM_ALLOWED_DOMAINS_MAX_LENGTH {
 		return NewLocAppError("Team.IsValid", "model.team.is_valid.domains.app_error", nil, "id="+o.Id)
 	}