Move Elasticsearch to source available 🎉 🎉 (#29015)

* Move Elasticsearch to source available

server/enterprise/elasticsearch/common/common.go

@@ -0,0 +1,375 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/url"
+	"regexp"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	MaxLineLength = 10000
+
+	URLRegexpRE       = `(\b|^)(?:https?:\/\/)?[a-zA-Z0-9-.]+\.[a-z]+(\s|\)?[a-zA-Z0-9\-._~:/?#\[\]@!$&'\(\)*\+,;=]*)(\b|$)`
+	URLMarkdownLinkRE = `(\[[^\]]+\]\([a-zA-Z0-9\-._~:/?#\[\]@!$&'\(\)*\+,;=]+\))`
+	EmailRE           = `^[^\s"]+@[^\s"]+$`
+
+	IndexBasePosts       = "posts"
+	IndexBasePosts_MONTH = IndexBasePosts + "month"
+	IndexBaseChannels    = "channels"
+	IndexBaseUsers       = "users"
+	IndexBaseFiles       = "files"
+
+	// At the moment, this number is hardcoded. If needed, we can expose
+	// this to the config.
+	BulkFlushInterval = 5 * time.Second
+)
+
+var (
+	urlRe          = regexp.MustCompile(URLRegexpRE)
+	markdownLinkRe = regexp.MustCompile(URLMarkdownLinkRE)
+)
+
+type ESPost struct {
+	Id          string   `json:"id"`
+	TeamId      string   `json:"team_id"`
+	ChannelId   string   `json:"channel_id"`
+	UserId      string   `json:"user_id"`
+	CreateAt    int64    `json:"create_at"`
+	Message     string   `json:"message"`
+	Type        string   `json:"type"`
+	Hashtags    []string `json:"hashtags"`
+	Attachments string   `json:"attachments"`
+	URLs        []string `json:"urls"`
+}
+
+type ESFile struct {
+	Id        string `json:"id"`
+	CreatorId string `json:"creator_id"`
+	ChannelId string `json:"channel_id"`
+	PostId    string `json:"post_id"`
+	CreateAt  int64  `json:"create_at"`
+	Content   string `json:"content"`
+	Extension string `json:"extension"`
+	Name      string `json:"name"`
+}
+
+type ESChannel struct {
+	Id            string            `json:"id"`
+	Type          model.ChannelType `json:"type"`
+	UserIDs       []string          `json:"user_ids"`
+	TeamId        string            `json:"team_id"`
+	TeamMemberIDs []string          `json:"team_member_ids"`
+	NameSuggest   []string          `json:"name_suggestions"`
+}
+
+type ESUser struct {
+	Id                         string   `json:"id"`
+	SuggestionsWithFullname    []string `json:"suggestions_with_fullname"`
+	SuggestionsWithoutFullname []string `json:"suggestions_without_fullname"`
+	DeleteAt                   int64    `json:"delete_at"`
+	Roles                      []string `json:"roles"`
+	TeamsIds                   []string `json:"team_id"`
+	ChannelsIds                []string `json:"channel_id"`
+}
+
+func ESPostFromPost(post *model.Post, teamId string) (*ESPost, error) {
+	p := &model.PostForIndexing{
+		TeamId: teamId,
+	}
+	err := post.ShallowCopy(&p.Post)
+	if err != nil {
+		return nil, err
+	}
+	return ESPostFromPostForIndexing(p), nil
+}
+
+func ESPostFromPostForIndexing(post *model.PostForIndexing) *ESPost {
+	searchPost := ESPost{
+		Id:        post.Id,
+		TeamId:    post.TeamId,
+		ChannelId: post.ChannelId,
+		UserId:    post.UserId,
+		CreateAt:  post.CreateAt,
+		Message:   post.Message,
+		Type:      post.Type,
+		Hashtags:  strings.Fields(post.Hashtags),
+	}
+
+	var searchAttachments []string
+
+	if attachments := post.GetProp("attachments"); attachments != nil {
+		attachmentsInterfaceArray, ok := attachments.([]any)
+		if ok {
+			for _, attachment := range attachmentsInterfaceArray {
+				if attachment != nil {
+					if attachmentText := attachment.(map[string]any)["text"]; attachmentText != nil {
+						searchAttachments = append(searchAttachments, attachmentText.(string))
+					}
+				}
+			}
+		}
+
+		attachmentsArray, ok := attachments.([]*model.SlackAttachment)
+		if ok {
+			for _, attachment := range attachmentsArray {
+				if attachment != nil {
+					searchAttachments = append(searchAttachments, attachment.Text)
+				}
+			}
+		}
+	}
+
+	searchPost.Attachments = strings.Join(searchAttachments, " ")
+
+	urls := extractURLsFromMessage(post.Message)
+	if len(urls) > 0 {
+		searchPost.URLs = urls
+	}
+
+	if searchPost.Type == "" {
+		searchPost.Type = "default"
+	}
+
+	return &searchPost
+}
+
+func extractURLsFromMessage(message string) []string {
+	message = markdownLinkRe.ReplaceAllString(message, "")
+	urls := urlRe.FindAllString(message, -1)
+
+	filteredURLs := make([]string, 0)
+	for _, u := range urls {
+		u = strings.TrimSpace(u)
+		urlToCheck := u
+		if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
+			urlToCheck = "http://" + u
+		}
+		parsedURL, err := url.Parse(urlToCheck)
+		if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" {
+			continue
+		}
+		filteredURLs = append(filteredURLs, u)
+	}
+
+	return filteredURLs
+}
+
+func splitFilenameWords(name string) string {
+	result := name
+	result = strings.ReplaceAll(result, "-", " ")
+	result = strings.ReplaceAll(result, ".", " ")
+	return result
+}
+
+func ESFileFromFileInfo(file *model.FileInfo, channelId string) *ESFile {
+	return &ESFile{
+		Id:        file.Id,
+		CreatorId: file.CreatorId,
+		ChannelId: channelId,
+		PostId:    file.PostId,
+		CreateAt:  file.CreateAt,
+		Content:   file.Content,
+		Extension: file.Extension,
+		Name:      file.Name + " " + splitFilenameWords(file.Name),
+	}
+}
+
+func ESFileFromFileForIndexing(file *model.FileForIndexing) *ESFile {
+	return &ESFile{
+		Id:        file.Id,
+		CreatorId: file.CreatorId,
+		ChannelId: file.ChannelId,
+		PostId:    file.PostId,
+		CreateAt:  file.CreateAt,
+		Content:   file.Content,
+		Extension: file.Extension,
+		Name:      file.Name + " " + splitFilenameWords(file.Name),
+	}
+}
+
+func ESChannelFromChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *ESChannel {
+	displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ")
+	nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"})
+
+	return &ESChannel{
+		Id:            channel.Id,
+		Type:          channel.Type,
+		UserIDs:       userIDs,
+		TeamId:        channel.TeamId,
+		TeamMemberIDs: teamMemberIDs,
+		NameSuggest:   append(displayNameInputs, nameInputs...),
+	}
+}
+
+func ESUserFromUserAndTeams(user *model.User, teamsIds, channelsIds []string) *ESUser {
+	usernameSuggestions := searchengine.GetSuggestionInputsSplitByMultiple(user.Username, []string{".", "-", "_"})
+
+	fullnameStrings := []string{}
+	if user.FirstName != "" {
+		fullnameStrings = append(fullnameStrings, user.FirstName)
+	}
+	if user.LastName != "" {
+		fullnameStrings = append(fullnameStrings, user.LastName)
+	}
+
+	fullnameSuggestions := []string{}
+	if len(fullnameStrings) > 0 {
+		fullname := strings.Join(fullnameStrings, " ")
+		fullnameSuggestions = searchengine.GetSuggestionInputsSplitBy(fullname, " ")
+	}
+
+	nicknameSuggestions := []string{}
+	if user.Nickname != "" {
+		nicknameSuggestions = searchengine.GetSuggestionInputsSplitBy(user.Nickname, " ")
+	}
+
+	usernameAndNicknameSuggestions := append(usernameSuggestions, nicknameSuggestions...)
+
+	return &ESUser{
+		Id:                         user.Id,
+		SuggestionsWithFullname:    append(usernameAndNicknameSuggestions, fullnameSuggestions...),
+		SuggestionsWithoutFullname: usernameAndNicknameSuggestions,
+		DeleteAt:                   user.DeleteAt,
+		Roles:                      user.GetRoles(),
+		TeamsIds:                   teamsIds,
+		ChannelsIds:                channelsIds,
+	}
+}
+
+func ESUserFromUserForIndexing(userForIndexing *model.UserForIndexing) *ESUser {
+	user := &model.User{
+		Id:        userForIndexing.Id,
+		Username:  userForIndexing.Username,
+		Nickname:  userForIndexing.Nickname,
+		FirstName: userForIndexing.FirstName,
+		Roles:     userForIndexing.Roles,
+		LastName:  userForIndexing.LastName,
+		CreateAt:  userForIndexing.CreateAt,
+		DeleteAt:  userForIndexing.DeleteAt,
+	}
+
+	return ESUserFromUserAndTeams(user, userForIndexing.TeamsIds, userForIndexing.ChannelsIds)
+}
+
+func BuildPostIndexName(aggregateAfterDays int, unaggregatedBase string, aggregatedBase string, now time.Time, createAt int64) string {
+	postTime := time.Unix(createAt/1000, 0)
+	aggregateCutoffTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -aggregateAfterDays+1)
+
+	if postTime.Before(aggregateCutoffTime) {
+		return fmt.Sprintf("%v_%d_%02d", aggregatedBase, postTime.Year(), postTime.Month())
+	}
+
+	return fmt.Sprintf("%v_%d_%02d_%02d", unaggregatedBase, postTime.Year(), postTime.Month(), postTime.Day())
+}
+
+func NumIndexWorkers() int {
+	const maxCPU = 4
+	if runtime.NumCPU() > maxCPU {
+		return maxCPU
+	}
+	return runtime.NumCPU()
+}
+
+// maxCertFileSizeBytes is an internal constant
+// used to limit file size of ClientCert, ClientKey and CA.
+const maxCertFileSizeBytes = 1_000_000 // 1MB
+
+func ReadFileSafely(fb filestore.FileBackend, path string) ([]byte, error) {
+	rd, err := fb.Reader(path)
+	if err != nil {
+		return nil, err
+	}
+	defer rd.Close()
+
+	type resp struct {
+		buf []byte
+		err error
+	}
+	ch := make(chan resp)
+
+	go func() {
+		buf, err := io.ReadAll(io.LimitReader(rd, maxCertFileSizeBytes))
+		ch <- resp{buf, err}
+	}()
+
+	select {
+	case got := <-ch:
+		return got.buf, got.err
+	case <-time.After(10 * time.Second): // Adding a timeout for the file read.
+		return nil, fmt.Errorf("timed out while reading file: %s", path)
+	}
+}
+
+func GetMatchesForHit(highlights map[string][]string) ([]string, error) {
+	matchMap := make(map[string]bool)
+
+	parseMatches := func(snippets []string) error {
+		// Highlighted matches are returned as an array of snippets of the post where
+		// each snippet has the highlighted text surrounded by html <em> tags
+		for _, snippet := range snippets {
+			decoder := xml.NewDecoder(strings.NewReader(snippet))
+			inMatch := false
+
+			for {
+				token, err := decoder.Token()
+				if err == io.EOF {
+					break
+				} else if err != nil {
+					return err
+				}
+
+				switch typed := token.(type) {
+				case xml.StartElement:
+					if typed.Name.Local == "em" {
+						inMatch = true
+					}
+				case xml.EndElement:
+					if typed.Name.Local == "em" {
+						inMatch = false
+					}
+				case xml.CharData:
+					if inMatch && len(typed) != 0 {
+						match := string(typed)
+						match = strings.Trim(match, "_*~")
+
+						matchMap[match] = true
+					}
+				}
+			}
+		}
+
+		return nil
+	}
+
+	if err := parseMatches(highlights["message"]); err != nil {
+		return nil, err
+	}
+	if err := parseMatches(highlights["attachments"]); err != nil {
+		return nil, err
+	}
+	if err := parseMatches(highlights["urls"]); err != nil {
+		return nil, err
+	}
+	if err := parseMatches(highlights["hashtags"]); err != nil {
+		return nil, err
+	}
+
+	var matches []string
+	for match := range matchMap {
+		matches = append(matches, match)
+	}
+
+	return matches, nil
+}

server/enterprise/elasticsearch/common/common_test.go

@@ -0,0 +1,132 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+func TestElasticsearchBuildPostIndexName(t *testing.T) {
+	now := time.Date(2017, 8, 14, 15, 16, 17, 123, time.Local)
+
+	sixDaysAgo := time.Date(2017, 8, 9, 12, 11, 10, 987, time.Local)
+	sevenDaysAgo := time.Date(2017, 8, 8, 11, 10, 9, 876, time.Local)
+	eightDaysAgo := time.Date(2017, 8, 7, 6, 5, 4, 321, time.Local)
+
+	sixMillis := sixDaysAgo.UnixNano() / int64(time.Millisecond)
+	sevenMillis := sevenDaysAgo.UnixNano() / int64(time.Millisecond)
+	eightMillis := eightDaysAgo.UnixNano() / int64(time.Millisecond)
+
+	aggregationCutoff := 7 // Aggregate monthly after 7 days.
+
+	sixName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, sixMillis)
+	sevenName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, sevenMillis)
+	eightName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, eightMillis)
+
+	assert.Equal(t, sixName, "posts_2017_08_09")
+	assert.Equal(t, sevenName, "posts_2017_08_08")
+	assert.Equal(t, eightName, "postsmonth_2017_08")
+}
+
+func TestESPostFromPostForIndexing(t *testing.T) {
+	// Create one with attachments in 'any' form.
+
+	post1 := model.PostForIndexing{
+		TeamId:         model.NewId(),
+		ParentCreateAt: nil,
+		Post: model.Post{
+			Id:        model.NewId(),
+			ChannelId: model.NewId(),
+			UserId:    model.NewId(),
+			CreateAt:  model.GetMillis(),
+			Message:   "message",
+			Type:      "",
+			Hashtags:  "",
+			Props: map[string]any{
+				"attachments": []any{
+					map[string]any{
+						"text": "text 1",
+					},
+				},
+			},
+		},
+	}
+
+	espost1 := ESPostFromPostForIndexing(&post1)
+
+	assert.Equal(t, post1.Id, espost1.Id)
+	assert.Equal(t, post1.TeamId, espost1.TeamId)
+	assert.Equal(t, post1.ChannelId, espost1.ChannelId)
+	assert.Equal(t, post1.UserId, espost1.UserId)
+	assert.Equal(t, post1.CreateAt, espost1.CreateAt)
+	assert.Equal(t, post1.Message, espost1.Message)
+	assert.Equal(t, "default", espost1.Type)
+	assert.Empty(t, espost1.Hashtags)
+	assert.Equal(t, "text 1", espost1.Attachments)
+
+	// Create one with attachments in model.SlackAttachment form.
+
+	post2 := model.PostForIndexing{
+		TeamId:         model.NewId(),
+		ParentCreateAt: nil,
+		Post: model.Post{
+			Id:        model.NewId(),
+			ChannelId: model.NewId(),
+			UserId:    model.NewId(),
+			CreateAt:  model.GetMillis(),
+			Message:   "message",
+			Type:      "slack_attachment",
+			Hashtags:  "#buh #boh",
+			Props: map[string]any{
+				"attachments": []*model.SlackAttachment{
+					{
+						Text: "text 2",
+					},
+				},
+			},
+		},
+	}
+
+	espost2 := ESPostFromPostForIndexing(&post2)
+
+	assert.Equal(t, post2.Id, espost2.Id)
+	assert.Equal(t, post2.TeamId, espost2.TeamId)
+	assert.Equal(t, post2.ChannelId, espost2.ChannelId)
+	assert.Equal(t, post2.UserId, espost2.UserId)
+	assert.Equal(t, post2.CreateAt, espost2.CreateAt)
+	assert.Equal(t, post2.Message, espost2.Message)
+	assert.Equal(t, "slack_attachment", espost2.Type)
+	assert.Len(t, espost2.Hashtags, 2)
+	assert.Equal(t, "text 2", espost2.Attachments)
+}
+
+func TestGetMatchesForHit(t *testing.T) {
+	snippets := map[string][]string{
+		"message": {
+			"<em>Apples</em> and oranges and <em>apple</em> and orange",
+			"Johnny <em>Appleseed</em>",
+			"That doesn't <em>apply</em> to me, and it doesn't <em>apply</em> to you.",
+		},
+		"hashtags": {
+			"This is an <em>#hashtag</em>",
+		},
+	}
+	expected := []string{
+		"Apples",
+		"apple",
+		"Appleseed",
+		"apply",
+		"#hashtag",
+	}
+
+	actual, err := GetMatchesForHit(snippets)
+	require.NoError(t, err)
+	require.ElementsMatch(t, expected, actual)
+}

server/enterprise/elasticsearch/common/indexing_job.go

@@ -0,0 +1,841 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	timeBetweenBatches = 100 * time.Millisecond
+
+	estimatedPostCount    = 10000000
+	estimatedChannelCount = 100000
+	estimatedFilesCount   = 100000
+	estimatedUserCount    = 10000
+)
+
+const (
+	indexOp  = "index"
+	deleteOp = "delete"
+)
+
+func NewIndexerWorker(name string,
+	jobServer *jobs.JobServer,
+	logger mlog.LoggerIFace,
+	fileBackend filestore.FileBackend,
+	licenseFn func() *model.License,
+	createBulkProcessorFn func() error,
+	addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error,
+	closeBulkProcessorFn func() error,
+) *IndexerWorker {
+	return &IndexerWorker{
+		name:                   name,
+		stoppedCh:              make(chan bool, 1),
+		jobs:                   make(chan model.Job),
+		jobServer:              jobServer,
+		logger:                 logger,
+		fileBackend:            fileBackend,
+		license:                licenseFn,
+		stopped:                true,
+		createBulkProcessor:    createBulkProcessorFn,
+		addItemToBulkProcessor: addItemToBulkProcessorFn,
+		closeBulkProcessor:     closeBulkProcessorFn,
+	}
+}
+
+type IndexerWorker struct {
+	name string
+	// stateMut protects stopCh and stopped and helps enforce
+	// ordering in case subsequent Run or Stop calls are made.
+	stateMut    sync.Mutex
+	stopCh      chan struct{}
+	stopped     bool
+	stoppedCh   chan bool
+	jobs        chan model.Job
+	jobServer   *jobs.JobServer
+	logger      mlog.LoggerIFace
+	fileBackend filestore.FileBackend
+
+	license func() *model.License
+
+	createBulkProcessor    func() error
+	closeBulkProcessor     func() error
+	addItemToBulkProcessor func(indexName, indexOp, docID string, body io.ReadSeeker) error
+}
+
+type IndexingProgress struct {
+	Now            time.Time
+	StartAtTime    int64
+	EndAtTime      int64
+	LastEntityTime int64
+
+	TotalPostsCount int64
+	DonePostsCount  int64
+	DonePosts       bool
+	LastPostID      string
+
+	TotalFilesCount int64
+	DoneFilesCount  int64
+	DoneFiles       bool
+	LastFileID      string
+
+	TotalChannelsCount int64
+	DoneChannelsCount  int64
+	DoneChannels       bool
+	LastChannelID      string
+
+	TotalUsersCount int64
+	DoneUsersCount  int64
+	DoneUsers       bool
+	LastUserID      string
+}
+
+func (ip *IndexingProgress) CurrentProgress() int64 {
+	current := ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount
+	total := ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalFilesCount + ip.TotalUsersCount
+	return current * 100 / total
+}
+
+func (ip *IndexingProgress) IsDone(job *model.Job) bool {
+	// an entity's progress is completed if it was specified not to be indexed, or if it's completed indexing.
+
+	donePosts := job.Data["index_posts"] == "false" || ip.DonePosts
+	doneChannels := job.Data["index_channels"] == "false" || ip.DoneChannels
+	doneUsers := job.Data["index_users"] == "false" || ip.DoneUsers
+	doneFiles := job.Data["index_files"] == "false" || ip.DoneFiles
+
+	return donePosts && doneChannels && doneUsers && doneFiles
+}
+
+func (worker *IndexerWorker) Run() {
+	worker.stateMut.Lock()
+	// We have to re-assign the stop channel again, because
+	// it might happen that the job was restarted due to a config change.
+	if worker.stopped {
+		worker.stopped = false
+		worker.stopCh = make(chan struct{})
+	} else {
+		worker.stateMut.Unlock()
+		return
+	}
+	// Run is called from a separate goroutine and doesn't return.
+	// So we cannot Unlock in a defer clause.
+	worker.stateMut.Unlock()
+
+	worker.logger.Debug("Worker Started")
+
+	defer func() {
+		worker.logger.Debug("Worker: Finished")
+		worker.stoppedCh <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stopCh:
+			worker.logger.Debug("Worker: Received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *IndexerWorker) Stop() {
+	worker.stateMut.Lock()
+	defer worker.stateMut.Unlock()
+
+	// Set to close, and if already closed before, then return.
+	if worker.stopped {
+		return
+	}
+	worker.stopped = true
+
+	worker.logger.Debug("Worker Stopping")
+	close(worker.stopCh)
+	<-worker.stoppedCh
+}
+
+func (worker *IndexerWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *IndexerWorker) IsEnabled(cfg *model.Config) bool {
+	if license := worker.license(); license == nil || !*license.Features.Elasticsearch {
+		return false
+	}
+
+	if *cfg.ElasticsearchSettings.EnableIndexing {
+		return true
+	}
+
+	return false
+}
+
+func (worker *IndexerWorker) initEntitiesToIndex(job *model.Job) {
+	// Specifying entities to index is optional, and even when specified, all entities need not be specified.
+	// This function parses the provided job data and sets enabled or disabled value for each entity,
+	// so that rest of the code can use job.Data as the source of truth to decide if an entity was
+	// to be indexed or not.
+
+	if job.Data == nil {
+		job.Data = model.StringMap{}
+	}
+
+	indexPostsRaw, ok := job.Data["index_posts"]
+	job.Data["index_posts"] = strconv.FormatBool(!ok || indexPostsRaw == "true")
+
+	indexChannelsRaw, ok := job.Data["index_channels"]
+	job.Data["index_channels"] = strconv.FormatBool(!ok || indexChannelsRaw == "true")
+
+	indexUsersRaw, ok := job.Data["index_users"]
+	job.Data["index_users"] = strconv.FormatBool(!ok || indexUsersRaw == "true")
+
+	indexFilesRaw, ok := job.Data["index_files"]
+	job.Data["index_files"] = strconv.FormatBool(!ok || indexFilesRaw == "true")
+}
+
+func (worker *IndexerWorker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	claimed, appErr := worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr))
+		return
+	}
+	if !claimed {
+		return
+	}
+
+	logger.Info("Worker: Indexing job claimed by worker")
+
+	err := worker.createBulkProcessor()
+	if err != nil {
+		worker.logger.Error("Worker: Failed to setup bulk processor", mlog.Err(err))
+		return
+	}
+
+	worker.initEntitiesToIndex(job)
+	progress, err := initProgress(logger, worker.jobServer, job)
+	if err != nil {
+		return
+	}
+
+	var cancelContext request.CTX = request.EmptyContext(worker.logger)
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	cancelContext = cancelContext.WithContext(cancelCtx)
+	go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
+
+	defer func() {
+		cancelCancelWatcher()
+		err := worker.closeBulkProcessor()
+		if err != nil {
+			logger.Warn("Error while closing the bulk indexer", mlog.Err(err), mlog.String("job_id", job.Id))
+		}
+	}()
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			logger.Info("Worker: Indexing job has been canceled via CancellationWatcher")
+			if err := worker.jobServer.SetJobCanceled(job); err != nil {
+				logger.Error("Worker: Failed to mark job as cancelled", mlog.Err(err))
+			}
+			return
+
+		case <-worker.stopCh:
+			logger.Info("Worker: Indexing has been canceled via Worker Stop. Setting the job back to pending.")
+			if err := worker.jobServer.SetJobPending(job); err != nil {
+				logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
+			}
+			return
+
+		case <-time.After(timeBetweenBatches):
+			var err *model.AppError
+			if progress, err = worker.IndexBatch(logger, progress, job); err != nil {
+				logger.Error("Worker: Failed to index batch for job", mlog.Err(err))
+				if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+					logger.Error("Worker: Failed to set job error", mlog.Err(err2), mlog.NamedErr("set_error", err))
+				}
+				return
+			}
+
+			// Storing the batch progress in metadata.
+			if job.Data == nil {
+				job.Data = make(model.StringMap)
+			}
+
+			job.Data["done_posts_count"] = strconv.FormatInt(progress.DonePostsCount, 10)
+			job.Data["done_channels_count"] = strconv.FormatInt(progress.DoneChannelsCount, 10)
+			job.Data["done_users_count"] = strconv.FormatInt(progress.DoneUsersCount, 10)
+			job.Data["done_files_count"] = strconv.FormatInt(progress.DoneFilesCount, 10)
+
+			job.Data["start_time"] = strconv.FormatInt(progress.LastEntityTime, 10)
+			job.Data["start_post_id"] = progress.LastPostID
+			job.Data["start_channel_id"] = progress.LastChannelID
+			job.Data["start_user_id"] = progress.LastUserID
+			job.Data["start_file_id"] = progress.LastFileID
+			job.Data["original_start_time"] = strconv.FormatInt(progress.StartAtTime, 10)
+			job.Data["end_time"] = strconv.FormatInt(progress.EndAtTime, 10)
+
+			if err := worker.jobServer.SetJobProgress(job, progress.CurrentProgress()); err != nil {
+				logger.Error("Worker: Failed to set progress for job", mlog.Err(err))
+				if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+					logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err))
+				}
+				return
+			}
+
+			if progress.IsDone(job) {
+				if err := worker.jobServer.SetJobSuccess(job); err != nil {
+					logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+					if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+						logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err))
+					}
+				}
+				logger.Info("Worker: Indexing job finished successfully")
+				return
+			}
+		}
+	}
+}
+
+func (worker *IndexerWorker) IndexBatch(logger mlog.LoggerIFace, progress IndexingProgress, job *model.Job) (IndexingProgress, *model.AppError) {
+	// an entity's batch is processed if it wasn't specified to be skipped, or if its completed indexing.
+
+	if job.Data["index_posts"] != "false" && !progress.DonePosts {
+		worker.logger.Debug("Worker: indexing post batch...")
+		return worker.IndexPostsBatch(logger, progress)
+	}
+
+	if job.Data["index_channels"] != "false" && !progress.DoneChannels {
+		worker.logger.Debug("Worker: indexing channels batch...")
+		return IndexChannelsBatch(logger, worker.jobServer.Config(), worker.jobServer.Store, worker.addItemToBulkProcessor, progress)
+	}
+
+	if job.Data["index_users"] != "false" && !progress.DoneUsers {
+		worker.logger.Debug("Worker: indexing users batch...")
+		return worker.IndexUsersBatch(logger, progress)
+	}
+
+	if job.Data["index_files"] != "false" && !progress.DoneFiles {
+		worker.logger.Debug("Worker: indexing files batch...")
+		return worker.IndexFilesBatch(logger, progress)
+	}
+
+	return progress, model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError)
+}
+
+func (worker *IndexerWorker) IndexPostsBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var posts []*model.PostForIndexing
+
+	tries := 0
+	for posts == nil {
+		var err error
+		posts, err = worker.jobServer.Store.Post().GetPostsBatchForIndexing(progress.LastEntityTime, progress.LastPostID, *worker.jobServer.Config().ElasticsearchSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexPostsBatch", "ent.elasticsearch.post.get_posts_batch_for_indexing.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			logger.Warn("Failed to get posts batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	// Handle zero messages.
+	if len(posts) == 0 {
+		progress.DonePosts = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastPost, err := worker.BulkIndexPosts(posts, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last post's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastPost.CreateAt {
+		progress.DonePosts = true
+		// We reset the last entity time to the beginning to begin
+		// indexing of the next set of entities (users, channels etc.)
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastPost.CreateAt
+	}
+
+	progress.LastPostID = lastPost.Id
+	progress.DonePostsCount += int64(len(posts))
+
+	return progress, nil
+}
+
+func (worker *IndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, progress IndexingProgress) (*model.Post, *model.AppError) {
+	for _, post := range posts {
+		indexName := BuildPostIndexName(*worker.jobServer.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			*worker.jobServer.Config().ElasticsearchSettings.IndexPrefix+IndexBasePosts,
+			*worker.jobServer.Config().ElasticsearchSettings.IndexPrefix+IndexBasePosts_MONTH, progress.Now, post.CreateAt)
+
+		if post.DeleteAt == 0 {
+			searchPost := ESPostFromPostForIndexing(post)
+
+			data, err := json.Marshal(searchPost)
+			if err != nil {
+				worker.logger.Warn("Failed to marshal JSON, skipping this post.", mlog.String("post_id", post.Id))
+				continue
+			}
+
+			err = worker.addItemToBulkProcessor(indexName, indexOp, searchPost.Id, bytes.NewReader(data))
+			if err != nil {
+				worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		} else {
+			err := worker.addItemToBulkProcessor(indexName, deleteOp, post.Id, nil)
+			if err != nil {
+				worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		}
+	}
+
+	return &posts[len(posts)-1].Post, nil
+}
+
+func (worker *IndexerWorker) IndexFilesBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var files []*model.FileForIndexing
+
+	tries := 0
+	for files == nil {
+		var err error
+		files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, progress.LastFileID, true, *worker.jobServer.Config().ElasticsearchSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexFilesBatch", "ent.elasticsearch.post.get_files_batch_for_indexing.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			logger.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	if len(files) == 0 {
+		progress.DoneFiles = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastFile, err := worker.BulkIndexFiles(files, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last file's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastFile.CreateAt {
+		progress.DoneFiles = true
+		// We reset the last entity time to the beginning to begin
+		// indexing of the next set of entities (users, channels etc.)
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastFile.CreateAt
+	}
+
+	progress.LastFileID = lastFile.Id
+	progress.DoneFilesCount += int64(len(files))
+
+	return progress, nil
+}
+
+func (worker *IndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (*model.FileInfo, *model.AppError) {
+	for _, file := range files {
+		indexName := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + IndexBaseFiles
+
+		if file.ShouldIndex() {
+			searchFile := ESFileFromFileForIndexing(file)
+
+			data, err := json.Marshal(searchFile)
+			if err != nil {
+				worker.logger.Warn("Failed to marshal JSON")
+				continue
+			}
+
+			err = worker.addItemToBulkProcessor(indexName, indexOp, searchFile.Id, bytes.NewReader(data))
+			if err != nil {
+				worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		} else {
+			err := worker.addItemToBulkProcessor(indexName, deleteOp, file.Id, nil)
+			if err != nil {
+				worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		}
+	}
+
+	return &files[len(files)-1].FileInfo, nil
+}
+
+func IndexChannelsBatch(logger mlog.LoggerIFace, config *model.Config, store store.Store, addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error, progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var channels []*model.Channel
+
+	tries := 0
+	for channels == nil {
+		var err error
+		channels, err = store.Channel().GetChannelsBatchForIndexing(progress.LastEntityTime, progress.LastChannelID, *config.ElasticsearchSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexerWorker.IndexChannelsBatch", "ent.elasticsearch.index_channels_batch.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			logger.Warn("Failed to get channels batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+		tries++
+	}
+
+	if len(channels) == 0 {
+		progress.DoneChannels = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastChannel, err := BulkIndexChannels(config, store, logger, addItemToBulkProcessorFn, channels, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last channel's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastChannel.CreateAt {
+		progress.DoneChannels = true
+		// We reset the last entity time to the beginning to begin
+		// indexing of the next set of entities (users etc.)
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastChannel.CreateAt
+	}
+
+	progress.LastChannelID = lastChannel.Id
+	progress.DoneChannelsCount += int64(len(channels))
+
+	return progress, nil
+}
+
+func BulkIndexChannels(config *model.Config,
+	store store.Store,
+	logger mlog.LoggerIFace,
+	addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error,
+	channels []*model.Channel,
+	progress IndexingProgress) (*model.Channel, *model.AppError) {
+	for _, channel := range channels {
+		indexName := *config.ElasticsearchSettings.IndexPrefix + IndexBaseChannels
+
+		if channel.DeleteAt == 0 {
+			var userIDs []string
+			var err error
+			if channel.Type == model.ChannelTypePrivate {
+				userIDs, err = store.Channel().GetAllChannelMemberIdsByChannelId(channel.Id)
+				if err != nil {
+					return nil, model.NewAppError("IndexerWorker.BulkIndexChannels", "ent.elasticsearch.getAllChannelMembers.error", nil, "", http.StatusInternalServerError).Wrap(err)
+				}
+			}
+
+			teamMemberIDs, err := store.Channel().GetTeamMembersForChannel(channel.Id)
+			if err != nil {
+				return nil, model.NewAppError("IndexerWorker.BulkIndexChannels", "ent.elasticsearch.getAllTeamMembers.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			searchChannel := ESChannelFromChannel(channel, userIDs, teamMemberIDs)
+
+			data, err := json.Marshal(searchChannel)
+			if err != nil {
+				logger.Warn("Failed to marshal JSON")
+				continue
+			}
+
+			err = addItemToBulkProcessorFn(indexName, indexOp, searchChannel.Id, bytes.NewReader(data))
+			if err != nil {
+				logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		} else {
+			err := addItemToBulkProcessorFn(indexName, deleteOp, channel.Id, nil)
+			if err != nil {
+				logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		}
+	}
+
+	return channels[len(channels)-1], nil
+}
+
+func (worker *IndexerWorker) IndexUsersBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var users []*model.UserForIndexing
+
+	tries := 0
+	for users == nil {
+		if usersBatch, err := worker.jobServer.Store.User().GetUsersBatchForIndexing(progress.LastEntityTime, progress.LastUserID, *worker.jobServer.Config().ElasticsearchSettings.BatchSize); err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexerWorker.IndexUsersBatch", "app.user.get_users_batch_for_indexing.get_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			logger.Warn("Failed to get users batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		} else {
+			users = usersBatch
+		}
+
+		tries++
+	}
+
+	if len(users) == 0 {
+		progress.DoneUsers = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastUser, err := worker.BulkIndexUsers(users, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last user's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastUser.CreateAt {
+		progress.DoneUsers = true
+		// We reset the last entity time to the beginning to begin
+		// indexing of the next set of entities in case they get added in the future.
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastUser.CreateAt
+	}
+	progress.LastUserID = lastUser.Id
+	progress.DoneUsersCount += int64(len(users))
+
+	return progress, nil
+}
+
+func (worker *IndexerWorker) BulkIndexUsers(users []*model.UserForIndexing, progress IndexingProgress) (*model.UserForIndexing, *model.AppError) {
+	for _, user := range users {
+		indexName := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + IndexBaseUsers
+
+		searchUser := ESUserFromUserForIndexing(user)
+
+		data, err := json.Marshal(searchUser)
+		if err != nil {
+			worker.logger.Warn("Failed to marshal JSON")
+			continue
+		}
+
+		err = worker.addItemToBulkProcessor(indexName, indexOp, searchUser.Id, bytes.NewReader(data))
+		if err != nil {
+			worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+		}
+	}
+
+	return users[len(users)-1], nil
+}
+
+func initProgress(logger mlog.LoggerIFace, jobServer *jobs.JobServer, job *model.Job) (IndexingProgress, error) {
+	progress := IndexingProgress{
+		Now:          time.Now(),
+		DonePosts:    false,
+		DoneChannels: false,
+		DoneUsers:    false,
+		DoneFiles:    false,
+		StartAtTime:  0,
+		EndAtTime:    model.GetMillis(),
+	}
+
+	progress, err := parseStartTime(logger, jobServer, progress, job)
+	if err != nil {
+		return progress, err
+	}
+
+	progress, err = parseEndTime(logger, jobServer, progress, job)
+	if err != nil {
+		return progress, err
+	}
+
+	progress = parseDoneCount(logger, progress, job)
+	progress = setStartEntityIDs(progress, job)
+	progress = setEntityCount(logger, jobServer, progress, job)
+
+	return progress, nil
+}
+
+func parseStartTime(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) (IndexingProgress, error) {
+	// Extract the start time, if it is set.
+	if startString, ok := job.Data["start_time"]; ok {
+		startInt, err := strconv.ParseInt(startString, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse start_time for job", mlog.String("start_time", startString), mlog.Err(err))
+			appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.parse_start_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := jobServer.SetJobError(job, appError); err != nil {
+				logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return progress, err
+		}
+		progress.StartAtTime = startInt
+	} else {
+		// Set start time to oldest entity (user, channel or post) in the database.
+		oldestEntityTime, err := jobServer.Store.Post().GetOldestEntityCreationTime()
+		if err != nil {
+			logger.Error("Worker: Failed to fetch oldest post for job.", mlog.String("start_time", startString), mlog.Err(err))
+			appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.get_oldest_entity.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := jobServer.SetJobError(job, appError); err != nil {
+				logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return progress, err
+		}
+		progress.StartAtTime = oldestEntityTime
+	}
+
+	progress.LastEntityTime = progress.StartAtTime
+	return progress, nil
+}
+
+func parseEndTime(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) (IndexingProgress, error) {
+	if endString, ok := job.Data["end_time"]; ok {
+		endInt, err := strconv.ParseInt(endString, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse end_time for job", mlog.String("end_time", endString), mlog.Err(err))
+			appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.parse_end_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := jobServer.SetJobError(job, appError); err != nil {
+				logger.Error("Worker: Failed to set job errorv", mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return progress, err
+		}
+		progress.EndAtTime = endInt
+	}
+
+	return progress, nil
+}
+
+func parseDoneCount(logger mlog.LoggerIFace, progress IndexingProgress, job *model.Job) IndexingProgress {
+	if count, ok := job.Data["done_posts_count"]; ok {
+		countInt, err := strconv.ParseInt(count, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse done_posts_count for job", mlog.String("done_posts_count", count), mlog.Err(err))
+		}
+		progress.DonePostsCount = countInt
+	}
+
+	if count, ok := job.Data["done_channels_count"]; ok {
+		countInt, err := strconv.ParseInt(count, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse done_channels_count for job", mlog.String("done_channels_count", count), mlog.Err(err))
+		}
+		progress.DoneChannelsCount = countInt
+	}
+
+	if count, ok := job.Data["done_users_count"]; ok {
+		countInt, err := strconv.ParseInt(count, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse done_users_count for job", mlog.String("done_users_count", count), mlog.Err(err))
+		}
+		progress.DoneUsersCount = countInt
+	}
+
+	if count, ok := job.Data["done_files_count"]; ok {
+		countInt, err := strconv.ParseInt(count, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse done_files_count for job", mlog.String("done_files_count", count), mlog.Err(err))
+		}
+		progress.DoneFilesCount = countInt
+	}
+
+	return progress
+}
+
+func setStartEntityIDs(progress IndexingProgress, job *model.Job) IndexingProgress {
+	if id, ok := job.Data["start_post_id"]; ok {
+		progress.LastPostID = id
+	}
+	if id, ok := job.Data["start_channel_id"]; ok {
+		progress.LastChannelID = id
+	}
+	if id, ok := job.Data["start_user_id"]; ok {
+		progress.LastUserID = id
+	}
+	if id, ok := job.Data["start_file_id"]; ok {
+		progress.LastFileID = id
+	}
+
+	return progress
+}
+
+func setEntityCount(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) IndexingProgress {
+	if job.Data["index_posts"] == "true" {
+		// Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry
+		// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
+		if count, err := jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}); err != nil {
+			logger.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedPostCount", estimatedPostCount), mlog.Err(err))
+			progress.TotalPostsCount = estimatedPostCount
+		} else {
+			progress.TotalPostsCount = count
+		}
+	}
+
+	if job.Data["index_channels"] == "true" {
+		// Same possible fail as above can happen when counting channels
+		if count, err := jobServer.Store.Channel().AnalyticsTypeCount("", ""); err != nil {
+			logger.Warn("Worker: Failed to fetch total channel count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedChannelCount", estimatedChannelCount), mlog.Err(err))
+			progress.TotalChannelsCount = estimatedChannelCount
+		} else {
+			progress.TotalChannelsCount = count
+		}
+	}
+
+	if job.Data["index_users"] == "true" {
+		// Same possible fail as above can happen when counting users
+		if count, err := jobServer.Store.User().Count(model.UserCountOptions{
+			IncludeBotAccounts: true, // This actually doesn't join with the bots table
+			// since ExcludeRegularUsers is set to false
+		}); err != nil {
+			logger.Warn("Worker: Failed to fetch total user count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedUserCount", estimatedUserCount), mlog.Err(err))
+			progress.TotalUsersCount = estimatedUserCount
+		} else {
+			progress.TotalUsersCount = count
+		}
+	}
+
+	if job.Data["index_files"] == "true" {
+		// Same possible fail as above can happen when counting files
+		if count, err := jobServer.Store.FileInfo().CountAll(); err != nil {
+			logger.Warn("Worker: Failed to fetch total files count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedFilesCount", estimatedFilesCount), mlog.Err(err))
+			progress.TotalFilesCount = estimatedFilesCount
+		} else {
+			progress.TotalFilesCount = count
+		}
+	}
+
+	return progress
+}

server/enterprise/elasticsearch/common/logger.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/mattermost/logr/v2"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+func NewLogger(backend string, mlogger mlog.LoggerIFace, trace bool) *Logger {
+	return &Logger{
+		backend: backend,
+		logger:  mlogger,
+		trace:   trace,
+	}
+}
+
+// Logger is a target to pass to the Logger instance of the search backend.
+type Logger struct {
+	backend string
+	trace   bool
+	logger  mlog.LoggerIFace
+}
+
+// LogRoundTrip prints the information about request and response.
+func (l *Logger) LogRoundTrip(req *http.Request, res *http.Response, err error, start time.Time, dur time.Duration) error {
+	// Set error level.
+	// 0 = debug, 1=warn, 2=error
+	var level int
+	switch {
+	case err != nil:
+		level = 2
+	case res != nil && res.StatusCode > 0 && res.StatusCode < 500:
+		level = 0
+	case res != nil && res.StatusCode > 499:
+		level = 2
+	default:
+		level = 2
+	}
+
+	// Capture fields.
+	fields := []mlog.Field{
+		mlog.String("method", req.Method),
+		mlog.Int("status_code", res.StatusCode),
+		mlog.String("duration", dur.String()),
+		mlog.String("url", req.URL.String()),
+	}
+
+	var logFn func(string, ...logr.Field)
+	switch level {
+	case 0:
+		logFn = l.logger.Debug
+	case 1:
+		logFn = l.logger.Warn
+	case 2:
+		logFn = l.logger.Error
+	}
+	logFn(l.backend+" request", fields...)
+
+	return nil
+}
+
+// RequestBodyEnabled makes the client pass request body to logger
+func (l *Logger) RequestBodyEnabled() bool { return l.trace }
+
+// ResponseBodyEnabled makes the client pass response body to logger
+func (l *Logger) ResponseBodyEnabled() bool { return false }
+
+func NewBulkIndexerLogger(mlogger mlog.LoggerIFace, name string) BulkIndexerDebugLogger {
+	return BulkIndexerDebugLogger{
+		logger: mlogger,
+		name:   name,
+	}
+}
+
+type BulkIndexerDebugLogger struct {
+	logger mlog.LoggerIFace
+	name   string
+}
+
+func (bl BulkIndexerDebugLogger) Printf(str string, params ...any) {
+	line := fmt.Sprintf(str, params...)
+	bl.logger.Debug(line, mlog.String("workername", bl.name))
+}

server/enterprise/elasticsearch/common/templates.go

@@ -0,0 +1,237 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"strconv"
+
+	"github.com/elastic/go-elasticsearch/v8/typedapi/indices/putindextemplate"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+func GetPostTemplate(cfg *model.Config) *putindextemplate.Request {
+	mappings := &types.TypeMapping{
+		Properties: map[string]types.Property{
+			"message": types.TextProperty{
+				Analyzer: model.NewPointer("mm_lowercaser"),
+				Type:     "text",
+			},
+			"attachments": types.TextProperty{
+				Analyzer: model.NewPointer("mm_lowercaser"),
+				Type:     "text",
+			},
+			"urls": types.TextProperty{
+				Analyzer: model.NewPointer("mm_url"),
+				Type:     "text",
+			},
+			"hashtags": types.KeywordProperty{
+				Type:       "keyword",
+				Normalizer: model.NewPointer("mm_hashtag"),
+				Store:      model.NewPointer(true),
+			},
+		},
+	}
+
+	return &putindextemplate.Request{
+		IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBasePosts + "*"},
+		Template: &types.IndexTemplateMapping{
+			Settings: &types.IndexSettings{
+				Index: &types.IndexSettings{
+					NumberOfShards:   strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexShards),
+					NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexReplicas),
+				},
+				Analysis: &types.IndexSettingsAnalysis{
+					CharFilter: map[string]types.CharFilter{
+						"leading_underscores": map[string]any{
+							"type":        "pattern_replace",
+							"pattern":     `(^|[\s\r\n])_`,
+							"replacement": "$1",
+						},
+						"trailing_underscores": map[string]any{
+							"type":        "pattern_replace",
+							"pattern":     `_([\s\r\n]|$)`,
+							"replacement": "$1",
+						},
+					},
+					Analyzer: map[string]types.Analyzer{
+						"mm_lowercaser": map[string]any{
+							"tokenizer": "icu_tokenizer",
+							"filter": []string{
+								"icu_normalizer",
+								"mm_snowball",
+								"mm_stop",
+							},
+							"char_filter": []string{
+								"leading_underscores",
+								"trailing_underscores",
+							},
+						},
+						"mm_url": map[string]any{
+							"tokenizer": "pattern",
+							"pattern":   "\\W",
+							"lowercase": true,
+						}},
+					Filter: map[string]types.TokenFilter{
+						"mm_snowball": map[string]any{
+							"type":     "snowball",
+							"language": "English",
+						},
+						"mm_stop": map[string]any{
+							"type":      "stop",
+							"stopwords": "_english_",
+						},
+					},
+					Normalizer: map[string]types.Normalizer{
+						"mm_hashtag": map[string]any{
+							"type":        "custom",
+							"char_filter": []string{},
+							"filter":      []string{"lowercase", "icu_normalizer"},
+						},
+					},
+				},
+			},
+			Mappings: mappings,
+		},
+	}
+}
+
+func GetFileInfoTemplate(cfg *model.Config) *putindextemplate.Request {
+	mappings := &types.TypeMapping{
+		Properties: map[string]types.Property{
+			"name": types.TextProperty{
+				Analyzer: model.NewPointer("mm_lowercaser"),
+				Type:     "text",
+			},
+			"content": types.TextProperty{
+				Analyzer: model.NewPointer("mm_lowercaser"),
+				Type:     "text",
+			},
+		},
+	}
+
+	return &putindextemplate.Request{
+		IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseFiles + "*"},
+		Template: &types.IndexTemplateMapping{
+			Settings: &types.IndexSettings{
+				Index: &types.IndexSettings{
+					NumberOfShards:   strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexShards),
+					NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexReplicas),
+				},
+				Analysis: &types.IndexSettingsAnalysis{
+					CharFilter: map[string]types.CharFilter{
+						"leading_underscores": map[string]any{
+							"type":        "pattern_replace",
+							"pattern":     `(^|[\s\r\n])_`,
+							"replacement": "$1",
+						},
+						"trailing_underscores": map[string]any{
+							"type":        "pattern_replace",
+							"pattern":     `_([\s\r\n]|$)`,
+							"replacement": "$1",
+						},
+					},
+					Analyzer: map[string]types.Analyzer{
+						"mm_lowercaser": map[string]any{
+							"tokenizer": "icu_tokenizer",
+							"filter": []string{
+								"icu_normalizer",
+								"mm_snowball",
+								"mm_stop",
+							},
+							"char_filter": []string{
+								"leading_underscores",
+								"trailing_underscores",
+							},
+						},
+					},
+					Filter: map[string]types.TokenFilter{
+						"mm_snowball": map[string]any{
+							"type":     "snowball",
+							"language": "English",
+						},
+						"mm_stop": map[string]any{
+							"type":      "stop",
+							"stopwords": "_english_",
+						},
+					},
+				},
+			},
+			Mappings: mappings,
+		},
+	}
+}
+
+func GetChannelTemplate(cfg *model.Config) *putindextemplate.Request {
+	mappings := &types.TypeMapping{
+		Properties: map[string]types.Property{
+			"name_suggestions": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"team_id": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"user_ids": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"team_member_ids": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"type": types.KeywordProperty{
+				Type: "keyword",
+			},
+		},
+	}
+
+	return &putindextemplate.Request{
+		IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseChannels + "*"},
+		Template: &types.IndexTemplateMapping{
+			Settings: &types.IndexSettings{
+				Index: &types.IndexSettings{
+					NumberOfShards:   strconv.Itoa(*cfg.ElasticsearchSettings.ChannelIndexShards),
+					NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.ChannelIndexReplicas),
+				},
+			},
+			Mappings: mappings,
+		},
+	}
+}
+
+func GetUserTemplate(cfg *model.Config) *putindextemplate.Request {
+	mappings := &types.TypeMapping{
+		Properties: map[string]types.Property{
+			"suggestions_with_fullname": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"suggestions_without_fullname": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"team_id": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"channel_id": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"delete_at": types.LongNumberProperty{
+				Type: "long",
+			},
+			"roles": types.KeywordProperty{
+				Type: "keyword",
+			},
+		},
+	}
+
+	return &putindextemplate.Request{
+		IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseUsers + "*"},
+		Template: &types.IndexTemplateMapping{
+			Settings: &types.IndexSettings{
+				Index: &types.IndexSettings{
+					NumberOfShards:   strconv.Itoa(*cfg.ElasticsearchSettings.UserIndexShards),
+					NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.UserIndexReplicas),
+				},
+			},
+			Mappings: mappings,
+		},
+	}
+}

server/enterprise/elasticsearch/common/test_helpers.go

@@ -0,0 +1,76 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func createPost(userId string, channelId string, message string) *model.Post {
+	post := &model.Post{
+		Message:       message,
+		ChannelId:     channelId,
+		PendingPostId: model.NewId() + ":" + fmt.Sprint(model.GetMillis()),
+		UserId:        userId,
+		CreateAt:      1000000,
+	}
+	post.PreSave()
+
+	return post
+}
+
+func createChannel(teamId, name, displayName string, channelType model.ChannelType) *model.Channel {
+	channel := &model.Channel{
+		TeamId:      teamId,
+		Type:        channelType,
+		Name:        name,
+		DisplayName: displayName,
+	}
+	channel.PreSave()
+
+	return channel
+}
+
+func createUser(username, nickname, firstName, lastName string) *model.User {
+	user := &model.User{
+		Username:  username,
+		Password:  username,
+		Nickname:  nickname,
+		FirstName: firstName,
+		LastName:  lastName,
+	}
+	if err := user.PreSave(); err != nil {
+		return nil
+	}
+
+	return user
+}
+
+func createFile(creatorID, channelID, postID, content, name, extension string) *model.FileInfo {
+	file := &model.FileInfo{
+		CreatorId: creatorID,
+		ChannelId: channelID,
+		PostId:    postID,
+		Content:   content,
+		Name:      name,
+		Extension: extension,
+	}
+	file.PreSave()
+
+	return file
+}
+
+func CheckMatchesEqual(t *testing.T, expected model.PostSearchMatches, actual map[string][]string) {
+	a := assert.New(t)
+
+	a.Len(actual, len(expected), "Received matches for a different number of posts")
+
+	for postId, expectedMatches := range expected {
+		a.ElementsMatch(expectedMatches, actual[postId], fmt.Sprintf("%v: expected %v, got %v", postId, expectedMatches, actual[postId]))
+	}
+}

server/enterprise/elasticsearch/common/test_suite.go

@@ -0,0 +1,789 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/store/searchtest"
+	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type CommonTestSuite struct {
+	suite.Suite
+
+	TH             *api4.TestHelper
+	ESImpl         searchengine.SearchEngineInterface
+	GetDocumentFn  func(index, documentID string) (bool, json.RawMessage, error)
+	CreateIndexFn  func(index string) error
+	GetIndexFn     func(indexPattern string) ([]string, error)
+	RefreshIndexFn func() error
+}
+
+func (c *CommonTestSuite) TestSearchStore() {
+	searchTestEngine := &searchtest.SearchTestEngine{
+		Driver: searchtest.EngineElasticSearch,
+	}
+
+	c.Run("TestSearchChannelStore", func() {
+		searchtest.TestSearchChannelStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine)
+	})
+
+	c.Run("TestSearchUserStore", func() {
+		searchtest.TestSearchUserStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine)
+	})
+
+	c.Run("TestSearchPostStore", func() {
+		searchtest.TestSearchPostStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine)
+	})
+
+	c.Run("TestSearchFileInfoStore", func() {
+		searchtest.TestSearchFileInfoStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine)
+	})
+}
+
+func (c *CommonTestSuite) TestIndexPost() {
+	testCases := []struct {
+		Name                string
+		Message             string
+		Hashtags            string
+		ExpectedAttachments string
+		ExpectedHashtags    []string
+		ExpectedURLs        []string
+	}{
+		{
+			Name:                "Should be able to index a plain message",
+			Message:             "Test message 1 2 3",
+			ExpectedAttachments: "",
+			ExpectedHashtags:    []string{},
+			ExpectedURLs:        []string(nil),
+		},
+		{
+			Name:                "Should be able to index hashtags",
+			Message:             "Test message #1234",
+			Hashtags:            "#1234",
+			ExpectedAttachments: "",
+			ExpectedHashtags:    []string{"#1234"},
+			ExpectedURLs:        []string(nil),
+		},
+		// TODO: actually send attachments
+		{
+			Name:                "Should be able to index attachments",
+			Message:             "Test message 1 2 3",
+			ExpectedAttachments: "",
+			ExpectedHashtags:    []string{},
+			ExpectedURLs:        []string(nil),
+		},
+		{
+			Name:                "Should be able to index urls",
+			Message:             "Test message www.mattermost.com http://www.mattermost.com [link](http://www.notindexed.com)",
+			ExpectedAttachments: "",
+			ExpectedHashtags:    []string{},
+			ExpectedURLs:        []string{"www.mattermost.com", "http://www.mattermost.com"},
+		},
+	}
+
+	for _, tc := range testCases {
+		c.Run(tc.Name, func() {
+			post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, tc.Message)
+			if tc.Hashtags != "" {
+				post.Hashtags = tc.Hashtags
+			}
+			c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+
+			c.NoError(c.RefreshIndexFn())
+			indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+				IndexBasePosts,
+				IndexBasePosts_MONTH,
+				time.Now(),
+				post.CreateAt,
+			)
+
+			found, source, err := c.GetDocumentFn(indexName, post.Id)
+			c.NoError(err)
+			c.True(found)
+
+			var esPost ESPost
+			err = json.Unmarshal(source, &esPost)
+			c.NoError(err)
+			c.NotNil(post)
+			c.Equal(tc.Message, post.Message)
+			c.Equal(tc.ExpectedAttachments, esPost.Attachments)
+			c.Equal(tc.ExpectedHashtags, esPost.Hashtags)
+			c.Equal(tc.ExpectedURLs, esPost.URLs)
+		})
+	}
+}
+
+func (c *CommonTestSuite) TestSearchPosts() {
+	// Create and index a post
+	post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+	c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+
+	c.NoError(c.RefreshIndexFn())
+	indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt)
+
+	// Check the post is there.
+	found, _, err := c.GetDocumentFn(indexName, post.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Do a search for that post.
+	channels := model.ChannelList{
+		c.TH.BasicChannel,
+	}
+
+	searchParams := []*model.SearchParams{
+		{
+			Terms:     post.Message,
+			IsHashtag: false,
+			OrTerms:   false,
+		},
+	}
+
+	// Check the post is found as expected
+	ids, matches, err := c.ESImpl.SearchPosts(channels, searchParams, 0, 20)
+	c.Nil(err)
+	c.Len(ids, 1)
+	c.Equal(ids[0], post.Id)
+	CheckMatchesEqual(c.T(), map[string][]string{
+		post.Id: {post.Message},
+	}, matches)
+
+	// Do a search that won't match anything.
+	searchParams = []*model.SearchParams{
+		{
+			Terms:     model.NewId(),
+			IsHashtag: false,
+			OrTerms:   false,
+		},
+	}
+
+	ids, matches, err = c.ESImpl.SearchPosts(channels, searchParams, 0, 20)
+	c.Nil(err)
+	c.Len(ids, 0)
+	c.Len(matches, 0)
+}
+
+func (c *CommonTestSuite) TestDeletePost() {
+	c.Require().NotNil(c.TH)
+
+	post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+	indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt)
+
+	// Index the post.
+	c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the post is there.
+	found, _, err := c.GetDocumentFn(indexName, post.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete the post.
+	c.Nil(c.ESImpl.DeletePost(post))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the post is not there.
+	found, _, err = c.GetDocumentFn(indexName, post.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestDeleteChannelPosts() {
+	c.Run("Should remove all the channel posts", func() {
+		channelPosts := make([]*model.Post, 0)
+		post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+		channelPosts = append(channelPosts, post)
+		post2 := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId())
+		post2.CreateAt = 1200000
+		channelPosts = append(channelPosts, post2)
+		post3 := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId())
+		post3.CreateAt = 1300000
+		channelPosts = append(channelPosts, post3)
+		postReply := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId())
+		postReply.RootId = post.Id
+		postReply.CreateAt = 1400000
+		channelPosts = append(channelPosts, postReply)
+		anotherPost := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel2.Id, model.NewId())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt)
+		for _, post := range channelPosts {
+			c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+		}
+		c.Nil(c.ESImpl.IndexPost(anotherPost, c.TH.BasicTeam.Id))
+		c.NoError(c.RefreshIndexFn())
+		for _, post := range channelPosts {
+			found, _, err := c.GetDocumentFn(indexName, post.Id)
+			c.NoError(err)
+			c.True(found)
+		}
+		c.Nil(c.ESImpl.DeleteChannelPosts(c.TH.Context, c.TH.BasicChannel.Id))
+		c.NoError(c.RefreshIndexFn())
+		for _, post := range channelPosts {
+			found, _, err := c.GetDocumentFn(indexName, post.Id)
+			// This is a difference in behavior between engines.
+			if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+				c.Error(err)
+			} else {
+				c.NoError(err)
+			}
+			c.False(found)
+		}
+
+		found, _, err := c.GetDocumentFn(indexName, anotherPost.Id)
+		c.NoError(err)
+		c.True(found)
+	})
+
+	c.Run("Should not remove other channels posts even if there was no posts to remove", func() {
+		postNotInChannel := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel2.Id, model.NewId())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts, IndexBasePosts_MONTH, time.Now(), postNotInChannel.CreateAt)
+		c.Nil(c.ESImpl.IndexPost(postNotInChannel, c.TH.BasicTeam.Id))
+		c.NoError(c.RefreshIndexFn())
+		c.Nil(c.ESImpl.DeleteChannelPosts(c.TH.Context, c.TH.BasicChannel.Id))
+		c.NoError(c.RefreshIndexFn())
+
+		found, _, err := c.GetDocumentFn(indexName, postNotInChannel.Id)
+		c.NoError(err)
+		c.True(found)
+	})
+}
+
+func (c *CommonTestSuite) TestDeleteUserPosts() {
+	c.Run("Should remove all the user posts", func() {
+		anotherTeam := c.TH.CreateTeam()
+		anotherTeamChannel := createChannel(anotherTeam.Id, "anotherteamchannel", "", model.ChannelTypeOpen)
+		userPosts := make([]*model.Post, 0)
+		post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+		userPosts = append(userPosts, post)
+		post2 := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel2.Id, model.NewId())
+		post2.CreateAt = 1200000
+		userPosts = append(userPosts, post2)
+		post3 := createPost(c.TH.BasicUser.Id, c.TH.BasicPrivateChannel.Id, model.NewId())
+		post3.CreateAt = 1300000
+		userPosts = append(userPosts, post3)
+		postReply := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+		postReply.RootId = post.Id
+		postReply.CreateAt = 1400000
+		userPosts = append(userPosts, postReply)
+		postAnotherTeam := createPost(c.TH.BasicUser.Id, anotherTeamChannel.Id, model.NewId())
+		postAnotherTeam.CreateAt = 1400000
+		userPosts = append(userPosts, postAnotherTeam)
+		anotherPost := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel2.Id, model.NewId())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt)
+		for _, post := range userPosts {
+			c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+		}
+		c.Nil(c.ESImpl.IndexPost(postAnotherTeam, anotherTeam.Id))
+		c.Nil(c.ESImpl.IndexPost(anotherPost, c.TH.BasicTeam.Id))
+		c.NoError(c.RefreshIndexFn())
+		for _, post := range userPosts {
+			found, _, err := c.GetDocumentFn(indexName, post.Id)
+			c.NoError(err)
+			c.True(found)
+		}
+		c.Nil(c.ESImpl.DeleteUserPosts(c.TH.Context, c.TH.BasicUser.Id))
+		c.NoError(c.RefreshIndexFn())
+		for _, post := range userPosts {
+			found, _, err := c.GetDocumentFn(indexName, post.Id)
+			// This is a difference in behavior between engines.
+			if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+				c.Error(err)
+			} else {
+				c.NoError(err)
+			}
+			c.False(found)
+		}
+		found, _, err := c.GetDocumentFn(indexName, anotherPost.Id)
+		c.NoError(err)
+		c.True(found)
+	})
+
+	c.Run("Should not remove other channels posts even if there was no posts to remove", func() {
+		postNotInChannel := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts, IndexBasePosts_MONTH, time.Now(), postNotInChannel.CreateAt)
+		c.Nil(c.ESImpl.IndexPost(postNotInChannel, c.TH.BasicTeam.Id))
+		c.NoError(c.RefreshIndexFn())
+		c.Nil(c.ESImpl.DeleteUserPosts(c.TH.Context, c.TH.BasicUser.Id))
+		c.NoError(c.RefreshIndexFn())
+		found, _, err := c.GetDocumentFn(indexName, postNotInChannel.Id)
+		c.NoError(err)
+		c.True(found)
+	})
+}
+
+func (c *CommonTestSuite) TestIndexChannel() {
+	// Create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the channel is there.
+	found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id)
+	c.NoError(err)
+	c.True(found)
+}
+
+func (c *CommonTestSuite) TestDeleteChannel() {
+	// Create and index a channel.
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the channel is there.
+	found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete the channel.
+	c.Nil(c.ESImpl.DeleteChannel(channel))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the channel is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestIndexUser() {
+	// Create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the user is there.
+	found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id)
+	c.NoError(err)
+	c.True(found)
+}
+
+func (c *CommonTestSuite) TestDeleteUser() {
+	// Create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the user is there.
+	found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete the user.
+	c.Nil(c.ESImpl.DeleteUser(user))
+	c.NoError(c.RefreshIndexFn())
+	// Check the user is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseUsers, user.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestTestConfig() {
+	c.Nil(c.ESImpl.TestConfig(c.TH.Context, c.TH.App.Config()))
+
+	originalConfig := c.TH.App.Config()
+	defer c.TH.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ElasticsearchSettings.ConnectionURL = *originalConfig.ElasticsearchSettings.ConnectionURL
+	})
+
+	c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.ConnectionURL = "example.com:12345" })
+	c.Error(c.ESImpl.TestConfig(c.TH.Context, c.TH.App.Config()))
+
+	// Passing a temp config which is different from the saved
+	// config should be taken correctly.
+	c.Nil(c.ESImpl.TestConfig(c.TH.Context, originalConfig))
+}
+
+func (c *CommonTestSuite) TestIndexFile() {
+	// First, create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	// Then, create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id}))
+
+	// Create and index a file
+	file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt")
+	c.Nil(c.ESImpl.IndexFile(file, channel.Id))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is there
+	found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id)
+	c.NoError(err)
+	c.True(found)
+}
+
+func (c *CommonTestSuite) TestDeleteFile() {
+	// First, create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	// Then, create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id}))
+
+	// Create and index a file
+	file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt")
+	c.Nil(c.ESImpl.IndexFile(file, channel.Id))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is there
+	found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete the file
+	c.Nil(c.ESImpl.DeleteFile(file.Id))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestDeleteUserFiles() {
+	// First, create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	// Then, create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id}))
+
+	// Create and index a file
+	file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt")
+	c.Nil(c.ESImpl.IndexFile(file, channel.Id))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is there
+	found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete file by creator
+	c.Nil(c.ESImpl.DeleteUserFiles(c.TH.Context, user.Id))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestDeletePostFiles() {
+	// First, create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	// Then, create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id}))
+
+	// Create and index a post
+	post := createPost(user.Id, channel.Id, "test post message")
+	c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+
+	// Create and index a file
+	file := createFile(user.Id, channel.Id, post.Id, "file contents", "testfile", "txt")
+	c.Nil(c.ESImpl.IndexFile(file, channel.Id))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is there
+	found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete file by post
+	c.Nil(c.ESImpl.DeletePostFiles(c.TH.Context, post.Id))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestElasticsearchDataRetentionDeleteIndexes() {
+	c.Nil(c.CreateIndexFn("posts_2017_09_15"))
+	c.Nil(c.CreateIndexFn("posts_2017_09_16"))
+	c.Nil(c.CreateIndexFn("posts_2017_09_17"))
+	c.Nil(c.CreateIndexFn("posts_2017_09_18"))
+	c.Nil(c.CreateIndexFn("posts_2017_09_19"))
+
+	c.Run("Should delete indexes using start of day cut off", func() {
+		c.Nil(c.ESImpl.DataRetentionDeleteIndexes(c.TH.Context, time.Date(2017, 9, 16, 0, 0, 0, 0, time.UTC)))
+
+		postIndexesResult, err := c.GetIndexFn("posts_*")
+		c.Nil(err)
+		if err == nil {
+			found1 := false
+			found2 := false
+			found3 := false
+			found4 := false
+			found5 := false
+
+			for _, index := range postIndexesResult {
+				if index == "posts_2017_09_15" {
+					found1 = true
+				} else if index == "posts_2017_09_16" {
+					found2 = true
+				} else if index == "posts_2017_09_17" {
+					found3 = true
+				} else if index == "posts_2017_09_18" {
+					found4 = true
+				} else if index == "posts_2017_09_19" {
+					found5 = true
+				}
+			}
+
+			c.False(found1)
+			c.False(found2)
+			c.True(found3)
+			c.True(found4)
+			c.True(found5)
+		}
+	})
+
+	c.Run("Should delete indexes when cut off is in hours", func() {
+		c.Nil(c.ESImpl.DataRetentionDeleteIndexes(c.TH.Context, time.Date(2017, 9, 18, 11, 6, 0, 0, time.UTC)))
+
+		postIndexesResult, err := c.GetIndexFn("posts_*")
+		c.Nil(err)
+		if err == nil {
+			found1 := false
+			found2 := false
+			found3 := false
+
+			for _, index := range postIndexesResult {
+				if index == "posts_2017_09_17" {
+					found1 = true
+				} else if index == "posts_2017_09_18" {
+					found2 = true
+				} else if index == "posts_2017_09_19" {
+					found3 = true
+				}
+			}
+
+			c.False(found1)
+			c.False(found2)
+			c.True(found3)
+		}
+	})
+}
+
+func (c *CommonTestSuite) TestPurgeIndexes() {
+	existingIndexPrefix := *c.TH.Server.Config().ElasticsearchSettings.IndexPrefix
+	defer c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = existingIndexPrefix })
+
+	c.Run("Should purge all indexes", func() {
+		// Create and index a user
+		user := createUser("test.user", "testuser", "Test", "User")
+		c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{}))
+
+		c.NoError(c.RefreshIndexFn())
+
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "test_" })
+
+		// index user with a new index prefix
+		c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{}))
+		c.NoError(c.RefreshIndexFn())
+
+		c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context))
+
+		found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id)
+		c.NoError(err)
+		c.True(found)
+
+		found, _, err = c.GetDocumentFn("test_"+IndexBaseUsers, user.Id)
+		if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+			c.False(found)
+		} else {
+			elasticErr := err.(*types.ElasticsearchError)
+			c.Equal(404, elasticErr.Status)
+		}
+	})
+
+	c.Run("Should not purge indexes defined to ignore", func() {
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "posts*" })
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "" })
+
+		// Create a user
+		user := createUser("test.user", "testuser", "Test", "User")
+
+		// Create and index a post
+		post := createPost(user.Id, c.TH.BasicChannel.Id, "Test")
+		c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+
+		c.NoError(c.RefreshIndexFn())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts,
+			IndexBasePosts_MONTH,
+			time.Now(),
+			post.CreateAt,
+		)
+
+		// We expect posts indexes to remain after purge
+		c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context))
+
+		found, _, err := c.GetDocumentFn(indexName, post.Id)
+		c.NoError(err)
+		c.True(found)
+
+		// Remove the ignore rule
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "" })
+
+		c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context))
+
+		// Validate the indexes are gone
+		found, _, err = c.GetDocumentFn(IndexBasePosts, post.Id)
+		if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+			c.False(found)
+		} else {
+			elasticErr := err.(*types.ElasticsearchError)
+			c.Equal(404, elasticErr.Status)
+		}
+	})
+}
+
+func (c *CommonTestSuite) TestPurgeIndexList() {
+	existingIndexPrefix := *c.TH.Server.Config().ElasticsearchSettings.IndexPrefix
+	defer c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = existingIndexPrefix })
+
+	c.Run("Should purge allowed index", func() {
+		// Create and index a channel
+		channel := createChannel("test.channel", "testuser", "Test", model.ChannelTypeOpen)
+		c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+		c.NoError(c.RefreshIndexFn())
+
+		// verify data is in Elasticsearch
+		found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		c.NoError(err)
+		c.True(found)
+
+		// now we'll purge
+		c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"}))
+
+		found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+			c.False(found)
+		} else {
+			elasticErr := err.(*types.ElasticsearchError)
+			c.Equal(404, elasticErr.Status)
+		}
+	})
+
+	c.Run("Should not purge indexes defined to ignore", func() {
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "channels" })
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "" })
+
+		channel := createChannel("test.channel", "testuser", "Test", model.ChannelTypeOpen)
+		c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+		c.NoError(c.RefreshIndexFn())
+
+		// verify data is in Elasticsearch
+		found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		c.NoError(err)
+		c.True(found)
+
+		// now we'll purge
+		c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"}))
+
+		// the channel should still be there because we ignored that index
+		found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		c.NoError(err)
+		c.True(found)
+
+		// Remove the ignore rule
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "" })
+
+		c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"}))
+
+		// now it should be gone as we're no longer ignoring it
+		found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+			c.False(found)
+		} else {
+			elasticErr := err.(*types.ElasticsearchError)
+			c.Equal(404, elasticErr.Status)
+		}
+	})
+}
+
+func (c *CommonTestSuite) TestSearchChannels() {
+	// Create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Channel Open", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{c.TH.BasicUser.Id, "otheruser"}))
+	channel2 := createChannel(c.TH.BasicTeam.Id, "channel", "Channel Private", model.ChannelTypePrivate)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel2, []string{c.TH.BasicUser.Id}, []string{c.TH.BasicUser.Id, "otheruser"}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Private channels should be returned for right user.
+	ids, appErr := c.ESImpl.SearchChannels("", c.TH.BasicUser.Id, "Channel", false)
+	c.Nil(appErr)
+	c.Len(ids, 2)
+
+	// No private channels if user is guest
+	ids, appErr = c.ESImpl.SearchChannels("", c.TH.BasicUser.Id, "Channel", true)
+	c.Nil(appErr)
+	c.Len(ids, 1)
+	c.Equal(channel.Id, ids[0])
+
+	// No Private channels should be returned for wrong user.
+	ids, appErr = c.ESImpl.SearchChannels("", "otheruser", "Channel", false)
+	c.Nil(appErr)
+	c.Len(ids, 1)
+	c.Equal(channel.Id, ids[0])
+}

server/enterprise/elasticsearch/common/version.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"strconv"
+	"strings"
+)
+
+func GetVersionComponents(version string) (int, int, int, error) {
+	spl := strings.Split(version, ".")
+	major, err := strconv.Atoi(spl[0])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	minor, err := strconv.Atoi(spl[1])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	patch, err := strconv.Atoi(spl[2])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	return major, minor, patch, nil
+}

server/enterprise/elasticsearch/common/version_test.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetVersionComponents(t *testing.T) {
+	testCases := []struct {
+		Name          string
+		Version       string
+		ExpectedMajor int
+		ExpectedMinor int
+		ExpectedPatch int
+		ExpectedError bool
+	}{
+		{
+			Name:          "Should error if version format is invalid",
+			Version:       "invalid",
+			ExpectedMajor: 0,
+			ExpectedMinor: 0,
+			ExpectedPatch: 0,
+			ExpectedError: true,
+		},
+		{
+			Name:          "Should work correctly if version has three valid components",
+			Version:       "7.2.3",
+			ExpectedMajor: 7,
+			ExpectedMinor: 2,
+			ExpectedPatch: 3,
+			ExpectedError: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			major, minor, patch, err := GetVersionComponents(tc.Version)
+			if tc.ExpectedError {
+				require.Error(t, err)
+			}
+			assert.Equal(t, tc.ExpectedMajor, major)
+			assert.Equal(t, tc.ExpectedMinor, minor)
+			assert.Equal(t, tc.ExpectedPatch, patch)
+		})
+	}
+}