Move Elasticsearch to source available 🎉 🎉 (#29015)

* Move Elasticsearch to source available

server/enterprise/elasticsearch/common/common.go

@@ -0,0 +1,375 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"net/url"
+	"regexp"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	MaxLineLength = 10000
+
+	URLRegexpRE       = `(\b|^)(?:https?:\/\/)?[a-zA-Z0-9-.]+\.[a-z]+(\s|\)?[a-zA-Z0-9\-._~:/?#\[\]@!$&'\(\)*\+,;=]*)(\b|$)`
+	URLMarkdownLinkRE = `(\[[^\]]+\]\([a-zA-Z0-9\-._~:/?#\[\]@!$&'\(\)*\+,;=]+\))`
+	EmailRE           = `^[^\s"]+@[^\s"]+$`
+
+	IndexBasePosts       = "posts"
+	IndexBasePosts_MONTH = IndexBasePosts + "month"
+	IndexBaseChannels    = "channels"
+	IndexBaseUsers       = "users"
+	IndexBaseFiles       = "files"
+
+	// At the moment, this number is hardcoded. If needed, we can expose
+	// this to the config.
+	BulkFlushInterval = 5 * time.Second
+)
+
+var (
+	urlRe          = regexp.MustCompile(URLRegexpRE)
+	markdownLinkRe = regexp.MustCompile(URLMarkdownLinkRE)
+)
+
+type ESPost struct {
+	Id          string   `json:"id"`
+	TeamId      string   `json:"team_id"`
+	ChannelId   string   `json:"channel_id"`
+	UserId      string   `json:"user_id"`
+	CreateAt    int64    `json:"create_at"`
+	Message     string   `json:"message"`
+	Type        string   `json:"type"`
+	Hashtags    []string `json:"hashtags"`
+	Attachments string   `json:"attachments"`
+	URLs        []string `json:"urls"`
+}
+
+type ESFile struct {
+	Id        string `json:"id"`
+	CreatorId string `json:"creator_id"`
+	ChannelId string `json:"channel_id"`
+	PostId    string `json:"post_id"`
+	CreateAt  int64  `json:"create_at"`
+	Content   string `json:"content"`
+	Extension string `json:"extension"`
+	Name      string `json:"name"`
+}
+
+type ESChannel struct {
+	Id            string            `json:"id"`
+	Type          model.ChannelType `json:"type"`
+	UserIDs       []string          `json:"user_ids"`
+	TeamId        string            `json:"team_id"`
+	TeamMemberIDs []string          `json:"team_member_ids"`
+	NameSuggest   []string          `json:"name_suggestions"`
+}
+
+type ESUser struct {
+	Id                         string   `json:"id"`
+	SuggestionsWithFullname    []string `json:"suggestions_with_fullname"`
+	SuggestionsWithoutFullname []string `json:"suggestions_without_fullname"`
+	DeleteAt                   int64    `json:"delete_at"`
+	Roles                      []string `json:"roles"`
+	TeamsIds                   []string `json:"team_id"`
+	ChannelsIds                []string `json:"channel_id"`
+}
+
+func ESPostFromPost(post *model.Post, teamId string) (*ESPost, error) {
+	p := &model.PostForIndexing{
+		TeamId: teamId,
+	}
+	err := post.ShallowCopy(&p.Post)
+	if err != nil {
+		return nil, err
+	}
+	return ESPostFromPostForIndexing(p), nil
+}
+
+func ESPostFromPostForIndexing(post *model.PostForIndexing) *ESPost {
+	searchPost := ESPost{
+		Id:        post.Id,
+		TeamId:    post.TeamId,
+		ChannelId: post.ChannelId,
+		UserId:    post.UserId,
+		CreateAt:  post.CreateAt,
+		Message:   post.Message,
+		Type:      post.Type,
+		Hashtags:  strings.Fields(post.Hashtags),
+	}
+
+	var searchAttachments []string
+
+	if attachments := post.GetProp("attachments"); attachments != nil {
+		attachmentsInterfaceArray, ok := attachments.([]any)
+		if ok {
+			for _, attachment := range attachmentsInterfaceArray {
+				if attachment != nil {
+					if attachmentText := attachment.(map[string]any)["text"]; attachmentText != nil {
+						searchAttachments = append(searchAttachments, attachmentText.(string))
+					}
+				}
+			}
+		}
+
+		attachmentsArray, ok := attachments.([]*model.SlackAttachment)
+		if ok {
+			for _, attachment := range attachmentsArray {
+				if attachment != nil {
+					searchAttachments = append(searchAttachments, attachment.Text)
+				}
+			}
+		}
+	}
+
+	searchPost.Attachments = strings.Join(searchAttachments, " ")
+
+	urls := extractURLsFromMessage(post.Message)
+	if len(urls) > 0 {
+		searchPost.URLs = urls
+	}
+
+	if searchPost.Type == "" {
+		searchPost.Type = "default"
+	}
+
+	return &searchPost
+}
+
+func extractURLsFromMessage(message string) []string {
+	message = markdownLinkRe.ReplaceAllString(message, "")
+	urls := urlRe.FindAllString(message, -1)
+
+	filteredURLs := make([]string, 0)
+	for _, u := range urls {
+		u = strings.TrimSpace(u)
+		urlToCheck := u
+		if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") {
+			urlToCheck = "http://" + u
+		}
+		parsedURL, err := url.Parse(urlToCheck)
+		if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" {
+			continue
+		}
+		filteredURLs = append(filteredURLs, u)
+	}
+
+	return filteredURLs
+}
+
+func splitFilenameWords(name string) string {
+	result := name
+	result = strings.ReplaceAll(result, "-", " ")
+	result = strings.ReplaceAll(result, ".", " ")
+	return result
+}
+
+func ESFileFromFileInfo(file *model.FileInfo, channelId string) *ESFile {
+	return &ESFile{
+		Id:        file.Id,
+		CreatorId: file.CreatorId,
+		ChannelId: channelId,
+		PostId:    file.PostId,
+		CreateAt:  file.CreateAt,
+		Content:   file.Content,
+		Extension: file.Extension,
+		Name:      file.Name + " " + splitFilenameWords(file.Name),
+	}
+}
+
+func ESFileFromFileForIndexing(file *model.FileForIndexing) *ESFile {
+	return &ESFile{
+		Id:        file.Id,
+		CreatorId: file.CreatorId,
+		ChannelId: file.ChannelId,
+		PostId:    file.PostId,
+		CreateAt:  file.CreateAt,
+		Content:   file.Content,
+		Extension: file.Extension,
+		Name:      file.Name + " " + splitFilenameWords(file.Name),
+	}
+}
+
+func ESChannelFromChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *ESChannel {
+	displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ")
+	nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"})
+
+	return &ESChannel{
+		Id:            channel.Id,
+		Type:          channel.Type,
+		UserIDs:       userIDs,
+		TeamId:        channel.TeamId,
+		TeamMemberIDs: teamMemberIDs,
+		NameSuggest:   append(displayNameInputs, nameInputs...),
+	}
+}
+
+func ESUserFromUserAndTeams(user *model.User, teamsIds, channelsIds []string) *ESUser {
+	usernameSuggestions := searchengine.GetSuggestionInputsSplitByMultiple(user.Username, []string{".", "-", "_"})
+
+	fullnameStrings := []string{}
+	if user.FirstName != "" {
+		fullnameStrings = append(fullnameStrings, user.FirstName)
+	}
+	if user.LastName != "" {
+		fullnameStrings = append(fullnameStrings, user.LastName)
+	}
+
+	fullnameSuggestions := []string{}
+	if len(fullnameStrings) > 0 {
+		fullname := strings.Join(fullnameStrings, " ")
+		fullnameSuggestions = searchengine.GetSuggestionInputsSplitBy(fullname, " ")
+	}
+
+	nicknameSuggestions := []string{}
+	if user.Nickname != "" {
+		nicknameSuggestions = searchengine.GetSuggestionInputsSplitBy(user.Nickname, " ")
+	}
+
+	usernameAndNicknameSuggestions := append(usernameSuggestions, nicknameSuggestions...)
+
+	return &ESUser{
+		Id:                         user.Id,
+		SuggestionsWithFullname:    append(usernameAndNicknameSuggestions, fullnameSuggestions...),
+		SuggestionsWithoutFullname: usernameAndNicknameSuggestions,
+		DeleteAt:                   user.DeleteAt,
+		Roles:                      user.GetRoles(),
+		TeamsIds:                   teamsIds,
+		ChannelsIds:                channelsIds,
+	}
+}
+
+func ESUserFromUserForIndexing(userForIndexing *model.UserForIndexing) *ESUser {
+	user := &model.User{
+		Id:        userForIndexing.Id,
+		Username:  userForIndexing.Username,
+		Nickname:  userForIndexing.Nickname,
+		FirstName: userForIndexing.FirstName,
+		Roles:     userForIndexing.Roles,
+		LastName:  userForIndexing.LastName,
+		CreateAt:  userForIndexing.CreateAt,
+		DeleteAt:  userForIndexing.DeleteAt,
+	}
+
+	return ESUserFromUserAndTeams(user, userForIndexing.TeamsIds, userForIndexing.ChannelsIds)
+}
+
+func BuildPostIndexName(aggregateAfterDays int, unaggregatedBase string, aggregatedBase string, now time.Time, createAt int64) string {
+	postTime := time.Unix(createAt/1000, 0)
+	aggregateCutoffTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -aggregateAfterDays+1)
+
+	if postTime.Before(aggregateCutoffTime) {
+		return fmt.Sprintf("%v_%d_%02d", aggregatedBase, postTime.Year(), postTime.Month())
+	}
+
+	return fmt.Sprintf("%v_%d_%02d_%02d", unaggregatedBase, postTime.Year(), postTime.Month(), postTime.Day())
+}
+
+func NumIndexWorkers() int {
+	const maxCPU = 4
+	if runtime.NumCPU() > maxCPU {
+		return maxCPU
+	}
+	return runtime.NumCPU()
+}
+
+// maxCertFileSizeBytes is an internal constant
+// used to limit file size of ClientCert, ClientKey and CA.
+const maxCertFileSizeBytes = 1_000_000 // 1MB
+
+func ReadFileSafely(fb filestore.FileBackend, path string) ([]byte, error) {
+	rd, err := fb.Reader(path)
+	if err != nil {
+		return nil, err
+	}
+	defer rd.Close()
+
+	type resp struct {
+		buf []byte
+		err error
+	}
+	ch := make(chan resp)
+
+	go func() {
+		buf, err := io.ReadAll(io.LimitReader(rd, maxCertFileSizeBytes))
+		ch <- resp{buf, err}
+	}()
+
+	select {
+	case got := <-ch:
+		return got.buf, got.err
+	case <-time.After(10 * time.Second): // Adding a timeout for the file read.
+		return nil, fmt.Errorf("timed out while reading file: %s", path)
+	}
+}
+
+func GetMatchesForHit(highlights map[string][]string) ([]string, error) {
+	matchMap := make(map[string]bool)
+
+	parseMatches := func(snippets []string) error {
+		// Highlighted matches are returned as an array of snippets of the post where
+		// each snippet has the highlighted text surrounded by html <em> tags
+		for _, snippet := range snippets {
+			decoder := xml.NewDecoder(strings.NewReader(snippet))
+			inMatch := false
+
+			for {
+				token, err := decoder.Token()
+				if err == io.EOF {
+					break
+				} else if err != nil {
+					return err
+				}
+
+				switch typed := token.(type) {
+				case xml.StartElement:
+					if typed.Name.Local == "em" {
+						inMatch = true
+					}
+				case xml.EndElement:
+					if typed.Name.Local == "em" {
+						inMatch = false
+					}
+				case xml.CharData:
+					if inMatch && len(typed) != 0 {
+						match := string(typed)
+						match = strings.Trim(match, "_*~")
+
+						matchMap[match] = true
+					}
+				}
+			}
+		}
+
+		return nil
+	}
+
+	if err := parseMatches(highlights["message"]); err != nil {
+		return nil, err
+	}
+	if err := parseMatches(highlights["attachments"]); err != nil {
+		return nil, err
+	}
+	if err := parseMatches(highlights["urls"]); err != nil {
+		return nil, err
+	}
+	if err := parseMatches(highlights["hashtags"]); err != nil {
+		return nil, err
+	}
+
+	var matches []string
+	for match := range matchMap {
+		matches = append(matches, match)
+	}
+
+	return matches, nil
+}

server/enterprise/elasticsearch/common/common_test.go

@@ -0,0 +1,132 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+func TestElasticsearchBuildPostIndexName(t *testing.T) {
+	now := time.Date(2017, 8, 14, 15, 16, 17, 123, time.Local)
+
+	sixDaysAgo := time.Date(2017, 8, 9, 12, 11, 10, 987, time.Local)
+	sevenDaysAgo := time.Date(2017, 8, 8, 11, 10, 9, 876, time.Local)
+	eightDaysAgo := time.Date(2017, 8, 7, 6, 5, 4, 321, time.Local)
+
+	sixMillis := sixDaysAgo.UnixNano() / int64(time.Millisecond)
+	sevenMillis := sevenDaysAgo.UnixNano() / int64(time.Millisecond)
+	eightMillis := eightDaysAgo.UnixNano() / int64(time.Millisecond)
+
+	aggregationCutoff := 7 // Aggregate monthly after 7 days.
+
+	sixName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, sixMillis)
+	sevenName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, sevenMillis)
+	eightName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, eightMillis)
+
+	assert.Equal(t, sixName, "posts_2017_08_09")
+	assert.Equal(t, sevenName, "posts_2017_08_08")
+	assert.Equal(t, eightName, "postsmonth_2017_08")
+}
+
+func TestESPostFromPostForIndexing(t *testing.T) {
+	// Create one with attachments in 'any' form.
+
+	post1 := model.PostForIndexing{
+		TeamId:         model.NewId(),
+		ParentCreateAt: nil,
+		Post: model.Post{
+			Id:        model.NewId(),
+			ChannelId: model.NewId(),
+			UserId:    model.NewId(),
+			CreateAt:  model.GetMillis(),
+			Message:   "message",
+			Type:      "",
+			Hashtags:  "",
+			Props: map[string]any{
+				"attachments": []any{
+					map[string]any{
+						"text": "text 1",
+					},
+				},
+			},
+		},
+	}
+
+	espost1 := ESPostFromPostForIndexing(&post1)
+
+	assert.Equal(t, post1.Id, espost1.Id)
+	assert.Equal(t, post1.TeamId, espost1.TeamId)
+	assert.Equal(t, post1.ChannelId, espost1.ChannelId)
+	assert.Equal(t, post1.UserId, espost1.UserId)
+	assert.Equal(t, post1.CreateAt, espost1.CreateAt)
+	assert.Equal(t, post1.Message, espost1.Message)
+	assert.Equal(t, "default", espost1.Type)
+	assert.Empty(t, espost1.Hashtags)
+	assert.Equal(t, "text 1", espost1.Attachments)
+
+	// Create one with attachments in model.SlackAttachment form.
+
+	post2 := model.PostForIndexing{
+		TeamId:         model.NewId(),
+		ParentCreateAt: nil,
+		Post: model.Post{
+			Id:        model.NewId(),
+			ChannelId: model.NewId(),
+			UserId:    model.NewId(),
+			CreateAt:  model.GetMillis(),
+			Message:   "message",
+			Type:      "slack_attachment",
+			Hashtags:  "#buh #boh",
+			Props: map[string]any{
+				"attachments": []*model.SlackAttachment{
+					{
+						Text: "text 2",
+					},
+				},
+			},
+		},
+	}
+
+	espost2 := ESPostFromPostForIndexing(&post2)
+
+	assert.Equal(t, post2.Id, espost2.Id)
+	assert.Equal(t, post2.TeamId, espost2.TeamId)
+	assert.Equal(t, post2.ChannelId, espost2.ChannelId)
+	assert.Equal(t, post2.UserId, espost2.UserId)
+	assert.Equal(t, post2.CreateAt, espost2.CreateAt)
+	assert.Equal(t, post2.Message, espost2.Message)
+	assert.Equal(t, "slack_attachment", espost2.Type)
+	assert.Len(t, espost2.Hashtags, 2)
+	assert.Equal(t, "text 2", espost2.Attachments)
+}
+
+func TestGetMatchesForHit(t *testing.T) {
+	snippets := map[string][]string{
+		"message": {
+			"<em>Apples</em> and oranges and <em>apple</em> and orange",
+			"Johnny <em>Appleseed</em>",
+			"That doesn't <em>apply</em> to me, and it doesn't <em>apply</em> to you.",
+		},
+		"hashtags": {
+			"This is an <em>#hashtag</em>",
+		},
+	}
+	expected := []string{
+		"Apples",
+		"apple",
+		"Appleseed",
+		"apply",
+		"#hashtag",
+	}
+
+	actual, err := GetMatchesForHit(snippets)
+	require.NoError(t, err)
+	require.ElementsMatch(t, expected, actual)
+}

server/enterprise/elasticsearch/common/indexing_job.go

@@ -0,0 +1,841 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	timeBetweenBatches = 100 * time.Millisecond
+
+	estimatedPostCount    = 10000000
+	estimatedChannelCount = 100000
+	estimatedFilesCount   = 100000
+	estimatedUserCount    = 10000
+)
+
+const (
+	indexOp  = "index"
+	deleteOp = "delete"
+)
+
+func NewIndexerWorker(name string,
+	jobServer *jobs.JobServer,
+	logger mlog.LoggerIFace,
+	fileBackend filestore.FileBackend,
+	licenseFn func() *model.License,
+	createBulkProcessorFn func() error,
+	addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error,
+	closeBulkProcessorFn func() error,
+) *IndexerWorker {
+	return &IndexerWorker{
+		name:                   name,
+		stoppedCh:              make(chan bool, 1),
+		jobs:                   make(chan model.Job),
+		jobServer:              jobServer,
+		logger:                 logger,
+		fileBackend:            fileBackend,
+		license:                licenseFn,
+		stopped:                true,
+		createBulkProcessor:    createBulkProcessorFn,
+		addItemToBulkProcessor: addItemToBulkProcessorFn,
+		closeBulkProcessor:     closeBulkProcessorFn,
+	}
+}
+
+type IndexerWorker struct {
+	name string
+	// stateMut protects stopCh and stopped and helps enforce
+	// ordering in case subsequent Run or Stop calls are made.
+	stateMut    sync.Mutex
+	stopCh      chan struct{}
+	stopped     bool
+	stoppedCh   chan bool
+	jobs        chan model.Job
+	jobServer   *jobs.JobServer
+	logger      mlog.LoggerIFace
+	fileBackend filestore.FileBackend
+
+	license func() *model.License
+
+	createBulkProcessor    func() error
+	closeBulkProcessor     func() error
+	addItemToBulkProcessor func(indexName, indexOp, docID string, body io.ReadSeeker) error
+}
+
+type IndexingProgress struct {
+	Now            time.Time
+	StartAtTime    int64
+	EndAtTime      int64
+	LastEntityTime int64
+
+	TotalPostsCount int64
+	DonePostsCount  int64
+	DonePosts       bool
+	LastPostID      string
+
+	TotalFilesCount int64
+	DoneFilesCount  int64
+	DoneFiles       bool
+	LastFileID      string
+
+	TotalChannelsCount int64
+	DoneChannelsCount  int64
+	DoneChannels       bool
+	LastChannelID      string
+
+	TotalUsersCount int64
+	DoneUsersCount  int64
+	DoneUsers       bool
+	LastUserID      string
+}
+
+func (ip *IndexingProgress) CurrentProgress() int64 {
+	current := ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount
+	total := ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalFilesCount + ip.TotalUsersCount
+	return current * 100 / total
+}
+
+func (ip *IndexingProgress) IsDone(job *model.Job) bool {
+	// an entity's progress is completed if it was specified not to be indexed, or if it's completed indexing.
+
+	donePosts := job.Data["index_posts"] == "false" || ip.DonePosts
+	doneChannels := job.Data["index_channels"] == "false" || ip.DoneChannels
+	doneUsers := job.Data["index_users"] == "false" || ip.DoneUsers
+	doneFiles := job.Data["index_files"] == "false" || ip.DoneFiles
+
+	return donePosts && doneChannels && doneUsers && doneFiles
+}
+
+func (worker *IndexerWorker) Run() {
+	worker.stateMut.Lock()
+	// We have to re-assign the stop channel again, because
+	// it might happen that the job was restarted due to a config change.
+	if worker.stopped {
+		worker.stopped = false
+		worker.stopCh = make(chan struct{})
+	} else {
+		worker.stateMut.Unlock()
+		return
+	}
+	// Run is called from a separate goroutine and doesn't return.
+	// So we cannot Unlock in a defer clause.
+	worker.stateMut.Unlock()
+
+	worker.logger.Debug("Worker Started")
+
+	defer func() {
+		worker.logger.Debug("Worker: Finished")
+		worker.stoppedCh <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stopCh:
+			worker.logger.Debug("Worker: Received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *IndexerWorker) Stop() {
+	worker.stateMut.Lock()
+	defer worker.stateMut.Unlock()
+
+	// Set to close, and if already closed before, then return.
+	if worker.stopped {
+		return
+	}
+	worker.stopped = true
+
+	worker.logger.Debug("Worker Stopping")
+	close(worker.stopCh)
+	<-worker.stoppedCh
+}
+
+func (worker *IndexerWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *IndexerWorker) IsEnabled(cfg *model.Config) bool {
+	if license := worker.license(); license == nil || !*license.Features.Elasticsearch {
+		return false
+	}
+
+	if *cfg.ElasticsearchSettings.EnableIndexing {
+		return true
+	}
+
+	return false
+}
+
+func (worker *IndexerWorker) initEntitiesToIndex(job *model.Job) {
+	// Specifying entities to index is optional, and even when specified, all entities need not be specified.
+	// This function parses the provided job data and sets enabled or disabled value for each entity,
+	// so that rest of the code can use job.Data as the source of truth to decide if an entity was
+	// to be indexed or not.
+
+	if job.Data == nil {
+		job.Data = model.StringMap{}
+	}
+
+	indexPostsRaw, ok := job.Data["index_posts"]
+	job.Data["index_posts"] = strconv.FormatBool(!ok || indexPostsRaw == "true")
+
+	indexChannelsRaw, ok := job.Data["index_channels"]
+	job.Data["index_channels"] = strconv.FormatBool(!ok || indexChannelsRaw == "true")
+
+	indexUsersRaw, ok := job.Data["index_users"]
+	job.Data["index_users"] = strconv.FormatBool(!ok || indexUsersRaw == "true")
+
+	indexFilesRaw, ok := job.Data["index_files"]
+	job.Data["index_files"] = strconv.FormatBool(!ok || indexFilesRaw == "true")
+}
+
+func (worker *IndexerWorker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	claimed, appErr := worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr))
+		return
+	}
+	if !claimed {
+		return
+	}
+
+	logger.Info("Worker: Indexing job claimed by worker")
+
+	err := worker.createBulkProcessor()
+	if err != nil {
+		worker.logger.Error("Worker: Failed to setup bulk processor", mlog.Err(err))
+		return
+	}
+
+	worker.initEntitiesToIndex(job)
+	progress, err := initProgress(logger, worker.jobServer, job)
+	if err != nil {
+		return
+	}
+
+	var cancelContext request.CTX = request.EmptyContext(worker.logger)
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	cancelContext = cancelContext.WithContext(cancelCtx)
+	go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
+
+	defer func() {
+		cancelCancelWatcher()
+		err := worker.closeBulkProcessor()
+		if err != nil {
+			logger.Warn("Error while closing the bulk indexer", mlog.Err(err), mlog.String("job_id", job.Id))
+		}
+	}()
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			logger.Info("Worker: Indexing job has been canceled via CancellationWatcher")
+			if err := worker.jobServer.SetJobCanceled(job); err != nil {
+				logger.Error("Worker: Failed to mark job as cancelled", mlog.Err(err))
+			}
+			return
+
+		case <-worker.stopCh:
+			logger.Info("Worker: Indexing has been canceled via Worker Stop. Setting the job back to pending.")
+			if err := worker.jobServer.SetJobPending(job); err != nil {
+				logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
+			}
+			return
+
+		case <-time.After(timeBetweenBatches):
+			var err *model.AppError
+			if progress, err = worker.IndexBatch(logger, progress, job); err != nil {
+				logger.Error("Worker: Failed to index batch for job", mlog.Err(err))
+				if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+					logger.Error("Worker: Failed to set job error", mlog.Err(err2), mlog.NamedErr("set_error", err))
+				}
+				return
+			}
+
+			// Storing the batch progress in metadata.
+			if job.Data == nil {
+				job.Data = make(model.StringMap)
+			}
+
+			job.Data["done_posts_count"] = strconv.FormatInt(progress.DonePostsCount, 10)
+			job.Data["done_channels_count"] = strconv.FormatInt(progress.DoneChannelsCount, 10)
+			job.Data["done_users_count"] = strconv.FormatInt(progress.DoneUsersCount, 10)
+			job.Data["done_files_count"] = strconv.FormatInt(progress.DoneFilesCount, 10)
+
+			job.Data["start_time"] = strconv.FormatInt(progress.LastEntityTime, 10)
+			job.Data["start_post_id"] = progress.LastPostID
+			job.Data["start_channel_id"] = progress.LastChannelID
+			job.Data["start_user_id"] = progress.LastUserID
+			job.Data["start_file_id"] = progress.LastFileID
+			job.Data["original_start_time"] = strconv.FormatInt(progress.StartAtTime, 10)
+			job.Data["end_time"] = strconv.FormatInt(progress.EndAtTime, 10)
+
+			if err := worker.jobServer.SetJobProgress(job, progress.CurrentProgress()); err != nil {
+				logger.Error("Worker: Failed to set progress for job", mlog.Err(err))
+				if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+					logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err))
+				}
+				return
+			}
+
+			if progress.IsDone(job) {
+				if err := worker.jobServer.SetJobSuccess(job); err != nil {
+					logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+					if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+						logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err))
+					}
+				}
+				logger.Info("Worker: Indexing job finished successfully")
+				return
+			}
+		}
+	}
+}
+
+func (worker *IndexerWorker) IndexBatch(logger mlog.LoggerIFace, progress IndexingProgress, job *model.Job) (IndexingProgress, *model.AppError) {
+	// an entity's batch is processed if it wasn't specified to be skipped, or if its completed indexing.
+
+	if job.Data["index_posts"] != "false" && !progress.DonePosts {
+		worker.logger.Debug("Worker: indexing post batch...")
+		return worker.IndexPostsBatch(logger, progress)
+	}
+
+	if job.Data["index_channels"] != "false" && !progress.DoneChannels {
+		worker.logger.Debug("Worker: indexing channels batch...")
+		return IndexChannelsBatch(logger, worker.jobServer.Config(), worker.jobServer.Store, worker.addItemToBulkProcessor, progress)
+	}
+
+	if job.Data["index_users"] != "false" && !progress.DoneUsers {
+		worker.logger.Debug("Worker: indexing users batch...")
+		return worker.IndexUsersBatch(logger, progress)
+	}
+
+	if job.Data["index_files"] != "false" && !progress.DoneFiles {
+		worker.logger.Debug("Worker: indexing files batch...")
+		return worker.IndexFilesBatch(logger, progress)
+	}
+
+	return progress, model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError)
+}
+
+func (worker *IndexerWorker) IndexPostsBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var posts []*model.PostForIndexing
+
+	tries := 0
+	for posts == nil {
+		var err error
+		posts, err = worker.jobServer.Store.Post().GetPostsBatchForIndexing(progress.LastEntityTime, progress.LastPostID, *worker.jobServer.Config().ElasticsearchSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexPostsBatch", "ent.elasticsearch.post.get_posts_batch_for_indexing.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			logger.Warn("Failed to get posts batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	// Handle zero messages.
+	if len(posts) == 0 {
+		progress.DonePosts = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastPost, err := worker.BulkIndexPosts(posts, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last post's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastPost.CreateAt {
+		progress.DonePosts = true
+		// We reset the last entity time to the beginning to begin
+		// indexing of the next set of entities (users, channels etc.)
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastPost.CreateAt
+	}
+
+	progress.LastPostID = lastPost.Id
+	progress.DonePostsCount += int64(len(posts))
+
+	return progress, nil
+}
+
+func (worker *IndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, progress IndexingProgress) (*model.Post, *model.AppError) {
+	for _, post := range posts {
+		indexName := BuildPostIndexName(*worker.jobServer.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			*worker.jobServer.Config().ElasticsearchSettings.IndexPrefix+IndexBasePosts,
+			*worker.jobServer.Config().ElasticsearchSettings.IndexPrefix+IndexBasePosts_MONTH, progress.Now, post.CreateAt)
+
+		if post.DeleteAt == 0 {
+			searchPost := ESPostFromPostForIndexing(post)
+
+			data, err := json.Marshal(searchPost)
+			if err != nil {
+				worker.logger.Warn("Failed to marshal JSON, skipping this post.", mlog.String("post_id", post.Id))
+				continue
+			}
+
+			err = worker.addItemToBulkProcessor(indexName, indexOp, searchPost.Id, bytes.NewReader(data))
+			if err != nil {
+				worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		} else {
+			err := worker.addItemToBulkProcessor(indexName, deleteOp, post.Id, nil)
+			if err != nil {
+				worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		}
+	}
+
+	return &posts[len(posts)-1].Post, nil
+}
+
+func (worker *IndexerWorker) IndexFilesBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var files []*model.FileForIndexing
+
+	tries := 0
+	for files == nil {
+		var err error
+		files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, progress.LastFileID, true, *worker.jobServer.Config().ElasticsearchSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexFilesBatch", "ent.elasticsearch.post.get_files_batch_for_indexing.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			logger.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	if len(files) == 0 {
+		progress.DoneFiles = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastFile, err := worker.BulkIndexFiles(files, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last file's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastFile.CreateAt {
+		progress.DoneFiles = true
+		// We reset the last entity time to the beginning to begin
+		// indexing of the next set of entities (users, channels etc.)
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastFile.CreateAt
+	}
+
+	progress.LastFileID = lastFile.Id
+	progress.DoneFilesCount += int64(len(files))
+
+	return progress, nil
+}
+
+func (worker *IndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (*model.FileInfo, *model.AppError) {
+	for _, file := range files {
+		indexName := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + IndexBaseFiles
+
+		if file.ShouldIndex() {
+			searchFile := ESFileFromFileForIndexing(file)
+
+			data, err := json.Marshal(searchFile)
+			if err != nil {
+				worker.logger.Warn("Failed to marshal JSON")
+				continue
+			}
+
+			err = worker.addItemToBulkProcessor(indexName, indexOp, searchFile.Id, bytes.NewReader(data))
+			if err != nil {
+				worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		} else {
+			err := worker.addItemToBulkProcessor(indexName, deleteOp, file.Id, nil)
+			if err != nil {
+				worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		}
+	}
+
+	return &files[len(files)-1].FileInfo, nil
+}
+
+func IndexChannelsBatch(logger mlog.LoggerIFace, config *model.Config, store store.Store, addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error, progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var channels []*model.Channel
+
+	tries := 0
+	for channels == nil {
+		var err error
+		channels, err = store.Channel().GetChannelsBatchForIndexing(progress.LastEntityTime, progress.LastChannelID, *config.ElasticsearchSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexerWorker.IndexChannelsBatch", "ent.elasticsearch.index_channels_batch.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			logger.Warn("Failed to get channels batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+		tries++
+	}
+
+	if len(channels) == 0 {
+		progress.DoneChannels = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastChannel, err := BulkIndexChannels(config, store, logger, addItemToBulkProcessorFn, channels, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last channel's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastChannel.CreateAt {
+		progress.DoneChannels = true
+		// We reset the last entity time to the beginning to begin
+		// indexing of the next set of entities (users etc.)
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastChannel.CreateAt
+	}
+
+	progress.LastChannelID = lastChannel.Id
+	progress.DoneChannelsCount += int64(len(channels))
+
+	return progress, nil
+}
+
+func BulkIndexChannels(config *model.Config,
+	store store.Store,
+	logger mlog.LoggerIFace,
+	addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error,
+	channels []*model.Channel,
+	progress IndexingProgress) (*model.Channel, *model.AppError) {
+	for _, channel := range channels {
+		indexName := *config.ElasticsearchSettings.IndexPrefix + IndexBaseChannels
+
+		if channel.DeleteAt == 0 {
+			var userIDs []string
+			var err error
+			if channel.Type == model.ChannelTypePrivate {
+				userIDs, err = store.Channel().GetAllChannelMemberIdsByChannelId(channel.Id)
+				if err != nil {
+					return nil, model.NewAppError("IndexerWorker.BulkIndexChannels", "ent.elasticsearch.getAllChannelMembers.error", nil, "", http.StatusInternalServerError).Wrap(err)
+				}
+			}
+
+			teamMemberIDs, err := store.Channel().GetTeamMembersForChannel(channel.Id)
+			if err != nil {
+				return nil, model.NewAppError("IndexerWorker.BulkIndexChannels", "ent.elasticsearch.getAllTeamMembers.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			searchChannel := ESChannelFromChannel(channel, userIDs, teamMemberIDs)
+
+			data, err := json.Marshal(searchChannel)
+			if err != nil {
+				logger.Warn("Failed to marshal JSON")
+				continue
+			}
+
+			err = addItemToBulkProcessorFn(indexName, indexOp, searchChannel.Id, bytes.NewReader(data))
+			if err != nil {
+				logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		} else {
+			err := addItemToBulkProcessorFn(indexName, deleteOp, channel.Id, nil)
+			if err != nil {
+				logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+			}
+		}
+	}
+
+	return channels[len(channels)-1], nil
+}
+
+func (worker *IndexerWorker) IndexUsersBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var users []*model.UserForIndexing
+
+	tries := 0
+	for users == nil {
+		if usersBatch, err := worker.jobServer.Store.User().GetUsersBatchForIndexing(progress.LastEntityTime, progress.LastUserID, *worker.jobServer.Config().ElasticsearchSettings.BatchSize); err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexerWorker.IndexUsersBatch", "app.user.get_users_batch_for_indexing.get_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			logger.Warn("Failed to get users batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		} else {
+			users = usersBatch
+		}
+
+		tries++
+	}
+
+	if len(users) == 0 {
+		progress.DoneUsers = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastUser, err := worker.BulkIndexUsers(users, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last user's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastUser.CreateAt {
+		progress.DoneUsers = true
+		// We reset the last entity time to the beginning to begin
+		// indexing of the next set of entities in case they get added in the future.
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastUser.CreateAt
+	}
+	progress.LastUserID = lastUser.Id
+	progress.DoneUsersCount += int64(len(users))
+
+	return progress, nil
+}
+
+func (worker *IndexerWorker) BulkIndexUsers(users []*model.UserForIndexing, progress IndexingProgress) (*model.UserForIndexing, *model.AppError) {
+	for _, user := range users {
+		indexName := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + IndexBaseUsers
+
+		searchUser := ESUserFromUserForIndexing(user)
+
+		data, err := json.Marshal(searchUser)
+		if err != nil {
+			worker.logger.Warn("Failed to marshal JSON")
+			continue
+		}
+
+		err = worker.addItemToBulkProcessor(indexName, indexOp, searchUser.Id, bytes.NewReader(data))
+		if err != nil {
+			worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName))
+		}
+	}
+
+	return users[len(users)-1], nil
+}
+
+func initProgress(logger mlog.LoggerIFace, jobServer *jobs.JobServer, job *model.Job) (IndexingProgress, error) {
+	progress := IndexingProgress{
+		Now:          time.Now(),
+		DonePosts:    false,
+		DoneChannels: false,
+		DoneUsers:    false,
+		DoneFiles:    false,
+		StartAtTime:  0,
+		EndAtTime:    model.GetMillis(),
+	}
+
+	progress, err := parseStartTime(logger, jobServer, progress, job)
+	if err != nil {
+		return progress, err
+	}
+
+	progress, err = parseEndTime(logger, jobServer, progress, job)
+	if err != nil {
+		return progress, err
+	}
+
+	progress = parseDoneCount(logger, progress, job)
+	progress = setStartEntityIDs(progress, job)
+	progress = setEntityCount(logger, jobServer, progress, job)
+
+	return progress, nil
+}
+
+func parseStartTime(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) (IndexingProgress, error) {
+	// Extract the start time, if it is set.
+	if startString, ok := job.Data["start_time"]; ok {
+		startInt, err := strconv.ParseInt(startString, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse start_time for job", mlog.String("start_time", startString), mlog.Err(err))
+			appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.parse_start_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := jobServer.SetJobError(job, appError); err != nil {
+				logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return progress, err
+		}
+		progress.StartAtTime = startInt
+	} else {
+		// Set start time to oldest entity (user, channel or post) in the database.
+		oldestEntityTime, err := jobServer.Store.Post().GetOldestEntityCreationTime()
+		if err != nil {
+			logger.Error("Worker: Failed to fetch oldest post for job.", mlog.String("start_time", startString), mlog.Err(err))
+			appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.get_oldest_entity.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := jobServer.SetJobError(job, appError); err != nil {
+				logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return progress, err
+		}
+		progress.StartAtTime = oldestEntityTime
+	}
+
+	progress.LastEntityTime = progress.StartAtTime
+	return progress, nil
+}
+
+func parseEndTime(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) (IndexingProgress, error) {
+	if endString, ok := job.Data["end_time"]; ok {
+		endInt, err := strconv.ParseInt(endString, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse end_time for job", mlog.String("end_time", endString), mlog.Err(err))
+			appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.parse_end_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := jobServer.SetJobError(job, appError); err != nil {
+				logger.Error("Worker: Failed to set job errorv", mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return progress, err
+		}
+		progress.EndAtTime = endInt
+	}
+
+	return progress, nil
+}
+
+func parseDoneCount(logger mlog.LoggerIFace, progress IndexingProgress, job *model.Job) IndexingProgress {
+	if count, ok := job.Data["done_posts_count"]; ok {
+		countInt, err := strconv.ParseInt(count, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse done_posts_count for job", mlog.String("done_posts_count", count), mlog.Err(err))
+		}
+		progress.DonePostsCount = countInt
+	}
+
+	if count, ok := job.Data["done_channels_count"]; ok {
+		countInt, err := strconv.ParseInt(count, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse done_channels_count for job", mlog.String("done_channels_count", count), mlog.Err(err))
+		}
+		progress.DoneChannelsCount = countInt
+	}
+
+	if count, ok := job.Data["done_users_count"]; ok {
+		countInt, err := strconv.ParseInt(count, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse done_users_count for job", mlog.String("done_users_count", count), mlog.Err(err))
+		}
+		progress.DoneUsersCount = countInt
+	}
+
+	if count, ok := job.Data["done_files_count"]; ok {
+		countInt, err := strconv.ParseInt(count, 10, 64)
+		if err != nil {
+			logger.Error("Worker: Failed to parse done_files_count for job", mlog.String("done_files_count", count), mlog.Err(err))
+		}
+		progress.DoneFilesCount = countInt
+	}
+
+	return progress
+}
+
+func setStartEntityIDs(progress IndexingProgress, job *model.Job) IndexingProgress {
+	if id, ok := job.Data["start_post_id"]; ok {
+		progress.LastPostID = id
+	}
+	if id, ok := job.Data["start_channel_id"]; ok {
+		progress.LastChannelID = id
+	}
+	if id, ok := job.Data["start_user_id"]; ok {
+		progress.LastUserID = id
+	}
+	if id, ok := job.Data["start_file_id"]; ok {
+		progress.LastFileID = id
+	}
+
+	return progress
+}
+
+func setEntityCount(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) IndexingProgress {
+	if job.Data["index_posts"] == "true" {
+		// Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry
+		// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
+		if count, err := jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}); err != nil {
+			logger.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedPostCount", estimatedPostCount), mlog.Err(err))
+			progress.TotalPostsCount = estimatedPostCount
+		} else {
+			progress.TotalPostsCount = count
+		}
+	}
+
+	if job.Data["index_channels"] == "true" {
+		// Same possible fail as above can happen when counting channels
+		if count, err := jobServer.Store.Channel().AnalyticsTypeCount("", ""); err != nil {
+			logger.Warn("Worker: Failed to fetch total channel count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedChannelCount", estimatedChannelCount), mlog.Err(err))
+			progress.TotalChannelsCount = estimatedChannelCount
+		} else {
+			progress.TotalChannelsCount = count
+		}
+	}
+
+	if job.Data["index_users"] == "true" {
+		// Same possible fail as above can happen when counting users
+		if count, err := jobServer.Store.User().Count(model.UserCountOptions{
+			IncludeBotAccounts: true, // This actually doesn't join with the bots table
+			// since ExcludeRegularUsers is set to false
+		}); err != nil {
+			logger.Warn("Worker: Failed to fetch total user count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedUserCount", estimatedUserCount), mlog.Err(err))
+			progress.TotalUsersCount = estimatedUserCount
+		} else {
+			progress.TotalUsersCount = count
+		}
+	}
+
+	if job.Data["index_files"] == "true" {
+		// Same possible fail as above can happen when counting files
+		if count, err := jobServer.Store.FileInfo().CountAll(); err != nil {
+			logger.Warn("Worker: Failed to fetch total files count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedFilesCount", estimatedFilesCount), mlog.Err(err))
+			progress.TotalFilesCount = estimatedFilesCount
+		} else {
+			progress.TotalFilesCount = count
+		}
+	}
+
+	return progress
+}

server/enterprise/elasticsearch/common/logger.go

@@ -0,0 +1,89 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/mattermost/logr/v2"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+)
+
+func NewLogger(backend string, mlogger mlog.LoggerIFace, trace bool) *Logger {
+	return &Logger{
+		backend: backend,
+		logger:  mlogger,
+		trace:   trace,
+	}
+}
+
+// Logger is a target to pass to the Logger instance of the search backend.
+type Logger struct {
+	backend string
+	trace   bool
+	logger  mlog.LoggerIFace
+}
+
+// LogRoundTrip prints the information about request and response.
+func (l *Logger) LogRoundTrip(req *http.Request, res *http.Response, err error, start time.Time, dur time.Duration) error {
+	// Set error level.
+	// 0 = debug, 1=warn, 2=error
+	var level int
+	switch {
+	case err != nil:
+		level = 2
+	case res != nil && res.StatusCode > 0 && res.StatusCode < 500:
+		level = 0
+	case res != nil && res.StatusCode > 499:
+		level = 2
+	default:
+		level = 2
+	}
+
+	// Capture fields.
+	fields := []mlog.Field{
+		mlog.String("method", req.Method),
+		mlog.Int("status_code", res.StatusCode),
+		mlog.String("duration", dur.String()),
+		mlog.String("url", req.URL.String()),
+	}
+
+	var logFn func(string, ...logr.Field)
+	switch level {
+	case 0:
+		logFn = l.logger.Debug
+	case 1:
+		logFn = l.logger.Warn
+	case 2:
+		logFn = l.logger.Error
+	}
+	logFn(l.backend+" request", fields...)
+
+	return nil
+}
+
+// RequestBodyEnabled makes the client pass request body to logger
+func (l *Logger) RequestBodyEnabled() bool { return l.trace }
+
+// ResponseBodyEnabled makes the client pass response body to logger
+func (l *Logger) ResponseBodyEnabled() bool { return false }
+
+func NewBulkIndexerLogger(mlogger mlog.LoggerIFace, name string) BulkIndexerDebugLogger {
+	return BulkIndexerDebugLogger{
+		logger: mlogger,
+		name:   name,
+	}
+}
+
+type BulkIndexerDebugLogger struct {
+	logger mlog.LoggerIFace
+	name   string
+}
+
+func (bl BulkIndexerDebugLogger) Printf(str string, params ...any) {
+	line := fmt.Sprintf(str, params...)
+	bl.logger.Debug(line, mlog.String("workername", bl.name))
+}

server/enterprise/elasticsearch/common/templates.go

@@ -0,0 +1,237 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"strconv"
+
+	"github.com/elastic/go-elasticsearch/v8/typedapi/indices/putindextemplate"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/mattermost/mattermost/server/public/model"
+)
+
+func GetPostTemplate(cfg *model.Config) *putindextemplate.Request {
+	mappings := &types.TypeMapping{
+		Properties: map[string]types.Property{
+			"message": types.TextProperty{
+				Analyzer: model.NewPointer("mm_lowercaser"),
+				Type:     "text",
+			},
+			"attachments": types.TextProperty{
+				Analyzer: model.NewPointer("mm_lowercaser"),
+				Type:     "text",
+			},
+			"urls": types.TextProperty{
+				Analyzer: model.NewPointer("mm_url"),
+				Type:     "text",
+			},
+			"hashtags": types.KeywordProperty{
+				Type:       "keyword",
+				Normalizer: model.NewPointer("mm_hashtag"),
+				Store:      model.NewPointer(true),
+			},
+		},
+	}
+
+	return &putindextemplate.Request{
+		IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBasePosts + "*"},
+		Template: &types.IndexTemplateMapping{
+			Settings: &types.IndexSettings{
+				Index: &types.IndexSettings{
+					NumberOfShards:   strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexShards),
+					NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexReplicas),
+				},
+				Analysis: &types.IndexSettingsAnalysis{
+					CharFilter: map[string]types.CharFilter{
+						"leading_underscores": map[string]any{
+							"type":        "pattern_replace",
+							"pattern":     `(^|[\s\r\n])_`,
+							"replacement": "$1",
+						},
+						"trailing_underscores": map[string]any{
+							"type":        "pattern_replace",
+							"pattern":     `_([\s\r\n]|$)`,
+							"replacement": "$1",
+						},
+					},
+					Analyzer: map[string]types.Analyzer{
+						"mm_lowercaser": map[string]any{
+							"tokenizer": "icu_tokenizer",
+							"filter": []string{
+								"icu_normalizer",
+								"mm_snowball",
+								"mm_stop",
+							},
+							"char_filter": []string{
+								"leading_underscores",
+								"trailing_underscores",
+							},
+						},
+						"mm_url": map[string]any{
+							"tokenizer": "pattern",
+							"pattern":   "\\W",
+							"lowercase": true,
+						}},
+					Filter: map[string]types.TokenFilter{
+						"mm_snowball": map[string]any{
+							"type":     "snowball",
+							"language": "English",
+						},
+						"mm_stop": map[string]any{
+							"type":      "stop",
+							"stopwords": "_english_",
+						},
+					},
+					Normalizer: map[string]types.Normalizer{
+						"mm_hashtag": map[string]any{
+							"type":        "custom",
+							"char_filter": []string{},
+							"filter":      []string{"lowercase", "icu_normalizer"},
+						},
+					},
+				},
+			},
+			Mappings: mappings,
+		},
+	}
+}
+
+func GetFileInfoTemplate(cfg *model.Config) *putindextemplate.Request {
+	mappings := &types.TypeMapping{
+		Properties: map[string]types.Property{
+			"name": types.TextProperty{
+				Analyzer: model.NewPointer("mm_lowercaser"),
+				Type:     "text",
+			},
+			"content": types.TextProperty{
+				Analyzer: model.NewPointer("mm_lowercaser"),
+				Type:     "text",
+			},
+		},
+	}
+
+	return &putindextemplate.Request{
+		IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseFiles + "*"},
+		Template: &types.IndexTemplateMapping{
+			Settings: &types.IndexSettings{
+				Index: &types.IndexSettings{
+					NumberOfShards:   strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexShards),
+					NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexReplicas),
+				},
+				Analysis: &types.IndexSettingsAnalysis{
+					CharFilter: map[string]types.CharFilter{
+						"leading_underscores": map[string]any{
+							"type":        "pattern_replace",
+							"pattern":     `(^|[\s\r\n])_`,
+							"replacement": "$1",
+						},
+						"trailing_underscores": map[string]any{
+							"type":        "pattern_replace",
+							"pattern":     `_([\s\r\n]|$)`,
+							"replacement": "$1",
+						},
+					},
+					Analyzer: map[string]types.Analyzer{
+						"mm_lowercaser": map[string]any{
+							"tokenizer": "icu_tokenizer",
+							"filter": []string{
+								"icu_normalizer",
+								"mm_snowball",
+								"mm_stop",
+							},
+							"char_filter": []string{
+								"leading_underscores",
+								"trailing_underscores",
+							},
+						},
+					},
+					Filter: map[string]types.TokenFilter{
+						"mm_snowball": map[string]any{
+							"type":     "snowball",
+							"language": "English",
+						},
+						"mm_stop": map[string]any{
+							"type":      "stop",
+							"stopwords": "_english_",
+						},
+					},
+				},
+			},
+			Mappings: mappings,
+		},
+	}
+}
+
+func GetChannelTemplate(cfg *model.Config) *putindextemplate.Request {
+	mappings := &types.TypeMapping{
+		Properties: map[string]types.Property{
+			"name_suggestions": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"team_id": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"user_ids": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"team_member_ids": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"type": types.KeywordProperty{
+				Type: "keyword",
+			},
+		},
+	}
+
+	return &putindextemplate.Request{
+		IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseChannels + "*"},
+		Template: &types.IndexTemplateMapping{
+			Settings: &types.IndexSettings{
+				Index: &types.IndexSettings{
+					NumberOfShards:   strconv.Itoa(*cfg.ElasticsearchSettings.ChannelIndexShards),
+					NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.ChannelIndexReplicas),
+				},
+			},
+			Mappings: mappings,
+		},
+	}
+}
+
+func GetUserTemplate(cfg *model.Config) *putindextemplate.Request {
+	mappings := &types.TypeMapping{
+		Properties: map[string]types.Property{
+			"suggestions_with_fullname": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"suggestions_without_fullname": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"team_id": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"channel_id": types.KeywordProperty{
+				Type: "keyword",
+			},
+			"delete_at": types.LongNumberProperty{
+				Type: "long",
+			},
+			"roles": types.KeywordProperty{
+				Type: "keyword",
+			},
+		},
+	}
+
+	return &putindextemplate.Request{
+		IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseUsers + "*"},
+		Template: &types.IndexTemplateMapping{
+			Settings: &types.IndexSettings{
+				Index: &types.IndexSettings{
+					NumberOfShards:   strconv.Itoa(*cfg.ElasticsearchSettings.UserIndexShards),
+					NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.UserIndexReplicas),
+				},
+			},
+			Mappings: mappings,
+		},
+	}
+}

server/enterprise/elasticsearch/common/test_helpers.go

@@ -0,0 +1,76 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/stretchr/testify/assert"
+)
+
+func createPost(userId string, channelId string, message string) *model.Post {
+	post := &model.Post{
+		Message:       message,
+		ChannelId:     channelId,
+		PendingPostId: model.NewId() + ":" + fmt.Sprint(model.GetMillis()),
+		UserId:        userId,
+		CreateAt:      1000000,
+	}
+	post.PreSave()
+
+	return post
+}
+
+func createChannel(teamId, name, displayName string, channelType model.ChannelType) *model.Channel {
+	channel := &model.Channel{
+		TeamId:      teamId,
+		Type:        channelType,
+		Name:        name,
+		DisplayName: displayName,
+	}
+	channel.PreSave()
+
+	return channel
+}
+
+func createUser(username, nickname, firstName, lastName string) *model.User {
+	user := &model.User{
+		Username:  username,
+		Password:  username,
+		Nickname:  nickname,
+		FirstName: firstName,
+		LastName:  lastName,
+	}
+	if err := user.PreSave(); err != nil {
+		return nil
+	}
+
+	return user
+}
+
+func createFile(creatorID, channelID, postID, content, name, extension string) *model.FileInfo {
+	file := &model.FileInfo{
+		CreatorId: creatorID,
+		ChannelId: channelID,
+		PostId:    postID,
+		Content:   content,
+		Name:      name,
+		Extension: extension,
+	}
+	file.PreSave()
+
+	return file
+}
+
+func CheckMatchesEqual(t *testing.T, expected model.PostSearchMatches, actual map[string][]string) {
+	a := assert.New(t)
+
+	a.Len(actual, len(expected), "Received matches for a different number of posts")
+
+	for postId, expectedMatches := range expected {
+		a.ElementsMatch(expectedMatches, actual[postId], fmt.Sprintf("%v: expected %v, got %v", postId, expectedMatches, actual[postId]))
+	}
+}

server/enterprise/elasticsearch/common/test_suite.go

@@ -0,0 +1,789 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/store/searchtest"
+	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
+
+	"github.com/stretchr/testify/suite"
+)
+
+type CommonTestSuite struct {
+	suite.Suite
+
+	TH             *api4.TestHelper
+	ESImpl         searchengine.SearchEngineInterface
+	GetDocumentFn  func(index, documentID string) (bool, json.RawMessage, error)
+	CreateIndexFn  func(index string) error
+	GetIndexFn     func(indexPattern string) ([]string, error)
+	RefreshIndexFn func() error
+}
+
+func (c *CommonTestSuite) TestSearchStore() {
+	searchTestEngine := &searchtest.SearchTestEngine{
+		Driver: searchtest.EngineElasticSearch,
+	}
+
+	c.Run("TestSearchChannelStore", func() {
+		searchtest.TestSearchChannelStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine)
+	})
+
+	c.Run("TestSearchUserStore", func() {
+		searchtest.TestSearchUserStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine)
+	})
+
+	c.Run("TestSearchPostStore", func() {
+		searchtest.TestSearchPostStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine)
+	})
+
+	c.Run("TestSearchFileInfoStore", func() {
+		searchtest.TestSearchFileInfoStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine)
+	})
+}
+
+func (c *CommonTestSuite) TestIndexPost() {
+	testCases := []struct {
+		Name                string
+		Message             string
+		Hashtags            string
+		ExpectedAttachments string
+		ExpectedHashtags    []string
+		ExpectedURLs        []string
+	}{
+		{
+			Name:                "Should be able to index a plain message",
+			Message:             "Test message 1 2 3",
+			ExpectedAttachments: "",
+			ExpectedHashtags:    []string{},
+			ExpectedURLs:        []string(nil),
+		},
+		{
+			Name:                "Should be able to index hashtags",
+			Message:             "Test message #1234",
+			Hashtags:            "#1234",
+			ExpectedAttachments: "",
+			ExpectedHashtags:    []string{"#1234"},
+			ExpectedURLs:        []string(nil),
+		},
+		// TODO: actually send attachments
+		{
+			Name:                "Should be able to index attachments",
+			Message:             "Test message 1 2 3",
+			ExpectedAttachments: "",
+			ExpectedHashtags:    []string{},
+			ExpectedURLs:        []string(nil),
+		},
+		{
+			Name:                "Should be able to index urls",
+			Message:             "Test message www.mattermost.com http://www.mattermost.com [link](http://www.notindexed.com)",
+			ExpectedAttachments: "",
+			ExpectedHashtags:    []string{},
+			ExpectedURLs:        []string{"www.mattermost.com", "http://www.mattermost.com"},
+		},
+	}
+
+	for _, tc := range testCases {
+		c.Run(tc.Name, func() {
+			post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, tc.Message)
+			if tc.Hashtags != "" {
+				post.Hashtags = tc.Hashtags
+			}
+			c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+
+			c.NoError(c.RefreshIndexFn())
+			indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+				IndexBasePosts,
+				IndexBasePosts_MONTH,
+				time.Now(),
+				post.CreateAt,
+			)
+
+			found, source, err := c.GetDocumentFn(indexName, post.Id)
+			c.NoError(err)
+			c.True(found)
+
+			var esPost ESPost
+			err = json.Unmarshal(source, &esPost)
+			c.NoError(err)
+			c.NotNil(post)
+			c.Equal(tc.Message, post.Message)
+			c.Equal(tc.ExpectedAttachments, esPost.Attachments)
+			c.Equal(tc.ExpectedHashtags, esPost.Hashtags)
+			c.Equal(tc.ExpectedURLs, esPost.URLs)
+		})
+	}
+}
+
+func (c *CommonTestSuite) TestSearchPosts() {
+	// Create and index a post
+	post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+	c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+
+	c.NoError(c.RefreshIndexFn())
+	indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt)
+
+	// Check the post is there.
+	found, _, err := c.GetDocumentFn(indexName, post.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Do a search for that post.
+	channels := model.ChannelList{
+		c.TH.BasicChannel,
+	}
+
+	searchParams := []*model.SearchParams{
+		{
+			Terms:     post.Message,
+			IsHashtag: false,
+			OrTerms:   false,
+		},
+	}
+
+	// Check the post is found as expected
+	ids, matches, err := c.ESImpl.SearchPosts(channels, searchParams, 0, 20)
+	c.Nil(err)
+	c.Len(ids, 1)
+	c.Equal(ids[0], post.Id)
+	CheckMatchesEqual(c.T(), map[string][]string{
+		post.Id: {post.Message},
+	}, matches)
+
+	// Do a search that won't match anything.
+	searchParams = []*model.SearchParams{
+		{
+			Terms:     model.NewId(),
+			IsHashtag: false,
+			OrTerms:   false,
+		},
+	}
+
+	ids, matches, err = c.ESImpl.SearchPosts(channels, searchParams, 0, 20)
+	c.Nil(err)
+	c.Len(ids, 0)
+	c.Len(matches, 0)
+}
+
+func (c *CommonTestSuite) TestDeletePost() {
+	c.Require().NotNil(c.TH)
+
+	post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+	indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt)
+
+	// Index the post.
+	c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the post is there.
+	found, _, err := c.GetDocumentFn(indexName, post.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete the post.
+	c.Nil(c.ESImpl.DeletePost(post))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the post is not there.
+	found, _, err = c.GetDocumentFn(indexName, post.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestDeleteChannelPosts() {
+	c.Run("Should remove all the channel posts", func() {
+		channelPosts := make([]*model.Post, 0)
+		post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+		channelPosts = append(channelPosts, post)
+		post2 := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId())
+		post2.CreateAt = 1200000
+		channelPosts = append(channelPosts, post2)
+		post3 := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId())
+		post3.CreateAt = 1300000
+		channelPosts = append(channelPosts, post3)
+		postReply := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId())
+		postReply.RootId = post.Id
+		postReply.CreateAt = 1400000
+		channelPosts = append(channelPosts, postReply)
+		anotherPost := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel2.Id, model.NewId())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt)
+		for _, post := range channelPosts {
+			c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+		}
+		c.Nil(c.ESImpl.IndexPost(anotherPost, c.TH.BasicTeam.Id))
+		c.NoError(c.RefreshIndexFn())
+		for _, post := range channelPosts {
+			found, _, err := c.GetDocumentFn(indexName, post.Id)
+			c.NoError(err)
+			c.True(found)
+		}
+		c.Nil(c.ESImpl.DeleteChannelPosts(c.TH.Context, c.TH.BasicChannel.Id))
+		c.NoError(c.RefreshIndexFn())
+		for _, post := range channelPosts {
+			found, _, err := c.GetDocumentFn(indexName, post.Id)
+			// This is a difference in behavior between engines.
+			if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+				c.Error(err)
+			} else {
+				c.NoError(err)
+			}
+			c.False(found)
+		}
+
+		found, _, err := c.GetDocumentFn(indexName, anotherPost.Id)
+		c.NoError(err)
+		c.True(found)
+	})
+
+	c.Run("Should not remove other channels posts even if there was no posts to remove", func() {
+		postNotInChannel := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel2.Id, model.NewId())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts, IndexBasePosts_MONTH, time.Now(), postNotInChannel.CreateAt)
+		c.Nil(c.ESImpl.IndexPost(postNotInChannel, c.TH.BasicTeam.Id))
+		c.NoError(c.RefreshIndexFn())
+		c.Nil(c.ESImpl.DeleteChannelPosts(c.TH.Context, c.TH.BasicChannel.Id))
+		c.NoError(c.RefreshIndexFn())
+
+		found, _, err := c.GetDocumentFn(indexName, postNotInChannel.Id)
+		c.NoError(err)
+		c.True(found)
+	})
+}
+
+func (c *CommonTestSuite) TestDeleteUserPosts() {
+	c.Run("Should remove all the user posts", func() {
+		anotherTeam := c.TH.CreateTeam()
+		anotherTeamChannel := createChannel(anotherTeam.Id, "anotherteamchannel", "", model.ChannelTypeOpen)
+		userPosts := make([]*model.Post, 0)
+		post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+		userPosts = append(userPosts, post)
+		post2 := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel2.Id, model.NewId())
+		post2.CreateAt = 1200000
+		userPosts = append(userPosts, post2)
+		post3 := createPost(c.TH.BasicUser.Id, c.TH.BasicPrivateChannel.Id, model.NewId())
+		post3.CreateAt = 1300000
+		userPosts = append(userPosts, post3)
+		postReply := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId())
+		postReply.RootId = post.Id
+		postReply.CreateAt = 1400000
+		userPosts = append(userPosts, postReply)
+		postAnotherTeam := createPost(c.TH.BasicUser.Id, anotherTeamChannel.Id, model.NewId())
+		postAnotherTeam.CreateAt = 1400000
+		userPosts = append(userPosts, postAnotherTeam)
+		anotherPost := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel2.Id, model.NewId())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt)
+		for _, post := range userPosts {
+			c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+		}
+		c.Nil(c.ESImpl.IndexPost(postAnotherTeam, anotherTeam.Id))
+		c.Nil(c.ESImpl.IndexPost(anotherPost, c.TH.BasicTeam.Id))
+		c.NoError(c.RefreshIndexFn())
+		for _, post := range userPosts {
+			found, _, err := c.GetDocumentFn(indexName, post.Id)
+			c.NoError(err)
+			c.True(found)
+		}
+		c.Nil(c.ESImpl.DeleteUserPosts(c.TH.Context, c.TH.BasicUser.Id))
+		c.NoError(c.RefreshIndexFn())
+		for _, post := range userPosts {
+			found, _, err := c.GetDocumentFn(indexName, post.Id)
+			// This is a difference in behavior between engines.
+			if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+				c.Error(err)
+			} else {
+				c.NoError(err)
+			}
+			c.False(found)
+		}
+		found, _, err := c.GetDocumentFn(indexName, anotherPost.Id)
+		c.NoError(err)
+		c.True(found)
+	})
+
+	c.Run("Should not remove other channels posts even if there was no posts to remove", func() {
+		postNotInChannel := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts, IndexBasePosts_MONTH, time.Now(), postNotInChannel.CreateAt)
+		c.Nil(c.ESImpl.IndexPost(postNotInChannel, c.TH.BasicTeam.Id))
+		c.NoError(c.RefreshIndexFn())
+		c.Nil(c.ESImpl.DeleteUserPosts(c.TH.Context, c.TH.BasicUser.Id))
+		c.NoError(c.RefreshIndexFn())
+		found, _, err := c.GetDocumentFn(indexName, postNotInChannel.Id)
+		c.NoError(err)
+		c.True(found)
+	})
+}
+
+func (c *CommonTestSuite) TestIndexChannel() {
+	// Create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the channel is there.
+	found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id)
+	c.NoError(err)
+	c.True(found)
+}
+
+func (c *CommonTestSuite) TestDeleteChannel() {
+	// Create and index a channel.
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the channel is there.
+	found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete the channel.
+	c.Nil(c.ESImpl.DeleteChannel(channel))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the channel is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestIndexUser() {
+	// Create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the user is there.
+	found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id)
+	c.NoError(err)
+	c.True(found)
+}
+
+func (c *CommonTestSuite) TestDeleteUser() {
+	// Create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the user is there.
+	found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete the user.
+	c.Nil(c.ESImpl.DeleteUser(user))
+	c.NoError(c.RefreshIndexFn())
+	// Check the user is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseUsers, user.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestTestConfig() {
+	c.Nil(c.ESImpl.TestConfig(c.TH.Context, c.TH.App.Config()))
+
+	originalConfig := c.TH.App.Config()
+	defer c.TH.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ElasticsearchSettings.ConnectionURL = *originalConfig.ElasticsearchSettings.ConnectionURL
+	})
+
+	c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.ConnectionURL = "example.com:12345" })
+	c.Error(c.ESImpl.TestConfig(c.TH.Context, c.TH.App.Config()))
+
+	// Passing a temp config which is different from the saved
+	// config should be taken correctly.
+	c.Nil(c.ESImpl.TestConfig(c.TH.Context, originalConfig))
+}
+
+func (c *CommonTestSuite) TestIndexFile() {
+	// First, create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	// Then, create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id}))
+
+	// Create and index a file
+	file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt")
+	c.Nil(c.ESImpl.IndexFile(file, channel.Id))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is there
+	found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id)
+	c.NoError(err)
+	c.True(found)
+}
+
+func (c *CommonTestSuite) TestDeleteFile() {
+	// First, create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	// Then, create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id}))
+
+	// Create and index a file
+	file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt")
+	c.Nil(c.ESImpl.IndexFile(file, channel.Id))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is there
+	found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete the file
+	c.Nil(c.ESImpl.DeleteFile(file.Id))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestDeleteUserFiles() {
+	// First, create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	// Then, create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id}))
+
+	// Create and index a file
+	file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt")
+	c.Nil(c.ESImpl.IndexFile(file, channel.Id))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is there
+	found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete file by creator
+	c.Nil(c.ESImpl.DeleteUserFiles(c.TH.Context, user.Id))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestDeletePostFiles() {
+	// First, create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+	// Then, create and index a user
+	user := createUser("test.user", "testuser", "Test", "User")
+	c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id}))
+
+	// Create and index a post
+	post := createPost(user.Id, channel.Id, "test post message")
+	c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+
+	// Create and index a file
+	file := createFile(user.Id, channel.Id, post.Id, "file contents", "testfile", "txt")
+	c.Nil(c.ESImpl.IndexFile(file, channel.Id))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is there
+	found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id)
+	c.NoError(err)
+	c.True(found)
+
+	// Delete file by post
+	c.Nil(c.ESImpl.DeletePostFiles(c.TH.Context, post.Id))
+	c.NoError(c.RefreshIndexFn())
+
+	// Check the file is not there.
+	found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id)
+	// This is a difference in behavior between engines.
+	if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+		c.Error(err)
+	} else {
+		c.NoError(err)
+	}
+	c.False(found)
+}
+
+func (c *CommonTestSuite) TestElasticsearchDataRetentionDeleteIndexes() {
+	c.Nil(c.CreateIndexFn("posts_2017_09_15"))
+	c.Nil(c.CreateIndexFn("posts_2017_09_16"))
+	c.Nil(c.CreateIndexFn("posts_2017_09_17"))
+	c.Nil(c.CreateIndexFn("posts_2017_09_18"))
+	c.Nil(c.CreateIndexFn("posts_2017_09_19"))
+
+	c.Run("Should delete indexes using start of day cut off", func() {
+		c.Nil(c.ESImpl.DataRetentionDeleteIndexes(c.TH.Context, time.Date(2017, 9, 16, 0, 0, 0, 0, time.UTC)))
+
+		postIndexesResult, err := c.GetIndexFn("posts_*")
+		c.Nil(err)
+		if err == nil {
+			found1 := false
+			found2 := false
+			found3 := false
+			found4 := false
+			found5 := false
+
+			for _, index := range postIndexesResult {
+				if index == "posts_2017_09_15" {
+					found1 = true
+				} else if index == "posts_2017_09_16" {
+					found2 = true
+				} else if index == "posts_2017_09_17" {
+					found3 = true
+				} else if index == "posts_2017_09_18" {
+					found4 = true
+				} else if index == "posts_2017_09_19" {
+					found5 = true
+				}
+			}
+
+			c.False(found1)
+			c.False(found2)
+			c.True(found3)
+			c.True(found4)
+			c.True(found5)
+		}
+	})
+
+	c.Run("Should delete indexes when cut off is in hours", func() {
+		c.Nil(c.ESImpl.DataRetentionDeleteIndexes(c.TH.Context, time.Date(2017, 9, 18, 11, 6, 0, 0, time.UTC)))
+
+		postIndexesResult, err := c.GetIndexFn("posts_*")
+		c.Nil(err)
+		if err == nil {
+			found1 := false
+			found2 := false
+			found3 := false
+
+			for _, index := range postIndexesResult {
+				if index == "posts_2017_09_17" {
+					found1 = true
+				} else if index == "posts_2017_09_18" {
+					found2 = true
+				} else if index == "posts_2017_09_19" {
+					found3 = true
+				}
+			}
+
+			c.False(found1)
+			c.False(found2)
+			c.True(found3)
+		}
+	})
+}
+
+func (c *CommonTestSuite) TestPurgeIndexes() {
+	existingIndexPrefix := *c.TH.Server.Config().ElasticsearchSettings.IndexPrefix
+	defer c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = existingIndexPrefix })
+
+	c.Run("Should purge all indexes", func() {
+		// Create and index a user
+		user := createUser("test.user", "testuser", "Test", "User")
+		c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{}))
+
+		c.NoError(c.RefreshIndexFn())
+
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "test_" })
+
+		// index user with a new index prefix
+		c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{}))
+		c.NoError(c.RefreshIndexFn())
+
+		c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context))
+
+		found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id)
+		c.NoError(err)
+		c.True(found)
+
+		found, _, err = c.GetDocumentFn("test_"+IndexBaseUsers, user.Id)
+		if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+			c.False(found)
+		} else {
+			elasticErr := err.(*types.ElasticsearchError)
+			c.Equal(404, elasticErr.Status)
+		}
+	})
+
+	c.Run("Should not purge indexes defined to ignore", func() {
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "posts*" })
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "" })
+
+		// Create a user
+		user := createUser("test.user", "testuser", "Test", "User")
+
+		// Create and index a post
+		post := createPost(user.Id, c.TH.BasicChannel.Id, "Test")
+		c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id))
+
+		c.NoError(c.RefreshIndexFn())
+		indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+			IndexBasePosts,
+			IndexBasePosts_MONTH,
+			time.Now(),
+			post.CreateAt,
+		)
+
+		// We expect posts indexes to remain after purge
+		c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context))
+
+		found, _, err := c.GetDocumentFn(indexName, post.Id)
+		c.NoError(err)
+		c.True(found)
+
+		// Remove the ignore rule
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "" })
+
+		c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context))
+
+		// Validate the indexes are gone
+		found, _, err = c.GetDocumentFn(IndexBasePosts, post.Id)
+		if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+			c.False(found)
+		} else {
+			elasticErr := err.(*types.ElasticsearchError)
+			c.Equal(404, elasticErr.Status)
+		}
+	})
+}
+
+func (c *CommonTestSuite) TestPurgeIndexList() {
+	existingIndexPrefix := *c.TH.Server.Config().ElasticsearchSettings.IndexPrefix
+	defer c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = existingIndexPrefix })
+
+	c.Run("Should purge allowed index", func() {
+		// Create and index a channel
+		channel := createChannel("test.channel", "testuser", "Test", model.ChannelTypeOpen)
+		c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+		c.NoError(c.RefreshIndexFn())
+
+		// verify data is in Elasticsearch
+		found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		c.NoError(err)
+		c.True(found)
+
+		// now we'll purge
+		c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"}))
+
+		found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+			c.False(found)
+		} else {
+			elasticErr := err.(*types.ElasticsearchError)
+			c.Equal(404, elasticErr.Status)
+		}
+	})
+
+	c.Run("Should not purge indexes defined to ignore", func() {
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "channels" })
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "" })
+
+		channel := createChannel("test.channel", "testuser", "Test", model.ChannelTypeOpen)
+		c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{}))
+
+		c.NoError(c.RefreshIndexFn())
+
+		// verify data is in Elasticsearch
+		found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		c.NoError(err)
+		c.True(found)
+
+		// now we'll purge
+		c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"}))
+
+		// the channel should still be there because we ignored that index
+		found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		c.NoError(err)
+		c.True(found)
+
+		// Remove the ignore rule
+		c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "" })
+
+		c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"}))
+
+		// now it should be gone as we're no longer ignoring it
+		found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id)
+		if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend {
+			c.False(found)
+		} else {
+			elasticErr := err.(*types.ElasticsearchError)
+			c.Equal(404, elasticErr.Status)
+		}
+	})
+}
+
+func (c *CommonTestSuite) TestSearchChannels() {
+	// Create and index a channel
+	channel := createChannel(c.TH.BasicTeam.Id, "channel", "Channel Open", model.ChannelTypeOpen)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{c.TH.BasicUser.Id, "otheruser"}))
+	channel2 := createChannel(c.TH.BasicTeam.Id, "channel", "Channel Private", model.ChannelTypePrivate)
+	c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel2, []string{c.TH.BasicUser.Id}, []string{c.TH.BasicUser.Id, "otheruser"}))
+
+	c.NoError(c.RefreshIndexFn())
+
+	// Private channels should be returned for right user.
+	ids, appErr := c.ESImpl.SearchChannels("", c.TH.BasicUser.Id, "Channel", false)
+	c.Nil(appErr)
+	c.Len(ids, 2)
+
+	// No private channels if user is guest
+	ids, appErr = c.ESImpl.SearchChannels("", c.TH.BasicUser.Id, "Channel", true)
+	c.Nil(appErr)
+	c.Len(ids, 1)
+	c.Equal(channel.Id, ids[0])
+
+	// No Private channels should be returned for wrong user.
+	ids, appErr = c.ESImpl.SearchChannels("", "otheruser", "Channel", false)
+	c.Nil(appErr)
+	c.Len(ids, 1)
+	c.Equal(channel.Id, ids[0])
+}

server/enterprise/elasticsearch/common/version.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"strconv"
+	"strings"
+)
+
+func GetVersionComponents(version string) (int, int, int, error) {
+	spl := strings.Split(version, ".")
+	major, err := strconv.Atoi(spl[0])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	minor, err := strconv.Atoi(spl[1])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	patch, err := strconv.Atoi(spl[2])
+	if err != nil {
+		return 0, 0, 0, err
+	}
+
+	return major, minor, patch, nil
+}

server/enterprise/elasticsearch/common/version_test.go

@@ -0,0 +1,51 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package common
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetVersionComponents(t *testing.T) {
+	testCases := []struct {
+		Name          string
+		Version       string
+		ExpectedMajor int
+		ExpectedMinor int
+		ExpectedPatch int
+		ExpectedError bool
+	}{
+		{
+			Name:          "Should error if version format is invalid",
+			Version:       "invalid",
+			ExpectedMajor: 0,
+			ExpectedMinor: 0,
+			ExpectedPatch: 0,
+			ExpectedError: true,
+		},
+		{
+			Name:          "Should work correctly if version has three valid components",
+			Version:       "7.2.3",
+			ExpectedMajor: 7,
+			ExpectedMinor: 2,
+			ExpectedPatch: 3,
+			ExpectedError: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Name, func(t *testing.T) {
+			major, minor, patch, err := GetVersionComponents(tc.Version)
+			if tc.ExpectedError {
+				require.Error(t, err)
+			}
+			assert.Equal(t, tc.ExpectedMajor, major)
+			assert.Equal(t, tc.ExpectedMinor, minor)
+			assert.Equal(t, tc.ExpectedPatch, patch)
+		})
+	}
+}

server/enterprise/elasticsearch/elasticsearch/aggregation_job.go

@@ -0,0 +1,330 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v8"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	aggregatorJobPollingInterval = 15 * time.Second
+	indexDeletionBatchSize       = 20
+)
+
+type ElasticsearchAggregatorInterfaceImpl struct {
+	Server *app.Server
+}
+
+type ElasticsearchAggregatorWorker struct {
+	name string
+	// stateMut protects stopCh and stopped and helps enforce
+	// ordering in case subsequent Run or Stop calls are made.
+	stateMut    sync.Mutex
+	stopCh      chan struct{}
+	stopped     bool
+	stoppedCh   chan bool
+	jobs        chan model.Job
+	jobServer   *jobs.JobServer
+	logger      mlog.LoggerIFace
+	fileBackend filestore.FileBackend
+
+	client  *elasticsearch.TypedClient
+	license func() *model.License
+}
+
+func (esi *ElasticsearchAggregatorInterfaceImpl) MakeWorker() model.Worker {
+	const workerName = "EnterpriseElasticsearchAggregator"
+	worker := ElasticsearchAggregatorWorker{
+		name:        workerName,
+		stoppedCh:   make(chan bool, 1),
+		jobs:        make(chan model.Job),
+		jobServer:   esi.Server.Jobs,
+		logger:      esi.Server.Jobs.Logger().With(mlog.String("worker_name", workerName)),
+		fileBackend: esi.Server.Platform().FileBackend(),
+		license:     esi.Server.License,
+		stopped:     true,
+	}
+
+	return &worker
+}
+
+func (worker *ElasticsearchAggregatorWorker) Run() {
+	worker.stateMut.Lock()
+	// We have to re-assign the stop channel again, because
+	// it might happen that the job was restarted due to a config change.
+	if worker.stopped {
+		worker.stopped = false
+		worker.stopCh = make(chan struct{})
+	} else {
+		worker.stateMut.Unlock()
+		return
+	}
+	// Run is called from a separate goroutine and doesn't return.
+	// So we cannot Unlock in a defer clause.
+	worker.stateMut.Unlock()
+
+	worker.logger.Debug("Worker Started")
+
+	defer func() {
+		worker.logger.Debug("Worker Finished")
+		worker.stoppedCh <- true
+	}()
+
+	client, err := createTypedClient(worker.logger, worker.jobServer.Config(), worker.fileBackend, false)
+	if err != nil {
+		worker.logger.Error("Worker Failed to Create Client", mlog.Err(err))
+		return
+	}
+
+	worker.client = client
+
+	for {
+		select {
+		case <-worker.stopCh:
+			worker.logger.Debug("Worker Received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *ElasticsearchAggregatorWorker) IsEnabled(cfg *model.Config) bool {
+	if license := worker.license(); license == nil || !*license.Features.Elasticsearch {
+		return false
+	}
+
+	if *cfg.ElasticsearchSettings.EnableIndexing {
+		return true
+	}
+
+	return false
+}
+
+func (worker *ElasticsearchAggregatorWorker) Stop() {
+	worker.stateMut.Lock()
+	defer worker.stateMut.Unlock()
+
+	// Set to close, and if already closed before, then return.
+	if worker.stopped {
+		return
+	}
+	worker.stopped = true
+
+	worker.logger.Debug("Worker Stopping")
+	close(worker.stopCh)
+	<-worker.stoppedCh
+}
+
+func (worker *ElasticsearchAggregatorWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *ElasticsearchAggregatorWorker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	claimed, appErr := worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr))
+		return
+	}
+
+	if !claimed {
+		return
+	}
+
+	logger.Info("Worker: Aggregation job claimed by worker")
+
+	var cancelContext request.CTX = request.EmptyContext(worker.logger)
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	cancelContext = cancelContext.WithContext(cancelCtx)
+	go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
+	defer cancelCancelWatcher()
+
+	rctx := request.EmptyContext(worker.logger)
+	now := time.Now()
+	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local)
+	cutoff := today.AddDate(0, 0, -*worker.jobServer.Config().ElasticsearchSettings.AggregatePostsAfterDays+1)
+
+	// Get all the daily Elasticsearch post indexes to work out which days aren't aggregated yet.
+	dateFormat := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_2006_01_02"
+	datedIndexes := []time.Time{}
+
+	postIndexesResult, err := worker.client.API.Indices.
+		Get(*worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_*").
+		Do(rctx.Context())
+	if err != nil {
+		appError := model.NewAppError("ElasticsearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.get_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		worker.setJobError(logger, job, appError)
+		return
+	}
+
+	for index := range postIndexesResult {
+		var indexDate time.Time
+		indexDate, err = time.Parse(dateFormat, index)
+		if err != nil {
+			logger.Warn("Failed to parse date from posts index. Ignoring index.", mlog.String("index", index))
+		} else {
+			datedIndexes = append(datedIndexes, indexDate)
+		}
+	}
+
+	// Work out how far back the reindexing (and index deletion) needs to go.
+	var oldestDay time.Time
+	oldestDayFound := false
+	indexesToPurge := []string{}
+	for _, date := range datedIndexes {
+		if date.Before(cutoff) {
+			logger.Debug("Worker: Post index identified for purging", mlog.Time("date", date))
+			indexesToPurge = append(indexesToPurge, date.Format(dateFormat))
+			if !oldestDayFound || oldestDay.After(date) {
+				oldestDay = date
+				oldestDayFound = true
+			}
+		} else {
+			logger.Debug("Worker: Post index is within the range to keep", mlog.Time("date", date))
+		}
+	}
+
+	if !oldestDayFound {
+		// Nothing to purge.
+		logger.Info("Worker: Aggregation job completed. Nothing to aggregate.")
+		worker.setJobSuccess(logger, job)
+		return
+	}
+
+	// Trigger a reindexing job with the appropriate dates.
+	reindexingStartDate := oldestDay
+	reindexingEndDate := cutoff
+
+	logger.Info("Worker: Aggregation job reindexing", mlog.String("start_date", reindexingStartDate.Format("2006-01-02")), mlog.String("end_date", reindexingEndDate.Format("2006-01-02")))
+
+	var indexJob *model.Job
+	if indexJob, appErr = worker.jobServer.CreateJob(
+		rctx,
+		model.JobTypeElasticsearchPostIndexing,
+		map[string]string{
+			"start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10),
+			"end_time":   strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10),
+		},
+	); appErr != nil {
+		logger.Error("Worker: Failed to create indexing job.", mlog.Err(appErr))
+		appError := model.NewAppError("ElasticsearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.create_index_job.error", nil, "", http.StatusInternalServerError).Wrap(appErr)
+		worker.setJobError(logger, job, appError)
+		return
+	}
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			logger.Info("Worker: Aggregation job has been canceled via CancellationWatcher")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-worker.stopCh:
+			logger.Info("Worker: Aggregation job has been canceled via Worker Stop")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-time.After(aggregatorJobPollingInterval):
+			// Get the details of the indexing job we are waiting on.
+			indexJob, err = worker.jobServer.Store.Job().Get(rctx, indexJob.Id)
+			if err != nil {
+				var appErr *model.AppError
+				var nfErr *store.ErrNotFound
+				switch {
+				case errors.As(err, &nfErr):
+					appErr = model.NewAppError("DoJob", "app.job.get.app_error", nil, "", http.StatusNotFound).Wrap(nfErr)
+				default:
+					appErr = model.NewAppError("DoJob", "app.job.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+				}
+				worker.setJobError(logger, job, appErr)
+				return
+			}
+
+			// Wait for the aggregation job to finish.
+			// On success, we delete the old indexes.
+			// Otherwise, fail the job.
+			switch indexJob.Status {
+			case model.JobStatusSuccess:
+				// We limit the number of indexes to delete at one shot.
+				// A minor side-effect of this is that the aggregation job status
+				// will be redundantly queried multiple times, but that's not a major bottleneck.
+				curWindow := indexesToPurge
+				deleteMore := false
+				if len(indexesToPurge) > indexDeletionBatchSize {
+					curWindow = indexesToPurge[:indexDeletionBatchSize]
+					indexesToPurge = indexesToPurge[indexDeletionBatchSize:]
+					deleteMore = true
+				}
+				// Delete indexes
+				if _, err = worker.client.Indices.Delete(strings.Join(curWindow, ",")).Do(rctx.Context()); err != nil {
+					appError := model.NewAppError("ElasticsearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.delete_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err)
+					logger.Error("Worker: Failed to delete indexes for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(appError))
+					worker.setJobError(logger, job, appError)
+					return
+				}
+
+				if !deleteMore {
+					// Job done. Set the status to success.
+					logger.Info("Worker: Aggregation job finished successfully")
+					worker.setJobSuccess(logger, job)
+					return
+				}
+			case model.JobStatusPending, model.JobStatusInProgress:
+				// Indexing job is in progress or pending. Update the progress of this job.
+				if err := worker.jobServer.SetJobProgress(job, indexJob.Progress); err != nil {
+					logger.Error("Worker: Failed to set progress for job", mlog.Err(err))
+					worker.setJobError(logger, job, err)
+					return
+				}
+			default:
+				// error case
+				appError := model.NewAppError("ElasticsearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.index_job_failed.error", nil, "", http.StatusInternalServerError)
+				logger.Error("Worker: Index aggregation job failed", mlog.Err(appError))
+				worker.setJobError(logger, job, appError)
+				return
+			}
+		}
+	}
+}
+
+func (worker *ElasticsearchAggregatorWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *ElasticsearchAggregatorWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("Worker: Failed to set job error", mlog.Err(err))
+	}
+}
+
+func (worker *ElasticsearchAggregatorWorker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobCanceled(job); err != nil {
+		logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
+	}
+}

server/enterprise/elasticsearch/elasticsearch/aggregation_job_test.go

@@ -0,0 +1,197 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+)
+
+func TestElasticsearchAggregation(t *testing.T) {
+	th := api4.SetupEnterpriseWithStoreMock(t)
+	rctx := request.TestContext(t)
+
+	mockUserStore := mocks.UserStore{}
+	mockUserStore.On("Count", mock.Anything).Return(int64(10), nil)
+	mockUserStore.On("GetAllProfiles", mock.Anything).Return(nil, nil)
+
+	mockPostStore := mocks.PostStore{}
+	mockPostStore.On("GetMaxPostSize").Return(65535, nil)
+
+	mockSystemStore := mocks.SystemStore{}
+	mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil)
+	mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil)
+	mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil)
+
+	mockJobStore := mocks.JobStore{}
+	mockJobStore.On("Save", mock.AnythingOfType("*model.Job")).Return(&model.Job{}, nil)
+	mockJobStore.On("UpdateStatus", mock.AnythingOfType("string"), model.JobStatusSuccess).Return(&model.Job{}, nil)
+	mockJobStore.On("Get", mock.AnythingOfType("*request.Context"), mock.AnythingOfType("string")).Return(&model.Job{
+		Status: model.JobStatusSuccess,
+	}, nil)
+	mockJobStore.On("UpdateStatusOptimistically",
+		mock.AnythingOfType("string"),
+		model.JobStatusPending,
+		model.JobStatusInProgress).Return(true, nil)
+	mockJobStore.On("GetAllByType", mock.AnythingOfType("string")).Return([]*model.Job{{
+		Id:     "abcxyz123",
+		Type:   "EnterpriseElasticsearchIndexer",
+		Status: model.JobStatusCanceled,
+	}}, nil)
+
+	mockStore := th.App.Srv().Platform().Store.(*mocks.Store)
+	mockStore.On("User").Return(&mockUserStore)
+	mockStore.On("Post").Return(&mockPostStore)
+	mockStore.On("System").Return(&mockSystemStore)
+	mockStore.On("Job").Return(&mockJobStore)
+	mockStore.On("GetDBSchemaVersion").Return(1, nil)
+
+	aggImpl := ElasticsearchAggregatorInterfaceImpl{Server: th.Server}
+
+	// Register search engine
+	th.App.SearchEngine().RegisterElasticsearchEngine(&ElasticsearchInterfaceImpl{
+		Platform: th.Server.Platform(),
+	})
+
+	// Set up the state for the tests.
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+		*cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1
+		*cfg.ElasticsearchSettings.AggregatePostsAfterDays = 1
+		*cfg.SqlSettings.DisableDatabaseSearch = true
+	})
+
+	esImpl := th.App.SearchEngine().ElasticsearchEngine
+	appErr := esImpl.Start()
+	if appErr != nil && appErr.Id != "ent.elasticsearch.start.already_started.app_error" {
+		require.Fail(t, "failed to start elasticsearch: %v", appErr)
+	}
+	require.Nil(t, esImpl.PurgeIndexes(rctx))
+
+	post := &model.Post{
+		Id:        model.NewId(),
+		ChannelId: "channel",
+		Message:   "hi",
+	}
+	for i := 0; i < indexDeletionBatchSize+1; i++ {
+		indexPost(t, th, esImpl.(*ElasticsearchInterfaceImpl),
+			post,
+			time.Now().Add(-time.Duration(4+i)*24*time.Hour))
+	}
+
+	job := &model.Job{
+		Id:     model.NewId(),
+		Type:   model.JobTypeElasticsearchPostAggregation,
+		Status: model.JobStatusPending,
+	}
+
+	_, err := th.Server.Store().Job().Save(job)
+	require.NoError(t, err)
+
+	worker := aggImpl.MakeWorker().(*ElasticsearchAggregatorWorker)
+	worker.client = createTestClient(t, th.Context, th.App.Config(), th.App.FileBackend())
+	worker.jobServer.Store = mockStore
+
+	indexingImpl := ElasticsearchIndexerInterfaceImpl{
+		Server: th.App.Srv(),
+	}
+	th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, indexingImpl.MakeWorker(), nil)
+
+	worker.DoJob(job)
+
+	// We assert the minimum number of calls to verify that
+	// batching is working correctly. Because job().Get() will happen
+	// in each iteration.
+	numCalls := 0
+	for _, call := range mockJobStore.Calls {
+		if call.Method == "Get" {
+			numCalls++
+		}
+	}
+	assert.GreaterOrEqual(t, numCalls, 8, "Unexpected number of Jobstore.Get calls")
+}
+
+func TestElasticsearchAggregationSkipDuringBulkIndexing(t *testing.T) {
+	th := api4.SetupEnterpriseWithStoreMock(t)
+
+	mockUserStore := mocks.UserStore{}
+	mockUserStore.On("Count", mock.Anything).Return(int64(10), nil)
+
+	mockPostStore := mocks.PostStore{}
+	mockPostStore.On("GetMaxPostSize").Return(65535, nil)
+
+	mockSystemStore := mocks.SystemStore{}
+	mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil)
+	mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil)
+	mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil)
+
+	mockJobStore := mocks.JobStore{}
+
+	mockStore := th.App.Srv().Platform().Store.(*mocks.Store)
+	mockStore.On("User").Return(&mockUserStore)
+	mockStore.On("Post").Return(&mockPostStore)
+	mockStore.On("System").Return(&mockSystemStore)
+	mockStore.On("Job").Return(&mockJobStore)
+	mockStore.On("GetDBSchemaVersion").Return(1, nil)
+
+	aggImpl := ElasticsearchAggregatorInterfaceImpl{Server: th.Server}
+	aggImpl.Server.Jobs.Store = mockStore
+
+	// Register search engine
+	th.App.SearchEngine().RegisterElasticsearchEngine(&ElasticsearchInterfaceImpl{
+		Platform: th.Server.Platform(),
+	})
+
+	// Set up the state for the tests.
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+		*cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1
+		*cfg.ElasticsearchSettings.AggregatePostsAfterDays = 1
+		*cfg.SqlSettings.DisableDatabaseSearch = true
+	})
+
+	sched := aggImpl.MakeScheduler()
+	// Pass pending jobs as true
+	job, appErr := sched.ScheduleJob(th.Context, th.App.Config(), true, nil)
+	require.Nil(t, job)
+	require.Nil(t, appErr)
+
+	mockJobStore.AssertNotCalled(t, "GetCountByStatusAndType")
+}
+
+func indexPost(t *testing.T, th *api4.TestHelper, esImpl *ElasticsearchInterfaceImpl, post *model.Post, createTime time.Time) { //nolint:unused
+	t.Helper()
+	indexName := common.BuildPostIndexName(*th.Server.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+		common.IndexBasePosts,
+		common.IndexBasePosts_MONTH,
+		createTime.Add(-1*24*time.Hour),
+		model.GetMillisForTime(createTime),
+	)
+	searchPost, err := common.ESPostFromPost(post, "teamID")
+	require.NoError(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(),
+		time.Duration(*esImpl.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second)
+	defer cancel()
+
+	_, err = esImpl.client.Index(indexName).
+		Id(post.Id).
+		Document(searchPost).
+		Do(ctx)
+	require.NoError(t, err)
+}

server/enterprise/elasticsearch/elasticsearch/aggregation_scheduler.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
+)
+
+type ElasticSearchAggregatorScheduler struct {
+	jobServer *jobs.JobServer
+	server    *app.Server
+}
+
+func (s *ElasticSearchAggregatorScheduler) Enabled(cfg *model.Config) bool {
+	if license := s.server.License(); license == nil || !*license.Features.Elasticsearch {
+		return false
+	}
+
+	if *cfg.ElasticsearchSettings.EnableIndexing {
+		return true
+	}
+
+	return false
+}
+
+func (s *ElasticSearchAggregatorScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	parsedTime, err := time.Parse("15:04", *cfg.ElasticsearchSettings.PostsAggregatorJobStartTime)
+	if err != nil {
+		s.server.Log().Error("Cannot determine next schedule time for elastic search post aggregator. PostsAggregatorJobStartTime config value is invalid.", mlog.Err(err))
+		return nil
+	}
+
+	return jobs.GenerateNextStartDateTime(now, parsedTime)
+}
+
+func (s *ElasticSearchAggregatorScheduler) ScheduleJob(rctx request.CTX, _ *model.Config, pendingJobs bool, _ *model.Job) (*model.Job, *model.AppError) {
+	if pendingJobs {
+		s.server.Log().Warn("An aggregator job is already running. Skipping.")
+		return nil, nil
+	}
+
+	// Don't schedule a job if we already have a running bulk indexing job
+	count, err := s.jobServer.Store.Job().GetCountByStatusAndType(model.JobStatusInProgress, model.JobTypeElasticsearchPostIndexing)
+	if err != nil {
+		return nil, model.NewAppError(
+			"ScheduleJob",
+			model.NoTranslation,
+			nil,
+			"",
+			http.StatusInternalServerError).Wrap(err)
+	}
+	if count > 0 {
+		return nil, nil
+	}
+
+	return s.jobServer.CreateJob(rctx, model.JobTypeElasticsearchPostAggregation, nil)
+}
+
+func (esi *ElasticsearchAggregatorInterfaceImpl) MakeScheduler() ejobs.Scheduler {
+	return &ElasticSearchAggregatorScheduler{
+		server:    esi.Server,
+		jobServer: esi.Server.Jobs,
+	}
+}

server/enterprise/elasticsearch/elasticsearch/bulk.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	elastic "github.com/elastic/go-elasticsearch/v8"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/core/bulk"
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+)
+
+type Bulk struct {
+	mut sync.Mutex
+
+	logger     mlog.LoggerIFace
+	client     *elastic.TypedClient
+	bulkClient *bulk.Bulk
+	settings   model.ElasticsearchSettings
+
+	quitFlusher   chan struct{}
+	quitFlusherWg sync.WaitGroup
+
+	pendingRequests int
+}
+
+func NewBulk(settings model.ElasticsearchSettings,
+	logger mlog.LoggerIFace,
+	client *elastic.TypedClient) *Bulk {
+	b := &Bulk{
+		settings:    settings,
+		logger:      logger,
+		client:      client,
+		bulkClient:  client.Bulk(),
+		quitFlusher: make(chan struct{}),
+	}
+
+	b.quitFlusherWg.Add(1)
+	go b.periodicFlusher()
+
+	return b
+}
+
+// IndexOp is a helper function to add an IndexOperation to the current bulk request.
+// doc argument can be a []byte, json.RawMessage or a struct.
+func (r *Bulk) IndexOp(op types.IndexOperation, doc any) error {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	if err := r.bulkClient.IndexOp(op, doc); err != nil {
+		return err
+	}
+
+	return r.flushIfNecessary()
+}
+
+// DeleteOp is a helper function to add a DeleteOperation to the current bulk request.
+func (r *Bulk) DeleteOp(op types.DeleteOperation) error {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	if err := r.bulkClient.DeleteOp(op); err != nil {
+		return err
+	}
+
+	return r.flushIfNecessary()
+}
+
+// flushIfNecessary flushes the pending buffer if needed.
+// It MUST be called with an already acquired mutex.
+func (r *Bulk) flushIfNecessary() error {
+	r.pendingRequests++
+
+	if r.pendingRequests > *r.settings.LiveIndexingBatchSize {
+		return r._flush()
+	}
+
+	return nil
+}
+
+func (r *Bulk) Stop() error {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+	r.logger.Info("Stopping Bulk processor")
+
+	if r.pendingRequests > 0 {
+		return r._flush()
+	}
+
+	close(r.quitFlusher)
+	r.quitFlusherWg.Wait()
+
+	return nil
+}
+
+func (r *Bulk) periodicFlusher() {
+	defer r.quitFlusherWg.Done()
+
+	for {
+		select {
+		case <-time.After(common.BulkFlushInterval):
+			r.mut.Lock()
+			if r.pendingRequests > 0 {
+				if err := r._flush(); err != nil {
+					r.logger.Warn("Error flushing live indexing buffer", mlog.Err(err))
+				}
+			}
+			r.mut.Unlock()
+		case <-r.quitFlusher:
+			return
+		}
+	}
+}
+
+// _flush MUST be called with an acquired lock.
+func (r *Bulk) _flush() error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*r.settings.RequestTimeoutSeconds)*time.Second)
+	defer cancel()
+
+	_, err := r.bulkClient.Do(ctx)
+	if err != nil {
+		return err
+	}
+	r.pendingRequests = 0
+
+	return nil
+}

server/enterprise/elasticsearch/elasticsearch/bulk_test.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBulkProcessor(t *testing.T) {
+	th := api4.SetupEnterprise(t)
+	defer th.TearDown()
+
+	client := createTestClient(t, th.Context, th.App.Config(), th.App.FileBackend())
+	bulk := NewBulk(th.App.Config().ElasticsearchSettings,
+		th.Server.Platform().Log(),
+		client)
+
+	post, err := common.ESPostFromPost(&model.Post{
+		Id:      model.NewId(),
+		Message: "hello world",
+	}, "myteam")
+	require.NoError(t, err)
+
+	err = bulk.IndexOp(types.IndexOperation{
+		Index_: model.NewPointer("myindex"),
+		Id_:    model.NewPointer(post.Id),
+	}, post)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, bulk.pendingRequests)
+
+	err = bulk.Stop()
+	require.NoError(t, err)
+
+	require.Equal(t, 0, bulk.pendingRequests)
+}

server/enterprise/elasticsearch/elasticsearch/common.go

@@ -0,0 +1,134 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"crypto/tls"
+	"net/http"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v8"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+func createTypedClient(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend, debugLogging bool) (*elasticsearch.TypedClient, *model.AppError) {
+	esCfg, appErr := createClientConfig(logger, cfg, fileBackend, debugLogging)
+	if appErr != nil {
+		return nil, appErr
+	}
+
+	client, err := elasticsearch.NewTypedClient(*esCfg)
+	if err != nil {
+		return nil, model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.connect_failed", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return client, nil
+}
+
+func createUntypedClient(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend) (*elasticsearch.Client, *model.AppError) {
+	esCfg, appErr := createClientConfig(logger, cfg, fileBackend, true)
+	if appErr != nil {
+		return nil, appErr
+	}
+
+	client, err := elasticsearch.NewClient(*esCfg)
+	if err != nil {
+		return nil, model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.connect_failed", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return client, nil
+}
+
+func createClientConfig(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend, debugLogging bool) (*elasticsearch.Config, *model.AppError) {
+	tp := http.DefaultTransport.(*http.Transport).Clone()
+	tp.TLSClientConfig = &tls.Config{
+		InsecureSkipVerify: *cfg.ElasticsearchSettings.SkipTLSVerification,
+	}
+
+	esCfg := &elasticsearch.Config{
+		Addresses:            []string{*cfg.ElasticsearchSettings.ConnectionURL},
+		RetryBackoff:         func(i int) time.Duration { return time.Duration(i) * 100 * time.Millisecond }, // A minimal backoff function
+		RetryOnStatus:        []int{502, 503, 504, 429},                                                      // Retry on 429 TooManyRequests statuses
+		MaxRetries:           3,
+		DiscoverNodesOnStart: *cfg.ElasticsearchSettings.Sniff,
+	}
+
+	if esCfg.DiscoverNodesOnStart {
+		esCfg.DiscoverNodesInterval = 30 * time.Second
+	}
+
+	if *cfg.ElasticsearchSettings.ClientCert != "" {
+		appErr := configureClientCertificate(tp.TLSClientConfig, cfg, fileBackend)
+		if appErr != nil {
+			return nil, appErr
+		}
+	}
+
+	// custom CA
+	if *cfg.ElasticsearchSettings.CA != "" {
+		appErr := configureCA(esCfg, cfg, fileBackend)
+		if appErr != nil {
+			return nil, appErr
+		}
+	}
+
+	esCfg.Transport = tp
+
+	if *cfg.ElasticsearchSettings.Username != "" {
+		esCfg.Username = *cfg.ElasticsearchSettings.Username
+		esCfg.Password = *cfg.ElasticsearchSettings.Password
+	}
+
+	// This is a compatibility mode from previous config settings.
+	// We have to conditionally enable debug logging due to
+	// https://github.com/elastic/elastic-transport-go/issues/22
+	if *cfg.ElasticsearchSettings.Trace == "all" && debugLogging {
+		esCfg.EnableDebugLogger = true
+	}
+
+	esCfg.Logger = common.NewLogger("Elasticsearch", logger, *cfg.ElasticsearchSettings.Trace == "all")
+
+	return esCfg, nil
+}
+
+func configureCA(esCfg *elasticsearch.Config, cfg *model.Config, fb filestore.FileBackend) *model.AppError {
+	// read the certificate authority (CA) file
+	clientCA, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.CA)
+	if err != nil {
+		return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.ca_cert_missing", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	esCfg.CACert = clientCA
+
+	return nil
+}
+
+func configureClientCertificate(tlsConfig *tls.Config, cfg *model.Config, fb filestore.FileBackend) *model.AppError {
+	// read the client certificate file
+	clientCert, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.ClientCert)
+	if err != nil {
+		return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_cert_missing", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// read the client key file
+	clientKey, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.ClientKey)
+	if err != nil {
+		return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_key_missing", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// load the client key and certificate
+	certificate, err := tls.X509KeyPair(clientCert, clientKey)
+	if err != nil {
+		return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_cert_malformed", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// update the TLS config
+	tlsConfig.Certificates = []tls.Certificate{certificate}
+
+	return nil
+}

server/enterprise/elasticsearch/elasticsearch/elasticsearch_test.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"context"
+	"encoding/json"
+	"testing"
+
+	elastic "github.com/elastic/go-elasticsearch/v8"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks"
+)
+
+type ElasticsearchInterfaceTestSuite struct {
+	common.CommonTestSuite
+
+	th          *api4.TestHelper
+	client      *elastic.TypedClient
+	ctx         context.Context
+	fileBackend filestore.FileBackend
+}
+
+func TestElasticsearchInterfaceTestSuite(t *testing.T) {
+	testSuite := &ElasticsearchInterfaceTestSuite{
+		CommonTestSuite: common.CommonTestSuite{},
+	}
+	suite.Run(t, testSuite)
+}
+
+func (s *ElasticsearchInterfaceTestSuite) SetupSuite() {
+	s.th = api4.SetupEnterprise(s.T()).InitBasic()
+	s.CommonTestSuite.TH = s.th
+	s.CommonTestSuite.GetDocumentFn = func(index, documentID string) (bool, json.RawMessage, error) {
+		resp, err := s.client.API.Get(index, documentID).Do(s.ctx)
+		if resp == nil {
+			return false, nil, err
+		}
+		return resp.Found, resp.Source_, err
+	}
+	s.CommonTestSuite.RefreshIndexFn = func() error {
+		_, err := s.client.Indices.Refresh().Do(context.Background())
+		return err
+	}
+	s.CommonTestSuite.CreateIndexFn = func(index string) error {
+		_, err := s.client.Indices.Create(index).Do(s.ctx)
+		return err
+	}
+	s.CommonTestSuite.GetIndexFn = func(indexPattern string) ([]string, error) {
+		res, err := s.client.Indices.Get(indexPattern).Do(s.ctx)
+		if err != nil {
+			return nil, err
+		}
+		var names []string
+		for name := range res {
+			names = append(names, name)
+		}
+		return names, nil
+	}
+
+	// Set up the state for the tests.
+	s.th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+		*cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1
+		*cfg.SqlSettings.DisableDatabaseSearch = true
+	})
+	s.th.App.Srv().SetLicense(model.NewTestLicense())
+
+	if s.fileBackend == nil {
+		s.fileBackend = &mocks.FileBackend{}
+	}
+
+	// Initialise other stuff for the test.
+	s.client = createTestClient(s.T(), s.th.Context, s.th.App.Config(), s.th.App.FileBackend())
+	s.ctx = context.Background()
+
+	// Register search engine
+	s.th.App.SearchEngine().RegisterElasticsearchEngine(&ElasticsearchInterfaceImpl{Platform: s.th.Server.Platform()})
+}
+
+func (s *ElasticsearchInterfaceTestSuite) SetupTest() {
+	s.CommonTestSuite.ESImpl = s.th.App.SearchEngine().ElasticsearchEngine
+
+	if s.CommonTestSuite.ESImpl.IsActive() {
+		appErr := s.CommonTestSuite.ESImpl.Stop()
+		s.Require().Nil(appErr)
+	}
+
+	s.Require().Nil(s.CommonTestSuite.ESImpl.Start())
+
+	s.Nil(s.CommonTestSuite.ESImpl.PurgeIndexes(s.th.Context))
+}

server/enterprise/elasticsearch/elasticsearch/indexing_job.go

@@ -0,0 +1,72 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+
+	"github.com/elastic/go-elasticsearch/v8/esutil"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+)
+
+type ElasticsearchIndexerInterfaceImpl struct {
+	Server        *app.Server
+	bulkProcessor esutil.BulkIndexer
+}
+
+func (esi *ElasticsearchIndexerInterfaceImpl) MakeWorker() model.Worker {
+	const workerName = "EnterpriseElasticsearchIndexer"
+
+	// Initializing logger
+	logger := esi.Server.Jobs.Logger().With(mlog.String("worker_name", workerName))
+
+	// Creating the client
+	client, appErr := createUntypedClient(logger, esi.Server.Jobs.Config(), esi.Server.Platform().FileBackend())
+	if appErr != nil {
+		logger.Error("Worker: Failed to Create Client", mlog.Err(appErr))
+		return nil
+	}
+
+	return common.NewIndexerWorker(workerName,
+		esi.Server.Jobs,
+		logger,
+		esi.Server.Platform().FileBackend(), esi.Server.License,
+		func() error {
+			// Creating the bulk indexer from the client.
+			biCfg := esutil.BulkIndexerConfig{
+				Client: client,
+				OnError: func(_ context.Context, err error) {
+					logger.Error("Error from elasticsearch bulk indexer", mlog.Err(err))
+				},
+				Timeout:    time.Duration(*esi.Server.Jobs.Config().ElasticsearchSettings.RequestTimeoutSeconds) * time.Second,
+				NumWorkers: common.NumIndexWorkers(),
+			}
+			if *esi.Server.Jobs.Config().ElasticsearchSettings.Trace == "all" {
+				biCfg.DebugLogger = common.NewBulkIndexerLogger(logger, workerName)
+			}
+			var err error
+			esi.bulkProcessor, err = esutil.NewBulkIndexer(biCfg)
+			return err
+		},
+		// Function to add an item in the bulk processor
+		func(indexName, indexOp, docID string, body io.ReadSeeker) error {
+			return esi.bulkProcessor.Add(context.Background(), esutil.BulkIndexerItem{
+				Index:      indexName,
+				Action:     indexOp,
+				DocumentID: docID,
+				Body:       body,
+			})
+		},
+		// Closing the bulk processor.
+		func() error {
+			return esi.bulkProcessor.Close(context.Background())
+		})
+}

server/enterprise/elasticsearch/elasticsearch/indexing_job_test.go

@@ -0,0 +1,94 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+)
+
+func TestElasticSearchIndexerJobIsEnabled(t *testing.T) {
+	t.Run("ElasticSearch feature is enabled then job is enabled", func(t *testing.T) {
+		th := api4.SetupEnterpriseWithStoreMock(t)
+		defer th.TearDown()
+
+		th.Server.SetLicense(model.NewTestLicense("elastic_search"))
+
+		esImpl := &ElasticsearchIndexerInterfaceImpl{
+			Server: th.Server,
+		}
+		worker := esImpl.MakeWorker()
+
+		config := &model.Config{
+			ElasticsearchSettings: model.ElasticsearchSettings{
+				EnableIndexing: model.NewPointer(true),
+			},
+		}
+
+		result := worker.IsEnabled(config)
+
+		assert.Equal(t, result, true)
+	})
+
+	t.Run("there is NO license then job is disabled", func(t *testing.T) {
+		th := api4.SetupEnterpriseWithStoreMock(t)
+		defer th.TearDown()
+
+		th.Server.SetLicense(nil)
+
+		esImpl := &ElasticsearchIndexerInterfaceImpl{
+			Server: th.Server,
+		}
+		worker := esImpl.MakeWorker()
+
+		config := &model.Config{
+			ElasticsearchSettings: model.ElasticsearchSettings{
+				EnableIndexing: model.NewPointer(true),
+			},
+		}
+
+		result := worker.IsEnabled(config)
+
+		assert.Equal(t, result, false)
+	})
+}
+
+func TestElasticSearchIndexerPending(t *testing.T) {
+	th := api4.SetupEnterprise(t).InitBasic()
+	defer th.TearDown()
+
+	// Set up the state for the tests.
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+		*cfg.SqlSettings.DisableDatabaseSearch = true
+	})
+	th.App.Srv().SetLicense(model.NewTestLicense())
+
+	impl := ElasticsearchIndexerInterfaceImpl{
+		Server: th.App.Srv(),
+	}
+
+	worker := impl.MakeWorker()
+	th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, worker, nil)
+
+	go worker.Run()
+
+	job, appErr := th.App.Srv().Jobs.CreateJob(th.Context, model.JobTypeElasticsearchPostIndexing, map[string]string{})
+	require.Nil(t, appErr)
+
+	worker.JobChannel() <- *job
+
+	worker.Stop()
+
+	job, err := th.App.Srv().Store().Job().Get(th.Context, job.Id)
+	require.NoError(t, err)
+	assert.Equal(t, job.Status, model.JobStatusPending)
+}

server/enterprise/elasticsearch/elasticsearch/main_test.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+	api4.SetMainHelper(mainHelper)
+
+	mainHelper.Main(m)
+}

server/enterprise/elasticsearch/elasticsearch/testlib.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v8"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks"
+)
+
+func createTestClient(t *testing.T, rctx request.CTX, cfg *model.Config, fileStore filestore.FileBackend) *elasticsearch.TypedClient {
+	t.Helper()
+
+	if fileStore == nil {
+		fileStore = &mocks.FileBackend{}
+	}
+
+	client, err := createTypedClient(rctx.Logger(), cfg, fileStore, true)
+	require.Nil(t, err)
+	return client
+}

server/enterprise/elasticsearch/init.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package elasticsearch
+
+import (
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/elasticsearch"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/opensearch"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+	"github.com/mattermost/mattermost/server/v8/channels/app/platform"
+	ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
+	"github.com/mattermost/mattermost/server/v8/platform/services/searchengine"
+)
+
+func init() {
+	platform.RegisterElasticsearchInterface(func(s *platform.PlatformService) searchengine.SearchEngineInterface {
+		if *s.Config().ElasticsearchSettings.Backend == model.ElasticsearchSettingsESBackend {
+			return &elasticsearch.ElasticsearchInterfaceImpl{Platform: s}
+		}
+		return &opensearch.OpensearchInterfaceImpl{Platform: s}
+	})
+	app.RegisterJobsElasticsearchIndexerInterface(func(s *app.Server) ejobs.IndexerJobInterface {
+		if *s.Config().ElasticsearchSettings.Backend == model.ElasticsearchSettingsESBackend {
+			return &elasticsearch.ElasticsearchIndexerInterfaceImpl{Server: s}
+		}
+		return &opensearch.OpensearchIndexerInterfaceImpl{Server: s}
+	})
+	app.RegisterJobsElasticsearchAggregatorInterface(func(s *app.Server) ejobs.ElasticsearchAggregatorInterface {
+		if *s.Config().ElasticsearchSettings.Backend == model.ElasticsearchSettingsESBackend {
+			return &elasticsearch.ElasticsearchAggregatorInterfaceImpl{Server: s}
+		}
+		return &opensearch.OpensearchAggregatorInterfaceImpl{Server: s}
+	})
+}

server/enterprise/elasticsearch/opensearch/aggregation_job.go

@@ -0,0 +1,331 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	"github.com/mattermost/mattermost/server/v8/channels/store"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+const (
+	aggregatorJobPollingInterval = 15 * time.Second
+	indexDeletionBatchSize       = 20
+)
+
+type OpensearchAggregatorInterfaceImpl struct {
+	Server *app.Server
+}
+
+type OpensearchAggregatorWorker struct {
+	name string
+	// stateMut protects stopCh and stopped and helps enforce
+	// ordering in case subsequent Run or Stop calls are made.
+	stateMut    sync.Mutex
+	stopCh      chan struct{}
+	stopped     bool
+	stoppedCh   chan bool
+	jobs        chan model.Job
+	jobServer   *jobs.JobServer
+	logger      mlog.LoggerIFace
+	fileBackend filestore.FileBackend
+
+	client  *opensearchapi.Client
+	license func() *model.License
+}
+
+func (esi *OpensearchAggregatorInterfaceImpl) MakeWorker() model.Worker {
+	const workerName = "EnterpriseOpensearchAggregator"
+	worker := OpensearchAggregatorWorker{
+		name:        workerName,
+		stoppedCh:   make(chan bool, 1),
+		jobs:        make(chan model.Job),
+		jobServer:   esi.Server.Jobs,
+		logger:      esi.Server.Jobs.Logger().With(mlog.String("worker_name", workerName)),
+		fileBackend: esi.Server.Platform().FileBackend(),
+		license:     esi.Server.License,
+		stopped:     true,
+	}
+
+	return &worker
+}
+
+func (worker *OpensearchAggregatorWorker) Run() {
+	worker.stateMut.Lock()
+	// We have to re-assign the stop channel again, because
+	// it might happen that the job was restarted due to a config change.
+	if worker.stopped {
+		worker.stopped = false
+		worker.stopCh = make(chan struct{})
+	} else {
+		worker.stateMut.Unlock()
+		return
+	}
+	// Run is called from a separate goroutine and doesn't return.
+	// So we cannot Unlock in a defer clause.
+	worker.stateMut.Unlock()
+
+	worker.logger.Debug("Worker Started")
+
+	defer func() {
+		worker.logger.Debug("Worker Finished")
+		worker.stoppedCh <- true
+	}()
+
+	client, err := createClient(worker.logger, worker.jobServer.Config(), worker.fileBackend, false)
+	if err != nil {
+		worker.logger.Error("Worker Failed to Create Client", mlog.Err(err))
+		return
+	}
+
+	worker.client = client
+
+	for {
+		select {
+		case <-worker.stopCh:
+			worker.logger.Debug("Worker Received stop signal")
+			return
+		case job := <-worker.jobs:
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *OpensearchAggregatorWorker) IsEnabled(cfg *model.Config) bool {
+	if license := worker.license(); license == nil || !*license.Features.Elasticsearch {
+		return false
+	}
+
+	if *cfg.ElasticsearchSettings.EnableIndexing {
+		return true
+	}
+
+	return false
+}
+
+func (worker *OpensearchAggregatorWorker) Stop() {
+	worker.stateMut.Lock()
+	defer worker.stateMut.Unlock()
+
+	// Set to close, and if already closed before, then return.
+	if worker.stopped {
+		return
+	}
+	worker.stopped = true
+
+	worker.logger.Debug("Worker Stopping")
+	close(worker.stopCh)
+	<-worker.stoppedCh
+}
+
+func (worker *OpensearchAggregatorWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *OpensearchAggregatorWorker) DoJob(job *model.Job) {
+	logger := worker.logger.With(jobs.JobLoggerFields(job)...)
+	logger.Debug("Worker: Received a new candidate job.")
+	defer worker.jobServer.HandleJobPanic(logger, job)
+
+	claimed, appErr := worker.jobServer.ClaimJob(job)
+	if appErr != nil {
+		logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr))
+		return
+	}
+
+	if !claimed {
+		return
+	}
+
+	logger.Info("Worker: Aggregation job claimed by worker")
+
+	var cancelContext request.CTX = request.EmptyContext(worker.logger)
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	cancelContext = cancelContext.WithContext(cancelCtx)
+	go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan)
+	defer cancelCancelWatcher()
+
+	rctx := request.EmptyContext(worker.logger)
+	now := time.Now()
+	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local)
+	cutoff := today.AddDate(0, 0, -*worker.jobServer.Config().ElasticsearchSettings.AggregatePostsAfterDays+1)
+
+	// Get all the daily Elasticsearch post indexes to work out which days aren't aggregated yet.
+	dateFormat := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_2006_01_02"
+	datedIndexes := []time.Time{}
+
+	postIndexesResult, err := worker.client.Indices.Get(rctx.Context(), opensearchapi.IndicesGetReq{
+		Indices: []string{*worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_*"},
+	})
+	if err != nil {
+		appError := model.NewAppError("OpensearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.get_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		worker.setJobError(logger, job, appError)
+		return
+	}
+
+	for index := range postIndexesResult.Indices {
+		var indexDate time.Time
+		indexDate, err = time.Parse(dateFormat, index)
+		if err != nil {
+			logger.Warn("Failed to parse date from posts index. Ignoring index.", mlog.String("index", index))
+		} else {
+			datedIndexes = append(datedIndexes, indexDate)
+		}
+	}
+
+	// Work out how far back the reindexing (and index deletion) needs to go.
+	var oldestDay time.Time
+	oldestDayFound := false
+	indexesToPurge := []string{}
+	for _, date := range datedIndexes {
+		if date.Before(cutoff) {
+			logger.Debug("Worker: Post index identified for purging", mlog.Time("date", date))
+			indexesToPurge = append(indexesToPurge, date.Format(dateFormat))
+			if !oldestDayFound || oldestDay.After(date) {
+				oldestDay = date
+				oldestDayFound = true
+			}
+		} else {
+			logger.Debug("Worker: Post index is within the range to keep", mlog.Time("date", date))
+		}
+	}
+
+	if !oldestDayFound {
+		// Nothing to purge.
+		logger.Info("Worker: Aggregation job completed. Nothing to aggregate.")
+		worker.setJobSuccess(logger, job)
+		return
+	}
+
+	// Trigger a reindexing job with the appropriate dates.
+	reindexingStartDate := oldestDay
+	reindexingEndDate := cutoff
+
+	logger.Info("Worker: Aggregation job reindexing", mlog.String("start_date", reindexingStartDate.Format("2006-01-02")), mlog.String("end_date", reindexingEndDate.Format("2006-01-02")))
+
+	var indexJob *model.Job
+	if indexJob, appErr = worker.jobServer.CreateJob(
+		rctx,
+		model.JobTypeElasticsearchPostIndexing,
+		map[string]string{
+			"start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10),
+			"end_time":   strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10),
+		},
+	); appErr != nil {
+		logger.Error("Worker: Failed to create indexing job.", mlog.Err(appErr))
+		appError := model.NewAppError("OpensearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.create_index_job.error", nil, "", http.StatusInternalServerError).Wrap(appErr)
+		worker.setJobError(logger, job, appError)
+		return
+	}
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			logger.Info("Worker: Aggregation job has been canceled via CancellationWatcher")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-worker.stopCh:
+			logger.Info("Worker: Aggregation job has been canceled via Worker Stop")
+			worker.setJobCanceled(logger, job)
+			return
+
+		case <-time.After(aggregatorJobPollingInterval):
+			// Get the details of the indexing job we are waiting on.
+			indexJob, err = worker.jobServer.Store.Job().Get(rctx, indexJob.Id)
+			if err != nil {
+				var appErr *model.AppError
+				var nfErr *store.ErrNotFound
+				switch {
+				case errors.As(err, &nfErr):
+					appErr = model.NewAppError("DoJob", "app.job.get.app_error", nil, "", http.StatusNotFound).Wrap(nfErr)
+				default:
+					appErr = model.NewAppError("DoJob", "app.job.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+				}
+				worker.setJobError(logger, job, appErr)
+				return
+			}
+
+			// Wait for the aggregation job to finish.
+			// On success, we delete the old indexes.
+			// Otherwise, fail the job.
+			switch indexJob.Status {
+			case model.JobStatusSuccess:
+				// We limit the number of indexes to delete at one shot.
+				// A minor side-effect of this is that the aggregation job status
+				// will be redundantly queried multiple times, but that's not a major bottleneck.
+				curWindow := indexesToPurge
+				deleteMore := false
+				if len(indexesToPurge) > indexDeletionBatchSize {
+					curWindow = indexesToPurge[:indexDeletionBatchSize]
+					indexesToPurge = indexesToPurge[indexDeletionBatchSize:]
+					deleteMore = true
+				}
+				// Delete indexes
+				if _, err = worker.client.Indices.Delete(rctx.Context(), opensearchapi.IndicesDeleteReq{
+					Indices: curWindow,
+				}); err != nil {
+					appError := model.NewAppError("OpensearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.delete_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err)
+					logger.Error("Worker: Failed to delete indexes for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(appError))
+					worker.setJobError(logger, job, appError)
+					return
+				}
+
+				if !deleteMore {
+					// Job done. Set the status to success.
+					logger.Info("Worker: Aggregation job finished successfully")
+					worker.setJobSuccess(logger, job)
+					return
+				}
+			case model.JobStatusPending, model.JobStatusInProgress:
+				// Indexing job is in progress or pending. Update the progress of this job.
+				if err := worker.jobServer.SetJobProgress(job, indexJob.Progress); err != nil {
+					logger.Error("Worker: Failed to set progress for job", mlog.Err(err))
+					worker.setJobError(logger, job, err)
+					return
+				}
+			default:
+				// error case
+				appError := model.NewAppError("OpensearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.index_job_failed.error", nil, "", http.StatusInternalServerError)
+				logger.Error("Worker: Index aggregation job failed", mlog.Err(appError))
+				worker.setJobError(logger, job, appError)
+				return
+			}
+		}
+	}
+}
+
+func (worker *OpensearchAggregatorWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobSuccess(job); err != nil {
+		logger.Error("Worker: Failed to set success for job", mlog.Err(err))
+		worker.setJobError(logger, job, err)
+	}
+}
+
+func (worker *OpensearchAggregatorWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) {
+	if err := worker.jobServer.SetJobError(job, appError); err != nil {
+		logger.Error("Worker: Failed to set job error", mlog.Err(err))
+	}
+}
+
+func (worker *OpensearchAggregatorWorker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) {
+	if err := worker.jobServer.SetJobCanceled(job); err != nil {
+		logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err))
+	}
+}

server/enterprise/elasticsearch/opensearch/aggregation_job_test.go

@@ -0,0 +1,223 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+)
+
+func TestElasticsearchAggregation(t *testing.T) {
+	if os.Getenv("IS_CI") == "true" {
+		os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://opensearch:9201")
+		os.Setenv("MM_ELASTICSEARCHSETTINGS_BACKEND", "opensearch")
+	}
+
+	defer func() {
+		if os.Getenv("IS_CI") == "true" {
+			os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://elasticsearch:9201")
+			os.Unsetenv("MM_ELASTICSEARCHSETTINGS_BACKEND")
+		}
+	}()
+
+	th := api4.SetupEnterpriseWithStoreMock(t)
+	rctx := request.TestContext(t)
+
+	mockUserStore := mocks.UserStore{}
+	mockUserStore.On("Count", mock.Anything).Return(int64(10), nil)
+	mockUserStore.On("GetAllProfiles", mock.Anything).Return(nil, nil)
+
+	mockPostStore := mocks.PostStore{}
+	mockPostStore.On("GetMaxPostSize").Return(65535, nil)
+
+	mockSystemStore := mocks.SystemStore{}
+	mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil)
+	mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil)
+	mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil)
+
+	mockJobStore := mocks.JobStore{}
+	mockJobStore.On("Save", mock.AnythingOfType("*model.Job")).Return(&model.Job{}, nil)
+	mockJobStore.On("UpdateStatus", mock.AnythingOfType("string"), model.JobStatusSuccess).Return(&model.Job{}, nil)
+	mockJobStore.On("Get", mock.AnythingOfType("*request.Context"), mock.AnythingOfType("string")).Return(&model.Job{
+		Status: model.JobStatusSuccess,
+	}, nil)
+	mockJobStore.On("UpdateStatusOptimistically",
+		mock.AnythingOfType("string"),
+		model.JobStatusPending,
+		model.JobStatusInProgress).Return(true, nil)
+	mockJobStore.On("GetAllByType", mock.AnythingOfType("string")).Return([]*model.Job{{
+		Id:     "abcxyz123",
+		Type:   "EnterpriseElasticsearchIndexer",
+		Status: model.JobStatusCanceled,
+	}}, nil)
+
+	mockStore := th.App.Srv().Platform().Store.(*mocks.Store)
+	mockStore.On("User").Return(&mockUserStore)
+	mockStore.On("Post").Return(&mockPostStore)
+	mockStore.On("System").Return(&mockSystemStore)
+	mockStore.On("Job").Return(&mockJobStore)
+	mockStore.On("GetDBSchemaVersion").Return(1, nil)
+
+	aggImpl := OpensearchAggregatorInterfaceImpl{Server: th.Server}
+
+	// Register search engine
+	th.App.SearchEngine().RegisterElasticsearchEngine(&OpensearchInterfaceImpl{
+		Platform: th.Server.Platform(),
+	})
+
+	// Set up the state for the tests.
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		if os.Getenv("IS_CI") == "true" {
+			*cfg.ElasticsearchSettings.ConnectionURL = "http://opensearch:9201"
+		} else {
+			*cfg.ElasticsearchSettings.ConnectionURL = "http://localhost:9201"
+		}
+		*cfg.ElasticsearchSettings.Backend = model.ElasticsearchSettingsOSBackend
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+		*cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1
+		*cfg.ElasticsearchSettings.AggregatePostsAfterDays = 1
+		*cfg.SqlSettings.DisableDatabaseSearch = true
+	})
+
+	esImpl := th.App.SearchEngine().ElasticsearchEngine
+	appErr := esImpl.Start()
+	if appErr != nil && appErr.Id != "ent.elasticsearch.start.already_started.app_error" {
+		require.Fail(t, "failed to start elasticsearch", appErr)
+	}
+	require.Nil(t, esImpl.PurgeIndexes(rctx))
+
+	post := &model.Post{
+		Id:        model.NewId(),
+		ChannelId: "channel",
+		Message:   "hi",
+	}
+	for i := 0; i < indexDeletionBatchSize+1; i++ {
+		indexPost(t, th, esImpl.(*OpensearchInterfaceImpl),
+			post,
+			time.Now().Add(-time.Duration(4+i)*24*time.Hour))
+	}
+
+	job := &model.Job{
+		Id:     model.NewId(),
+		Type:   model.JobTypeElasticsearchPostAggregation,
+		Status: model.JobStatusPending,
+	}
+
+	_, err := th.Server.Store().Job().Save(job)
+	require.NoError(t, err)
+
+	worker := aggImpl.MakeWorker().(*OpensearchAggregatorWorker)
+	worker.client = createTestClient(t, th.Context, th.App.Config(), th.App.FileBackend())
+	worker.jobServer.Store = mockStore
+
+	indexingImpl := OpensearchIndexerInterfaceImpl{
+		Server: th.App.Srv(),
+	}
+	th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, indexingImpl.MakeWorker(), nil)
+
+	worker.DoJob(job)
+
+	// We assert the minimum number of calls to verify that
+	// batching is working correctly. Because job().Get() will happen
+	// in each iteration.
+	numCalls := 0
+	for _, call := range mockJobStore.Calls {
+		if call.Method == "Get" {
+			numCalls++
+		}
+	}
+	assert.GreaterOrEqual(t, numCalls, 8, "Unexpected number of Jobstore.Get calls")
+}
+
+func TestElasticsearchAggregationSkipDuringBulkIndexing(t *testing.T) {
+	th := api4.SetupEnterpriseWithStoreMock(t)
+
+	mockUserStore := mocks.UserStore{}
+	mockUserStore.On("Count", mock.Anything).Return(int64(10), nil)
+
+	mockPostStore := mocks.PostStore{}
+	mockPostStore.On("GetMaxPostSize").Return(65535, nil)
+
+	mockSystemStore := mocks.SystemStore{}
+	mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil)
+	mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil)
+	mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil)
+
+	mockJobStore := mocks.JobStore{}
+
+	mockStore := th.App.Srv().Platform().Store.(*mocks.Store)
+	mockStore.On("User").Return(&mockUserStore)
+	mockStore.On("Post").Return(&mockPostStore)
+	mockStore.On("System").Return(&mockSystemStore)
+	mockStore.On("Job").Return(&mockJobStore)
+	mockStore.On("GetDBSchemaVersion").Return(1, nil)
+
+	aggImpl := OpensearchAggregatorInterfaceImpl{Server: th.Server}
+	aggImpl.Server.Jobs.Store = mockStore
+
+	// Register search engine
+	th.App.SearchEngine().RegisterElasticsearchEngine(&OpensearchInterfaceImpl{
+		Platform: th.Server.Platform(),
+	})
+
+	// Set up the state for the tests.
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+		*cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1
+		*cfg.ElasticsearchSettings.AggregatePostsAfterDays = 1
+		*cfg.SqlSettings.DisableDatabaseSearch = true
+	})
+
+	sched := aggImpl.MakeScheduler()
+	// Pass pending jobs as true
+	job, appErr := sched.ScheduleJob(th.Context, th.App.Config(), true, nil)
+	require.Nil(t, job)
+	require.Nil(t, appErr)
+
+	mockJobStore.AssertNotCalled(t, "GetCountByStatusAndType")
+}
+
+func indexPost(t *testing.T, th *api4.TestHelper, esImpl *OpensearchInterfaceImpl, post *model.Post, createTime time.Time) { //nolint:unused
+	t.Helper()
+	indexName := common.BuildPostIndexName(*th.Server.Config().ElasticsearchSettings.AggregatePostsAfterDays,
+		common.IndexBasePosts,
+		common.IndexBasePosts_MONTH,
+		createTime.Add(-1*24*time.Hour),
+		model.GetMillisForTime(createTime),
+	)
+	searchPost, err := common.ESPostFromPost(post, "teamID")
+	require.NoError(t, err)
+	ctx, cancel := context.WithTimeout(context.Background(),
+		time.Duration(*esImpl.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second)
+	defer cancel()
+
+	buf, err := json.Marshal(searchPost)
+	require.NoError(t, err)
+
+	_, err = esImpl.client.Index(ctx, opensearchapi.IndexReq{
+		Index:      indexName,
+		DocumentID: post.Id,
+		Body:       bytes.NewReader(buf),
+	})
+	require.NoError(t, err)
+}

server/enterprise/elasticsearch/opensearch/aggregation_scheduler.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+	"github.com/mattermost/mattermost/server/v8/channels/jobs"
+	ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs"
+)
+
+type OpenSearchAggregatorScheduler struct {
+	jobServer *jobs.JobServer
+	server    *app.Server
+}
+
+func (s *OpenSearchAggregatorScheduler) Enabled(cfg *model.Config) bool {
+	if license := s.server.License(); license == nil || !*license.Features.Elasticsearch {
+		return false
+	}
+
+	if *cfg.ElasticsearchSettings.EnableIndexing {
+		return true
+	}
+
+	return false
+}
+
+func (s *OpenSearchAggregatorScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time {
+	parsedTime, err := time.Parse("15:04", *cfg.ElasticsearchSettings.PostsAggregatorJobStartTime)
+	if err != nil {
+		s.server.Log().Error("Cannot determine next schedule time for opensearch post aggregator. PostsAggregatorJobStartTime config value is invalid.", mlog.Err(err))
+		return nil
+	}
+
+	return jobs.GenerateNextStartDateTime(now, parsedTime)
+}
+
+func (s *OpenSearchAggregatorScheduler) ScheduleJob(rctx request.CTX, _ *model.Config, pendingJobs bool, _ *model.Job) (*model.Job, *model.AppError) {
+	if pendingJobs {
+		s.server.Log().Warn("An aggregator job is already running. Skipping.")
+		return nil, nil
+	}
+
+	// Don't schedule a job if we already have a running bulk indexing job
+	count, err := s.jobServer.Store.Job().GetCountByStatusAndType(model.JobStatusInProgress, model.JobTypeElasticsearchPostIndexing)
+	if err != nil {
+		return nil, model.NewAppError(
+			"ScheduleJob",
+			model.NoTranslation,
+			nil,
+			"",
+			http.StatusInternalServerError).Wrap(err)
+	}
+	if count > 0 {
+		return nil, nil
+	}
+
+	return s.jobServer.CreateJob(rctx, model.JobTypeElasticsearchPostAggregation, nil)
+}
+
+func (esi *OpensearchAggregatorInterfaceImpl) MakeScheduler() ejobs.Scheduler {
+	return &OpenSearchAggregatorScheduler{
+		server:    esi.Server,
+		jobServer: esi.Server.Jobs,
+	}
+}

server/enterprise/elasticsearch/opensearch/bulk.go

@@ -0,0 +1,162 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"sync"
+	"time"
+
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+)
+
+type Bulk struct {
+	mut sync.Mutex
+	buf *bytes.Buffer
+
+	logger   mlog.LoggerIFace
+	client   *opensearchapi.Client
+	settings model.ElasticsearchSettings
+
+	quitFlusher   chan struct{}
+	quitFlusherWg sync.WaitGroup
+
+	pendingRequests int
+}
+
+func NewBulk(settings model.ElasticsearchSettings,
+	logger mlog.LoggerIFace,
+	client *opensearchapi.Client) *Bulk {
+	b := &Bulk{
+		settings:    settings,
+		logger:      logger,
+		client:      client,
+		quitFlusher: make(chan struct{}),
+		buf:         &bytes.Buffer{},
+	}
+
+	b.quitFlusherWg.Add(1)
+	go b.periodicFlusher()
+
+	return b
+}
+
+// IndexOp is a helper function to add an IndexOperation to the current bulk request.
+// doc argument can be a []byte, json.RawMessage or a struct.
+func (r *Bulk) IndexOp(op *types.IndexOperation, doc any) error {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	operation := types.OperationContainer{Index: op}
+	header, err := json.Marshal(operation)
+	if err != nil {
+		return err
+	}
+
+	r.buf.Write(header)
+	r.buf.Write([]byte("\n"))
+
+	switch v := doc.(type) {
+	case []byte:
+		r.buf.Write(v)
+	case json.RawMessage:
+		r.buf.Write(v)
+	default:
+		body, err := json.Marshal(doc)
+		if err != nil {
+			return err
+		}
+		r.buf.Write(body)
+	}
+
+	r.buf.Write([]byte("\n"))
+
+	return r.flushIfNecessary()
+}
+
+// DeleteOp is a helper function to add a DeleteOperation to the current bulk request.
+func (r *Bulk) DeleteOp(op *types.DeleteOperation) error {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+
+	operation := types.OperationContainer{Delete: op}
+	header, err := json.Marshal(operation)
+	if err != nil {
+		return err
+	}
+
+	r.buf.Write(header)
+	r.buf.Write([]byte("\n"))
+
+	return r.flushIfNecessary()
+}
+
+// flushIfNecessary flushes the pending buffer if needed.
+// It MUST be called with an already acquired mutex.
+func (r *Bulk) flushIfNecessary() error {
+	r.pendingRequests++
+
+	if r.pendingRequests > *r.settings.LiveIndexingBatchSize {
+		return r._flush()
+	}
+
+	return nil
+}
+
+func (r *Bulk) Stop() error {
+	r.mut.Lock()
+	defer r.mut.Unlock()
+	r.logger.Info("Stopping Bulk processor")
+
+	if r.pendingRequests > 0 {
+		return r._flush()
+	}
+
+	close(r.quitFlusher)
+	r.quitFlusherWg.Wait()
+
+	return nil
+}
+
+func (r *Bulk) periodicFlusher() {
+	defer r.quitFlusherWg.Done()
+
+	for {
+		select {
+		case <-time.After(common.BulkFlushInterval):
+			r.mut.Lock()
+			if r.pendingRequests > 0 {
+				if err := r._flush(); err != nil {
+					r.logger.Warn("Error flushing live indexing buffer", mlog.Err(err))
+				}
+			}
+			r.mut.Unlock()
+		case <-r.quitFlusher:
+			return
+		}
+	}
+}
+
+// _flush MUST be called with an acquired lock.
+func (r *Bulk) _flush() error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*r.settings.RequestTimeoutSeconds)*time.Second)
+	defer cancel()
+
+	_, err := r.client.Bulk(ctx, opensearchapi.BulkReq{
+		Body: bytes.NewReader(r.buf.Bytes()),
+	})
+	if err != nil {
+		return err
+	}
+	r.buf.Reset()
+	r.pendingRequests = 0
+
+	return nil
+}

server/enterprise/elasticsearch/opensearch/bulk_test.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"os"
+	"testing"
+
+	"github.com/elastic/go-elasticsearch/v8/typedapi/types"
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBulkProcessor(t *testing.T) {
+	th := api4.SetupEnterprise(t)
+	defer th.TearDown()
+
+	if os.Getenv("IS_CI") == "true" {
+		os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://opensearch:9201")
+		os.Setenv("MM_ELASTICSEARCHSETTINGS_BACKEND", "opensearch")
+	}
+
+	defer func() {
+		if os.Getenv("IS_CI") == "true" {
+			os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://elasticsearch:9201")
+			os.Unsetenv("MM_ELASTICSEARCHSETTINGS_BACKEND")
+		}
+	}()
+
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		if os.Getenv("IS_CI") == "true" {
+			*cfg.ElasticsearchSettings.ConnectionURL = "http://opensearch:9201"
+		} else {
+			*cfg.ElasticsearchSettings.ConnectionURL = "http://localhost:9201"
+		}
+		*cfg.ElasticsearchSettings.Backend = model.ElasticsearchSettingsOSBackend
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+	})
+
+	client := createTestClient(t, th.Context, th.App.Config(), th.App.FileBackend())
+	bulk := NewBulk(th.App.Config().ElasticsearchSettings,
+		th.Server.Platform().Log(),
+		client)
+
+	post, err := common.ESPostFromPost(&model.Post{
+		Id:      model.NewId(),
+		Message: "hello world",
+	}, "myteam")
+	require.NoError(t, err)
+
+	err = bulk.IndexOp(&types.IndexOperation{
+		Index_: model.NewPointer("myindex"),
+		Id_:    model.NewPointer(post.Id),
+	}, post)
+	require.NoError(t, err)
+
+	require.Equal(t, 1, bulk.pendingRequests)
+
+	err = bulk.Stop()
+	require.NoError(t, err)
+
+	require.Equal(t, 0, bulk.pendingRequests)
+}

server/enterprise/elasticsearch/opensearch/common.go

@@ -0,0 +1,124 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"crypto/tls"
+	"net/http"
+	"time"
+
+	"github.com/opensearch-project/opensearch-go/v4"
+	"github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+)
+
+func createClient(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend, debugLogging bool) (*opensearchapi.Client, *model.AppError) {
+	esCfg, appErr := createClientConfig(logger, cfg, fileBackend, debugLogging)
+	if appErr != nil {
+		return nil, appErr
+	}
+
+	client, err := opensearchapi.NewClient(*esCfg)
+	if err != nil {
+		return nil, model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.connect_failed", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	return client, nil
+}
+
+func createClientConfig(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend, debugLogging bool) (*opensearchapi.Config, *model.AppError) {
+	tp := http.DefaultTransport.(*http.Transport).Clone()
+	tp.TLSClientConfig = &tls.Config{
+		InsecureSkipVerify: *cfg.ElasticsearchSettings.SkipTLSVerification,
+	}
+
+	osCfg := &opensearchapi.Config{
+		Client: opensearch.Config{
+			Addresses:            []string{*cfg.ElasticsearchSettings.ConnectionURL},
+			RetryBackoff:         func(i int) time.Duration { return time.Duration(i) * 100 * time.Millisecond }, // A minimal backoff function
+			RetryOnStatus:        []int{502, 503, 504, 429},                                                      // Retry on 429 TooManyRequests statuses
+			MaxRetries:           3,
+			DiscoverNodesOnStart: *cfg.ElasticsearchSettings.Sniff,
+		},
+	}
+
+	if osCfg.Client.DiscoverNodesOnStart {
+		osCfg.Client.DiscoverNodesInterval = 30 * time.Second
+	}
+
+	if *cfg.ElasticsearchSettings.ClientCert != "" {
+		appErr := configureClientCertificate(tp.TLSClientConfig, cfg, fileBackend)
+		if appErr != nil {
+			return nil, appErr
+		}
+	}
+
+	// custom CA
+	if *cfg.ElasticsearchSettings.CA != "" {
+		appErr := configureCA(&osCfg.Client, cfg, fileBackend)
+		if appErr != nil {
+			return nil, appErr
+		}
+	}
+
+	osCfg.Client.Transport = tp
+
+	if *cfg.ElasticsearchSettings.Username != "" {
+		osCfg.Client.Username = *cfg.ElasticsearchSettings.Username
+		osCfg.Client.Password = *cfg.ElasticsearchSettings.Password
+	}
+
+	// This is a compatibility mode from previous config settings.
+	// We have to conditionally enable debug logging due to
+	// https://github.com/elastic/elastic-transport-go/issues/22
+	// Although, this is opensearch, the issue is the same.
+	if *cfg.ElasticsearchSettings.Trace == "all" && debugLogging {
+		osCfg.Client.EnableDebugLogger = true
+	}
+
+	osCfg.Client.Logger = common.NewLogger("Opensearch", logger, *cfg.ElasticsearchSettings.Trace == "all")
+
+	return osCfg, nil
+}
+
+func configureCA(esCfg *opensearch.Config, cfg *model.Config, fb filestore.FileBackend) *model.AppError {
+	// read the certificate authority (CA) file
+	clientCA, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.CA)
+	if err != nil {
+		return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.ca_cert_missing", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	esCfg.CACert = clientCA
+
+	return nil
+}
+
+func configureClientCertificate(tlsConfig *tls.Config, cfg *model.Config, fb filestore.FileBackend) *model.AppError {
+	// read the client certificate file
+	clientCert, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.ClientCert)
+	if err != nil {
+		return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_cert_missing", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// read the client key file
+	clientKey, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.ClientKey)
+	if err != nil {
+		return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_key_missing", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// load the client key and certificate
+	certificate, err := tls.X509KeyPair(clientCert, clientKey)
+	if err != nil {
+		return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_cert_malformed", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// update the TLS config
+	tlsConfig.Certificates = []tls.Certificate{certificate}
+
+	return nil
+}

server/enterprise/elasticsearch/opensearch/indexing_job.go

@@ -0,0 +1,73 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+
+	"github.com/opensearch-project/opensearch-go/v4/opensearchutil"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/mlog"
+	"github.com/mattermost/mattermost/server/v8/channels/app"
+)
+
+type OpensearchIndexerInterfaceImpl struct {
+	Server        *app.Server
+	bulkProcessor opensearchutil.BulkIndexer
+}
+
+func (esi *OpensearchIndexerInterfaceImpl) MakeWorker() model.Worker {
+	const workerName = "EnterpriseOpensearchIndexer"
+
+	// Initializing logger
+	logger := esi.Server.Jobs.Logger().With(mlog.String("worker_name", workerName))
+
+	// Creating the client
+	client, appErr := createClient(logger, esi.Server.Jobs.Config(), esi.Server.Platform().FileBackend(), true)
+	if appErr != nil {
+		logger.Error("Worker: Failed to Create Client", mlog.Err(appErr))
+		return nil
+	}
+
+	return common.NewIndexerWorker(workerName,
+		esi.Server.Jobs,
+		logger,
+		esi.Server.Platform().FileBackend(),
+		esi.Server.License,
+		func() error {
+			// Creating the bulk indexer from the client.
+			biCfg := opensearchutil.BulkIndexerConfig{
+				Client: client,
+				OnError: func(_ context.Context, err error) {
+					logger.Error("Error from opensearch bulk indexer", mlog.Err(err))
+				},
+				Timeout:    time.Duration(*esi.Server.Jobs.Config().ElasticsearchSettings.RequestTimeoutSeconds) * time.Second,
+				NumWorkers: common.NumIndexWorkers(),
+			}
+			if *esi.Server.Jobs.Config().ElasticsearchSettings.Trace == "all" {
+				biCfg.DebugLogger = common.NewBulkIndexerLogger(logger, workerName)
+			}
+			var err error
+			esi.bulkProcessor, err = opensearchutil.NewBulkIndexer(biCfg)
+			return err
+		},
+		// Function to add an item in the bulk processor
+		func(indexName, indexOp, docID string, body io.ReadSeeker) error {
+			return esi.bulkProcessor.Add(context.Background(), opensearchutil.BulkIndexerItem{
+				Index:      indexName,
+				Action:     indexOp,
+				DocumentID: docID,
+				Body:       body,
+			})
+		},
+		// Closing the bulk processor
+		func() error {
+			return esi.bulkProcessor.Close(context.Background())
+		})
+}

server/enterprise/elasticsearch/opensearch/indexing_job_test.go

@@ -0,0 +1,94 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+)
+
+func TestOpenSearchIndexerJobIsEnabled(t *testing.T) {
+	t.Run("ElasticSearch feature is enabled then job is enabled", func(t *testing.T) {
+		th := api4.SetupEnterpriseWithStoreMock(t)
+		defer th.TearDown()
+
+		th.Server.SetLicense(model.NewTestLicense("elastic_search"))
+
+		osImpl := &OpensearchIndexerInterfaceImpl{
+			Server: th.Server,
+		}
+		worker := osImpl.MakeWorker()
+
+		config := &model.Config{
+			ElasticsearchSettings: model.ElasticsearchSettings{
+				EnableIndexing: model.NewPointer(true),
+			},
+		}
+
+		result := worker.IsEnabled(config)
+
+		assert.Equal(t, result, true)
+	})
+
+	t.Run("there is NO license then job is disabled", func(t *testing.T) {
+		th := api4.SetupEnterpriseWithStoreMock(t)
+		defer th.TearDown()
+
+		th.Server.SetLicense(nil)
+
+		osImpl := &OpensearchIndexerInterfaceImpl{
+			Server: th.Server,
+		}
+		worker := osImpl.MakeWorker()
+
+		config := &model.Config{
+			ElasticsearchSettings: model.ElasticsearchSettings{
+				EnableIndexing: model.NewPointer(true),
+			},
+		}
+
+		result := worker.IsEnabled(config)
+
+		assert.Equal(t, result, false)
+	})
+}
+
+func TestOpenSearchIndexerPending(t *testing.T) {
+	th := api4.SetupEnterprise(t).InitBasic()
+	defer th.TearDown()
+
+	// Set up the state for the tests.
+	th.App.UpdateConfig(func(cfg *model.Config) {
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+		*cfg.SqlSettings.DisableDatabaseSearch = true
+	})
+	th.App.Srv().SetLicense(model.NewTestLicense())
+
+	impl := OpensearchIndexerInterfaceImpl{
+		Server: th.App.Srv(),
+	}
+
+	worker := impl.MakeWorker()
+	th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, worker, nil)
+
+	go worker.Run()
+
+	job, appErr := th.App.Srv().Jobs.CreateJob(th.Context, model.JobTypeElasticsearchPostIndexing, map[string]string{})
+	require.Nil(t, appErr)
+
+	worker.JobChannel() <- *job
+
+	worker.Stop()
+
+	job, err := th.App.Srv().Store().Job().Get(th.Context, job.Id)
+	require.NoError(t, err)
+	assert.Equal(t, job.Status, model.JobStatusPending)
+}

server/enterprise/elasticsearch/opensearch/main_test.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/channels/testlib"
+)
+
+var mainHelper *testlib.MainHelper
+
+func TestMain(m *testing.M) {
+	mainHelper = testlib.NewMainHelper()
+	defer mainHelper.Close()
+	api4.SetMainHelper(mainHelper)
+
+	mainHelper.Main(m)
+}

server/enterprise/elasticsearch/opensearch/opensearch_test.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/v8/channels/api4"
+	"github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks"
+)
+
+type OpensearchInterfaceTestSuite struct {
+	common.CommonTestSuite
+
+	th          *api4.TestHelper
+	client      *opensearchapi.Client
+	ctx         context.Context
+	fileBackend filestore.FileBackend
+}
+
+func TestOpensearchInterfaceTestSuite(t *testing.T) {
+	testSuite := &OpensearchInterfaceTestSuite{
+		CommonTestSuite: common.CommonTestSuite{},
+	}
+	suite.Run(t, testSuite)
+}
+
+func (s *OpensearchInterfaceTestSuite) SetupSuite() {
+	if os.Getenv("IS_CI") == "true" {
+		os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://opensearch:9201")
+		os.Setenv("MM_ELASTICSEARCHSETTINGS_BACKEND", "opensearch")
+	}
+
+	s.th = api4.SetupEnterprise(s.T()).InitBasic()
+	s.CommonTestSuite.TH = s.th
+	s.CommonTestSuite.GetDocumentFn = func(index, documentID string) (bool, json.RawMessage, error) {
+		resp, err := s.client.Document.Get(s.ctx, opensearchapi.DocumentGetReq{
+			Index:      index,
+			DocumentID: documentID,
+		})
+		if resp == nil {
+			return false, nil, err
+		}
+		return resp.Found, resp.Source, err
+	}
+	s.CommonTestSuite.RefreshIndexFn = func() error {
+		_, err := s.client.Indices.Refresh(context.Background(), nil)
+		return err
+	}
+	s.CommonTestSuite.CreateIndexFn = func(index string) error {
+		_, err := s.client.Indices.Create(s.ctx, opensearchapi.IndicesCreateReq{
+			Index: index,
+		})
+		return err
+	}
+	s.CommonTestSuite.GetIndexFn = func(indexPattern string) ([]string, error) {
+		res, err := s.client.Indices.Get(s.ctx, opensearchapi.IndicesGetReq{
+			Indices: []string{indexPattern},
+		})
+		if err != nil {
+			return nil, err
+		}
+		var names []string
+		for name := range res.Indices {
+			names = append(names, name)
+		}
+		return names, nil
+	}
+
+	// Set up the state for the tests.
+	s.th.App.UpdateConfig(func(cfg *model.Config) {
+		if os.Getenv("IS_CI") == "true" {
+			*cfg.ElasticsearchSettings.ConnectionURL = "http://opensearch:9201"
+		} else {
+			*cfg.ElasticsearchSettings.ConnectionURL = "http://localhost:9201"
+		}
+		*cfg.ElasticsearchSettings.Backend = model.ElasticsearchSettingsOSBackend
+		*cfg.ElasticsearchSettings.EnableIndexing = true
+		*cfg.ElasticsearchSettings.EnableSearching = true
+		*cfg.ElasticsearchSettings.EnableAutocomplete = true
+		*cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1
+		*cfg.SqlSettings.DisableDatabaseSearch = true
+	})
+	s.th.App.Srv().SetLicense(model.NewTestLicense())
+
+	if s.fileBackend == nil {
+		s.fileBackend = &mocks.FileBackend{}
+	}
+
+	// Initialise other stuff for the test.
+	s.client = createTestClient(s.T(), s.th.Context, s.th.App.Config(), s.th.App.FileBackend())
+	s.ctx = context.Background()
+
+	// Register search engine
+	s.th.App.SearchEngine().RegisterElasticsearchEngine(&OpensearchInterfaceImpl{Platform: s.th.Server.Platform()})
+}
+
+func (s *OpensearchInterfaceTestSuite) TearDownSuite() {
+	if os.Getenv("IS_CI") == "true" {
+		os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://elasticsearch:9201")
+		os.Unsetenv("MM_ELASTICSEARCHSETTINGS_BACKEND")
+	}
+}
+
+func (s *OpensearchInterfaceTestSuite) SetupTest() {
+	s.CommonTestSuite.ESImpl = s.th.App.SearchEngine().ElasticsearchEngine
+
+	if s.CommonTestSuite.ESImpl.IsActive() {
+		appErr := s.CommonTestSuite.ESImpl.Stop()
+		s.Require().Nil(appErr)
+	}
+
+	s.Require().Nil(s.CommonTestSuite.ESImpl.Start())
+
+	s.Nil(s.CommonTestSuite.ESImpl.PurgeIndexes(s.th.Context))
+}

server/enterprise/elasticsearch/opensearch/testlib.go

@@ -0,0 +1,28 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.enterprise for license information.
+
+package opensearch
+
+import (
+	"testing"
+
+	"github.com/opensearch-project/opensearch-go/v4/opensearchapi"
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost/server/public/model"
+	"github.com/mattermost/mattermost/server/public/shared/request"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
+	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks"
+)
+
+func createTestClient(t *testing.T, rctx request.CTX, cfg *model.Config, fileStore filestore.FileBackend) *opensearchapi.Client {
+	t.Helper()
+
+	if fileStore == nil {
+		fileStore = &mocks.FileBackend{}
+	}
+
+	client, err := createClient(rctx.Logger(), cfg, fileStore, true)
+	require.Nil(t, err)
+	return client
+}

server/enterprise/external_imports.go

@@ -15,8 +15,6 @@ import (
 	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/enterprise/data_retention"
 	// Needed to ensure the init() method in the EE gets run
-	_ "github.com/mattermost/enterprise/elasticsearch"
-	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/enterprise/ldap"
 	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/enterprise/cloud"

server/enterprise/local_imports.go

@@ -16,4 +16,6 @@ import (
 	_ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/csv_export"
 	// Needed to ensure the init() method in the EE gets run
 	_ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export"
+	// Needed to ensure the init() method in the EE gets run
+	_ "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch"
 )

server/go.mod

@@ -8,7 +8,7 @@ require (
 	code.sajari.com/docconv/v2 v2.0.0-pre.4
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/avct/uasurfer v0.0.0-20240501094946-ca0c4d1e541b
-	github.com/aws/aws-sdk-go v1.55.0
+	github.com/aws/aws-sdk-go v1.55.5
 	github.com/blang/semver/v4 v4.0.0
 	github.com/blevesearch/bleve/v2 v2.4.1
 	github.com/cespare/xxhash/v2 v2.3.0
@@ -51,6 +51,7 @@ require (
 	github.com/microcosm-cc/bluemonday v1.0.27
 	github.com/minio/minio-go/v7 v7.0.74
 	github.com/oov/psd v0.0.0-20220121172623-5db5eafcecbb
+	github.com/opensearch-project/opensearch-go/v4 v4.3.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.19.1

server/go.sum

@@ -47,8 +47,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/avct/uasurfer v0.0.0-20240501094946-ca0c4d1e541b h1:F1IDheTR2BqSIznXwfgxursfutFj5pNezhneejTPUYQ=
 github.com/avct/uasurfer v0.0.0-20240501094946-ca0c4d1e541b/go.mod h1:s+GCtuP4kZNxh1WGoqdWI1+PbluBcycrMMWuKQ9e5Nk=
-github.com/aws/aws-sdk-go v1.55.0 h1:hVALKPjXz33kP1R9nTyJpUK7qF59dO2mleQxUW9mCVE=
-github.com/aws/aws-sdk-go v1.55.0/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
+github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
+github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
 github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
@@ -440,6 +440,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k=
 github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY=
 github.com/oov/psd v0.0.0-20220121172623-5db5eafcecbb h1:JF9kOhBBk4WPF7luXFu5yR+WgaFm9L/KiHJHhU9vDwA=
 github.com/oov/psd v0.0.0-20220121172623-5db5eafcecbb/go.mod h1:GHI1bnmAcbp96z6LNfBJvtrjxhaXGkbsk967utPlvL8=
+github.com/opensearch-project/opensearch-go/v4 v4.3.0 h1:gmQ+ILFJW6AJimivf+lHGVqCS2SCr/PBBf2Qr1xOCgE=
+github.com/opensearch-project/opensearch-go/v4 v4.3.0/go.mod h1:+w6KAvEX3S0fVVmZciNLN0CkXhxxem26+F6Y7DoPp04=
 github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
@@ -616,6 +618,8 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
+github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/tinylib/msgp v1.2.0 h1:0uKB/662twsVBpYUPbokj4sTSKhWFKB7LopO2kWK8lY=
 github.com/tinylib/msgp v1.2.0/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -637,6 +641,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU
 github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
+github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI=
+github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw=
 github.com/wiggin77/merror v1.0.5 h1:P+lzicsn4vPMycAf2mFf7Zk6G9eco5N+jB1qJ2XW3ME=
 github.com/wiggin77/merror v1.0.5/go.mod h1:H2ETSu7/bPE0Ymf4bEwdUoo73OOEkdClnoRisfw0Nm0=
 github.com/wiggin77/srslog v1.0.1 h1:gA2XjSMy3DrRdX9UqLuDtuVAAshb8bE1NhX1YK0Qe+8=