Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

server/platform/services/searchengine/bleveengine/bleve.go

@@ -0,0 +1,328 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"net/http"
+	"os"
+	"path/filepath"
+	"reflect"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/analysis/analyzer/keyword"
+	"github.com/blevesearch/bleve/v2/analysis/analyzer/standard"
+	"github.com/blevesearch/bleve/v2/mapping"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	EngineName   = "bleve"
+	PostIndex    = "posts"
+	FileIndex    = "files"
+	UserIndex    = "users"
+	ChannelIndex = "channels"
+)
+
+type BleveEngine struct {
+	PostIndex    bleve.Index
+	FileIndex    bleve.Index
+	UserIndex    bleve.Index
+	ChannelIndex bleve.Index
+	Mutex        sync.RWMutex
+	ready        int32
+	cfg          *model.Config
+	indexSync    bool
+}
+
+var keywordMapping *mapping.FieldMapping
+var standardMapping *mapping.FieldMapping
+var dateMapping *mapping.FieldMapping
+
+func init() {
+	keywordMapping = bleve.NewTextFieldMapping()
+	keywordMapping.Analyzer = keyword.Name
+
+	standardMapping = bleve.NewTextFieldMapping()
+	standardMapping.Analyzer = standard.Name
+
+	dateMapping = bleve.NewNumericFieldMapping()
+}
+
+func getChannelIndexMapping() *mapping.IndexMappingImpl {
+	channelMapping := bleve.NewDocumentMapping()
+	channelMapping.AddFieldMappingsAt("Id", keywordMapping)
+	channelMapping.AddFieldMappingsAt("Type", keywordMapping)
+	channelMapping.AddFieldMappingsAt("TeamId", keywordMapping)
+	channelMapping.AddFieldMappingsAt("NameSuggest", keywordMapping)
+	channelMapping.AddFieldMappingsAt("UserIDs", keywordMapping)
+	channelMapping.AddFieldMappingsAt("TeamMemberIDs", keywordMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("_default", channelMapping)
+
+	return indexMapping
+}
+
+func getPostIndexMapping() *mapping.IndexMappingImpl {
+	postMapping := bleve.NewDocumentMapping()
+	postMapping.AddFieldMappingsAt("Id", keywordMapping)
+	postMapping.AddFieldMappingsAt("TeamId", keywordMapping)
+	postMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
+	postMapping.AddFieldMappingsAt("UserId", keywordMapping)
+	postMapping.AddFieldMappingsAt("CreateAt", dateMapping)
+	postMapping.AddFieldMappingsAt("Message", standardMapping)
+	postMapping.AddFieldMappingsAt("Type", keywordMapping)
+	postMapping.AddFieldMappingsAt("Hashtags", standardMapping)
+	postMapping.AddFieldMappingsAt("Attachments", standardMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("_default", postMapping)
+
+	return indexMapping
+}
+
+func getFileIndexMapping() *mapping.IndexMappingImpl {
+	fileMapping := bleve.NewDocumentMapping()
+	fileMapping.AddFieldMappingsAt("Id", keywordMapping)
+	fileMapping.AddFieldMappingsAt("CreatorId", keywordMapping)
+	fileMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
+	fileMapping.AddFieldMappingsAt("CreateAt", dateMapping)
+	fileMapping.AddFieldMappingsAt("Name", standardMapping)
+	fileMapping.AddFieldMappingsAt("Content", standardMapping)
+	fileMapping.AddFieldMappingsAt("Extension", keywordMapping)
+	fileMapping.AddFieldMappingsAt("Content", standardMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("_default", fileMapping)
+
+	return indexMapping
+}
+
+func getUserIndexMapping() *mapping.IndexMappingImpl {
+	userMapping := bleve.NewDocumentMapping()
+	userMapping.AddFieldMappingsAt("Id", keywordMapping)
+	userMapping.AddFieldMappingsAt("SuggestionsWithFullname", keywordMapping)
+	userMapping.AddFieldMappingsAt("SuggestionsWithoutFullname", keywordMapping)
+	userMapping.AddFieldMappingsAt("TeamsIds", keywordMapping)
+	userMapping.AddFieldMappingsAt("ChannelsIds", keywordMapping)
+
+	indexMapping := bleve.NewIndexMapping()
+	indexMapping.AddDocumentMapping("_default", userMapping)
+
+	return indexMapping
+}
+
+func NewBleveEngine(cfg *model.Config) *BleveEngine {
+	return &BleveEngine{
+		cfg: cfg,
+	}
+}
+
+func (b *BleveEngine) getIndexDir(indexName string) string {
+	return filepath.Join(*b.cfg.BleveSettings.IndexDir, indexName+".bleve")
+}
+
+func (b *BleveEngine) createOrOpenIndex(indexName string, mapping *mapping.IndexMappingImpl) (bleve.Index, error) {
+	indexPath := b.getIndexDir(indexName)
+	if index, err := bleve.Open(indexPath); err == nil {
+		return index, nil
+	}
+
+	index, err := bleve.NewUsing(indexPath, mapping, "scorch", "scorch", map[string]any{
+		"forceSegmentType":    "zap",
+		"forceSegmentVersion": 15,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return index, nil
+}
+
+func (b *BleveEngine) openIndexes() *model.AppError {
+	if atomic.LoadInt32(&b.ready) != 0 {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.already_started.error", nil, "", http.StatusInternalServerError)
+	}
+
+	var err error
+	b.PostIndex, err = b.createOrOpenIndex(PostIndex, getPostIndexMapping())
+	if err != nil {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	b.FileIndex, err = b.createOrOpenIndex(FileIndex, getFileIndexMapping())
+	if err != nil {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.create_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	b.UserIndex, err = b.createOrOpenIndex(UserIndex, getUserIndexMapping())
+	if err != nil {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	b.ChannelIndex, err = b.createOrOpenIndex(ChannelIndex, getChannelIndexMapping())
+	if err != nil {
+		return model.NewAppError("Bleveengine.Start", "bleveengine.create_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	atomic.StoreInt32(&b.ready, 1)
+	return nil
+}
+
+func (b *BleveEngine) Start() *model.AppError {
+	if !*b.cfg.BleveSettings.EnableIndexing || *b.cfg.BleveSettings.IndexDir == "" {
+		return nil
+	}
+
+	b.Mutex.Lock()
+	defer b.Mutex.Unlock()
+
+	mlog.Info("EXPERIMENTAL: Starting Bleve")
+
+	return b.openIndexes()
+}
+
+func (b *BleveEngine) closeIndexes() *model.AppError {
+	if b.IsActive() {
+		if err := b.PostIndex.Close(); err != nil {
+			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		if err := b.FileIndex.Close(); err != nil {
+			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		if err := b.UserIndex.Close(); err != nil {
+			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+
+		if err := b.ChannelIndex.Close(); err != nil {
+			return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+		}
+	}
+
+	atomic.StoreInt32(&b.ready, 0)
+	return nil
+}
+
+func (b *BleveEngine) Stop() *model.AppError {
+	b.Mutex.Lock()
+	defer b.Mutex.Unlock()
+
+	mlog.Info("Stopping Bleve")
+
+	return b.closeIndexes()
+}
+
+func (b *BleveEngine) IsActive() bool {
+	return atomic.LoadInt32(&b.ready) == 1
+}
+
+func (b *BleveEngine) IsIndexingSync() bool {
+	return b.indexSync
+}
+
+func (b *BleveEngine) RefreshIndexes() *model.AppError {
+	return nil
+}
+
+func (b *BleveEngine) GetVersion() int {
+	return 0
+}
+
+func (b *BleveEngine) GetFullVersion() string {
+	return "0"
+}
+
+func (b *BleveEngine) GetPlugins() []string {
+	return []string{}
+}
+
+func (b *BleveEngine) GetName() string {
+	return EngineName
+}
+
+func (b *BleveEngine) TestConfig(cfg *model.Config) *model.AppError {
+	return nil
+}
+
+func (b *BleveEngine) deleteIndexes() *model.AppError {
+	if err := os.RemoveAll(b.getIndexDir(PostIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_post_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if err := os.RemoveAll(b.getIndexDir(UserIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_user_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if err := os.RemoveAll(b.getIndexDir(ChannelIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_channel_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	if err := os.RemoveAll(b.getIndexDir(FileIndex)); err != nil {
+		return model.NewAppError("Bleveengine.PurgeIndexes", "bleveengine.purge_file_index.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) PurgeIndexes() *model.AppError {
+	if *b.cfg.BleveSettings.IndexDir == "" {
+		return nil
+	}
+
+	b.Mutex.Lock()
+	defer b.Mutex.Unlock()
+
+	mlog.Info("PurgeIndexes Bleve")
+	if err := b.closeIndexes(); err != nil {
+		return err
+	}
+
+	if err := b.deleteIndexes(); err != nil {
+		return err
+	}
+
+	return b.openIndexes()
+}
+
+func (b *BleveEngine) DataRetentionDeleteIndexes(cutoff time.Time) *model.AppError {
+	return nil
+}
+
+func (b *BleveEngine) IsAutocompletionEnabled() bool {
+	return *b.cfg.BleveSettings.EnableAutocomplete
+}
+
+func (b *BleveEngine) IsIndexingEnabled() bool {
+	return *b.cfg.BleveSettings.EnableIndexing
+}
+
+func (b *BleveEngine) IsSearchEnabled() bool {
+	return *b.cfg.BleveSettings.EnableSearching
+}
+
+func (b *BleveEngine) UpdateConfig(cfg *model.Config) {
+	b.Mutex.Lock()
+	defer b.Mutex.Unlock()
+
+	if reflect.DeepEqual(cfg.BleveSettings, b.cfg.BleveSettings) {
+		return
+	}
+
+	mlog.Info("UpdateConf Bleve")
+
+	if *cfg.BleveSettings.EnableIndexing != *b.cfg.BleveSettings.EnableIndexing || *cfg.BleveSettings.IndexDir != *b.cfg.BleveSettings.IndexDir {
+		if err := b.closeIndexes(); err != nil {
+			mlog.Error("Error closing Bleve indexes to update the config", mlog.Err(err))
+			return
+		}
+		b.cfg = cfg
+		if err := b.openIndexes(); err != nil {
+			mlog.Error("Error opening Bleve indexes after updating the config", mlog.Err(err))
+		}
+		return
+	}
+	b.cfg = cfg
+}

server/platform/services/searchengine/bleveengine/bleve_test.go

@@ -0,0 +1,226 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"os"
+	"testing"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/searchlayer"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/searchtest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/sqlstore"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/testlib"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine"
+)
+
+type BleveEngineTestSuite struct {
+	suite.Suite
+
+	SQLSettings  *model.SqlSettings
+	SQLStore     *sqlstore.SqlStore
+	SearchEngine *searchengine.Broker
+	Store        *searchlayer.SearchStore
+	BleveEngine  *BleveEngine
+	IndexDir     string
+}
+
+func TestBleveEngineTestSuite(t *testing.T) {
+	suite.Run(t, new(BleveEngineTestSuite))
+}
+
+func (s *BleveEngineTestSuite) setupIndexes() {
+	indexDir, err := os.MkdirTemp("", "mmbleve")
+	if err != nil {
+		s.Require().FailNow("Cannot setup bleveengine tests: %s", err.Error())
+	}
+	s.IndexDir = indexDir
+}
+
+func (s *BleveEngineTestSuite) setupStore() {
+	driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
+	if driverName == "" {
+		driverName = model.DatabaseDriverPostgres
+	}
+	s.SQLSettings = storetest.MakeSqlSettings(driverName, false)
+	s.SQLStore = sqlstore.New(*s.SQLSettings, nil)
+
+	cfg := &model.Config{}
+	cfg.SetDefaults()
+	cfg.BleveSettings.EnableIndexing = model.NewBool(true)
+	cfg.BleveSettings.EnableSearching = model.NewBool(true)
+	cfg.BleveSettings.EnableAutocomplete = model.NewBool(true)
+	cfg.BleveSettings.IndexDir = model.NewString(s.IndexDir)
+	cfg.SqlSettings.DisableDatabaseSearch = model.NewBool(true)
+
+	s.SearchEngine = searchengine.NewBroker(cfg)
+	s.Store = searchlayer.NewSearchLayer(&testlib.TestStore{Store: s.SQLStore}, s.SearchEngine, cfg)
+
+	s.BleveEngine = NewBleveEngine(cfg)
+	s.BleveEngine.indexSync = true
+	s.SearchEngine.RegisterBleveEngine(s.BleveEngine)
+	if err := s.BleveEngine.Start(); err != nil {
+		s.Require().FailNow("Cannot start bleveengine: %s", err.Error())
+	}
+}
+
+func (s *BleveEngineTestSuite) SetupSuite() {
+	s.setupIndexes()
+	s.setupStore()
+}
+
+func (s *BleveEngineTestSuite) TearDownSuite() {
+	os.RemoveAll(s.IndexDir)
+	s.SQLStore.Close()
+	storetest.CleanupSqlSettings(s.SQLSettings)
+}
+
+func (s *BleveEngineTestSuite) TestBleveSearchStoreTests() {
+	searchTestEngine := &searchtest.SearchTestEngine{
+		Driver: searchtest.EngineBleve,
+	}
+
+	s.Run("TestSearchChannelStore", func() {
+		searchtest.TestSearchChannelStore(s.T(), s.Store, searchTestEngine)
+	})
+
+	s.Run("TestSearchUserStore", func() {
+		searchtest.TestSearchUserStore(s.T(), s.Store, searchTestEngine)
+	})
+
+	s.Run("TestSearchPostStore", func() {
+		searchtest.TestSearchPostStore(s.T(), s.Store, searchTestEngine)
+	})
+
+	s.Run("TestSearchFileInfoStore", func() {
+		searchtest.TestSearchFileInfoStore(s.T(), s.Store, searchTestEngine)
+	})
+}
+
+func (s *BleveEngineTestSuite) TestDeleteChannelPosts() {
+	s.Run("Should remove all the posts that belongs to a channel", func() {
+		s.BleveEngine.PurgeIndexes()
+		teamID := model.NewId()
+		userID := model.NewId()
+		channelID := model.NewId()
+		channelToAvoidID := model.NewId()
+		for i := 0; i < 10; i++ {
+			post := createPost(userID, channelID)
+			appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+			require.Nil(s.T(), appErr)
+		}
+		postToAvoid := createPost(userID, channelToAvoidID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
+		require.Nil(s.T(), appErr)
+
+		s.SearchEngine.BleveEngine.DeleteChannelPosts(channelID)
+
+		doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), postToAvoid.Id, doc.ID())
+		numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), 1, int(numberDocs))
+	})
+
+	s.Run("Shouldn't do anything if there is not posts for the selected channel", func() {
+		s.BleveEngine.PurgeIndexes()
+		teamID := model.NewId()
+		userID := model.NewId()
+		channelID := model.NewId()
+		channelToDeleteID := model.NewId()
+		post := createPost(userID, channelID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+		require.Nil(s.T(), appErr)
+
+		s.SearchEngine.BleveEngine.DeleteChannelPosts(channelToDeleteID)
+
+		_, err := s.BleveEngine.PostIndex.Document(post.Id)
+		require.NoError(s.T(), err)
+		numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), 1, int(numberDocs))
+	})
+}
+
+func (s *BleveEngineTestSuite) TestDeleteUserPosts() {
+	s.Run("Should remove all the posts that belongs to a user", func() {
+		s.BleveEngine.PurgeIndexes()
+		teamID := model.NewId()
+		userID := model.NewId()
+		userToAvoidID := model.NewId()
+		channelID := model.NewId()
+		for i := 0; i < 10; i++ {
+			post := createPost(userID, channelID)
+			appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+			require.Nil(s.T(), appErr)
+		}
+		postToAvoid := createPost(userToAvoidID, channelID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
+		require.Nil(s.T(), appErr)
+
+		s.SearchEngine.BleveEngine.DeleteUserPosts(userID)
+
+		doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), postToAvoid.Id, doc.ID())
+		numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), 1, int(numberDocs))
+	})
+
+	s.Run("Shouldn't do anything if there is not posts for the selected user", func() {
+		s.BleveEngine.PurgeIndexes()
+		teamID := model.NewId()
+		userID := model.NewId()
+		userToDeleteID := model.NewId()
+		channelID := model.NewId()
+		post := createPost(userID, channelID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+		require.Nil(s.T(), appErr)
+
+		s.SearchEngine.BleveEngine.DeleteUserPosts(userToDeleteID)
+
+		_, err := s.BleveEngine.PostIndex.Document(post.Id)
+		require.NoError(s.T(), err)
+		numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+		require.NoError(s.T(), err)
+		require.Equal(s.T(), 1, int(numberDocs))
+	})
+}
+
+func (s *BleveEngineTestSuite) TestDeletePosts() {
+	s.BleveEngine.PurgeIndexes()
+	teamID := model.NewId()
+	userID := model.NewId()
+	userToAvoidID := model.NewId()
+	channelID := model.NewId()
+	for i := 0; i < 10; i++ {
+		post := createPost(userID, channelID)
+		appErr := s.SearchEngine.BleveEngine.IndexPost(post, teamID)
+		require.Nil(s.T(), appErr)
+	}
+	postToAvoid := createPost(userToAvoidID, channelID)
+	appErr := s.SearchEngine.BleveEngine.IndexPost(postToAvoid, teamID)
+	require.Nil(s.T(), appErr)
+
+	query := bleve.NewTermQuery(userID)
+	query.SetField("UserId")
+	search := bleve.NewSearchRequest(query)
+	count, err := s.BleveEngine.deletePosts(search, 1)
+	require.NoError(s.T(), err)
+	require.Equal(s.T(), 10, int(count))
+
+	doc, err := s.BleveEngine.PostIndex.Document(postToAvoid.Id)
+	require.NoError(s.T(), err)
+	require.Equal(s.T(), postToAvoid.Id, doc.ID())
+	numberDocs, err := s.BleveEngine.PostIndex.DocCount()
+	require.NoError(s.T(), err)
+	require.Equal(s.T(), 1, int(numberDocs))
+}

server/platform/services/searchengine/bleveengine/common.go

@@ -0,0 +1,163 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"strings"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine"
+)
+
+type BLVChannel struct {
+	Id            string
+	Type          model.ChannelType
+	UserIDs       []string
+	TeamId        []string
+	TeamMemberIDs []string
+	NameSuggest   []string
+}
+
+type BLVUser struct {
+	Id                         string
+	SuggestionsWithFullname    []string
+	SuggestionsWithoutFullname []string
+	TeamsIds                   []string
+	ChannelsIds                []string
+}
+
+type BLVPost struct {
+	Id          string
+	TeamId      string
+	ChannelId   string
+	UserId      string
+	CreateAt    int64
+	Message     string
+	Type        string
+	Hashtags    []string
+	Attachments string
+}
+
+type BLVFile struct {
+	Id        string
+	CreatorId string
+	ChannelId string
+	CreateAt  int64
+	Name      string
+	Content   string
+	Extension string
+}
+
+func BLVChannelFromChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *BLVChannel {
+	displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ")
+	nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"})
+
+	return &BLVChannel{
+		Id:            channel.Id,
+		Type:          channel.Type,
+		TeamId:        []string{channel.TeamId},
+		NameSuggest:   append(displayNameInputs, nameInputs...),
+		UserIDs:       userIDs,
+		TeamMemberIDs: teamMemberIDs,
+	}
+}
+
+func BLVUserFromUserAndTeams(user *model.User, teamsIds, channelsIds []string) *BLVUser {
+	usernameSuggestions := searchengine.GetSuggestionInputsSplitByMultiple(user.Username, []string{".", "-", "_"})
+
+	fullnameStrings := []string{}
+	if user.FirstName != "" {
+		fullnameStrings = append(fullnameStrings, user.FirstName)
+	}
+	if user.LastName != "" {
+		fullnameStrings = append(fullnameStrings, user.LastName)
+	}
+
+	fullnameSuggestions := []string{}
+	if len(fullnameStrings) > 0 {
+		fullname := strings.Join(fullnameStrings, " ")
+		fullnameSuggestions = searchengine.GetSuggestionInputsSplitBy(fullname, " ")
+	}
+
+	nicknameSuggestions := []string{}
+	if user.Nickname != "" {
+		nicknameSuggestions = searchengine.GetSuggestionInputsSplitBy(user.Nickname, " ")
+	}
+
+	usernameAndNicknameSuggestions := append(usernameSuggestions, nicknameSuggestions...)
+
+	return &BLVUser{
+		Id:                         user.Id,
+		SuggestionsWithFullname:    append(usernameAndNicknameSuggestions, fullnameSuggestions...),
+		SuggestionsWithoutFullname: usernameAndNicknameSuggestions,
+		TeamsIds:                   teamsIds,
+		ChannelsIds:                channelsIds,
+	}
+}
+
+func BLVUserFromUserForIndexing(userForIndexing *model.UserForIndexing) *BLVUser {
+	user := &model.User{
+		Id:        userForIndexing.Id,
+		Username:  userForIndexing.Username,
+		Nickname:  userForIndexing.Nickname,
+		FirstName: userForIndexing.FirstName,
+		LastName:  userForIndexing.LastName,
+		CreateAt:  userForIndexing.CreateAt,
+		DeleteAt:  userForIndexing.DeleteAt,
+	}
+
+	return BLVUserFromUserAndTeams(user, userForIndexing.TeamsIds, userForIndexing.ChannelsIds)
+}
+
+func BLVPostFromPost(post *model.Post, teamId string) *BLVPost {
+	p := &model.PostForIndexing{
+		TeamId: teamId,
+	}
+	post.ShallowCopy(&p.Post)
+	return BLVPostFromPostForIndexing(p)
+}
+
+func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost {
+	return &BLVPost{
+		Id:        post.Id,
+		TeamId:    post.TeamId,
+		ChannelId: post.ChannelId,
+		UserId:    post.UserId,
+		CreateAt:  post.CreateAt,
+		Message:   post.Message,
+		Type:      post.Type,
+		Hashtags:  strings.Fields(post.Hashtags),
+	}
+}
+
+func splitFilenameWords(name string) string {
+	result := name
+	result = strings.ReplaceAll(result, "-", " ")
+	result = strings.ReplaceAll(result, ".", " ")
+	return result
+}
+
+func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile {
+	return &BLVFile{
+		Id:        fileInfo.Id,
+		ChannelId: channelId,
+		CreatorId: fileInfo.CreatorId,
+		CreateAt:  fileInfo.CreateAt,
+		Content:   fileInfo.Content,
+		Extension: fileInfo.Extension,
+		Name:      fileInfo.Name + " " + splitFilenameWords(fileInfo.Name),
+	}
+}
+
+func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile {
+	return &BLVFile{
+		Id:        file.Id,
+		ChannelId: file.ChannelId,
+		CreatorId: file.CreatorId,
+		CreateAt:  file.CreateAt,
+		Content:   file.Content,
+		Extension: file.Extension,
+		Name:      file.Name + " " + splitFilenameWords(file.Name),
+	}
+}

server/platform/services/searchengine/bleveengine/indexer/indexing_job.go

@@ -0,0 +1,614 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package indexer
+
+import (
+	"context"
+	"net/http"
+	"strconv"
+	"sync/atomic"
+	"time"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine/bleveengine"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const (
+	timeBetweenBatches = 100 * time.Millisecond
+
+	estimatedPostCount    = 10000000
+	estimatedFilesCount   = 100000
+	estimatedChannelCount = 100000
+	estimatedUserCount    = 10000
+)
+
+type BleveIndexerWorker struct {
+	name      string
+	stop      chan struct{}
+	stopped   chan bool
+	jobs      chan model.Job
+	jobServer *jobs.JobServer
+	engine    *bleveengine.BleveEngine
+	closed    int32
+}
+
+func MakeWorker(jobServer *jobs.JobServer, engine *bleveengine.BleveEngine) model.Worker {
+	if engine == nil {
+		return nil
+	}
+	return &BleveIndexerWorker{
+		name:      "BleveIndexer",
+		stop:      make(chan struct{}),
+		stopped:   make(chan bool, 1),
+		jobs:      make(chan model.Job),
+		jobServer: jobServer,
+		engine:    engine,
+	}
+}
+
+type IndexingProgress struct {
+	Now            time.Time
+	StartAtTime    int64
+	EndAtTime      int64
+	LastEntityTime int64
+
+	TotalPostsCount int64
+	DonePostsCount  int64
+	DonePosts       bool
+	LastPostID      string
+
+	TotalFilesCount int64
+	DoneFilesCount  int64
+	DoneFiles       bool
+	LastFileID      string
+
+	TotalChannelsCount int64
+	DoneChannelsCount  int64
+	DoneChannels       bool
+	LastChannelID      string
+
+	TotalUsersCount int64
+	DoneUsersCount  int64
+	DoneUsers       bool
+	LastUserID      string
+}
+
+func (ip *IndexingProgress) CurrentProgress() int64 {
+	return (ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount) * 100 / (ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalUsersCount + ip.TotalFilesCount)
+}
+
+func (ip *IndexingProgress) IsDone() bool {
+	return ip.DonePosts && ip.DoneChannels && ip.DoneUsers && ip.DoneFiles
+}
+
+func (worker *BleveIndexerWorker) JobChannel() chan<- model.Job {
+	return worker.jobs
+}
+
+func (worker *BleveIndexerWorker) IsEnabled(cfg *model.Config) bool {
+	return true
+}
+
+func (worker *BleveIndexerWorker) Run() {
+	// Set to open if closed before. We are not bothered about multiple opens.
+	if atomic.CompareAndSwapInt32(&worker.closed, 1, 0) {
+		worker.stop = make(chan struct{})
+	}
+	mlog.Debug("Worker Started", mlog.String("workername", worker.name))
+
+	defer func() {
+		mlog.Debug("Worker: Finished", mlog.String("workername", worker.name))
+		worker.stopped <- true
+	}()
+
+	for {
+		select {
+		case <-worker.stop:
+			mlog.Debug("Worker: Received stop signal", mlog.String("workername", worker.name))
+			return
+		case job := <-worker.jobs:
+			mlog.Debug("Worker: Received a new candidate job.", mlog.String("workername", worker.name))
+			worker.DoJob(&job)
+		}
+	}
+}
+
+func (worker *BleveIndexerWorker) Stop() {
+	// Set to close, and if already closed before, then return.
+	if !atomic.CompareAndSwapInt32(&worker.closed, 0, 1) {
+		return
+	}
+	mlog.Debug("Worker Stopping", mlog.String("workername", worker.name))
+	close(worker.stop)
+	<-worker.stopped
+}
+
+func (worker *BleveIndexerWorker) DoJob(job *model.Job) {
+	claimed, err := worker.jobServer.ClaimJob(job)
+	if err != nil {
+		mlog.Warn("Worker: Error occurred while trying to claim job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		return
+	}
+	if !claimed {
+		return
+	}
+
+	mlog.Info("Worker: Indexing job claimed by worker", mlog.String("workername", worker.name), mlog.String("job_id", job.Id))
+
+	if !worker.engine.IsActive() {
+		appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.engine_inactive", nil, "", http.StatusInternalServerError)
+		if err := worker.jobServer.SetJobError(job, appError); err != nil {
+			mlog.Error("Worker: Failed to run job as ")
+		}
+		return
+	}
+
+	progress := IndexingProgress{
+		Now:          time.Now(),
+		DonePosts:    false,
+		DoneChannels: false,
+		DoneUsers:    false,
+		DoneFiles:    false,
+		StartAtTime:  0,
+		EndAtTime:    model.GetMillis(),
+	}
+
+	// Extract the start and end times, if they are set.
+	if startString, ok := job.Data["start_time"]; ok {
+		startInt, err := strconv.ParseInt(startString, 10, 64)
+		if err != nil {
+			mlog.Error("Worker: Failed to parse start_time for job", mlog.String("workername", worker.name), mlog.String("start_time", startString), mlog.String("job_id", job.Id), mlog.Err(err))
+			appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.parse_start_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := worker.jobServer.SetJobError(job, appError); err != nil {
+				mlog.Error("Worker: Failed to set job error", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return
+		}
+		progress.StartAtTime = startInt
+	} else {
+		// Set start time to oldest entity in the database.
+		// A user or a channel may be created before any post.
+		oldestEntityCreationTime, err := worker.jobServer.Store.Post().GetOldestEntityCreationTime()
+		if err != nil {
+			mlog.Error("Worker: Failed to fetch oldest entity for job.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.String("start_time", startString), mlog.Err(err))
+			appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.get_oldest_entity.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := worker.jobServer.SetJobError(job, appError); err != nil {
+				mlog.Error("Worker: Failed to set job error", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return
+		}
+		progress.StartAtTime = oldestEntityCreationTime
+	}
+	progress.LastEntityTime = progress.StartAtTime
+
+	if endString, ok := job.Data["end_time"]; ok {
+		endInt, err := strconv.ParseInt(endString, 10, 64)
+		if err != nil {
+			mlog.Error("Worker: Failed to parse end_time for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.String("end_time", endString), mlog.Err(err))
+			appError := model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.do_job.parse_end_time.error", nil, "", http.StatusInternalServerError).Wrap(err)
+			if err := worker.jobServer.SetJobError(job, appError); err != nil {
+				mlog.Error("Worker: Failed to set job errorv", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err), mlog.NamedErr("set_error", appError))
+			}
+			return
+		}
+		progress.EndAtTime = endInt
+	}
+
+	if id, ok := job.Data["start_post_id"]; ok {
+		progress.LastPostID = id
+	}
+	if id, ok := job.Data["start_channel_id"]; ok {
+		progress.LastChannelID = id
+	}
+	if id, ok := job.Data["start_user_id"]; ok {
+		progress.LastUserID = id
+	}
+	if id, ok := job.Data["start_file_id"]; ok {
+		progress.LastFileID = id
+	}
+
+	// Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry
+	// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
+	if count, err := worker.jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}); err != nil {
+		mlog.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalPostsCount = estimatedPostCount
+	} else {
+		progress.TotalPostsCount = count
+	}
+
+	// Same possible fail as above can happen when counting channels
+	if count, err := worker.jobServer.Store.Channel().AnalyticsTypeCount("", ""); err != nil {
+		mlog.Warn("Worker: Failed to fetch total channel count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalChannelsCount = estimatedChannelCount
+	} else {
+		progress.TotalChannelsCount = count
+	}
+
+	// Same possible fail as above can happen when counting users
+	if count, err := worker.jobServer.Store.User().Count(model.UserCountOptions{
+		IncludeBotAccounts: true, // This actually doesn't join with the bots table
+		// since ExcludeRegularUsers is set to false
+	}); err != nil {
+		mlog.Warn("Worker: Failed to fetch total user count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalUsersCount = estimatedUserCount
+	} else {
+		progress.TotalUsersCount = count
+	}
+
+	// Counting all files may fail or timeout when the file_info table is large. If this happens, log a warning, but carry
+	// on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate.
+	if count, err := worker.jobServer.Store.FileInfo().CountAll(); err != nil {
+		mlog.Warn("Worker: Failed to fetch total file info count for job. An estimated value will be used for progress reporting.", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+		progress.TotalFilesCount = estimatedFilesCount
+	} else {
+		progress.TotalFilesCount = count
+	}
+
+	cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background())
+	cancelWatcherChan := make(chan struct{}, 1)
+	go worker.jobServer.CancellationWatcher(cancelCtx, job.Id, cancelWatcherChan)
+
+	defer cancelCancelWatcher()
+
+	for {
+		select {
+		case <-cancelWatcherChan:
+			mlog.Info("Worker: Indexing job has been canceled via CancellationWatcher", mlog.String("workername", worker.name), mlog.String("job_id", job.Id))
+			if err := worker.jobServer.SetJobCanceled(job); err != nil {
+				mlog.Error("Worker: Failed to mark job as cancelled", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+			}
+			return
+
+		case <-worker.stop:
+			mlog.Info("Worker: Indexing has been canceled via Worker Stop", mlog.String("workername", worker.name), mlog.String("job_id", job.Id))
+			if err := worker.jobServer.SetJobCanceled(job); err != nil {
+				mlog.Error("Worker: Failed to mark job as canceled", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+			}
+			return
+
+		case <-time.After(timeBetweenBatches):
+			var err *model.AppError
+			if progress, err = worker.IndexBatch(progress); err != nil {
+				mlog.Error("Worker: Failed to index batch for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+				if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+					mlog.Error("Worker: Failed to set job error", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err2), mlog.NamedErr("set_error", err))
+				}
+				return
+			}
+
+			// Storing the batch progress in metadata.
+			if job.Data == nil {
+				job.Data = make(model.StringMap)
+			}
+
+			job.Data["start_time"] = strconv.FormatInt(progress.LastEntityTime, 10)
+			job.Data["start_post_id"] = progress.LastPostID
+			job.Data["start_channel_id"] = progress.LastChannelID
+			job.Data["start_user_id"] = progress.LastUserID
+			job.Data["start_file_id"] = progress.LastFileID
+			job.Data["original_start_time"] = strconv.FormatInt(progress.StartAtTime, 10)
+			job.Data["end_time"] = strconv.FormatInt(progress.EndAtTime, 10)
+
+			if err := worker.jobServer.SetJobProgress(job, progress.CurrentProgress()); err != nil {
+				mlog.Error("Worker: Failed to set progress for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+				if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+					mlog.Error("Worker: Failed to set error for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err2), mlog.NamedErr("set_error", err))
+				}
+				return
+			}
+
+			if progress.IsDone() {
+				if err := worker.jobServer.SetJobSuccess(job); err != nil {
+					mlog.Error("Worker: Failed to set success for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err))
+					if err2 := worker.jobServer.SetJobError(job, err); err2 != nil {
+						mlog.Error("Worker: Failed to set error for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(err2), mlog.NamedErr("set_error", err))
+					}
+				}
+				mlog.Info("Worker: Indexing job finished successfully", mlog.String("workername", worker.name), mlog.String("job_id", job.Id))
+				return
+			}
+		}
+	}
+}
+
+func (worker *BleveIndexerWorker) IndexBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	if !progress.DonePosts {
+		return worker.IndexPostsBatch(progress)
+	}
+	if !progress.DoneChannels {
+		return worker.IndexChannelsBatch(progress)
+	}
+	if !progress.DoneUsers {
+		return worker.IndexUsersBatch(progress)
+	}
+	if !progress.DoneFiles {
+		return worker.IndexFilesBatch(progress)
+	}
+	return progress, model.NewAppError("BleveIndexerWorker", "bleveengine.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError)
+}
+
+func (worker *BleveIndexerWorker) IndexPostsBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var posts []*model.PostForIndexing
+
+	tries := 0
+	for posts == nil {
+		var err error
+		posts, err = worker.jobServer.Store.Post().GetPostsBatchForIndexing(progress.LastEntityTime, progress.LastPostID, *worker.jobServer.Config().BleveSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexPostsBatch", "app.post.get_posts_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			mlog.Warn("Failed to get posts batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	// Handle zero messages.
+	if len(posts) == 0 {
+		progress.DonePosts = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastPost, err := worker.BulkIndexPosts(posts, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last post's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastPost.CreateAt {
+		progress.DonePosts = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastPost.CreateAt
+	}
+
+	progress.LastPostID = lastPost.Id
+	progress.DonePostsCount += int64(len(posts))
+
+	return progress, nil
+}
+
+func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, progress IndexingProgress) (*model.Post, *model.AppError) {
+	batch := worker.engine.PostIndex.NewBatch()
+
+	for _, post := range posts {
+		if post.DeleteAt == 0 {
+			searchPost := bleveengine.BLVPostFromPostForIndexing(post)
+			batch.Index(searchPost.Id, searchPost)
+		} else {
+			batch.Delete(post.Id)
+		}
+	}
+
+	worker.engine.Mutex.RLock()
+	defer worker.engine.Mutex.RUnlock()
+
+	if err := worker.engine.PostIndex.Batch(batch); err != nil {
+		return nil, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_posts.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return &posts[len(posts)-1].Post, nil
+}
+
+func (worker *BleveIndexerWorker) IndexFilesBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var files []*model.FileForIndexing
+
+	tries := 0
+	for files == nil {
+		var err error
+		files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, progress.LastFileID, *worker.jobServer.Config().BleveSettings.BatchSize)
+		if err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexFilesBatch", "app.post.get_files_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			mlog.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+
+		tries++
+	}
+
+	if len(files) == 0 {
+		progress.DoneFiles = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastFile, err := worker.BulkIndexFiles(files, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last file's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastFile.CreateAt {
+		progress.DoneFiles = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastFile.CreateAt
+	}
+
+	progress.LastFileID = lastFile.Id
+	progress.DoneFilesCount += int64(len(files))
+
+	return progress, nil
+}
+
+func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (*model.FileInfo, *model.AppError) {
+	batch := worker.engine.FileIndex.NewBatch()
+
+	for _, file := range files {
+		if file.DeleteAt == 0 {
+			searchFile := bleveengine.BLVFileFromFileForIndexing(file)
+			batch.Index(searchFile.Id, searchFile)
+		} else {
+			batch.Delete(file.Id)
+		}
+	}
+
+	worker.engine.Mutex.RLock()
+	defer worker.engine.Mutex.RUnlock()
+
+	if err := worker.engine.FileIndex.Batch(batch); err != nil {
+		return nil, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_files.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return &files[len(files)-1].FileInfo, nil
+}
+
+func (worker *BleveIndexerWorker) IndexChannelsBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var channels []*model.Channel
+
+	tries := 0
+	for channels == nil {
+		var nErr error
+		channels, nErr = worker.jobServer.Store.Channel().GetChannelsBatchForIndexing(progress.LastEntityTime, progress.LastChannelID, *worker.jobServer.Config().BleveSettings.BatchSize)
+		if nErr != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("BleveIndexerWorker.IndexChannelsBatch", "app.channel.get_channels_batch_for_indexing.get.app_error", nil, "", http.StatusInternalServerError).Wrap(nErr)
+			}
+
+			mlog.Warn("Failed to get channels batch for indexing. Retrying.", mlog.Err(nErr))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		}
+		tries++
+	}
+
+	if len(channels) == 0 {
+		progress.DoneChannels = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastChannel, err := worker.BulkIndexChannels(channels, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last channel's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastChannel.CreateAt {
+		progress.DoneChannels = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastChannel.CreateAt
+	}
+
+	progress.LastChannelID = lastChannel.Id
+	progress.DoneChannelsCount += int64(len(channels))
+
+	return progress, nil
+}
+
+func (worker *BleveIndexerWorker) BulkIndexChannels(channels []*model.Channel, progress IndexingProgress) (*model.Channel, *model.AppError) {
+	batch := worker.engine.ChannelIndex.NewBatch()
+
+	for _, channel := range channels {
+		if channel.DeleteAt == 0 {
+			var userIDs []string
+			var err error
+			if channel.Type == model.ChannelTypePrivate {
+				userIDs, err = worker.jobServer.Store.Channel().GetAllChannelMembersById(channel.Id)
+				if err != nil {
+					return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+				}
+			}
+
+			// Get teamMember ids from channelid
+			teamMemberIDs, err := worker.jobServer.Store.Channel().GetTeamMembersForChannel(channel.Id)
+			if err != nil {
+				return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+
+			searchChannel := bleveengine.BLVChannelFromChannel(channel, userIDs, teamMemberIDs)
+			batch.Index(searchChannel.Id, searchChannel)
+		} else {
+			batch.Delete(channel.Id)
+		}
+	}
+
+	worker.engine.Mutex.RLock()
+	defer worker.engine.Mutex.RUnlock()
+
+	if err := worker.engine.ChannelIndex.Batch(batch); err != nil {
+		return nil, model.NewAppError("BleveIndexerWorker.BulkIndexChannels", "bleveengine.indexer.do_job.bulk_index_channels.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return channels[len(channels)-1], nil
+}
+
+func (worker *BleveIndexerWorker) IndexUsersBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
+	var users []*model.UserForIndexing
+
+	tries := 0
+	for users == nil {
+		if usersBatch, err := worker.jobServer.Store.User().GetUsersBatchForIndexing(progress.LastEntityTime, progress.LastUserID, *worker.jobServer.Config().BleveSettings.BatchSize); err != nil {
+			if tries >= 10 {
+				return progress, model.NewAppError("IndexUsersBatch", "app.user.get_users_batch_for_indexing.get_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			}
+			mlog.Warn("Failed to get users batch for indexing. Retrying.", mlog.Err(err))
+
+			// Wait a bit before trying again.
+			time.Sleep(15 * time.Second)
+		} else {
+			users = usersBatch
+		}
+
+		tries++
+	}
+
+	if len(users) == 0 {
+		progress.DoneUsers = true
+		progress.LastEntityTime = progress.StartAtTime
+		return progress, nil
+	}
+
+	lastUser, err := worker.BulkIndexUsers(users, progress)
+	if err != nil {
+		return progress, err
+	}
+
+	// Our exit condition is when the last user's createAt reaches the initial endAtTime
+	// set during job creation.
+	if progress.EndAtTime <= lastUser.CreateAt {
+		progress.DoneUsers = true
+		progress.LastEntityTime = progress.StartAtTime
+	} else {
+		progress.LastEntityTime = lastUser.CreateAt
+	}
+	progress.LastUserID = lastUser.Id
+	progress.DoneUsersCount += int64(len(users))
+
+	return progress, nil
+}
+
+func (worker *BleveIndexerWorker) BulkIndexUsers(users []*model.UserForIndexing, progress IndexingProgress) (*model.UserForIndexing, *model.AppError) {
+	batch := worker.engine.UserIndex.NewBatch()
+
+	for _, user := range users {
+		if user.DeleteAt == 0 {
+			searchUser := bleveengine.BLVUserFromUserForIndexing(user)
+			batch.Index(searchUser.Id, searchUser)
+		} else {
+			batch.Delete(user.Id)
+		}
+	}
+
+	worker.engine.Mutex.RLock()
+	defer worker.engine.Mutex.RUnlock()
+
+	if err := worker.engine.UserIndex.Batch(batch); err != nil {
+		return nil, model.NewAppError("BleveIndexerWorker.BulkIndexUsers", "bleveengine.indexer.do_job.bulk_index_users.batch_error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return users[len(users)-1], nil
+}

server/platform/services/searchengine/bleveengine/indexer/indexing_job_test.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package indexer
+
+import (
+	"errors"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/channels/jobs"
+	"github.com/mattermost/mattermost-server/v6/server/channels/store/storetest"
+	"github.com/mattermost/mattermost-server/v6/server/channels/utils/testutils"
+	"github.com/mattermost/mattermost-server/v6/server/platform/services/searchengine/bleveengine"
+)
+
+func TestBleveIndexer(t *testing.T) {
+	mockStore := &storetest.Store{}
+	defer mockStore.AssertExpectations(t)
+
+	t.Run("Call GetOldestEntityCreationTime for the first indexing call", func(t *testing.T) {
+		job := &model.Job{
+			Id:       model.NewId(),
+			CreateAt: model.GetMillis(),
+			Status:   model.JobStatusPending,
+			Type:     model.JobTypeBlevePostIndexing,
+		}
+
+		mockStore.JobStore.On("UpdateStatusOptimistically", job.Id, model.JobStatusPending, model.JobStatusInProgress).Return(true, nil)
+		mockStore.JobStore.On("UpdateOptimistically", job, model.JobStatusInProgress).Return(true, nil)
+		mockStore.PostStore.On("GetOldestEntityCreationTime").Return(int64(1), errors.New("")) // intentionally return error to return from function
+
+		tempDir, err := os.MkdirTemp("", "setupConfigFile")
+		require.NoError(t, err)
+
+		t.Cleanup(func() {
+			os.RemoveAll(tempDir)
+		})
+
+		cfg := &model.Config{
+			BleveSettings: model.BleveSettings{
+				EnableIndexing: model.NewBool(true),
+				IndexDir:       model.NewString(tempDir),
+			},
+		}
+
+		jobServer := &jobs.JobServer{
+			Store: mockStore,
+			ConfigService: &testutils.StaticConfigService{
+				Cfg: cfg,
+			},
+		}
+
+		bleveEngine := bleveengine.NewBleveEngine(cfg)
+		aErr := bleveEngine.Start()
+		require.Nil(t, aErr)
+
+		worker := &BleveIndexerWorker{
+			jobServer: jobServer,
+			engine:    bleveEngine,
+		}
+
+		worker.DoJob(job)
+	})
+}

server/platform/services/searchengine/bleveengine/search.go

@@ -0,0 +1,879 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/blevesearch/bleve/v2"
+	"github.com/blevesearch/bleve/v2/search/query"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+	"github.com/mattermost/mattermost-server/v6/server/platform/shared/mlog"
+)
+
+const DeletePostsBatchSize = 500
+const DeleteFilesBatchSize = 500
+
+func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	blvPost := BLVPostFromPost(post, teamId)
+	if err := b.PostIndex.Index(blvPost.Id, blvPost); err != nil {
+		return model.NewAppError("Bleveengine.IndexPost", "bleveengine.index_post.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
+	channelQueries := []query.Query{}
+	for _, channel := range channels {
+		channelIdQ := bleve.NewTermQuery(channel.Id)
+		channelIdQ.SetField("ChannelId")
+		channelQueries = append(channelQueries, channelIdQ)
+	}
+	channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
+
+	var termQueries []query.Query
+	var notTermQueries []query.Query
+	var filters []query.Query
+	var notFilters []query.Query
+
+	typeQ := bleve.NewTermQuery("")
+	typeQ.SetField("Type")
+	filters = append(filters, typeQ)
+
+	for i, params := range searchParams {
+		var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
+		if searchParams[0].OrTerms {
+			termOperator = query.MatchQueryOperatorOr
+		}
+
+		// Date, channels and FromUsers filters come in all
+		// searchParams iteration, and as they are global to the
+		// query, we only need to process them once
+		if i == 0 {
+			if len(params.InChannels) > 0 {
+				inChannels := []query.Query{}
+				for _, channelId := range params.InChannels {
+					channelQ := bleve.NewTermQuery(channelId)
+					channelQ.SetField("ChannelId")
+					inChannels = append(inChannels, channelQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
+			}
+
+			if len(params.ExcludedChannels) > 0 {
+				excludedChannels := []query.Query{}
+				for _, channelId := range params.ExcludedChannels {
+					channelQ := bleve.NewTermQuery(channelId)
+					channelQ.SetField("ChannelId")
+					excludedChannels = append(excludedChannels, channelQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
+			}
+
+			if len(params.FromUsers) > 0 {
+				fromUsers := []query.Query{}
+				for _, userId := range params.FromUsers {
+					userQ := bleve.NewTermQuery(userId)
+					userQ.SetField("UserId")
+					fromUsers = append(fromUsers, userQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
+			}
+
+			if len(params.ExcludedUsers) > 0 {
+				excludedUsers := []query.Query{}
+				for _, userId := range params.ExcludedUsers {
+					userQ := bleve.NewTermQuery(userId)
+					userQ.SetField("UserId")
+					excludedUsers = append(excludedUsers, userQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
+			}
+
+			if params.OnDate != "" {
+				before, after := params.GetOnDateMillis()
+				beforeFloat64 := float64(before)
+				afterFloat64 := float64(after)
+				onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
+				onDateQ.SetField("CreateAt")
+				filters = append(filters, onDateQ)
+			} else {
+				if params.AfterDate != "" || params.BeforeDate != "" {
+					var min, max *float64
+					if params.AfterDate != "" {
+						minf := float64(params.GetAfterDateMillis())
+						min = &minf
+					}
+
+					if params.BeforeDate != "" {
+						maxf := float64(params.GetBeforeDateMillis())
+						max = &maxf
+					}
+
+					dateQ := bleve.NewNumericRangeQuery(min, max)
+					dateQ.SetField("CreateAt")
+					filters = append(filters, dateQ)
+				}
+
+				if params.ExcludedAfterDate != "" {
+					minf := float64(params.GetExcludedAfterDateMillis())
+					dateQ := bleve.NewNumericRangeQuery(&minf, nil)
+					dateQ.SetField("CreateAt")
+					notFilters = append(notFilters, dateQ)
+				}
+
+				if params.ExcludedBeforeDate != "" {
+					maxf := float64(params.GetExcludedBeforeDateMillis())
+					dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
+					dateQ.SetField("CreateAt")
+					notFilters = append(notFilters, dateQ)
+				}
+
+				if params.ExcludedDate != "" {
+					before, after := params.GetExcludedDateMillis()
+					beforef := float64(before)
+					afterf := float64(after)
+					onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
+					onDateQ.SetField("CreateAt")
+					notFilters = append(notFilters, onDateQ)
+				}
+			}
+		}
+
+		if params.IsHashtag {
+			if params.Terms != "" {
+				hashtagQ := bleve.NewMatchQuery(params.Terms)
+				hashtagQ.SetField("Hashtags")
+				hashtagQ.SetOperator(termOperator)
+				termQueries = append(termQueries, hashtagQ)
+			} else if params.ExcludedTerms != "" {
+				hashtagQ := bleve.NewMatchQuery(params.ExcludedTerms)
+				hashtagQ.SetField("Hashtags")
+				hashtagQ.SetOperator(termOperator)
+				notTermQueries = append(notTermQueries, hashtagQ)
+			}
+		} else {
+			if params.Terms != "" {
+				terms := []string{}
+				for _, term := range strings.Split(params.Terms, " ") {
+					if strings.HasSuffix(term, "*") {
+						messageQ := bleve.NewWildcardQuery(term)
+						messageQ.SetField("Message")
+						termQueries = append(termQueries, messageQ)
+					} else {
+						terms = append(terms, term)
+					}
+				}
+
+				if len(terms) > 0 {
+					messageQ := bleve.NewMatchQuery(strings.Join(terms, " "))
+					messageQ.SetField("Message")
+					messageQ.SetOperator(termOperator)
+					termQueries = append(termQueries, messageQ)
+				}
+			}
+
+			if params.ExcludedTerms != "" {
+				messageQ := bleve.NewMatchQuery(params.ExcludedTerms)
+				messageQ.SetField("Message")
+				messageQ.SetOperator(termOperator)
+				notTermQueries = append(notTermQueries, messageQ)
+			}
+		}
+	}
+
+	allTermsQ := bleve.NewBooleanQuery()
+	allTermsQ.AddMustNot(notTermQueries...)
+	if searchParams[0].OrTerms {
+		allTermsQ.AddShould(termQueries...)
+	} else {
+		allTermsQ.AddMust(termQueries...)
+	}
+
+	query := bleve.NewBooleanQuery()
+	query.AddMust(channelDisjunctionQ)
+
+	if len(termQueries) > 0 || len(notTermQueries) > 0 {
+		query.AddMust(allTermsQ)
+	}
+
+	if len(filters) > 0 {
+		query.AddMust(bleve.NewConjunctionQuery(filters...))
+	}
+	if len(notFilters) > 0 {
+		query.AddMustNot(notFilters...)
+	}
+
+	search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
+	search.SortBy([]string{"-CreateAt"})
+	results, err := b.PostIndex.Search(search)
+	if err != nil {
+		return nil, nil, model.NewAppError("Bleveengine.SearchPosts", "bleveengine.search_posts.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	postIds := []string{}
+	matches := model.PostSearchMatches{}
+
+	for _, r := range results.Hits {
+		postIds = append(postIds, r.ID)
+	}
+
+	return postIds, matches, nil
+}
+
+func (b *BleveEngine) deletePosts(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
+	resultsCount := int64(0)
+
+	for {
+		// As we are deleting the posts after fetching them, we need to keep
+		// From fixed always to 0
+		searchRequest.From = 0
+		searchRequest.Size = batchSize
+		results, err := b.PostIndex.Search(searchRequest)
+		if err != nil {
+			return -1, err
+		}
+		batch := b.PostIndex.NewBatch()
+		for _, post := range results.Hits {
+			batch.Delete(post.ID)
+		}
+		if err := b.PostIndex.Batch(batch); err != nil {
+			return -1, err
+		}
+		resultsCount += int64(results.Hits.Len())
+		if results.Hits.Len() < batchSize {
+			break
+		}
+	}
+
+	return resultsCount, nil
+}
+
+func (b *BleveEngine) DeleteChannelPosts(channelID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	query := bleve.NewTermQuery(channelID)
+	query.SetField("ChannelId")
+	search := bleve.NewSearchRequest(query)
+	deleted, err := b.deletePosts(search, DeletePostsBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeleteChannelPosts",
+			"bleveengine.delete_channel_posts.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Posts for channel deleted", mlog.String("channel_id", channelID), mlog.Int64("deleted", deleted))
+
+	return nil
+}
+
+func (b *BleveEngine) DeleteUserPosts(userID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	query := bleve.NewTermQuery(userID)
+	query.SetField("UserId")
+	search := bleve.NewSearchRequest(query)
+	deleted, err := b.deletePosts(search, DeletePostsBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeleteUserPosts",
+			"bleveengine.delete_user_posts.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Posts for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted))
+
+	return nil
+}
+
+func (b *BleveEngine) DeletePost(post *model.Post) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	if err := b.PostIndex.Delete(post.Id); err != nil {
+		return model.NewAppError("Bleveengine.DeletePost", "bleveengine.delete_post.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) IndexChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	blvChannel := BLVChannelFromChannel(channel, userIDs, teamMemberIDs)
+	if err := b.ChannelIndex.Index(blvChannel.Id, blvChannel); err != nil {
+		return model.NewAppError("Bleveengine.IndexChannel", "bleveengine.index_channel.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) SearchChannels(teamId, userID, term string, isGuest bool) ([]string, *model.AppError) {
+	// This query essentially boils down to (if teamID is passed):
+	// match teamID == <>
+	// AND
+	// match term == <>
+	// AND
+	// match (channelType != 'P' || (<> in userIDs && channelType == 'P'))
+
+	// (or if teamID is not passed)
+	// <> in teamMemberIds
+	// AND
+	// match term == <>
+	// AND
+	// match (channelType != 'P' || (<> in userIDs && channelType == 'P'))
+
+	// (or if isGuest is true)
+	// <> in teamMemberIds
+	// AND
+	// match term == <>
+	// AND
+	// match (<> in userIDs)
+
+	queries := []query.Query{}
+	if teamId != "" {
+		teamIdQ := bleve.NewTermQuery(teamId)
+		teamIdQ.SetField("TeamId")
+		queries = append(queries, teamIdQ)
+	} else {
+		teamMemberQ := bleve.NewTermQuery(userID)
+		teamMemberQ.SetField("TeamMemberIDs")
+		queries = append(queries, teamMemberQ)
+	}
+
+	if isGuest {
+		userQ := bleve.NewBooleanQuery()
+		userIDQ := bleve.NewTermQuery(userID)
+		userIDQ.SetField("UserIDs")
+		userQ.AddMust(userIDQ)
+		queries = append(queries, userIDQ)
+	} else {
+		boolNotPrivate := bleve.NewBooleanQuery()
+		privateQ := bleve.NewTermQuery(string(model.ChannelTypePrivate))
+		privateQ.SetField("Type")
+		boolNotPrivate.AddMustNot(privateQ)
+
+		userQ := bleve.NewBooleanQuery()
+		userIDQ := bleve.NewTermQuery(userID)
+		userIDQ.SetField("UserIDs")
+		userQ.AddMust(userIDQ)
+		userQ.AddMust(privateQ)
+
+		channelTypeQ := bleve.NewDisjunctionQuery()
+		channelTypeQ.AddQuery(boolNotPrivate)
+		channelTypeQ.AddQuery(userQ) // userID && 'p'
+		queries = append(queries, channelTypeQ)
+	}
+
+	if term != "" {
+		nameSuggestQ := bleve.NewPrefixQuery(strings.ToLower(term))
+		nameSuggestQ.SetField("NameSuggest")
+		queries = append(queries, nameSuggestQ)
+	}
+
+	query := bleve.NewSearchRequest(bleve.NewConjunctionQuery(queries...))
+	query.Size = model.ChannelSearchDefaultLimit
+	results, err := b.ChannelIndex.Search(query)
+	if err != nil {
+		return nil, model.NewAppError("Bleveengine.SearchChannels", "bleveengine.search_channels.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	channelIds := []string{}
+	for _, result := range results.Hits {
+		channelIds = append(channelIds, result.ID)
+	}
+
+	return channelIds, nil
+}
+
+func (b *BleveEngine) DeleteChannel(channel *model.Channel) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	if err := b.ChannelIndex.Delete(channel.Id); err != nil {
+		return model.NewAppError("Bleveengine.DeleteChannel", "bleveengine.delete_channel.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) IndexUser(user *model.User, teamsIds, channelsIds []string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	blvUser := BLVUserFromUserAndTeams(user, teamsIds, channelsIds)
+	if err := b.UserIndex.Index(blvUser.Id, blvUser); err != nil {
+		return model.NewAppError("Bleveengine.IndexUser", "bleveengine.index_user.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) {
+	if restrictedToChannels != nil && len(restrictedToChannels) == 0 {
+		return []string{}, []string{}, nil
+	}
+
+	// users in channel
+	var queries []query.Query
+	if term != "" {
+		termQ := bleve.NewPrefixQuery(strings.ToLower(term))
+		if options.AllowFullNames {
+			termQ.SetField("SuggestionsWithFullname")
+		} else {
+			termQ.SetField("SuggestionsWithoutFullname")
+		}
+		queries = append(queries, termQ)
+	}
+
+	channelIdQ := bleve.NewTermQuery(channelId)
+	channelIdQ.SetField("ChannelsIds")
+	queries = append(queries, channelIdQ)
+
+	query := bleve.NewConjunctionQuery(queries...)
+
+	uchanSearch := bleve.NewSearchRequest(query)
+	uchanSearch.Size = options.Limit
+	uchan, err := b.UserIndex.Search(uchanSearch)
+	if err != nil {
+		return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.uchan.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	// users not in channel
+	boolQ := bleve.NewBooleanQuery()
+
+	if term != "" {
+		termQ := bleve.NewPrefixQuery(strings.ToLower(term))
+		if options.AllowFullNames {
+			termQ.SetField("SuggestionsWithFullname")
+		} else {
+			termQ.SetField("SuggestionsWithoutFullname")
+		}
+		boolQ.AddMust(termQ)
+	}
+
+	teamIdQ := bleve.NewTermQuery(teamId)
+	teamIdQ.SetField("TeamsIds")
+	boolQ.AddMust(teamIdQ)
+
+	outsideChannelIdQ := bleve.NewTermQuery(channelId)
+	outsideChannelIdQ.SetField("ChannelsIds")
+	boolQ.AddMustNot(outsideChannelIdQ)
+
+	if len(restrictedToChannels) > 0 {
+		restrictedChannelsQ := bleve.NewDisjunctionQuery()
+		for _, channelId := range restrictedToChannels {
+			restrictedChannelQ := bleve.NewTermQuery(channelId)
+			restrictedChannelsQ.AddQuery(restrictedChannelQ)
+		}
+		boolQ.AddMust(restrictedChannelsQ)
+	}
+
+	nuchanSearch := bleve.NewSearchRequest(boolQ)
+	nuchanSearch.Size = options.Limit
+	nuchan, err := b.UserIndex.Search(nuchanSearch)
+	if err != nil {
+		return nil, nil, model.NewAppError("Bleveengine.SearchUsersInChannel", "bleveengine.search_users_in_channel.nuchan.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	uchanIds := []string{}
+	for _, result := range uchan.Hits {
+		uchanIds = append(uchanIds, result.ID)
+	}
+
+	nuchanIds := []string{}
+	for _, result := range nuchan.Hits {
+		nuchanIds = append(nuchanIds, result.ID)
+	}
+
+	return uchanIds, nuchanIds, nil
+}
+
+func (b *BleveEngine) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) {
+	if restrictedToChannels != nil && len(restrictedToChannels) == 0 {
+		return []string{}, nil
+	}
+
+	var rootQ query.Query
+	if term == "" && teamId == "" && restrictedToChannels == nil {
+		rootQ = bleve.NewMatchAllQuery()
+	} else {
+		boolQ := bleve.NewBooleanQuery()
+
+		if term != "" {
+			termQ := bleve.NewPrefixQuery(strings.ToLower(term))
+			if options.AllowFullNames {
+				termQ.SetField("SuggestionsWithFullname")
+			} else {
+				termQ.SetField("SuggestionsWithoutFullname")
+			}
+			boolQ.AddMust(termQ)
+		}
+
+		if len(restrictedToChannels) > 0 {
+			// restricted channels are already filtered by team, so we
+			// can search only those matches
+			restrictedChannelsQ := []query.Query{}
+			for _, channelId := range restrictedToChannels {
+				channelIdQ := bleve.NewTermQuery(channelId)
+				channelIdQ.SetField("ChannelsIds")
+				restrictedChannelsQ = append(restrictedChannelsQ, channelIdQ)
+			}
+			boolQ.AddMust(bleve.NewDisjunctionQuery(restrictedChannelsQ...))
+		} else {
+			// this means that we only need to restrict by team
+			if teamId != "" {
+				teamIdQ := bleve.NewTermQuery(teamId)
+				teamIdQ.SetField("TeamsIds")
+				boolQ.AddMust(teamIdQ)
+			}
+		}
+
+		rootQ = boolQ
+	}
+
+	search := bleve.NewSearchRequest(rootQ)
+	search.Size = options.Limit
+	results, err := b.UserIndex.Search(search)
+	if err != nil {
+		return nil, model.NewAppError("Bleveengine.SearchUsersInTeam", "bleveengine.search_users_in_team.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	usersIds := []string{}
+	for _, r := range results.Hits {
+		usersIds = append(usersIds, r.ID)
+	}
+
+	return usersIds, nil
+}
+
+func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	if err := b.UserIndex.Delete(user.Id); err != nil {
+		return model.NewAppError("Bleveengine.DeleteUser", "bleveengine.delete_user.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	blvFile := BLVFileFromFileInfo(file, channelId)
+	if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil {
+		return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) {
+	channelQueries := []query.Query{}
+	for _, channel := range channels {
+		channelIdQ := bleve.NewTermQuery(channel.Id)
+		channelIdQ.SetField("ChannelId")
+		channelQueries = append(channelQueries, channelIdQ)
+	}
+	channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
+
+	var termQueries []query.Query
+	var notTermQueries []query.Query
+	var filters []query.Query
+	var notFilters []query.Query
+
+	for i, params := range searchParams {
+		var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
+		if searchParams[0].OrTerms {
+			termOperator = query.MatchQueryOperatorOr
+		}
+
+		// Date, channels and FromUsers filters come in all
+		// searchParams iteration, and as they are global to the
+		// query, we only need to process them once
+		if i == 0 {
+			if len(params.InChannels) > 0 {
+				inChannels := []query.Query{}
+				for _, channelId := range params.InChannels {
+					channelQ := bleve.NewTermQuery(channelId)
+					channelQ.SetField("ChannelId")
+					inChannels = append(inChannels, channelQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
+			}
+
+			if len(params.ExcludedChannels) > 0 {
+				excludedChannels := []query.Query{}
+				for _, channelId := range params.ExcludedChannels {
+					channelQ := bleve.NewTermQuery(channelId)
+					channelQ.SetField("ChannelId")
+					excludedChannels = append(excludedChannels, channelQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
+			}
+
+			if len(params.FromUsers) > 0 {
+				fromUsers := []query.Query{}
+				for _, userId := range params.FromUsers {
+					userQ := bleve.NewTermQuery(userId)
+					userQ.SetField("CreatorId")
+					fromUsers = append(fromUsers, userQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
+			}
+
+			if len(params.ExcludedUsers) > 0 {
+				excludedUsers := []query.Query{}
+				for _, userId := range params.ExcludedUsers {
+					userQ := bleve.NewTermQuery(userId)
+					userQ.SetField("CreatorId")
+					excludedUsers = append(excludedUsers, userQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
+			}
+
+			if len(params.Extensions) > 0 {
+				extensions := []query.Query{}
+				for _, extension := range params.Extensions {
+					extensionQ := bleve.NewTermQuery(extension)
+					extensionQ.SetField("Extension")
+					extensions = append(extensions, extensionQ)
+				}
+				filters = append(filters, bleve.NewDisjunctionQuery(extensions...))
+			}
+
+			if len(params.ExcludedExtensions) > 0 {
+				excludedExtensions := []query.Query{}
+				for _, extension := range params.ExcludedExtensions {
+					extensionQ := bleve.NewTermQuery(extension)
+					extensionQ.SetField("Extension")
+					excludedExtensions = append(excludedExtensions, extensionQ)
+				}
+				notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...))
+			}
+
+			if params.OnDate != "" {
+				before, after := params.GetOnDateMillis()
+				beforeFloat64 := float64(before)
+				afterFloat64 := float64(after)
+				onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
+				onDateQ.SetField("CreateAt")
+				filters = append(filters, onDateQ)
+			} else {
+				if params.AfterDate != "" || params.BeforeDate != "" {
+					var min, max *float64
+					if params.AfterDate != "" {
+						minf := float64(params.GetAfterDateMillis())
+						min = &minf
+					}
+
+					if params.BeforeDate != "" {
+						maxf := float64(params.GetBeforeDateMillis())
+						max = &maxf
+					}
+
+					dateQ := bleve.NewNumericRangeQuery(min, max)
+					dateQ.SetField("CreateAt")
+					filters = append(filters, dateQ)
+				}
+
+				if params.ExcludedAfterDate != "" {
+					minf := float64(params.GetExcludedAfterDateMillis())
+					dateQ := bleve.NewNumericRangeQuery(&minf, nil)
+					dateQ.SetField("CreateAt")
+					notFilters = append(notFilters, dateQ)
+				}
+
+				if params.ExcludedBeforeDate != "" {
+					maxf := float64(params.GetExcludedBeforeDateMillis())
+					dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
+					dateQ.SetField("CreateAt")
+					notFilters = append(notFilters, dateQ)
+				}
+
+				if params.ExcludedDate != "" {
+					before, after := params.GetExcludedDateMillis()
+					beforef := float64(before)
+					afterf := float64(after)
+					onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
+					onDateQ.SetField("CreateAt")
+					notFilters = append(notFilters, onDateQ)
+				}
+			}
+		}
+
+		if params.Terms != "" {
+			terms := []string{}
+			for _, term := range strings.Split(params.Terms, " ") {
+				if strings.HasSuffix(term, "*") {
+					nameQ := bleve.NewWildcardQuery(term)
+					nameQ.SetField("Name")
+					contentQ := bleve.NewWildcardQuery(term)
+					contentQ.SetField("Content")
+					termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
+				} else {
+					terms = append(terms, term)
+				}
+			}
+
+			if len(terms) > 0 {
+				nameQ := bleve.NewMatchQuery(strings.Join(terms, " "))
+				nameQ.SetField("Name")
+				nameQ.SetOperator(termOperator)
+				contentQ := bleve.NewMatchQuery(strings.Join(terms, " "))
+				contentQ.SetField("Content")
+				contentQ.SetOperator(termOperator)
+				termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
+			}
+		}
+
+		if params.ExcludedTerms != "" {
+			nameQ := bleve.NewMatchQuery(params.ExcludedTerms)
+			nameQ.SetField("Name")
+			nameQ.SetOperator(termOperator)
+			contentQ := bleve.NewMatchQuery(params.ExcludedTerms)
+			contentQ.SetField("Content")
+			contentQ.SetOperator(termOperator)
+			notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
+		}
+	}
+
+	allTermsQ := bleve.NewBooleanQuery()
+	allTermsQ.AddMustNot(notTermQueries...)
+	if searchParams[0].OrTerms {
+		allTermsQ.AddShould(termQueries...)
+	} else {
+		allTermsQ.AddMust(termQueries...)
+	}
+
+	query := bleve.NewBooleanQuery()
+	query.AddMust(channelDisjunctionQ)
+
+	if len(termQueries) > 0 || len(notTermQueries) > 0 {
+		query.AddMust(allTermsQ)
+	}
+
+	if len(filters) > 0 {
+		query.AddMust(bleve.NewConjunctionQuery(filters...))
+	}
+	if len(notFilters) > 0 {
+		query.AddMustNot(notFilters...)
+	}
+
+	search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
+	search.SortBy([]string{"-CreateAt"})
+	results, err := b.FileIndex.Search(search)
+	if err != nil {
+		return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+
+	fileIds := []string{}
+
+	for _, r := range results.Hits {
+		fileIds = append(fileIds, r.ID)
+	}
+
+	return fileIds, nil
+}
+
+func (b *BleveEngine) DeleteFile(fileID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	if err := b.FileIndex.Delete(fileID); err != nil {
+		return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, "", http.StatusInternalServerError).Wrap(err)
+	}
+	return nil
+}
+
+func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
+	resultsCount := int64(0)
+
+	for {
+		// As we are deleting the files after fetching them, we need to keep
+		// From fixed always to 0
+		searchRequest.From = 0
+		searchRequest.Size = batchSize
+		results, err := b.FileIndex.Search(searchRequest)
+		if err != nil {
+			return -1, err
+		}
+		batch := b.FileIndex.NewBatch()
+		for _, file := range results.Hits {
+			batch.Delete(file.ID)
+		}
+		if err := b.FileIndex.Batch(batch); err != nil {
+			return -1, err
+		}
+		resultsCount += int64(results.Hits.Len())
+		if results.Hits.Len() < batchSize {
+			break
+		}
+	}
+
+	return resultsCount, nil
+}
+
+func (b *BleveEngine) DeleteUserFiles(userID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	query := bleve.NewTermQuery(userID)
+	query.SetField("CreatorId")
+	search := bleve.NewSearchRequest(query)
+	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeleteUserFiles",
+			"bleveengine.delete_user_files.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted))
+
+	return nil
+}
+
+func (b *BleveEngine) DeletePostFiles(postID string) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	query := bleve.NewTermQuery(postID)
+	query.SetField("PostId")
+	search := bleve.NewSearchRequest(query)
+	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeletePostFiles",
+			"bleveengine.delete_post_files.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int64("deleted", deleted))
+
+	return nil
+}
+
+func (b *BleveEngine) DeleteFilesBatch(endTime, limit int64) *model.AppError {
+	b.Mutex.RLock()
+	defer b.Mutex.RUnlock()
+
+	endTimeFloat := float64(endTime)
+	query := bleve.NewNumericRangeQuery(nil, &endTimeFloat)
+	query.SetField("CreateAt")
+	search := bleve.NewSearchRequestOptions(query, int(limit), 0, false)
+	search.SortBy([]string{"-CreateAt"})
+
+	deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
+	if err != nil {
+		return model.NewAppError("Bleveengine.DeleteFilesBatch",
+			"bleveengine.delete_files_batch.error", nil,
+			err.Error(), http.StatusInternalServerError)
+	}
+
+	mlog.Info("Files in batch deleted", mlog.Int64("endTime", endTime), mlog.Int64("limit", limit), mlog.Int64("deleted", deleted))
+
+	return nil
+}

server/platform/services/searchengine/bleveengine/testlib.go

@@ -0,0 +1,23 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package bleveengine
+
+import (
+	"fmt"
+
+	"github.com/mattermost/mattermost-server/v6/model"
+)
+
+func createPost(userId string, channelId string) *model.Post {
+	post := &model.Post{
+		Message:       model.NewRandomString(15),
+		ChannelId:     channelId,
+		PendingPostId: model.NewId() + ":" + fmt.Sprint(model.GetMillis()),
+		UserId:        userId,
+		CreateAt:      1000000,
+	}
+	post.PreSave()
+
+	return post
+}