[MM-20979] Add first implementation of the Bleve search engine (#14562)

* [MM-20979] Add first implementation of the Bleve search engine

* Fix i18n

* Migrate searchengine utils tests

* Fix linter

* Don't add allTermsQ if both termQueries and notTermQueries are empty

* Fix test that should work if user is system admin

* Modify naming according to review comments

* Abstract getIndexDir function

* Extracting bleve engine name as a constant

* Merge both Indexer interfaces into one

* Add worker stopped message

* Allow worker to be started/stopped with config change

* Use constants for index names

* Modify test order

* Fix linter

* Trying to unlock the CI

jobs/interfaces/indexer_interface.go

@@ -7,11 +7,6 @@ import (
 	"github.com/mattermost/mattermost-server/v5/model"
 )
 
-type SearchEngineIndexerInterface interface {
+type IndexerJobInterface interface {
 	MakeWorker() model.Worker
 }
-
-type SearchEngineAggregatorInterface interface {
-	MakeWorker() model.Worker
-	MakeScheduler() model.Scheduler
-}

jobs/jobs_watcher.go

@@ -100,6 +100,13 @@ func (watcher *Watcher) PollAndNotify() {
 				default:
 				}
 			}
+		} else if job.Type == model.JOB_TYPE_BLEVE_POST_INDEXING {
+			if watcher.workers.BleveIndexing != nil {
+				select {
+				case watcher.workers.BleveIndexing.JobChannel() <- *job:
+				default:
+				}
+			}
 		} else if job.Type == model.JOB_TYPE_LDAP_SYNC {
 			if watcher.workers.LdapSync != nil {
 				select {

jobs/server.go

@@ -20,10 +20,11 @@ type JobServer struct {
 	DataRetentionJob        ejobs.DataRetentionJobInterface
 	MessageExportJob        ejobs.MessageExportJobInterface
 	ElasticsearchAggregator ejobs.ElasticsearchAggregatorInterface
-	ElasticsearchIndexer    ejobs.ElasticsearchIndexerInterface
+	ElasticsearchIndexer    tjobs.IndexerJobInterface
 	LdapSync                ejobs.LdapSyncInterface
 	Migrations              tjobs.MigrationsJobInterface
 	Plugins                 tjobs.PluginsJobInterface
+	BleveIndexer            tjobs.IndexerJobInterface
 }
 
 func NewJobServer(configService configservice.ConfigService, store store.Store) *JobServer {

jobs/workers.go

@@ -23,6 +23,7 @@ type Workers struct {
 	LdapSync                 model.Worker
 	Migrations               model.Worker
 	Plugins                  model.Worker
+	BleveIndexing            model.Worker
 
 	listenerId string
 }
@@ -61,6 +62,10 @@ func (srv *JobServer) InitWorkers() *Workers {
 		workers.Plugins = pluginsInterface.MakeWorker()
 	}
 
+	if bleveIndexerInterface := srv.BleveIndexer; bleveIndexerInterface != nil {
+		workers.BleveIndexing = bleveIndexerInterface.MakeWorker()
+	}
+
 	return workers
 }
 
@@ -96,6 +101,10 @@ func (workers *Workers) Start() *Workers {
 			go workers.Plugins.Run()
 		}
 
+		if workers.BleveIndexing != nil && *workers.ConfigService.Config().BleveSettings.EnableIndexing && *workers.ConfigService.Config().BleveSettings.IndexDir != "" {
+			go workers.BleveIndexing.Run()
+		}
+
 		go workers.Watcher.Start()
 	})
 
@@ -146,6 +155,14 @@ func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *m
 			workers.LdapSync.Stop()
 		}
 	}
+
+	if workers.BleveIndexing != nil {
+		if !*oldConfig.BleveSettings.EnableIndexing && *newConfig.BleveSettings.EnableIndexing {
+			go workers.BleveIndexing.Run()
+		} else if *oldConfig.BleveSettings.EnableIndexing && !*newConfig.BleveSettings.EnableIndexing {
+			workers.BleveIndexing.Stop()
+		}
+	}
 }
 
 func (workers *Workers) Stop() *Workers {
@@ -181,6 +198,10 @@ func (workers *Workers) Stop() *Workers {
 		workers.Plugins.Stop()
 	}
 
+	if workers.BleveIndexing != nil && *workers.ConfigService.Config().BleveSettings.EnableIndexing {
+		workers.BleveIndexing.Stop()
+	}
+
 	mlog.Info("Stopped workers")
 
 	return workers