[MM-20979] Add first implementation of the Bleve search engine (#14562)

* [MM-20979] Add first implementation of the Bleve search engine

* Fix i18n

* Migrate searchengine utils tests

* Fix linter

* Don't add allTermsQ if both termQueries and notTermQueries are empty

* Fix test that should work if user is system admin

* Modify naming according to review comments

* Abstract getIndexDir function

* Extracting bleve engine name as a constant

* Merge both Indexer interfaces into one

* Add worker stopped message

* Allow worker to be started/stopped with config change

* Use constants for index names

* Modify test order

* Fix linter

* Trying to unlock the CI

model/client4.go

@@ -353,6 +353,10 @@ func (c *Client4) GetElasticsearchRoute() string {
 	return "/elasticsearch"
 }
 
+func (c *Client4) GetBleveRoute() string {
+	return fmt.Sprintf("/bleve")
+}
+
 func (c *Client4) GetCommandsRoute() string {
 	return "/commands"
 }
@@ -4075,6 +4079,18 @@ func (c *Client4) PurgeElasticsearchIndexes() (bool, *Response) {
 	return CheckStatusOK(r), BuildResponse(r)
 }
 
+// Bleve Section
+
+// PurgeBleveIndexes immediately deletes all Bleve indexes.
+func (c *Client4) PurgeBleveIndexes() (bool, *Response) {
+	r, err := c.DoApiPost(c.GetBleveRoute()+"/purge_indexes", "")
+	if err != nil {
+		return false, BuildErrorResponse(r, err)
+	}
+	defer closeBody(r)
+	return CheckStatusOK(r), BuildResponse(r)
+}
+
 // Data Retention Section
 
 // GetDataRetentionPolicy will get the current server data retention policy details.

model/config.go

@@ -181,6 +181,9 @@ const (
 	ELASTICSEARCH_SETTINGS_DEFAULT_BULK_INDEXING_TIME_WINDOW_SECONDS = 3600
 	ELASTICSEARCH_SETTINGS_DEFAULT_REQUEST_TIMEOUT_SECONDS           = 30
 
+	BLEVE_SETTINGS_DEFAULT_INDEX_DIR                         = ""
+	BLEVE_SETTINGS_DEFAULT_BULK_INDEXING_TIME_WINDOW_SECONDS = 3600
+
 	DATA_RETENTION_SETTINGS_DEFAULT_MESSAGE_RETENTION_DAYS  = 365
 	DATA_RETENTION_SETTINGS_DEFAULT_FILE_RETENTION_DAYS     = 365
 	DATA_RETENTION_SETTINGS_DEFAULT_DELETION_JOB_START_TIME = "02:00"
@@ -2402,6 +2405,36 @@ func (s *ElasticsearchSettings) SetDefaults() {
 	}
 }
 
+type BleveSettings struct {
+	IndexDir                      *string
+	EnableIndexing                *bool
+	EnableSearching               *bool
+	EnableAutocomplete            *bool
+	BulkIndexingTimeWindowSeconds *int
+}
+
+func (bs *BleveSettings) SetDefaults() {
+	if bs.IndexDir == nil {
+		bs.IndexDir = NewString(BLEVE_SETTINGS_DEFAULT_INDEX_DIR)
+	}
+
+	if bs.EnableIndexing == nil {
+		bs.EnableIndexing = NewBool(false)
+	}
+
+	if bs.EnableSearching == nil {
+		bs.EnableSearching = NewBool(false)
+	}
+
+	if bs.EnableAutocomplete == nil {
+		bs.EnableAutocomplete = NewBool(false)
+	}
+
+	if bs.BulkIndexingTimeWindowSeconds == nil {
+		bs.BulkIndexingTimeWindowSeconds = NewInt(BLEVE_SETTINGS_DEFAULT_BULK_INDEXING_TIME_WINDOW_SECONDS)
+	}
+}
+
 type DataRetentionSettings struct {
 	EnableMessageDeletion *bool
 	EnableFileDeletion    *bool
@@ -2704,6 +2737,7 @@ type Config struct {
 	ExperimentalSettings      ExperimentalSettings
 	AnalyticsSettings         AnalyticsSettings
 	ElasticsearchSettings     ElasticsearchSettings
+	BleveSettings             BleveSettings
 	DataRetentionSettings     DataRetentionSettings
 	MessageExportSettings     MessageExportSettings
 	JobSettings               JobSettings
@@ -2785,6 +2819,7 @@ func (o *Config) SetDefaults() {
 	o.ComplianceSettings.SetDefaults()
 	o.LocalizationSettings.SetDefaults()
 	o.ElasticsearchSettings.SetDefaults()
+	o.BleveSettings.SetDefaults()
 	o.NativeAppSettings.SetDefaults()
 	o.DataRetentionSettings.SetDefaults()
 	o.RateLimitSettings.SetDefaults()
@@ -2851,6 +2886,10 @@ func (o *Config) IsValid() *AppError {
 		return err
 	}
 
+	if err := o.BleveSettings.isValid(); err != nil {
+		return err
+	}
+
 	if err := o.DataRetentionSettings.isValid(); err != nil {
 		return err
 	}
@@ -3234,6 +3273,26 @@ func (s *ElasticsearchSettings) isValid() *AppError {
 	return nil
 }
 
+func (bs *BleveSettings) isValid() *AppError {
+	if *bs.EnableIndexing {
+		if len(*bs.IndexDir) == 0 {
+			return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.filename.app_error", nil, "", http.StatusBadRequest)
+		}
+	} else {
+		if *bs.EnableSearching {
+			return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.enable_searching.app_error", nil, "", http.StatusBadRequest)
+		}
+		if *bs.EnableAutocomplete {
+			return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.enable_autocomplete.app_error", nil, "", http.StatusBadRequest)
+		}
+	}
+	if *bs.BulkIndexingTimeWindowSeconds < 1 {
+		return NewAppError("Config.IsValid", "model.config.is_valid.bleve_search.bulk_indexing_time_window_seconds.app_error", nil, "", http.StatusBadRequest)
+	}
+
+	return nil
+}
+
 func (s *DataRetentionSettings) isValid() *AppError {
 	if *s.MessageRetentionDays <= 0 {
 		return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.message_retention_days_too_low.app_error", nil, "", http.StatusBadRequest)

model/job.go

@@ -15,6 +15,7 @@ const (
 	JOB_TYPE_MESSAGE_EXPORT                 = "message_export"
 	JOB_TYPE_ELASTICSEARCH_POST_INDEXING    = "elasticsearch_post_indexing"
 	JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION = "elasticsearch_post_aggregation"
+	JOB_TYPE_BLEVE_POST_INDEXING            = "bleve_post_indexing"
 	JOB_TYPE_LDAP_SYNC                      = "ldap_sync"
 	JOB_TYPE_MIGRATIONS                     = "migrations"
 	JOB_TYPE_PLUGINS                        = "plugins"
@@ -53,6 +54,7 @@ func (j *Job) IsValid() *AppError {
 	case JOB_TYPE_DATA_RETENTION:
 	case JOB_TYPE_ELASTICSEARCH_POST_INDEXING:
 	case JOB_TYPE_ELASTICSEARCH_POST_AGGREGATION:
+	case JOB_TYPE_BLEVE_POST_INDEXING:
 	case JOB_TYPE_LDAP_SYNC:
 	case JOB_TYPE_MESSAGE_EXPORT:
 	case JOB_TYPE_MIGRATIONS: