diff --git a/server/enterprise/elasticsearch/common/common.go b/server/enterprise/elasticsearch/common/common.go new file mode 100644 index 0000000000..77e9f0b400 --- /dev/null +++ b/server/enterprise/elasticsearch/common/common.go @@ -0,0 +1,375 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "encoding/xml" + "fmt" + "io" + "net/url" + "regexp" + "runtime" + "strings" + "time" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" +) + +const ( + MaxLineLength = 10000 + + URLRegexpRE = `(\b|^)(?:https?:\/\/)?[a-zA-Z0-9-.]+\.[a-z]+(\s|\)?[a-zA-Z0-9\-._~:/?#\[\]@!$&'\(\)*\+,;=]*)(\b|$)` + URLMarkdownLinkRE = `(\[[^\]]+\]\([a-zA-Z0-9\-._~:/?#\[\]@!$&'\(\)*\+,;=]+\))` + EmailRE = `^[^\s"]+@[^\s"]+$` + + IndexBasePosts = "posts" + IndexBasePosts_MONTH = IndexBasePosts + "month" + IndexBaseChannels = "channels" + IndexBaseUsers = "users" + IndexBaseFiles = "files" + + // At the moment, this number is hardcoded. If needed, we can expose + // this to the config. + BulkFlushInterval = 5 * time.Second +) + +var ( + urlRe = regexp.MustCompile(URLRegexpRE) + markdownLinkRe = regexp.MustCompile(URLMarkdownLinkRE) +) + +type ESPost struct { + Id string `json:"id"` + TeamId string `json:"team_id"` + ChannelId string `json:"channel_id"` + UserId string `json:"user_id"` + CreateAt int64 `json:"create_at"` + Message string `json:"message"` + Type string `json:"type"` + Hashtags []string `json:"hashtags"` + Attachments string `json:"attachments"` + URLs []string `json:"urls"` +} + +type ESFile struct { + Id string `json:"id"` + CreatorId string `json:"creator_id"` + ChannelId string `json:"channel_id"` + PostId string `json:"post_id"` + CreateAt int64 `json:"create_at"` + Content string `json:"content"` + Extension string `json:"extension"` + Name string `json:"name"` +} + +type ESChannel struct { + Id string `json:"id"` + Type model.ChannelType `json:"type"` + UserIDs []string `json:"user_ids"` + TeamId string `json:"team_id"` + TeamMemberIDs []string `json:"team_member_ids"` + NameSuggest []string `json:"name_suggestions"` +} + +type ESUser struct { + Id string `json:"id"` + SuggestionsWithFullname []string `json:"suggestions_with_fullname"` + SuggestionsWithoutFullname []string `json:"suggestions_without_fullname"` + DeleteAt int64 `json:"delete_at"` + Roles []string `json:"roles"` + TeamsIds []string `json:"team_id"` + ChannelsIds []string `json:"channel_id"` +} + +func ESPostFromPost(post *model.Post, teamId string) (*ESPost, error) { + p := &model.PostForIndexing{ + TeamId: teamId, + } + err := post.ShallowCopy(&p.Post) + if err != nil { + return nil, err + } + return ESPostFromPostForIndexing(p), nil +} + +func ESPostFromPostForIndexing(post *model.PostForIndexing) *ESPost { + searchPost := ESPost{ + Id: post.Id, + TeamId: post.TeamId, + ChannelId: post.ChannelId, + UserId: post.UserId, + CreateAt: post.CreateAt, + Message: post.Message, + Type: post.Type, + Hashtags: strings.Fields(post.Hashtags), + } + + var searchAttachments []string + + if attachments := post.GetProp("attachments"); attachments != nil { + attachmentsInterfaceArray, ok := attachments.([]any) + if ok { + for _, attachment := range attachmentsInterfaceArray { + if attachment != nil { + if attachmentText := attachment.(map[string]any)["text"]; attachmentText != nil { + searchAttachments = append(searchAttachments, attachmentText.(string)) + } + } + } + } + + attachmentsArray, ok := attachments.([]*model.SlackAttachment) + if ok { + for _, attachment := range attachmentsArray { + if attachment != nil { + searchAttachments = append(searchAttachments, attachment.Text) + } + } + } + } + + searchPost.Attachments = strings.Join(searchAttachments, " ") + + urls := extractURLsFromMessage(post.Message) + if len(urls) > 0 { + searchPost.URLs = urls + } + + if searchPost.Type == "" { + searchPost.Type = "default" + } + + return &searchPost +} + +func extractURLsFromMessage(message string) []string { + message = markdownLinkRe.ReplaceAllString(message, "") + urls := urlRe.FindAllString(message, -1) + + filteredURLs := make([]string, 0) + for _, u := range urls { + u = strings.TrimSpace(u) + urlToCheck := u + if !strings.HasPrefix(u, "http://") && !strings.HasPrefix(u, "https://") { + urlToCheck = "http://" + u + } + parsedURL, err := url.Parse(urlToCheck) + if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" { + continue + } + filteredURLs = append(filteredURLs, u) + } + + return filteredURLs +} + +func splitFilenameWords(name string) string { + result := name + result = strings.ReplaceAll(result, "-", " ") + result = strings.ReplaceAll(result, ".", " ") + return result +} + +func ESFileFromFileInfo(file *model.FileInfo, channelId string) *ESFile { + return &ESFile{ + Id: file.Id, + CreatorId: file.CreatorId, + ChannelId: channelId, + PostId: file.PostId, + CreateAt: file.CreateAt, + Content: file.Content, + Extension: file.Extension, + Name: file.Name + " " + splitFilenameWords(file.Name), + } +} + +func ESFileFromFileForIndexing(file *model.FileForIndexing) *ESFile { + return &ESFile{ + Id: file.Id, + CreatorId: file.CreatorId, + ChannelId: file.ChannelId, + PostId: file.PostId, + CreateAt: file.CreateAt, + Content: file.Content, + Extension: file.Extension, + Name: file.Name + " " + splitFilenameWords(file.Name), + } +} + +func ESChannelFromChannel(channel *model.Channel, userIDs, teamMemberIDs []string) *ESChannel { + displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ") + nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"}) + + return &ESChannel{ + Id: channel.Id, + Type: channel.Type, + UserIDs: userIDs, + TeamId: channel.TeamId, + TeamMemberIDs: teamMemberIDs, + NameSuggest: append(displayNameInputs, nameInputs...), + } +} + +func ESUserFromUserAndTeams(user *model.User, teamsIds, channelsIds []string) *ESUser { + usernameSuggestions := searchengine.GetSuggestionInputsSplitByMultiple(user.Username, []string{".", "-", "_"}) + + fullnameStrings := []string{} + if user.FirstName != "" { + fullnameStrings = append(fullnameStrings, user.FirstName) + } + if user.LastName != "" { + fullnameStrings = append(fullnameStrings, user.LastName) + } + + fullnameSuggestions := []string{} + if len(fullnameStrings) > 0 { + fullname := strings.Join(fullnameStrings, " ") + fullnameSuggestions = searchengine.GetSuggestionInputsSplitBy(fullname, " ") + } + + nicknameSuggestions := []string{} + if user.Nickname != "" { + nicknameSuggestions = searchengine.GetSuggestionInputsSplitBy(user.Nickname, " ") + } + + usernameAndNicknameSuggestions := append(usernameSuggestions, nicknameSuggestions...) + + return &ESUser{ + Id: user.Id, + SuggestionsWithFullname: append(usernameAndNicknameSuggestions, fullnameSuggestions...), + SuggestionsWithoutFullname: usernameAndNicknameSuggestions, + DeleteAt: user.DeleteAt, + Roles: user.GetRoles(), + TeamsIds: teamsIds, + ChannelsIds: channelsIds, + } +} + +func ESUserFromUserForIndexing(userForIndexing *model.UserForIndexing) *ESUser { + user := &model.User{ + Id: userForIndexing.Id, + Username: userForIndexing.Username, + Nickname: userForIndexing.Nickname, + FirstName: userForIndexing.FirstName, + Roles: userForIndexing.Roles, + LastName: userForIndexing.LastName, + CreateAt: userForIndexing.CreateAt, + DeleteAt: userForIndexing.DeleteAt, + } + + return ESUserFromUserAndTeams(user, userForIndexing.TeamsIds, userForIndexing.ChannelsIds) +} + +func BuildPostIndexName(aggregateAfterDays int, unaggregatedBase string, aggregatedBase string, now time.Time, createAt int64) string { + postTime := time.Unix(createAt/1000, 0) + aggregateCutoffTime := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local).AddDate(0, 0, -aggregateAfterDays+1) + + if postTime.Before(aggregateCutoffTime) { + return fmt.Sprintf("%v_%d_%02d", aggregatedBase, postTime.Year(), postTime.Month()) + } + + return fmt.Sprintf("%v_%d_%02d_%02d", unaggregatedBase, postTime.Year(), postTime.Month(), postTime.Day()) +} + +func NumIndexWorkers() int { + const maxCPU = 4 + if runtime.NumCPU() > maxCPU { + return maxCPU + } + return runtime.NumCPU() +} + +// maxCertFileSizeBytes is an internal constant +// used to limit file size of ClientCert, ClientKey and CA. +const maxCertFileSizeBytes = 1_000_000 // 1MB + +func ReadFileSafely(fb filestore.FileBackend, path string) ([]byte, error) { + rd, err := fb.Reader(path) + if err != nil { + return nil, err + } + defer rd.Close() + + type resp struct { + buf []byte + err error + } + ch := make(chan resp) + + go func() { + buf, err := io.ReadAll(io.LimitReader(rd, maxCertFileSizeBytes)) + ch <- resp{buf, err} + }() + + select { + case got := <-ch: + return got.buf, got.err + case <-time.After(10 * time.Second): // Adding a timeout for the file read. + return nil, fmt.Errorf("timed out while reading file: %s", path) + } +} + +func GetMatchesForHit(highlights map[string][]string) ([]string, error) { + matchMap := make(map[string]bool) + + parseMatches := func(snippets []string) error { + // Highlighted matches are returned as an array of snippets of the post where + // each snippet has the highlighted text surrounded by html tags + for _, snippet := range snippets { + decoder := xml.NewDecoder(strings.NewReader(snippet)) + inMatch := false + + for { + token, err := decoder.Token() + if err == io.EOF { + break + } else if err != nil { + return err + } + + switch typed := token.(type) { + case xml.StartElement: + if typed.Name.Local == "em" { + inMatch = true + } + case xml.EndElement: + if typed.Name.Local == "em" { + inMatch = false + } + case xml.CharData: + if inMatch && len(typed) != 0 { + match := string(typed) + match = strings.Trim(match, "_*~") + + matchMap[match] = true + } + } + } + } + + return nil + } + + if err := parseMatches(highlights["message"]); err != nil { + return nil, err + } + if err := parseMatches(highlights["attachments"]); err != nil { + return nil, err + } + if err := parseMatches(highlights["urls"]); err != nil { + return nil, err + } + if err := parseMatches(highlights["hashtags"]); err != nil { + return nil, err + } + + var matches []string + for match := range matchMap { + matches = append(matches, match) + } + + return matches, nil +} diff --git a/server/enterprise/elasticsearch/common/common_test.go b/server/enterprise/elasticsearch/common/common_test.go new file mode 100644 index 0000000000..763785c575 --- /dev/null +++ b/server/enterprise/elasticsearch/common/common_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost/server/public/model" +) + +func TestElasticsearchBuildPostIndexName(t *testing.T) { + now := time.Date(2017, 8, 14, 15, 16, 17, 123, time.Local) + + sixDaysAgo := time.Date(2017, 8, 9, 12, 11, 10, 987, time.Local) + sevenDaysAgo := time.Date(2017, 8, 8, 11, 10, 9, 876, time.Local) + eightDaysAgo := time.Date(2017, 8, 7, 6, 5, 4, 321, time.Local) + + sixMillis := sixDaysAgo.UnixNano() / int64(time.Millisecond) + sevenMillis := sevenDaysAgo.UnixNano() / int64(time.Millisecond) + eightMillis := eightDaysAgo.UnixNano() / int64(time.Millisecond) + + aggregationCutoff := 7 // Aggregate monthly after 7 days. + + sixName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, sixMillis) + sevenName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, sevenMillis) + eightName := BuildPostIndexName(aggregationCutoff, IndexBasePosts, IndexBasePosts_MONTH, now, eightMillis) + + assert.Equal(t, sixName, "posts_2017_08_09") + assert.Equal(t, sevenName, "posts_2017_08_08") + assert.Equal(t, eightName, "postsmonth_2017_08") +} + +func TestESPostFromPostForIndexing(t *testing.T) { + // Create one with attachments in 'any' form. + + post1 := model.PostForIndexing{ + TeamId: model.NewId(), + ParentCreateAt: nil, + Post: model.Post{ + Id: model.NewId(), + ChannelId: model.NewId(), + UserId: model.NewId(), + CreateAt: model.GetMillis(), + Message: "message", + Type: "", + Hashtags: "", + Props: map[string]any{ + "attachments": []any{ + map[string]any{ + "text": "text 1", + }, + }, + }, + }, + } + + espost1 := ESPostFromPostForIndexing(&post1) + + assert.Equal(t, post1.Id, espost1.Id) + assert.Equal(t, post1.TeamId, espost1.TeamId) + assert.Equal(t, post1.ChannelId, espost1.ChannelId) + assert.Equal(t, post1.UserId, espost1.UserId) + assert.Equal(t, post1.CreateAt, espost1.CreateAt) + assert.Equal(t, post1.Message, espost1.Message) + assert.Equal(t, "default", espost1.Type) + assert.Empty(t, espost1.Hashtags) + assert.Equal(t, "text 1", espost1.Attachments) + + // Create one with attachments in model.SlackAttachment form. + + post2 := model.PostForIndexing{ + TeamId: model.NewId(), + ParentCreateAt: nil, + Post: model.Post{ + Id: model.NewId(), + ChannelId: model.NewId(), + UserId: model.NewId(), + CreateAt: model.GetMillis(), + Message: "message", + Type: "slack_attachment", + Hashtags: "#buh #boh", + Props: map[string]any{ + "attachments": []*model.SlackAttachment{ + { + Text: "text 2", + }, + }, + }, + }, + } + + espost2 := ESPostFromPostForIndexing(&post2) + + assert.Equal(t, post2.Id, espost2.Id) + assert.Equal(t, post2.TeamId, espost2.TeamId) + assert.Equal(t, post2.ChannelId, espost2.ChannelId) + assert.Equal(t, post2.UserId, espost2.UserId) + assert.Equal(t, post2.CreateAt, espost2.CreateAt) + assert.Equal(t, post2.Message, espost2.Message) + assert.Equal(t, "slack_attachment", espost2.Type) + assert.Len(t, espost2.Hashtags, 2) + assert.Equal(t, "text 2", espost2.Attachments) +} + +func TestGetMatchesForHit(t *testing.T) { + snippets := map[string][]string{ + "message": { + "Apples and oranges and apple and orange", + "Johnny Appleseed", + "That doesn't apply to me, and it doesn't apply to you.", + }, + "hashtags": { + "This is an #hashtag", + }, + } + expected := []string{ + "Apples", + "apple", + "Appleseed", + "apply", + "#hashtag", + } + + actual, err := GetMatchesForHit(snippets) + require.NoError(t, err) + require.ElementsMatch(t, expected, actual) +} diff --git a/server/enterprise/elasticsearch/common/indexing_job.go b/server/enterprise/elasticsearch/common/indexing_job.go new file mode 100644 index 0000000000..cb0cc48105 --- /dev/null +++ b/server/enterprise/elasticsearch/common/indexing_job.go @@ -0,0 +1,841 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "strconv" + "sync" + "time" + + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/store" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/v8/channels/jobs" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" +) + +const ( + timeBetweenBatches = 100 * time.Millisecond + + estimatedPostCount = 10000000 + estimatedChannelCount = 100000 + estimatedFilesCount = 100000 + estimatedUserCount = 10000 +) + +const ( + indexOp = "index" + deleteOp = "delete" +) + +func NewIndexerWorker(name string, + jobServer *jobs.JobServer, + logger mlog.LoggerIFace, + fileBackend filestore.FileBackend, + licenseFn func() *model.License, + createBulkProcessorFn func() error, + addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error, + closeBulkProcessorFn func() error, +) *IndexerWorker { + return &IndexerWorker{ + name: name, + stoppedCh: make(chan bool, 1), + jobs: make(chan model.Job), + jobServer: jobServer, + logger: logger, + fileBackend: fileBackend, + license: licenseFn, + stopped: true, + createBulkProcessor: createBulkProcessorFn, + addItemToBulkProcessor: addItemToBulkProcessorFn, + closeBulkProcessor: closeBulkProcessorFn, + } +} + +type IndexerWorker struct { + name string + // stateMut protects stopCh and stopped and helps enforce + // ordering in case subsequent Run or Stop calls are made. + stateMut sync.Mutex + stopCh chan struct{} + stopped bool + stoppedCh chan bool + jobs chan model.Job + jobServer *jobs.JobServer + logger mlog.LoggerIFace + fileBackend filestore.FileBackend + + license func() *model.License + + createBulkProcessor func() error + closeBulkProcessor func() error + addItemToBulkProcessor func(indexName, indexOp, docID string, body io.ReadSeeker) error +} + +type IndexingProgress struct { + Now time.Time + StartAtTime int64 + EndAtTime int64 + LastEntityTime int64 + + TotalPostsCount int64 + DonePostsCount int64 + DonePosts bool + LastPostID string + + TotalFilesCount int64 + DoneFilesCount int64 + DoneFiles bool + LastFileID string + + TotalChannelsCount int64 + DoneChannelsCount int64 + DoneChannels bool + LastChannelID string + + TotalUsersCount int64 + DoneUsersCount int64 + DoneUsers bool + LastUserID string +} + +func (ip *IndexingProgress) CurrentProgress() int64 { + current := ip.DonePostsCount + ip.DoneChannelsCount + ip.DoneUsersCount + ip.DoneFilesCount + total := ip.TotalPostsCount + ip.TotalChannelsCount + ip.TotalFilesCount + ip.TotalUsersCount + return current * 100 / total +} + +func (ip *IndexingProgress) IsDone(job *model.Job) bool { + // an entity's progress is completed if it was specified not to be indexed, or if it's completed indexing. + + donePosts := job.Data["index_posts"] == "false" || ip.DonePosts + doneChannels := job.Data["index_channels"] == "false" || ip.DoneChannels + doneUsers := job.Data["index_users"] == "false" || ip.DoneUsers + doneFiles := job.Data["index_files"] == "false" || ip.DoneFiles + + return donePosts && doneChannels && doneUsers && doneFiles +} + +func (worker *IndexerWorker) Run() { + worker.stateMut.Lock() + // We have to re-assign the stop channel again, because + // it might happen that the job was restarted due to a config change. + if worker.stopped { + worker.stopped = false + worker.stopCh = make(chan struct{}) + } else { + worker.stateMut.Unlock() + return + } + // Run is called from a separate goroutine and doesn't return. + // So we cannot Unlock in a defer clause. + worker.stateMut.Unlock() + + worker.logger.Debug("Worker Started") + + defer func() { + worker.logger.Debug("Worker: Finished") + worker.stoppedCh <- true + }() + + for { + select { + case <-worker.stopCh: + worker.logger.Debug("Worker: Received stop signal") + return + case job := <-worker.jobs: + worker.DoJob(&job) + } + } +} + +func (worker *IndexerWorker) Stop() { + worker.stateMut.Lock() + defer worker.stateMut.Unlock() + + // Set to close, and if already closed before, then return. + if worker.stopped { + return + } + worker.stopped = true + + worker.logger.Debug("Worker Stopping") + close(worker.stopCh) + <-worker.stoppedCh +} + +func (worker *IndexerWorker) JobChannel() chan<- model.Job { + return worker.jobs +} + +func (worker *IndexerWorker) IsEnabled(cfg *model.Config) bool { + if license := worker.license(); license == nil || !*license.Features.Elasticsearch { + return false + } + + if *cfg.ElasticsearchSettings.EnableIndexing { + return true + } + + return false +} + +func (worker *IndexerWorker) initEntitiesToIndex(job *model.Job) { + // Specifying entities to index is optional, and even when specified, all entities need not be specified. + // This function parses the provided job data and sets enabled or disabled value for each entity, + // so that rest of the code can use job.Data as the source of truth to decide if an entity was + // to be indexed or not. + + if job.Data == nil { + job.Data = model.StringMap{} + } + + indexPostsRaw, ok := job.Data["index_posts"] + job.Data["index_posts"] = strconv.FormatBool(!ok || indexPostsRaw == "true") + + indexChannelsRaw, ok := job.Data["index_channels"] + job.Data["index_channels"] = strconv.FormatBool(!ok || indexChannelsRaw == "true") + + indexUsersRaw, ok := job.Data["index_users"] + job.Data["index_users"] = strconv.FormatBool(!ok || indexUsersRaw == "true") + + indexFilesRaw, ok := job.Data["index_files"] + job.Data["index_files"] = strconv.FormatBool(!ok || indexFilesRaw == "true") +} + +func (worker *IndexerWorker) DoJob(job *model.Job) { + logger := worker.logger.With(jobs.JobLoggerFields(job)...) + logger.Debug("Worker: Received a new candidate job.") + defer worker.jobServer.HandleJobPanic(logger, job) + + claimed, appErr := worker.jobServer.ClaimJob(job) + if appErr != nil { + logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr)) + return + } + if !claimed { + return + } + + logger.Info("Worker: Indexing job claimed by worker") + + err := worker.createBulkProcessor() + if err != nil { + worker.logger.Error("Worker: Failed to setup bulk processor", mlog.Err(err)) + return + } + + worker.initEntitiesToIndex(job) + progress, err := initProgress(logger, worker.jobServer, job) + if err != nil { + return + } + + var cancelContext request.CTX = request.EmptyContext(worker.logger) + cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background()) + cancelWatcherChan := make(chan struct{}, 1) + cancelContext = cancelContext.WithContext(cancelCtx) + go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan) + + defer func() { + cancelCancelWatcher() + err := worker.closeBulkProcessor() + if err != nil { + logger.Warn("Error while closing the bulk indexer", mlog.Err(err), mlog.String("job_id", job.Id)) + } + }() + + for { + select { + case <-cancelWatcherChan: + logger.Info("Worker: Indexing job has been canceled via CancellationWatcher") + if err := worker.jobServer.SetJobCanceled(job); err != nil { + logger.Error("Worker: Failed to mark job as cancelled", mlog.Err(err)) + } + return + + case <-worker.stopCh: + logger.Info("Worker: Indexing has been canceled via Worker Stop. Setting the job back to pending.") + if err := worker.jobServer.SetJobPending(job); err != nil { + logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err)) + } + return + + case <-time.After(timeBetweenBatches): + var err *model.AppError + if progress, err = worker.IndexBatch(logger, progress, job); err != nil { + logger.Error("Worker: Failed to index batch for job", mlog.Err(err)) + if err2 := worker.jobServer.SetJobError(job, err); err2 != nil { + logger.Error("Worker: Failed to set job error", mlog.Err(err2), mlog.NamedErr("set_error", err)) + } + return + } + + // Storing the batch progress in metadata. + if job.Data == nil { + job.Data = make(model.StringMap) + } + + job.Data["done_posts_count"] = strconv.FormatInt(progress.DonePostsCount, 10) + job.Data["done_channels_count"] = strconv.FormatInt(progress.DoneChannelsCount, 10) + job.Data["done_users_count"] = strconv.FormatInt(progress.DoneUsersCount, 10) + job.Data["done_files_count"] = strconv.FormatInt(progress.DoneFilesCount, 10) + + job.Data["start_time"] = strconv.FormatInt(progress.LastEntityTime, 10) + job.Data["start_post_id"] = progress.LastPostID + job.Data["start_channel_id"] = progress.LastChannelID + job.Data["start_user_id"] = progress.LastUserID + job.Data["start_file_id"] = progress.LastFileID + job.Data["original_start_time"] = strconv.FormatInt(progress.StartAtTime, 10) + job.Data["end_time"] = strconv.FormatInt(progress.EndAtTime, 10) + + if err := worker.jobServer.SetJobProgress(job, progress.CurrentProgress()); err != nil { + logger.Error("Worker: Failed to set progress for job", mlog.Err(err)) + if err2 := worker.jobServer.SetJobError(job, err); err2 != nil { + logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err)) + } + return + } + + if progress.IsDone(job) { + if err := worker.jobServer.SetJobSuccess(job); err != nil { + logger.Error("Worker: Failed to set success for job", mlog.Err(err)) + if err2 := worker.jobServer.SetJobError(job, err); err2 != nil { + logger.Error("Worker: Failed to set error for job", mlog.Err(err2), mlog.NamedErr("set_error", err)) + } + } + logger.Info("Worker: Indexing job finished successfully") + return + } + } + } +} + +func (worker *IndexerWorker) IndexBatch(logger mlog.LoggerIFace, progress IndexingProgress, job *model.Job) (IndexingProgress, *model.AppError) { + // an entity's batch is processed if it wasn't specified to be skipped, or if its completed indexing. + + if job.Data["index_posts"] != "false" && !progress.DonePosts { + worker.logger.Debug("Worker: indexing post batch...") + return worker.IndexPostsBatch(logger, progress) + } + + if job.Data["index_channels"] != "false" && !progress.DoneChannels { + worker.logger.Debug("Worker: indexing channels batch...") + return IndexChannelsBatch(logger, worker.jobServer.Config(), worker.jobServer.Store, worker.addItemToBulkProcessor, progress) + } + + if job.Data["index_users"] != "false" && !progress.DoneUsers { + worker.logger.Debug("Worker: indexing users batch...") + return worker.IndexUsersBatch(logger, progress) + } + + if job.Data["index_files"] != "false" && !progress.DoneFiles { + worker.logger.Debug("Worker: indexing files batch...") + return worker.IndexFilesBatch(logger, progress) + } + + return progress, model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.index_batch.nothing_left_to_index.error", nil, "", http.StatusInternalServerError) +} + +func (worker *IndexerWorker) IndexPostsBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) { + var posts []*model.PostForIndexing + + tries := 0 + for posts == nil { + var err error + posts, err = worker.jobServer.Store.Post().GetPostsBatchForIndexing(progress.LastEntityTime, progress.LastPostID, *worker.jobServer.Config().ElasticsearchSettings.BatchSize) + if err != nil { + if tries >= 10 { + return progress, model.NewAppError("IndexPostsBatch", "ent.elasticsearch.post.get_posts_batch_for_indexing.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + logger.Warn("Failed to get posts batch for indexing. Retrying.", mlog.Err(err)) + + // Wait a bit before trying again. + time.Sleep(15 * time.Second) + } + + tries++ + } + + // Handle zero messages. + if len(posts) == 0 { + progress.DonePosts = true + progress.LastEntityTime = progress.StartAtTime + return progress, nil + } + + lastPost, err := worker.BulkIndexPosts(posts, progress) + if err != nil { + return progress, err + } + + // Our exit condition is when the last post's createAt reaches the initial endAtTime + // set during job creation. + if progress.EndAtTime <= lastPost.CreateAt { + progress.DonePosts = true + // We reset the last entity time to the beginning to begin + // indexing of the next set of entities (users, channels etc.) + progress.LastEntityTime = progress.StartAtTime + } else { + progress.LastEntityTime = lastPost.CreateAt + } + + progress.LastPostID = lastPost.Id + progress.DonePostsCount += int64(len(posts)) + + return progress, nil +} + +func (worker *IndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, progress IndexingProgress) (*model.Post, *model.AppError) { + for _, post := range posts { + indexName := BuildPostIndexName(*worker.jobServer.Config().ElasticsearchSettings.AggregatePostsAfterDays, + *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix+IndexBasePosts, + *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix+IndexBasePosts_MONTH, progress.Now, post.CreateAt) + + if post.DeleteAt == 0 { + searchPost := ESPostFromPostForIndexing(post) + + data, err := json.Marshal(searchPost) + if err != nil { + worker.logger.Warn("Failed to marshal JSON, skipping this post.", mlog.String("post_id", post.Id)) + continue + } + + err = worker.addItemToBulkProcessor(indexName, indexOp, searchPost.Id, bytes.NewReader(data)) + if err != nil { + worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName)) + } + } else { + err := worker.addItemToBulkProcessor(indexName, deleteOp, post.Id, nil) + if err != nil { + worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName)) + } + } + } + + return &posts[len(posts)-1].Post, nil +} + +func (worker *IndexerWorker) IndexFilesBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) { + var files []*model.FileForIndexing + + tries := 0 + for files == nil { + var err error + files, err = worker.jobServer.Store.FileInfo().GetFilesBatchForIndexing(progress.LastEntityTime, progress.LastFileID, true, *worker.jobServer.Config().ElasticsearchSettings.BatchSize) + if err != nil { + if tries >= 10 { + return progress, model.NewAppError("IndexFilesBatch", "ent.elasticsearch.post.get_files_batch_for_indexing.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + logger.Warn("Failed to get files batch for indexing. Retrying.", mlog.Err(err)) + + // Wait a bit before trying again. + time.Sleep(15 * time.Second) + } + + tries++ + } + + if len(files) == 0 { + progress.DoneFiles = true + progress.LastEntityTime = progress.StartAtTime + return progress, nil + } + + lastFile, err := worker.BulkIndexFiles(files, progress) + if err != nil { + return progress, err + } + + // Our exit condition is when the last file's createAt reaches the initial endAtTime + // set during job creation. + if progress.EndAtTime <= lastFile.CreateAt { + progress.DoneFiles = true + // We reset the last entity time to the beginning to begin + // indexing of the next set of entities (users, channels etc.) + progress.LastEntityTime = progress.StartAtTime + } else { + progress.LastEntityTime = lastFile.CreateAt + } + + progress.LastFileID = lastFile.Id + progress.DoneFilesCount += int64(len(files)) + + return progress, nil +} + +func (worker *IndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (*model.FileInfo, *model.AppError) { + for _, file := range files { + indexName := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + IndexBaseFiles + + if file.ShouldIndex() { + searchFile := ESFileFromFileForIndexing(file) + + data, err := json.Marshal(searchFile) + if err != nil { + worker.logger.Warn("Failed to marshal JSON") + continue + } + + err = worker.addItemToBulkProcessor(indexName, indexOp, searchFile.Id, bytes.NewReader(data)) + if err != nil { + worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName)) + } + } else { + err := worker.addItemToBulkProcessor(indexName, deleteOp, file.Id, nil) + if err != nil { + worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName)) + } + } + } + + return &files[len(files)-1].FileInfo, nil +} + +func IndexChannelsBatch(logger mlog.LoggerIFace, config *model.Config, store store.Store, addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error, progress IndexingProgress) (IndexingProgress, *model.AppError) { + var channels []*model.Channel + + tries := 0 + for channels == nil { + var err error + channels, err = store.Channel().GetChannelsBatchForIndexing(progress.LastEntityTime, progress.LastChannelID, *config.ElasticsearchSettings.BatchSize) + if err != nil { + if tries >= 10 { + return progress, model.NewAppError("IndexerWorker.IndexChannelsBatch", "ent.elasticsearch.index_channels_batch.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + logger.Warn("Failed to get channels batch for indexing. Retrying.", mlog.Err(err)) + + // Wait a bit before trying again. + time.Sleep(15 * time.Second) + } + tries++ + } + + if len(channels) == 0 { + progress.DoneChannels = true + progress.LastEntityTime = progress.StartAtTime + return progress, nil + } + + lastChannel, err := BulkIndexChannels(config, store, logger, addItemToBulkProcessorFn, channels, progress) + if err != nil { + return progress, err + } + + // Our exit condition is when the last channel's createAt reaches the initial endAtTime + // set during job creation. + if progress.EndAtTime <= lastChannel.CreateAt { + progress.DoneChannels = true + // We reset the last entity time to the beginning to begin + // indexing of the next set of entities (users etc.) + progress.LastEntityTime = progress.StartAtTime + } else { + progress.LastEntityTime = lastChannel.CreateAt + } + + progress.LastChannelID = lastChannel.Id + progress.DoneChannelsCount += int64(len(channels)) + + return progress, nil +} + +func BulkIndexChannels(config *model.Config, + store store.Store, + logger mlog.LoggerIFace, + addItemToBulkProcessorFn func(indexName string, indexOp string, docID string, body io.ReadSeeker) error, + channels []*model.Channel, + progress IndexingProgress) (*model.Channel, *model.AppError) { + for _, channel := range channels { + indexName := *config.ElasticsearchSettings.IndexPrefix + IndexBaseChannels + + if channel.DeleteAt == 0 { + var userIDs []string + var err error + if channel.Type == model.ChannelTypePrivate { + userIDs, err = store.Channel().GetAllChannelMemberIdsByChannelId(channel.Id) + if err != nil { + return nil, model.NewAppError("IndexerWorker.BulkIndexChannels", "ent.elasticsearch.getAllChannelMembers.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + } + + teamMemberIDs, err := store.Channel().GetTeamMembersForChannel(channel.Id) + if err != nil { + return nil, model.NewAppError("IndexerWorker.BulkIndexChannels", "ent.elasticsearch.getAllTeamMembers.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + searchChannel := ESChannelFromChannel(channel, userIDs, teamMemberIDs) + + data, err := json.Marshal(searchChannel) + if err != nil { + logger.Warn("Failed to marshal JSON") + continue + } + + err = addItemToBulkProcessorFn(indexName, indexOp, searchChannel.Id, bytes.NewReader(data)) + if err != nil { + logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName)) + } + } else { + err := addItemToBulkProcessorFn(indexName, deleteOp, channel.Id, nil) + if err != nil { + logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName)) + } + } + } + + return channels[len(channels)-1], nil +} + +func (worker *IndexerWorker) IndexUsersBatch(logger mlog.LoggerIFace, progress IndexingProgress) (IndexingProgress, *model.AppError) { + var users []*model.UserForIndexing + + tries := 0 + for users == nil { + if usersBatch, err := worker.jobServer.Store.User().GetUsersBatchForIndexing(progress.LastEntityTime, progress.LastUserID, *worker.jobServer.Config().ElasticsearchSettings.BatchSize); err != nil { + if tries >= 10 { + return progress, model.NewAppError("IndexerWorker.IndexUsersBatch", "app.user.get_users_batch_for_indexing.get_users.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + logger.Warn("Failed to get users batch for indexing. Retrying.", mlog.Err(err)) + + // Wait a bit before trying again. + time.Sleep(15 * time.Second) + } else { + users = usersBatch + } + + tries++ + } + + if len(users) == 0 { + progress.DoneUsers = true + progress.LastEntityTime = progress.StartAtTime + return progress, nil + } + + lastUser, err := worker.BulkIndexUsers(users, progress) + if err != nil { + return progress, err + } + + // Our exit condition is when the last user's createAt reaches the initial endAtTime + // set during job creation. + if progress.EndAtTime <= lastUser.CreateAt { + progress.DoneUsers = true + // We reset the last entity time to the beginning to begin + // indexing of the next set of entities in case they get added in the future. + progress.LastEntityTime = progress.StartAtTime + } else { + progress.LastEntityTime = lastUser.CreateAt + } + progress.LastUserID = lastUser.Id + progress.DoneUsersCount += int64(len(users)) + + return progress, nil +} + +func (worker *IndexerWorker) BulkIndexUsers(users []*model.UserForIndexing, progress IndexingProgress) (*model.UserForIndexing, *model.AppError) { + for _, user := range users { + indexName := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + IndexBaseUsers + + searchUser := ESUserFromUserForIndexing(user) + + data, err := json.Marshal(searchUser) + if err != nil { + worker.logger.Warn("Failed to marshal JSON") + continue + } + + err = worker.addItemToBulkProcessor(indexName, indexOp, searchUser.Id, bytes.NewReader(data)) + if err != nil { + worker.logger.Warn("Failed to add item to bulk processor", mlog.String("indexName", indexName)) + } + } + + return users[len(users)-1], nil +} + +func initProgress(logger mlog.LoggerIFace, jobServer *jobs.JobServer, job *model.Job) (IndexingProgress, error) { + progress := IndexingProgress{ + Now: time.Now(), + DonePosts: false, + DoneChannels: false, + DoneUsers: false, + DoneFiles: false, + StartAtTime: 0, + EndAtTime: model.GetMillis(), + } + + progress, err := parseStartTime(logger, jobServer, progress, job) + if err != nil { + return progress, err + } + + progress, err = parseEndTime(logger, jobServer, progress, job) + if err != nil { + return progress, err + } + + progress = parseDoneCount(logger, progress, job) + progress = setStartEntityIDs(progress, job) + progress = setEntityCount(logger, jobServer, progress, job) + + return progress, nil +} + +func parseStartTime(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) (IndexingProgress, error) { + // Extract the start time, if it is set. + if startString, ok := job.Data["start_time"]; ok { + startInt, err := strconv.ParseInt(startString, 10, 64) + if err != nil { + logger.Error("Worker: Failed to parse start_time for job", mlog.String("start_time", startString), mlog.Err(err)) + appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.parse_start_time.error", nil, "", http.StatusInternalServerError).Wrap(err) + if err := jobServer.SetJobError(job, appError); err != nil { + logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError)) + } + return progress, err + } + progress.StartAtTime = startInt + } else { + // Set start time to oldest entity (user, channel or post) in the database. + oldestEntityTime, err := jobServer.Store.Post().GetOldestEntityCreationTime() + if err != nil { + logger.Error("Worker: Failed to fetch oldest post for job.", mlog.String("start_time", startString), mlog.Err(err)) + appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.get_oldest_entity.error", nil, "", http.StatusInternalServerError).Wrap(err) + if err := jobServer.SetJobError(job, appError); err != nil { + logger.Error("Worker: Failed to set job error", mlog.Err(err), mlog.NamedErr("set_error", appError)) + } + return progress, err + } + progress.StartAtTime = oldestEntityTime + } + + progress.LastEntityTime = progress.StartAtTime + return progress, nil +} + +func parseEndTime(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) (IndexingProgress, error) { + if endString, ok := job.Data["end_time"]; ok { + endInt, err := strconv.ParseInt(endString, 10, 64) + if err != nil { + logger.Error("Worker: Failed to parse end_time for job", mlog.String("end_time", endString), mlog.Err(err)) + appError := model.NewAppError("IndexerWorker", "ent.elasticsearch.indexer.do_job.parse_end_time.error", nil, "", http.StatusInternalServerError).Wrap(err) + if err := jobServer.SetJobError(job, appError); err != nil { + logger.Error("Worker: Failed to set job errorv", mlog.Err(err), mlog.NamedErr("set_error", appError)) + } + return progress, err + } + progress.EndAtTime = endInt + } + + return progress, nil +} + +func parseDoneCount(logger mlog.LoggerIFace, progress IndexingProgress, job *model.Job) IndexingProgress { + if count, ok := job.Data["done_posts_count"]; ok { + countInt, err := strconv.ParseInt(count, 10, 64) + if err != nil { + logger.Error("Worker: Failed to parse done_posts_count for job", mlog.String("done_posts_count", count), mlog.Err(err)) + } + progress.DonePostsCount = countInt + } + + if count, ok := job.Data["done_channels_count"]; ok { + countInt, err := strconv.ParseInt(count, 10, 64) + if err != nil { + logger.Error("Worker: Failed to parse done_channels_count for job", mlog.String("done_channels_count", count), mlog.Err(err)) + } + progress.DoneChannelsCount = countInt + } + + if count, ok := job.Data["done_users_count"]; ok { + countInt, err := strconv.ParseInt(count, 10, 64) + if err != nil { + logger.Error("Worker: Failed to parse done_users_count for job", mlog.String("done_users_count", count), mlog.Err(err)) + } + progress.DoneUsersCount = countInt + } + + if count, ok := job.Data["done_files_count"]; ok { + countInt, err := strconv.ParseInt(count, 10, 64) + if err != nil { + logger.Error("Worker: Failed to parse done_files_count for job", mlog.String("done_files_count", count), mlog.Err(err)) + } + progress.DoneFilesCount = countInt + } + + return progress +} + +func setStartEntityIDs(progress IndexingProgress, job *model.Job) IndexingProgress { + if id, ok := job.Data["start_post_id"]; ok { + progress.LastPostID = id + } + if id, ok := job.Data["start_channel_id"]; ok { + progress.LastChannelID = id + } + if id, ok := job.Data["start_user_id"]; ok { + progress.LastUserID = id + } + if id, ok := job.Data["start_file_id"]; ok { + progress.LastFileID = id + } + + return progress +} + +func setEntityCount(logger mlog.LoggerIFace, jobServer *jobs.JobServer, progress IndexingProgress, job *model.Job) IndexingProgress { + if job.Data["index_posts"] == "true" { + // Counting all posts may fail or timeout when the posts table is large. If this happens, log a warning, but carry + // on with the indexing job anyway. The only issue is that the progress % reporting will be inaccurate. + if count, err := jobServer.Store.Post().AnalyticsPostCount(&model.PostCountOptions{}); err != nil { + logger.Warn("Worker: Failed to fetch total post count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedPostCount", estimatedPostCount), mlog.Err(err)) + progress.TotalPostsCount = estimatedPostCount + } else { + progress.TotalPostsCount = count + } + } + + if job.Data["index_channels"] == "true" { + // Same possible fail as above can happen when counting channels + if count, err := jobServer.Store.Channel().AnalyticsTypeCount("", ""); err != nil { + logger.Warn("Worker: Failed to fetch total channel count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedChannelCount", estimatedChannelCount), mlog.Err(err)) + progress.TotalChannelsCount = estimatedChannelCount + } else { + progress.TotalChannelsCount = count + } + } + + if job.Data["index_users"] == "true" { + // Same possible fail as above can happen when counting users + if count, err := jobServer.Store.User().Count(model.UserCountOptions{ + IncludeBotAccounts: true, // This actually doesn't join with the bots table + // since ExcludeRegularUsers is set to false + }); err != nil { + logger.Warn("Worker: Failed to fetch total user count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedUserCount", estimatedUserCount), mlog.Err(err)) + progress.TotalUsersCount = estimatedUserCount + } else { + progress.TotalUsersCount = count + } + } + + if job.Data["index_files"] == "true" { + // Same possible fail as above can happen when counting files + if count, err := jobServer.Store.FileInfo().CountAll(); err != nil { + logger.Warn("Worker: Failed to fetch total files count for job. An estimated value will be used for progress reporting.", mlog.Int("estimatedFilesCount", estimatedFilesCount), mlog.Err(err)) + progress.TotalFilesCount = estimatedFilesCount + } else { + progress.TotalFilesCount = count + } + } + + return progress +} diff --git a/server/enterprise/elasticsearch/common/logger.go b/server/enterprise/elasticsearch/common/logger.go new file mode 100644 index 0000000000..ee36375ad2 --- /dev/null +++ b/server/enterprise/elasticsearch/common/logger.go @@ -0,0 +1,89 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "fmt" + "net/http" + "time" + + "github.com/mattermost/logr/v2" + "github.com/mattermost/mattermost/server/public/shared/mlog" +) + +func NewLogger(backend string, mlogger mlog.LoggerIFace, trace bool) *Logger { + return &Logger{ + backend: backend, + logger: mlogger, + trace: trace, + } +} + +// Logger is a target to pass to the Logger instance of the search backend. +type Logger struct { + backend string + trace bool + logger mlog.LoggerIFace +} + +// LogRoundTrip prints the information about request and response. +func (l *Logger) LogRoundTrip(req *http.Request, res *http.Response, err error, start time.Time, dur time.Duration) error { + // Set error level. + // 0 = debug, 1=warn, 2=error + var level int + switch { + case err != nil: + level = 2 + case res != nil && res.StatusCode > 0 && res.StatusCode < 500: + level = 0 + case res != nil && res.StatusCode > 499: + level = 2 + default: + level = 2 + } + + // Capture fields. + fields := []mlog.Field{ + mlog.String("method", req.Method), + mlog.Int("status_code", res.StatusCode), + mlog.String("duration", dur.String()), + mlog.String("url", req.URL.String()), + } + + var logFn func(string, ...logr.Field) + switch level { + case 0: + logFn = l.logger.Debug + case 1: + logFn = l.logger.Warn + case 2: + logFn = l.logger.Error + } + logFn(l.backend+" request", fields...) + + return nil +} + +// RequestBodyEnabled makes the client pass request body to logger +func (l *Logger) RequestBodyEnabled() bool { return l.trace } + +// ResponseBodyEnabled makes the client pass response body to logger +func (l *Logger) ResponseBodyEnabled() bool { return false } + +func NewBulkIndexerLogger(mlogger mlog.LoggerIFace, name string) BulkIndexerDebugLogger { + return BulkIndexerDebugLogger{ + logger: mlogger, + name: name, + } +} + +type BulkIndexerDebugLogger struct { + logger mlog.LoggerIFace + name string +} + +func (bl BulkIndexerDebugLogger) Printf(str string, params ...any) { + line := fmt.Sprintf(str, params...) + bl.logger.Debug(line, mlog.String("workername", bl.name)) +} diff --git a/server/enterprise/elasticsearch/common/templates.go b/server/enterprise/elasticsearch/common/templates.go new file mode 100644 index 0000000000..f0612603b2 --- /dev/null +++ b/server/enterprise/elasticsearch/common/templates.go @@ -0,0 +1,237 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "strconv" + + "github.com/elastic/go-elasticsearch/v8/typedapi/indices/putindextemplate" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/mattermost/mattermost/server/public/model" +) + +func GetPostTemplate(cfg *model.Config) *putindextemplate.Request { + mappings := &types.TypeMapping{ + Properties: map[string]types.Property{ + "message": types.TextProperty{ + Analyzer: model.NewPointer("mm_lowercaser"), + Type: "text", + }, + "attachments": types.TextProperty{ + Analyzer: model.NewPointer("mm_lowercaser"), + Type: "text", + }, + "urls": types.TextProperty{ + Analyzer: model.NewPointer("mm_url"), + Type: "text", + }, + "hashtags": types.KeywordProperty{ + Type: "keyword", + Normalizer: model.NewPointer("mm_hashtag"), + Store: model.NewPointer(true), + }, + }, + } + + return &putindextemplate.Request{ + IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBasePosts + "*"}, + Template: &types.IndexTemplateMapping{ + Settings: &types.IndexSettings{ + Index: &types.IndexSettings{ + NumberOfShards: strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexShards), + NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexReplicas), + }, + Analysis: &types.IndexSettingsAnalysis{ + CharFilter: map[string]types.CharFilter{ + "leading_underscores": map[string]any{ + "type": "pattern_replace", + "pattern": `(^|[\s\r\n])_`, + "replacement": "$1", + }, + "trailing_underscores": map[string]any{ + "type": "pattern_replace", + "pattern": `_([\s\r\n]|$)`, + "replacement": "$1", + }, + }, + Analyzer: map[string]types.Analyzer{ + "mm_lowercaser": map[string]any{ + "tokenizer": "icu_tokenizer", + "filter": []string{ + "icu_normalizer", + "mm_snowball", + "mm_stop", + }, + "char_filter": []string{ + "leading_underscores", + "trailing_underscores", + }, + }, + "mm_url": map[string]any{ + "tokenizer": "pattern", + "pattern": "\\W", + "lowercase": true, + }}, + Filter: map[string]types.TokenFilter{ + "mm_snowball": map[string]any{ + "type": "snowball", + "language": "English", + }, + "mm_stop": map[string]any{ + "type": "stop", + "stopwords": "_english_", + }, + }, + Normalizer: map[string]types.Normalizer{ + "mm_hashtag": map[string]any{ + "type": "custom", + "char_filter": []string{}, + "filter": []string{"lowercase", "icu_normalizer"}, + }, + }, + }, + }, + Mappings: mappings, + }, + } +} + +func GetFileInfoTemplate(cfg *model.Config) *putindextemplate.Request { + mappings := &types.TypeMapping{ + Properties: map[string]types.Property{ + "name": types.TextProperty{ + Analyzer: model.NewPointer("mm_lowercaser"), + Type: "text", + }, + "content": types.TextProperty{ + Analyzer: model.NewPointer("mm_lowercaser"), + Type: "text", + }, + }, + } + + return &putindextemplate.Request{ + IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseFiles + "*"}, + Template: &types.IndexTemplateMapping{ + Settings: &types.IndexSettings{ + Index: &types.IndexSettings{ + NumberOfShards: strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexShards), + NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.PostIndexReplicas), + }, + Analysis: &types.IndexSettingsAnalysis{ + CharFilter: map[string]types.CharFilter{ + "leading_underscores": map[string]any{ + "type": "pattern_replace", + "pattern": `(^|[\s\r\n])_`, + "replacement": "$1", + }, + "trailing_underscores": map[string]any{ + "type": "pattern_replace", + "pattern": `_([\s\r\n]|$)`, + "replacement": "$1", + }, + }, + Analyzer: map[string]types.Analyzer{ + "mm_lowercaser": map[string]any{ + "tokenizer": "icu_tokenizer", + "filter": []string{ + "icu_normalizer", + "mm_snowball", + "mm_stop", + }, + "char_filter": []string{ + "leading_underscores", + "trailing_underscores", + }, + }, + }, + Filter: map[string]types.TokenFilter{ + "mm_snowball": map[string]any{ + "type": "snowball", + "language": "English", + }, + "mm_stop": map[string]any{ + "type": "stop", + "stopwords": "_english_", + }, + }, + }, + }, + Mappings: mappings, + }, + } +} + +func GetChannelTemplate(cfg *model.Config) *putindextemplate.Request { + mappings := &types.TypeMapping{ + Properties: map[string]types.Property{ + "name_suggestions": types.KeywordProperty{ + Type: "keyword", + }, + "team_id": types.KeywordProperty{ + Type: "keyword", + }, + "user_ids": types.KeywordProperty{ + Type: "keyword", + }, + "team_member_ids": types.KeywordProperty{ + Type: "keyword", + }, + "type": types.KeywordProperty{ + Type: "keyword", + }, + }, + } + + return &putindextemplate.Request{ + IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseChannels + "*"}, + Template: &types.IndexTemplateMapping{ + Settings: &types.IndexSettings{ + Index: &types.IndexSettings{ + NumberOfShards: strconv.Itoa(*cfg.ElasticsearchSettings.ChannelIndexShards), + NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.ChannelIndexReplicas), + }, + }, + Mappings: mappings, + }, + } +} + +func GetUserTemplate(cfg *model.Config) *putindextemplate.Request { + mappings := &types.TypeMapping{ + Properties: map[string]types.Property{ + "suggestions_with_fullname": types.KeywordProperty{ + Type: "keyword", + }, + "suggestions_without_fullname": types.KeywordProperty{ + Type: "keyword", + }, + "team_id": types.KeywordProperty{ + Type: "keyword", + }, + "channel_id": types.KeywordProperty{ + Type: "keyword", + }, + "delete_at": types.LongNumberProperty{ + Type: "long", + }, + "roles": types.KeywordProperty{ + Type: "keyword", + }, + }, + } + + return &putindextemplate.Request{ + IndexPatterns: []string{*cfg.ElasticsearchSettings.IndexPrefix + IndexBaseUsers + "*"}, + Template: &types.IndexTemplateMapping{ + Settings: &types.IndexSettings{ + Index: &types.IndexSettings{ + NumberOfShards: strconv.Itoa(*cfg.ElasticsearchSettings.UserIndexShards), + NumberOfReplicas: strconv.Itoa(*cfg.ElasticsearchSettings.UserIndexReplicas), + }, + }, + Mappings: mappings, + }, + } +} diff --git a/server/enterprise/elasticsearch/common/test_helpers.go b/server/enterprise/elasticsearch/common/test_helpers.go new file mode 100644 index 0000000000..3c53784c1b --- /dev/null +++ b/server/enterprise/elasticsearch/common/test_helpers.go @@ -0,0 +1,76 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "fmt" + "testing" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/stretchr/testify/assert" +) + +func createPost(userId string, channelId string, message string) *model.Post { + post := &model.Post{ + Message: message, + ChannelId: channelId, + PendingPostId: model.NewId() + ":" + fmt.Sprint(model.GetMillis()), + UserId: userId, + CreateAt: 1000000, + } + post.PreSave() + + return post +} + +func createChannel(teamId, name, displayName string, channelType model.ChannelType) *model.Channel { + channel := &model.Channel{ + TeamId: teamId, + Type: channelType, + Name: name, + DisplayName: displayName, + } + channel.PreSave() + + return channel +} + +func createUser(username, nickname, firstName, lastName string) *model.User { + user := &model.User{ + Username: username, + Password: username, + Nickname: nickname, + FirstName: firstName, + LastName: lastName, + } + if err := user.PreSave(); err != nil { + return nil + } + + return user +} + +func createFile(creatorID, channelID, postID, content, name, extension string) *model.FileInfo { + file := &model.FileInfo{ + CreatorId: creatorID, + ChannelId: channelID, + PostId: postID, + Content: content, + Name: name, + Extension: extension, + } + file.PreSave() + + return file +} + +func CheckMatchesEqual(t *testing.T, expected model.PostSearchMatches, actual map[string][]string) { + a := assert.New(t) + + a.Len(actual, len(expected), "Received matches for a different number of posts") + + for postId, expectedMatches := range expected { + a.ElementsMatch(expectedMatches, actual[postId], fmt.Sprintf("%v: expected %v, got %v", postId, expectedMatches, actual[postId])) + } +} diff --git a/server/enterprise/elasticsearch/common/test_suite.go b/server/enterprise/elasticsearch/common/test_suite.go new file mode 100644 index 0000000000..539c8cb232 --- /dev/null +++ b/server/enterprise/elasticsearch/common/test_suite.go @@ -0,0 +1,789 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "encoding/json" + "time" + + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/channels/store/searchtest" + "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" + + "github.com/stretchr/testify/suite" +) + +type CommonTestSuite struct { + suite.Suite + + TH *api4.TestHelper + ESImpl searchengine.SearchEngineInterface + GetDocumentFn func(index, documentID string) (bool, json.RawMessage, error) + CreateIndexFn func(index string) error + GetIndexFn func(indexPattern string) ([]string, error) + RefreshIndexFn func() error +} + +func (c *CommonTestSuite) TestSearchStore() { + searchTestEngine := &searchtest.SearchTestEngine{ + Driver: searchtest.EngineElasticSearch, + } + + c.Run("TestSearchChannelStore", func() { + searchtest.TestSearchChannelStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine) + }) + + c.Run("TestSearchUserStore", func() { + searchtest.TestSearchUserStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine) + }) + + c.Run("TestSearchPostStore", func() { + searchtest.TestSearchPostStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine) + }) + + c.Run("TestSearchFileInfoStore", func() { + searchtest.TestSearchFileInfoStore(c.T(), c.TH.App.Srv().Store(), searchTestEngine) + }) +} + +func (c *CommonTestSuite) TestIndexPost() { + testCases := []struct { + Name string + Message string + Hashtags string + ExpectedAttachments string + ExpectedHashtags []string + ExpectedURLs []string + }{ + { + Name: "Should be able to index a plain message", + Message: "Test message 1 2 3", + ExpectedAttachments: "", + ExpectedHashtags: []string{}, + ExpectedURLs: []string(nil), + }, + { + Name: "Should be able to index hashtags", + Message: "Test message #1234", + Hashtags: "#1234", + ExpectedAttachments: "", + ExpectedHashtags: []string{"#1234"}, + ExpectedURLs: []string(nil), + }, + // TODO: actually send attachments + { + Name: "Should be able to index attachments", + Message: "Test message 1 2 3", + ExpectedAttachments: "", + ExpectedHashtags: []string{}, + ExpectedURLs: []string(nil), + }, + { + Name: "Should be able to index urls", + Message: "Test message www.mattermost.com http://www.mattermost.com [link](http://www.notindexed.com)", + ExpectedAttachments: "", + ExpectedHashtags: []string{}, + ExpectedURLs: []string{"www.mattermost.com", "http://www.mattermost.com"}, + }, + } + + for _, tc := range testCases { + c.Run(tc.Name, func() { + post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, tc.Message) + if tc.Hashtags != "" { + post.Hashtags = tc.Hashtags + } + c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id)) + + c.NoError(c.RefreshIndexFn()) + indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, + IndexBasePosts, + IndexBasePosts_MONTH, + time.Now(), + post.CreateAt, + ) + + found, source, err := c.GetDocumentFn(indexName, post.Id) + c.NoError(err) + c.True(found) + + var esPost ESPost + err = json.Unmarshal(source, &esPost) + c.NoError(err) + c.NotNil(post) + c.Equal(tc.Message, post.Message) + c.Equal(tc.ExpectedAttachments, esPost.Attachments) + c.Equal(tc.ExpectedHashtags, esPost.Hashtags) + c.Equal(tc.ExpectedURLs, esPost.URLs) + }) + } +} + +func (c *CommonTestSuite) TestSearchPosts() { + // Create and index a post + post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId()) + c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id)) + + c.NoError(c.RefreshIndexFn()) + indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt) + + // Check the post is there. + found, _, err := c.GetDocumentFn(indexName, post.Id) + c.NoError(err) + c.True(found) + + // Do a search for that post. + channels := model.ChannelList{ + c.TH.BasicChannel, + } + + searchParams := []*model.SearchParams{ + { + Terms: post.Message, + IsHashtag: false, + OrTerms: false, + }, + } + + // Check the post is found as expected + ids, matches, err := c.ESImpl.SearchPosts(channels, searchParams, 0, 20) + c.Nil(err) + c.Len(ids, 1) + c.Equal(ids[0], post.Id) + CheckMatchesEqual(c.T(), map[string][]string{ + post.Id: {post.Message}, + }, matches) + + // Do a search that won't match anything. + searchParams = []*model.SearchParams{ + { + Terms: model.NewId(), + IsHashtag: false, + OrTerms: false, + }, + } + + ids, matches, err = c.ESImpl.SearchPosts(channels, searchParams, 0, 20) + c.Nil(err) + c.Len(ids, 0) + c.Len(matches, 0) +} + +func (c *CommonTestSuite) TestDeletePost() { + c.Require().NotNil(c.TH) + + post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId()) + indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt) + + // Index the post. + c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id)) + c.NoError(c.RefreshIndexFn()) + + // Check the post is there. + found, _, err := c.GetDocumentFn(indexName, post.Id) + c.NoError(err) + c.True(found) + + // Delete the post. + c.Nil(c.ESImpl.DeletePost(post)) + c.NoError(c.RefreshIndexFn()) + + // Check the post is not there. + found, _, err = c.GetDocumentFn(indexName, post.Id) + // This is a difference in behavior between engines. + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.Error(err) + } else { + c.NoError(err) + } + c.False(found) +} + +func (c *CommonTestSuite) TestDeleteChannelPosts() { + c.Run("Should remove all the channel posts", func() { + channelPosts := make([]*model.Post, 0) + post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId()) + channelPosts = append(channelPosts, post) + post2 := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId()) + post2.CreateAt = 1200000 + channelPosts = append(channelPosts, post2) + post3 := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId()) + post3.CreateAt = 1300000 + channelPosts = append(channelPosts, post3) + postReply := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId()) + postReply.RootId = post.Id + postReply.CreateAt = 1400000 + channelPosts = append(channelPosts, postReply) + anotherPost := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel2.Id, model.NewId()) + indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, + IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt) + for _, post := range channelPosts { + c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id)) + } + c.Nil(c.ESImpl.IndexPost(anotherPost, c.TH.BasicTeam.Id)) + c.NoError(c.RefreshIndexFn()) + for _, post := range channelPosts { + found, _, err := c.GetDocumentFn(indexName, post.Id) + c.NoError(err) + c.True(found) + } + c.Nil(c.ESImpl.DeleteChannelPosts(c.TH.Context, c.TH.BasicChannel.Id)) + c.NoError(c.RefreshIndexFn()) + for _, post := range channelPosts { + found, _, err := c.GetDocumentFn(indexName, post.Id) + // This is a difference in behavior between engines. + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.Error(err) + } else { + c.NoError(err) + } + c.False(found) + } + + found, _, err := c.GetDocumentFn(indexName, anotherPost.Id) + c.NoError(err) + c.True(found) + }) + + c.Run("Should not remove other channels posts even if there was no posts to remove", func() { + postNotInChannel := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel2.Id, model.NewId()) + indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, + IndexBasePosts, IndexBasePosts_MONTH, time.Now(), postNotInChannel.CreateAt) + c.Nil(c.ESImpl.IndexPost(postNotInChannel, c.TH.BasicTeam.Id)) + c.NoError(c.RefreshIndexFn()) + c.Nil(c.ESImpl.DeleteChannelPosts(c.TH.Context, c.TH.BasicChannel.Id)) + c.NoError(c.RefreshIndexFn()) + + found, _, err := c.GetDocumentFn(indexName, postNotInChannel.Id) + c.NoError(err) + c.True(found) + }) +} + +func (c *CommonTestSuite) TestDeleteUserPosts() { + c.Run("Should remove all the user posts", func() { + anotherTeam := c.TH.CreateTeam() + anotherTeamChannel := createChannel(anotherTeam.Id, "anotherteamchannel", "", model.ChannelTypeOpen) + userPosts := make([]*model.Post, 0) + post := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId()) + userPosts = append(userPosts, post) + post2 := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel2.Id, model.NewId()) + post2.CreateAt = 1200000 + userPosts = append(userPosts, post2) + post3 := createPost(c.TH.BasicUser.Id, c.TH.BasicPrivateChannel.Id, model.NewId()) + post3.CreateAt = 1300000 + userPosts = append(userPosts, post3) + postReply := createPost(c.TH.BasicUser.Id, c.TH.BasicChannel.Id, model.NewId()) + postReply.RootId = post.Id + postReply.CreateAt = 1400000 + userPosts = append(userPosts, postReply) + postAnotherTeam := createPost(c.TH.BasicUser.Id, anotherTeamChannel.Id, model.NewId()) + postAnotherTeam.CreateAt = 1400000 + userPosts = append(userPosts, postAnotherTeam) + anotherPost := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel2.Id, model.NewId()) + indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, + IndexBasePosts, IndexBasePosts_MONTH, time.Now(), post.CreateAt) + for _, post := range userPosts { + c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id)) + } + c.Nil(c.ESImpl.IndexPost(postAnotherTeam, anotherTeam.Id)) + c.Nil(c.ESImpl.IndexPost(anotherPost, c.TH.BasicTeam.Id)) + c.NoError(c.RefreshIndexFn()) + for _, post := range userPosts { + found, _, err := c.GetDocumentFn(indexName, post.Id) + c.NoError(err) + c.True(found) + } + c.Nil(c.ESImpl.DeleteUserPosts(c.TH.Context, c.TH.BasicUser.Id)) + c.NoError(c.RefreshIndexFn()) + for _, post := range userPosts { + found, _, err := c.GetDocumentFn(indexName, post.Id) + // This is a difference in behavior between engines. + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.Error(err) + } else { + c.NoError(err) + } + c.False(found) + } + found, _, err := c.GetDocumentFn(indexName, anotherPost.Id) + c.NoError(err) + c.True(found) + }) + + c.Run("Should not remove other channels posts even if there was no posts to remove", func() { + postNotInChannel := createPost(c.TH.BasicUser2.Id, c.TH.BasicChannel.Id, model.NewId()) + indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, + IndexBasePosts, IndexBasePosts_MONTH, time.Now(), postNotInChannel.CreateAt) + c.Nil(c.ESImpl.IndexPost(postNotInChannel, c.TH.BasicTeam.Id)) + c.NoError(c.RefreshIndexFn()) + c.Nil(c.ESImpl.DeleteUserPosts(c.TH.Context, c.TH.BasicUser.Id)) + c.NoError(c.RefreshIndexFn()) + found, _, err := c.GetDocumentFn(indexName, postNotInChannel.Id) + c.NoError(err) + c.True(found) + }) +} + +func (c *CommonTestSuite) TestIndexChannel() { + // Create and index a channel + channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{})) + + c.NoError(c.RefreshIndexFn()) + + // Check the channel is there. + found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id) + c.NoError(err) + c.True(found) +} + +func (c *CommonTestSuite) TestDeleteChannel() { + // Create and index a channel. + channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{})) + + c.NoError(c.RefreshIndexFn()) + + // Check the channel is there. + found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id) + c.NoError(err) + c.True(found) + + // Delete the channel. + c.Nil(c.ESImpl.DeleteChannel(channel)) + c.NoError(c.RefreshIndexFn()) + + // Check the channel is not there. + found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id) + // This is a difference in behavior between engines. + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.Error(err) + } else { + c.NoError(err) + } + c.False(found) +} + +func (c *CommonTestSuite) TestIndexUser() { + // Create and index a user + user := createUser("test.user", "testuser", "Test", "User") + c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{})) + + c.NoError(c.RefreshIndexFn()) + + // Check the user is there. + found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id) + c.NoError(err) + c.True(found) +} + +func (c *CommonTestSuite) TestDeleteUser() { + // Create and index a user + user := createUser("test.user", "testuser", "Test", "User") + c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{})) + + c.NoError(c.RefreshIndexFn()) + + // Check the user is there. + found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id) + c.NoError(err) + c.True(found) + + // Delete the user. + c.Nil(c.ESImpl.DeleteUser(user)) + c.NoError(c.RefreshIndexFn()) + // Check the user is not there. + found, _, err = c.GetDocumentFn(IndexBaseUsers, user.Id) + // This is a difference in behavior between engines. + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.Error(err) + } else { + c.NoError(err) + } + c.False(found) +} + +func (c *CommonTestSuite) TestTestConfig() { + c.Nil(c.ESImpl.TestConfig(c.TH.Context, c.TH.App.Config())) + + originalConfig := c.TH.App.Config() + defer c.TH.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ElasticsearchSettings.ConnectionURL = *originalConfig.ElasticsearchSettings.ConnectionURL + }) + + c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.ConnectionURL = "example.com:12345" }) + c.Error(c.ESImpl.TestConfig(c.TH.Context, c.TH.App.Config())) + + // Passing a temp config which is different from the saved + // config should be taken correctly. + c.Nil(c.ESImpl.TestConfig(c.TH.Context, originalConfig)) +} + +func (c *CommonTestSuite) TestIndexFile() { + // First, create and index a channel + channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{})) + + // Then, create and index a user + user := createUser("test.user", "testuser", "Test", "User") + c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id})) + + // Create and index a file + file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt") + c.Nil(c.ESImpl.IndexFile(file, channel.Id)) + + c.NoError(c.RefreshIndexFn()) + + // Check the file is there + found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id) + c.NoError(err) + c.True(found) +} + +func (c *CommonTestSuite) TestDeleteFile() { + // First, create and index a channel + channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{})) + + // Then, create and index a user + user := createUser("test.user", "testuser", "Test", "User") + c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id})) + + // Create and index a file + file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt") + c.Nil(c.ESImpl.IndexFile(file, channel.Id)) + + c.NoError(c.RefreshIndexFn()) + + // Check the file is there + found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id) + c.NoError(err) + c.True(found) + + // Delete the file + c.Nil(c.ESImpl.DeleteFile(file.Id)) + c.NoError(c.RefreshIndexFn()) + + // Check the file is not there. + found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id) + // This is a difference in behavior between engines. + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.Error(err) + } else { + c.NoError(err) + } + c.False(found) +} + +func (c *CommonTestSuite) TestDeleteUserFiles() { + // First, create and index a channel + channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{})) + + // Then, create and index a user + user := createUser("test.user", "testuser", "Test", "User") + c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id})) + + // Create and index a file + file := createFile(user.Id, channel.Id, "", "file contents", "testfile", "txt") + c.Nil(c.ESImpl.IndexFile(file, channel.Id)) + + c.NoError(c.RefreshIndexFn()) + + // Check the file is there + found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id) + c.NoError(err) + c.True(found) + + // Delete file by creator + c.Nil(c.ESImpl.DeleteUserFiles(c.TH.Context, user.Id)) + c.NoError(c.RefreshIndexFn()) + + // Check the file is not there. + found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id) + // This is a difference in behavior between engines. + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.Error(err) + } else { + c.NoError(err) + } + c.False(found) +} + +func (c *CommonTestSuite) TestDeletePostFiles() { + // First, create and index a channel + channel := createChannel(c.TH.BasicTeam.Id, "channel", "Test Channel", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{})) + + // Then, create and index a user + user := createUser("test.user", "testuser", "Test", "User") + c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{c.TH.BasicTeam.Id}, []string{channel.Id})) + + // Create and index a post + post := createPost(user.Id, channel.Id, "test post message") + c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id)) + + // Create and index a file + file := createFile(user.Id, channel.Id, post.Id, "file contents", "testfile", "txt") + c.Nil(c.ESImpl.IndexFile(file, channel.Id)) + + c.NoError(c.RefreshIndexFn()) + + // Check the file is there + found, _, err := c.GetDocumentFn(IndexBaseFiles, file.Id) + c.NoError(err) + c.True(found) + + // Delete file by post + c.Nil(c.ESImpl.DeletePostFiles(c.TH.Context, post.Id)) + c.NoError(c.RefreshIndexFn()) + + // Check the file is not there. + found, _, err = c.GetDocumentFn(IndexBaseFiles, file.Id) + // This is a difference in behavior between engines. + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.Error(err) + } else { + c.NoError(err) + } + c.False(found) +} + +func (c *CommonTestSuite) TestElasticsearchDataRetentionDeleteIndexes() { + c.Nil(c.CreateIndexFn("posts_2017_09_15")) + c.Nil(c.CreateIndexFn("posts_2017_09_16")) + c.Nil(c.CreateIndexFn("posts_2017_09_17")) + c.Nil(c.CreateIndexFn("posts_2017_09_18")) + c.Nil(c.CreateIndexFn("posts_2017_09_19")) + + c.Run("Should delete indexes using start of day cut off", func() { + c.Nil(c.ESImpl.DataRetentionDeleteIndexes(c.TH.Context, time.Date(2017, 9, 16, 0, 0, 0, 0, time.UTC))) + + postIndexesResult, err := c.GetIndexFn("posts_*") + c.Nil(err) + if err == nil { + found1 := false + found2 := false + found3 := false + found4 := false + found5 := false + + for _, index := range postIndexesResult { + if index == "posts_2017_09_15" { + found1 = true + } else if index == "posts_2017_09_16" { + found2 = true + } else if index == "posts_2017_09_17" { + found3 = true + } else if index == "posts_2017_09_18" { + found4 = true + } else if index == "posts_2017_09_19" { + found5 = true + } + } + + c.False(found1) + c.False(found2) + c.True(found3) + c.True(found4) + c.True(found5) + } + }) + + c.Run("Should delete indexes when cut off is in hours", func() { + c.Nil(c.ESImpl.DataRetentionDeleteIndexes(c.TH.Context, time.Date(2017, 9, 18, 11, 6, 0, 0, time.UTC))) + + postIndexesResult, err := c.GetIndexFn("posts_*") + c.Nil(err) + if err == nil { + found1 := false + found2 := false + found3 := false + + for _, index := range postIndexesResult { + if index == "posts_2017_09_17" { + found1 = true + } else if index == "posts_2017_09_18" { + found2 = true + } else if index == "posts_2017_09_19" { + found3 = true + } + } + + c.False(found1) + c.False(found2) + c.True(found3) + } + }) +} + +func (c *CommonTestSuite) TestPurgeIndexes() { + existingIndexPrefix := *c.TH.Server.Config().ElasticsearchSettings.IndexPrefix + defer c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = existingIndexPrefix }) + + c.Run("Should purge all indexes", func() { + // Create and index a user + user := createUser("test.user", "testuser", "Test", "User") + c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{})) + + c.NoError(c.RefreshIndexFn()) + + c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "test_" }) + + // index user with a new index prefix + c.Nil(c.ESImpl.IndexUser(c.TH.Context, user, []string{}, []string{})) + c.NoError(c.RefreshIndexFn()) + + c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context)) + + found, _, err := c.GetDocumentFn(IndexBaseUsers, user.Id) + c.NoError(err) + c.True(found) + + found, _, err = c.GetDocumentFn("test_"+IndexBaseUsers, user.Id) + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.False(found) + } else { + elasticErr := err.(*types.ElasticsearchError) + c.Equal(404, elasticErr.Status) + } + }) + + c.Run("Should not purge indexes defined to ignore", func() { + c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "posts*" }) + c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "" }) + + // Create a user + user := createUser("test.user", "testuser", "Test", "User") + + // Create and index a post + post := createPost(user.Id, c.TH.BasicChannel.Id, "Test") + c.Nil(c.ESImpl.IndexPost(post, c.TH.BasicTeam.Id)) + + c.NoError(c.RefreshIndexFn()) + indexName := BuildPostIndexName(*c.TH.App.Config().ElasticsearchSettings.AggregatePostsAfterDays, + IndexBasePosts, + IndexBasePosts_MONTH, + time.Now(), + post.CreateAt, + ) + + // We expect posts indexes to remain after purge + c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context)) + + found, _, err := c.GetDocumentFn(indexName, post.Id) + c.NoError(err) + c.True(found) + + // Remove the ignore rule + c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "" }) + + c.Nil(c.ESImpl.PurgeIndexes(c.TH.Context)) + + // Validate the indexes are gone + found, _, err = c.GetDocumentFn(IndexBasePosts, post.Id) + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.False(found) + } else { + elasticErr := err.(*types.ElasticsearchError) + c.Equal(404, elasticErr.Status) + } + }) +} + +func (c *CommonTestSuite) TestPurgeIndexList() { + existingIndexPrefix := *c.TH.Server.Config().ElasticsearchSettings.IndexPrefix + defer c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = existingIndexPrefix }) + + c.Run("Should purge allowed index", func() { + // Create and index a channel + channel := createChannel("test.channel", "testuser", "Test", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{})) + + c.NoError(c.RefreshIndexFn()) + + // verify data is in Elasticsearch + found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id) + c.NoError(err) + c.True(found) + + // now we'll purge + c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"})) + + found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id) + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.False(found) + } else { + elasticErr := err.(*types.ElasticsearchError) + c.Equal(404, elasticErr.Status) + } + }) + + c.Run("Should not purge indexes defined to ignore", func() { + c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "channels" }) + c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IndexPrefix = "" }) + + channel := createChannel("test.channel", "testuser", "Test", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{})) + + c.NoError(c.RefreshIndexFn()) + + // verify data is in Elasticsearch + found, _, err := c.GetDocumentFn(IndexBaseChannels, channel.Id) + c.NoError(err) + c.True(found) + + // now we'll purge + c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"})) + + // the channel should still be there because we ignored that index + found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id) + c.NoError(err) + c.True(found) + + // Remove the ignore rule + c.TH.App.UpdateConfig(func(cfg *model.Config) { *cfg.ElasticsearchSettings.IgnoredPurgeIndexes = "" }) + + c.Nil(c.ESImpl.PurgeIndexList(c.TH.Context, []string{"channels"})) + + // now it should be gone as we're no longer ignoring it + found, _, err = c.GetDocumentFn(IndexBaseChannels, channel.Id) + if c.ESImpl.GetName() == model.ElasticsearchSettingsOSBackend { + c.False(found) + } else { + elasticErr := err.(*types.ElasticsearchError) + c.Equal(404, elasticErr.Status) + } + }) +} + +func (c *CommonTestSuite) TestSearchChannels() { + // Create and index a channel + channel := createChannel(c.TH.BasicTeam.Id, "channel", "Channel Open", model.ChannelTypeOpen) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel, []string{}, []string{c.TH.BasicUser.Id, "otheruser"})) + channel2 := createChannel(c.TH.BasicTeam.Id, "channel", "Channel Private", model.ChannelTypePrivate) + c.Nil(c.ESImpl.IndexChannel(c.TH.Context, channel2, []string{c.TH.BasicUser.Id}, []string{c.TH.BasicUser.Id, "otheruser"})) + + c.NoError(c.RefreshIndexFn()) + + // Private channels should be returned for right user. + ids, appErr := c.ESImpl.SearchChannels("", c.TH.BasicUser.Id, "Channel", false) + c.Nil(appErr) + c.Len(ids, 2) + + // No private channels if user is guest + ids, appErr = c.ESImpl.SearchChannels("", c.TH.BasicUser.Id, "Channel", true) + c.Nil(appErr) + c.Len(ids, 1) + c.Equal(channel.Id, ids[0]) + + // No Private channels should be returned for wrong user. + ids, appErr = c.ESImpl.SearchChannels("", "otheruser", "Channel", false) + c.Nil(appErr) + c.Len(ids, 1) + c.Equal(channel.Id, ids[0]) +} diff --git a/server/enterprise/elasticsearch/common/version.go b/server/enterprise/elasticsearch/common/version.go new file mode 100644 index 0000000000..6f9347e572 --- /dev/null +++ b/server/enterprise/elasticsearch/common/version.go @@ -0,0 +1,27 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "strconv" + "strings" +) + +func GetVersionComponents(version string) (int, int, int, error) { + spl := strings.Split(version, ".") + major, err := strconv.Atoi(spl[0]) + if err != nil { + return 0, 0, 0, err + } + minor, err := strconv.Atoi(spl[1]) + if err != nil { + return 0, 0, 0, err + } + patch, err := strconv.Atoi(spl[2]) + if err != nil { + return 0, 0, 0, err + } + + return major, minor, patch, nil +} diff --git a/server/enterprise/elasticsearch/common/version_test.go b/server/enterprise/elasticsearch/common/version_test.go new file mode 100644 index 0000000000..a02fd57352 --- /dev/null +++ b/server/enterprise/elasticsearch/common/version_test.go @@ -0,0 +1,51 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetVersionComponents(t *testing.T) { + testCases := []struct { + Name string + Version string + ExpectedMajor int + ExpectedMinor int + ExpectedPatch int + ExpectedError bool + }{ + { + Name: "Should error if version format is invalid", + Version: "invalid", + ExpectedMajor: 0, + ExpectedMinor: 0, + ExpectedPatch: 0, + ExpectedError: true, + }, + { + Name: "Should work correctly if version has three valid components", + Version: "7.2.3", + ExpectedMajor: 7, + ExpectedMinor: 2, + ExpectedPatch: 3, + ExpectedError: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + major, minor, patch, err := GetVersionComponents(tc.Version) + if tc.ExpectedError { + require.Error(t, err) + } + assert.Equal(t, tc.ExpectedMajor, major) + assert.Equal(t, tc.ExpectedMinor, minor) + assert.Equal(t, tc.ExpectedPatch, patch) + }) + } +} diff --git a/server/enterprise/elasticsearch/elasticsearch/aggregation_job.go b/server/enterprise/elasticsearch/elasticsearch/aggregation_job.go new file mode 100644 index 0000000000..6ef99b54cc --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/aggregation_job.go @@ -0,0 +1,330 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/elastic/go-elasticsearch/v8" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/app" + "github.com/mattermost/mattermost/server/v8/channels/jobs" + "github.com/mattermost/mattermost/server/v8/channels/store" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" +) + +const ( + aggregatorJobPollingInterval = 15 * time.Second + indexDeletionBatchSize = 20 +) + +type ElasticsearchAggregatorInterfaceImpl struct { + Server *app.Server +} + +type ElasticsearchAggregatorWorker struct { + name string + // stateMut protects stopCh and stopped and helps enforce + // ordering in case subsequent Run or Stop calls are made. + stateMut sync.Mutex + stopCh chan struct{} + stopped bool + stoppedCh chan bool + jobs chan model.Job + jobServer *jobs.JobServer + logger mlog.LoggerIFace + fileBackend filestore.FileBackend + + client *elasticsearch.TypedClient + license func() *model.License +} + +func (esi *ElasticsearchAggregatorInterfaceImpl) MakeWorker() model.Worker { + const workerName = "EnterpriseElasticsearchAggregator" + worker := ElasticsearchAggregatorWorker{ + name: workerName, + stoppedCh: make(chan bool, 1), + jobs: make(chan model.Job), + jobServer: esi.Server.Jobs, + logger: esi.Server.Jobs.Logger().With(mlog.String("worker_name", workerName)), + fileBackend: esi.Server.Platform().FileBackend(), + license: esi.Server.License, + stopped: true, + } + + return &worker +} + +func (worker *ElasticsearchAggregatorWorker) Run() { + worker.stateMut.Lock() + // We have to re-assign the stop channel again, because + // it might happen that the job was restarted due to a config change. + if worker.stopped { + worker.stopped = false + worker.stopCh = make(chan struct{}) + } else { + worker.stateMut.Unlock() + return + } + // Run is called from a separate goroutine and doesn't return. + // So we cannot Unlock in a defer clause. + worker.stateMut.Unlock() + + worker.logger.Debug("Worker Started") + + defer func() { + worker.logger.Debug("Worker Finished") + worker.stoppedCh <- true + }() + + client, err := createTypedClient(worker.logger, worker.jobServer.Config(), worker.fileBackend, false) + if err != nil { + worker.logger.Error("Worker Failed to Create Client", mlog.Err(err)) + return + } + + worker.client = client + + for { + select { + case <-worker.stopCh: + worker.logger.Debug("Worker Received stop signal") + return + case job := <-worker.jobs: + worker.DoJob(&job) + } + } +} + +func (worker *ElasticsearchAggregatorWorker) IsEnabled(cfg *model.Config) bool { + if license := worker.license(); license == nil || !*license.Features.Elasticsearch { + return false + } + + if *cfg.ElasticsearchSettings.EnableIndexing { + return true + } + + return false +} + +func (worker *ElasticsearchAggregatorWorker) Stop() { + worker.stateMut.Lock() + defer worker.stateMut.Unlock() + + // Set to close, and if already closed before, then return. + if worker.stopped { + return + } + worker.stopped = true + + worker.logger.Debug("Worker Stopping") + close(worker.stopCh) + <-worker.stoppedCh +} + +func (worker *ElasticsearchAggregatorWorker) JobChannel() chan<- model.Job { + return worker.jobs +} + +func (worker *ElasticsearchAggregatorWorker) DoJob(job *model.Job) { + logger := worker.logger.With(jobs.JobLoggerFields(job)...) + logger.Debug("Worker: Received a new candidate job.") + defer worker.jobServer.HandleJobPanic(logger, job) + + claimed, appErr := worker.jobServer.ClaimJob(job) + if appErr != nil { + logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr)) + return + } + + if !claimed { + return + } + + logger.Info("Worker: Aggregation job claimed by worker") + + var cancelContext request.CTX = request.EmptyContext(worker.logger) + cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background()) + cancelWatcherChan := make(chan struct{}, 1) + cancelContext = cancelContext.WithContext(cancelCtx) + go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan) + defer cancelCancelWatcher() + + rctx := request.EmptyContext(worker.logger) + now := time.Now() + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local) + cutoff := today.AddDate(0, 0, -*worker.jobServer.Config().ElasticsearchSettings.AggregatePostsAfterDays+1) + + // Get all the daily Elasticsearch post indexes to work out which days aren't aggregated yet. + dateFormat := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_2006_01_02" + datedIndexes := []time.Time{} + + postIndexesResult, err := worker.client.API.Indices. + Get(*worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_*"). + Do(rctx.Context()) + if err != nil { + appError := model.NewAppError("ElasticsearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.get_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err) + worker.setJobError(logger, job, appError) + return + } + + for index := range postIndexesResult { + var indexDate time.Time + indexDate, err = time.Parse(dateFormat, index) + if err != nil { + logger.Warn("Failed to parse date from posts index. Ignoring index.", mlog.String("index", index)) + } else { + datedIndexes = append(datedIndexes, indexDate) + } + } + + // Work out how far back the reindexing (and index deletion) needs to go. + var oldestDay time.Time + oldestDayFound := false + indexesToPurge := []string{} + for _, date := range datedIndexes { + if date.Before(cutoff) { + logger.Debug("Worker: Post index identified for purging", mlog.Time("date", date)) + indexesToPurge = append(indexesToPurge, date.Format(dateFormat)) + if !oldestDayFound || oldestDay.After(date) { + oldestDay = date + oldestDayFound = true + } + } else { + logger.Debug("Worker: Post index is within the range to keep", mlog.Time("date", date)) + } + } + + if !oldestDayFound { + // Nothing to purge. + logger.Info("Worker: Aggregation job completed. Nothing to aggregate.") + worker.setJobSuccess(logger, job) + return + } + + // Trigger a reindexing job with the appropriate dates. + reindexingStartDate := oldestDay + reindexingEndDate := cutoff + + logger.Info("Worker: Aggregation job reindexing", mlog.String("start_date", reindexingStartDate.Format("2006-01-02")), mlog.String("end_date", reindexingEndDate.Format("2006-01-02"))) + + var indexJob *model.Job + if indexJob, appErr = worker.jobServer.CreateJob( + rctx, + model.JobTypeElasticsearchPostIndexing, + map[string]string{ + "start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10), + "end_time": strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10), + }, + ); appErr != nil { + logger.Error("Worker: Failed to create indexing job.", mlog.Err(appErr)) + appError := model.NewAppError("ElasticsearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.create_index_job.error", nil, "", http.StatusInternalServerError).Wrap(appErr) + worker.setJobError(logger, job, appError) + return + } + + for { + select { + case <-cancelWatcherChan: + logger.Info("Worker: Aggregation job has been canceled via CancellationWatcher") + worker.setJobCanceled(logger, job) + return + + case <-worker.stopCh: + logger.Info("Worker: Aggregation job has been canceled via Worker Stop") + worker.setJobCanceled(logger, job) + return + + case <-time.After(aggregatorJobPollingInterval): + // Get the details of the indexing job we are waiting on. + indexJob, err = worker.jobServer.Store.Job().Get(rctx, indexJob.Id) + if err != nil { + var appErr *model.AppError + var nfErr *store.ErrNotFound + switch { + case errors.As(err, &nfErr): + appErr = model.NewAppError("DoJob", "app.job.get.app_error", nil, "", http.StatusNotFound).Wrap(nfErr) + default: + appErr = model.NewAppError("DoJob", "app.job.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + worker.setJobError(logger, job, appErr) + return + } + + // Wait for the aggregation job to finish. + // On success, we delete the old indexes. + // Otherwise, fail the job. + switch indexJob.Status { + case model.JobStatusSuccess: + // We limit the number of indexes to delete at one shot. + // A minor side-effect of this is that the aggregation job status + // will be redundantly queried multiple times, but that's not a major bottleneck. + curWindow := indexesToPurge + deleteMore := false + if len(indexesToPurge) > indexDeletionBatchSize { + curWindow = indexesToPurge[:indexDeletionBatchSize] + indexesToPurge = indexesToPurge[indexDeletionBatchSize:] + deleteMore = true + } + // Delete indexes + if _, err = worker.client.Indices.Delete(strings.Join(curWindow, ",")).Do(rctx.Context()); err != nil { + appError := model.NewAppError("ElasticsearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.delete_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err) + logger.Error("Worker: Failed to delete indexes for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(appError)) + worker.setJobError(logger, job, appError) + return + } + + if !deleteMore { + // Job done. Set the status to success. + logger.Info("Worker: Aggregation job finished successfully") + worker.setJobSuccess(logger, job) + return + } + case model.JobStatusPending, model.JobStatusInProgress: + // Indexing job is in progress or pending. Update the progress of this job. + if err := worker.jobServer.SetJobProgress(job, indexJob.Progress); err != nil { + logger.Error("Worker: Failed to set progress for job", mlog.Err(err)) + worker.setJobError(logger, job, err) + return + } + default: + // error case + appError := model.NewAppError("ElasticsearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.index_job_failed.error", nil, "", http.StatusInternalServerError) + logger.Error("Worker: Index aggregation job failed", mlog.Err(appError)) + worker.setJobError(logger, job, appError) + return + } + } + } +} + +func (worker *ElasticsearchAggregatorWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) { + if err := worker.jobServer.SetJobSuccess(job); err != nil { + logger.Error("Worker: Failed to set success for job", mlog.Err(err)) + worker.setJobError(logger, job, err) + } +} + +func (worker *ElasticsearchAggregatorWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) { + if err := worker.jobServer.SetJobError(job, appError); err != nil { + logger.Error("Worker: Failed to set job error", mlog.Err(err)) + } +} + +func (worker *ElasticsearchAggregatorWorker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) { + if err := worker.jobServer.SetJobCanceled(job); err != nil { + logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err)) + } +} diff --git a/server/enterprise/elasticsearch/elasticsearch/aggregation_job_test.go b/server/enterprise/elasticsearch/elasticsearch/aggregation_job_test.go new file mode 100644 index 0000000000..e4bcfb6892 --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/aggregation_job_test.go @@ -0,0 +1,197 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" +) + +func TestElasticsearchAggregation(t *testing.T) { + th := api4.SetupEnterpriseWithStoreMock(t) + rctx := request.TestContext(t) + + mockUserStore := mocks.UserStore{} + mockUserStore.On("Count", mock.Anything).Return(int64(10), nil) + mockUserStore.On("GetAllProfiles", mock.Anything).Return(nil, nil) + + mockPostStore := mocks.PostStore{} + mockPostStore.On("GetMaxPostSize").Return(65535, nil) + + mockSystemStore := mocks.SystemStore{} + mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil) + mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil) + mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil) + + mockJobStore := mocks.JobStore{} + mockJobStore.On("Save", mock.AnythingOfType("*model.Job")).Return(&model.Job{}, nil) + mockJobStore.On("UpdateStatus", mock.AnythingOfType("string"), model.JobStatusSuccess).Return(&model.Job{}, nil) + mockJobStore.On("Get", mock.AnythingOfType("*request.Context"), mock.AnythingOfType("string")).Return(&model.Job{ + Status: model.JobStatusSuccess, + }, nil) + mockJobStore.On("UpdateStatusOptimistically", + mock.AnythingOfType("string"), + model.JobStatusPending, + model.JobStatusInProgress).Return(true, nil) + mockJobStore.On("GetAllByType", mock.AnythingOfType("string")).Return([]*model.Job{{ + Id: "abcxyz123", + Type: "EnterpriseElasticsearchIndexer", + Status: model.JobStatusCanceled, + }}, nil) + + mockStore := th.App.Srv().Platform().Store.(*mocks.Store) + mockStore.On("User").Return(&mockUserStore) + mockStore.On("Post").Return(&mockPostStore) + mockStore.On("System").Return(&mockSystemStore) + mockStore.On("Job").Return(&mockJobStore) + mockStore.On("GetDBSchemaVersion").Return(1, nil) + + aggImpl := ElasticsearchAggregatorInterfaceImpl{Server: th.Server} + + // Register search engine + th.App.SearchEngine().RegisterElasticsearchEngine(&ElasticsearchInterfaceImpl{ + Platform: th.Server.Platform(), + }) + + // Set up the state for the tests. + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + *cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1 + *cfg.ElasticsearchSettings.AggregatePostsAfterDays = 1 + *cfg.SqlSettings.DisableDatabaseSearch = true + }) + + esImpl := th.App.SearchEngine().ElasticsearchEngine + appErr := esImpl.Start() + if appErr != nil && appErr.Id != "ent.elasticsearch.start.already_started.app_error" { + require.Fail(t, "failed to start elasticsearch: %v", appErr) + } + require.Nil(t, esImpl.PurgeIndexes(rctx)) + + post := &model.Post{ + Id: model.NewId(), + ChannelId: "channel", + Message: "hi", + } + for i := 0; i < indexDeletionBatchSize+1; i++ { + indexPost(t, th, esImpl.(*ElasticsearchInterfaceImpl), + post, + time.Now().Add(-time.Duration(4+i)*24*time.Hour)) + } + + job := &model.Job{ + Id: model.NewId(), + Type: model.JobTypeElasticsearchPostAggregation, + Status: model.JobStatusPending, + } + + _, err := th.Server.Store().Job().Save(job) + require.NoError(t, err) + + worker := aggImpl.MakeWorker().(*ElasticsearchAggregatorWorker) + worker.client = createTestClient(t, th.Context, th.App.Config(), th.App.FileBackend()) + worker.jobServer.Store = mockStore + + indexingImpl := ElasticsearchIndexerInterfaceImpl{ + Server: th.App.Srv(), + } + th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, indexingImpl.MakeWorker(), nil) + + worker.DoJob(job) + + // We assert the minimum number of calls to verify that + // batching is working correctly. Because job().Get() will happen + // in each iteration. + numCalls := 0 + for _, call := range mockJobStore.Calls { + if call.Method == "Get" { + numCalls++ + } + } + assert.GreaterOrEqual(t, numCalls, 8, "Unexpected number of Jobstore.Get calls") +} + +func TestElasticsearchAggregationSkipDuringBulkIndexing(t *testing.T) { + th := api4.SetupEnterpriseWithStoreMock(t) + + mockUserStore := mocks.UserStore{} + mockUserStore.On("Count", mock.Anything).Return(int64(10), nil) + + mockPostStore := mocks.PostStore{} + mockPostStore.On("GetMaxPostSize").Return(65535, nil) + + mockSystemStore := mocks.SystemStore{} + mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil) + mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil) + mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil) + + mockJobStore := mocks.JobStore{} + + mockStore := th.App.Srv().Platform().Store.(*mocks.Store) + mockStore.On("User").Return(&mockUserStore) + mockStore.On("Post").Return(&mockPostStore) + mockStore.On("System").Return(&mockSystemStore) + mockStore.On("Job").Return(&mockJobStore) + mockStore.On("GetDBSchemaVersion").Return(1, nil) + + aggImpl := ElasticsearchAggregatorInterfaceImpl{Server: th.Server} + aggImpl.Server.Jobs.Store = mockStore + + // Register search engine + th.App.SearchEngine().RegisterElasticsearchEngine(&ElasticsearchInterfaceImpl{ + Platform: th.Server.Platform(), + }) + + // Set up the state for the tests. + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + *cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1 + *cfg.ElasticsearchSettings.AggregatePostsAfterDays = 1 + *cfg.SqlSettings.DisableDatabaseSearch = true + }) + + sched := aggImpl.MakeScheduler() + // Pass pending jobs as true + job, appErr := sched.ScheduleJob(th.Context, th.App.Config(), true, nil) + require.Nil(t, job) + require.Nil(t, appErr) + + mockJobStore.AssertNotCalled(t, "GetCountByStatusAndType") +} + +func indexPost(t *testing.T, th *api4.TestHelper, esImpl *ElasticsearchInterfaceImpl, post *model.Post, createTime time.Time) { //nolint:unused + t.Helper() + indexName := common.BuildPostIndexName(*th.Server.Config().ElasticsearchSettings.AggregatePostsAfterDays, + common.IndexBasePosts, + common.IndexBasePosts_MONTH, + createTime.Add(-1*24*time.Hour), + model.GetMillisForTime(createTime), + ) + searchPost, err := common.ESPostFromPost(post, "teamID") + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), + time.Duration(*esImpl.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = esImpl.client.Index(indexName). + Id(post.Id). + Document(searchPost). + Do(ctx) + require.NoError(t, err) +} diff --git a/server/enterprise/elasticsearch/elasticsearch/aggregation_scheduler.go b/server/enterprise/elasticsearch/elasticsearch/aggregation_scheduler.go new file mode 100644 index 0000000000..553b043071 --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/aggregation_scheduler.go @@ -0,0 +1,73 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "net/http" + "time" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/app" + "github.com/mattermost/mattermost/server/v8/channels/jobs" + ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs" +) + +type ElasticSearchAggregatorScheduler struct { + jobServer *jobs.JobServer + server *app.Server +} + +func (s *ElasticSearchAggregatorScheduler) Enabled(cfg *model.Config) bool { + if license := s.server.License(); license == nil || !*license.Features.Elasticsearch { + return false + } + + if *cfg.ElasticsearchSettings.EnableIndexing { + return true + } + + return false +} + +func (s *ElasticSearchAggregatorScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { + parsedTime, err := time.Parse("15:04", *cfg.ElasticsearchSettings.PostsAggregatorJobStartTime) + if err != nil { + s.server.Log().Error("Cannot determine next schedule time for elastic search post aggregator. PostsAggregatorJobStartTime config value is invalid.", mlog.Err(err)) + return nil + } + + return jobs.GenerateNextStartDateTime(now, parsedTime) +} + +func (s *ElasticSearchAggregatorScheduler) ScheduleJob(rctx request.CTX, _ *model.Config, pendingJobs bool, _ *model.Job) (*model.Job, *model.AppError) { + if pendingJobs { + s.server.Log().Warn("An aggregator job is already running. Skipping.") + return nil, nil + } + + // Don't schedule a job if we already have a running bulk indexing job + count, err := s.jobServer.Store.Job().GetCountByStatusAndType(model.JobStatusInProgress, model.JobTypeElasticsearchPostIndexing) + if err != nil { + return nil, model.NewAppError( + "ScheduleJob", + model.NoTranslation, + nil, + "", + http.StatusInternalServerError).Wrap(err) + } + if count > 0 { + return nil, nil + } + + return s.jobServer.CreateJob(rctx, model.JobTypeElasticsearchPostAggregation, nil) +} + +func (esi *ElasticsearchAggregatorInterfaceImpl) MakeScheduler() ejobs.Scheduler { + return &ElasticSearchAggregatorScheduler{ + server: esi.Server, + jobServer: esi.Server.Jobs, + } +} diff --git a/server/enterprise/elasticsearch/elasticsearch/bulk.go b/server/enterprise/elasticsearch/elasticsearch/bulk.go new file mode 100644 index 0000000000..9488e68e8f --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/bulk.go @@ -0,0 +1,133 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "context" + "sync" + "time" + + elastic "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/bulk" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" +) + +type Bulk struct { + mut sync.Mutex + + logger mlog.LoggerIFace + client *elastic.TypedClient + bulkClient *bulk.Bulk + settings model.ElasticsearchSettings + + quitFlusher chan struct{} + quitFlusherWg sync.WaitGroup + + pendingRequests int +} + +func NewBulk(settings model.ElasticsearchSettings, + logger mlog.LoggerIFace, + client *elastic.TypedClient) *Bulk { + b := &Bulk{ + settings: settings, + logger: logger, + client: client, + bulkClient: client.Bulk(), + quitFlusher: make(chan struct{}), + } + + b.quitFlusherWg.Add(1) + go b.periodicFlusher() + + return b +} + +// IndexOp is a helper function to add an IndexOperation to the current bulk request. +// doc argument can be a []byte, json.RawMessage or a struct. +func (r *Bulk) IndexOp(op types.IndexOperation, doc any) error { + r.mut.Lock() + defer r.mut.Unlock() + + if err := r.bulkClient.IndexOp(op, doc); err != nil { + return err + } + + return r.flushIfNecessary() +} + +// DeleteOp is a helper function to add a DeleteOperation to the current bulk request. +func (r *Bulk) DeleteOp(op types.DeleteOperation) error { + r.mut.Lock() + defer r.mut.Unlock() + + if err := r.bulkClient.DeleteOp(op); err != nil { + return err + } + + return r.flushIfNecessary() +} + +// flushIfNecessary flushes the pending buffer if needed. +// It MUST be called with an already acquired mutex. +func (r *Bulk) flushIfNecessary() error { + r.pendingRequests++ + + if r.pendingRequests > *r.settings.LiveIndexingBatchSize { + return r._flush() + } + + return nil +} + +func (r *Bulk) Stop() error { + r.mut.Lock() + defer r.mut.Unlock() + r.logger.Info("Stopping Bulk processor") + + if r.pendingRequests > 0 { + return r._flush() + } + + close(r.quitFlusher) + r.quitFlusherWg.Wait() + + return nil +} + +func (r *Bulk) periodicFlusher() { + defer r.quitFlusherWg.Done() + + for { + select { + case <-time.After(common.BulkFlushInterval): + r.mut.Lock() + if r.pendingRequests > 0 { + if err := r._flush(); err != nil { + r.logger.Warn("Error flushing live indexing buffer", mlog.Err(err)) + } + } + r.mut.Unlock() + case <-r.quitFlusher: + return + } + } +} + +// _flush MUST be called with an acquired lock. +func (r *Bulk) _flush() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*r.settings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err := r.bulkClient.Do(ctx) + if err != nil { + return err + } + r.pendingRequests = 0 + + return nil +} diff --git a/server/enterprise/elasticsearch/elasticsearch/bulk_test.go b/server/enterprise/elasticsearch/elasticsearch/bulk_test.go new file mode 100644 index 0000000000..0970dbddb2 --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/bulk_test.go @@ -0,0 +1,43 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "testing" + + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/stretchr/testify/require" +) + +func TestBulkProcessor(t *testing.T) { + th := api4.SetupEnterprise(t) + defer th.TearDown() + + client := createTestClient(t, th.Context, th.App.Config(), th.App.FileBackend()) + bulk := NewBulk(th.App.Config().ElasticsearchSettings, + th.Server.Platform().Log(), + client) + + post, err := common.ESPostFromPost(&model.Post{ + Id: model.NewId(), + Message: "hello world", + }, "myteam") + require.NoError(t, err) + + err = bulk.IndexOp(types.IndexOperation{ + Index_: model.NewPointer("myindex"), + Id_: model.NewPointer(post.Id), + }, post) + require.NoError(t, err) + + require.Equal(t, 1, bulk.pendingRequests) + + err = bulk.Stop() + require.NoError(t, err) + + require.Equal(t, 0, bulk.pendingRequests) +} diff --git a/server/enterprise/elasticsearch/elasticsearch/common.go b/server/enterprise/elasticsearch/elasticsearch/common.go new file mode 100644 index 0000000000..c857b0e6c0 --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/common.go @@ -0,0 +1,134 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "crypto/tls" + "net/http" + "time" + + "github.com/elastic/go-elasticsearch/v8" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" +) + +func createTypedClient(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend, debugLogging bool) (*elasticsearch.TypedClient, *model.AppError) { + esCfg, appErr := createClientConfig(logger, cfg, fileBackend, debugLogging) + if appErr != nil { + return nil, appErr + } + + client, err := elasticsearch.NewTypedClient(*esCfg) + if err != nil { + return nil, model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.connect_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return client, nil +} + +func createUntypedClient(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend) (*elasticsearch.Client, *model.AppError) { + esCfg, appErr := createClientConfig(logger, cfg, fileBackend, true) + if appErr != nil { + return nil, appErr + } + + client, err := elasticsearch.NewClient(*esCfg) + if err != nil { + return nil, model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.connect_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return client, nil +} + +func createClientConfig(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend, debugLogging bool) (*elasticsearch.Config, *model.AppError) { + tp := http.DefaultTransport.(*http.Transport).Clone() + tp.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: *cfg.ElasticsearchSettings.SkipTLSVerification, + } + + esCfg := &elasticsearch.Config{ + Addresses: []string{*cfg.ElasticsearchSettings.ConnectionURL}, + RetryBackoff: func(i int) time.Duration { return time.Duration(i) * 100 * time.Millisecond }, // A minimal backoff function + RetryOnStatus: []int{502, 503, 504, 429}, // Retry on 429 TooManyRequests statuses + MaxRetries: 3, + DiscoverNodesOnStart: *cfg.ElasticsearchSettings.Sniff, + } + + if esCfg.DiscoverNodesOnStart { + esCfg.DiscoverNodesInterval = 30 * time.Second + } + + if *cfg.ElasticsearchSettings.ClientCert != "" { + appErr := configureClientCertificate(tp.TLSClientConfig, cfg, fileBackend) + if appErr != nil { + return nil, appErr + } + } + + // custom CA + if *cfg.ElasticsearchSettings.CA != "" { + appErr := configureCA(esCfg, cfg, fileBackend) + if appErr != nil { + return nil, appErr + } + } + + esCfg.Transport = tp + + if *cfg.ElasticsearchSettings.Username != "" { + esCfg.Username = *cfg.ElasticsearchSettings.Username + esCfg.Password = *cfg.ElasticsearchSettings.Password + } + + // This is a compatibility mode from previous config settings. + // We have to conditionally enable debug logging due to + // https://github.com/elastic/elastic-transport-go/issues/22 + if *cfg.ElasticsearchSettings.Trace == "all" && debugLogging { + esCfg.EnableDebugLogger = true + } + + esCfg.Logger = common.NewLogger("Elasticsearch", logger, *cfg.ElasticsearchSettings.Trace == "all") + + return esCfg, nil +} + +func configureCA(esCfg *elasticsearch.Config, cfg *model.Config, fb filestore.FileBackend) *model.AppError { + // read the certificate authority (CA) file + clientCA, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.CA) + if err != nil { + return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.ca_cert_missing", nil, "", http.StatusInternalServerError).Wrap(err) + } + + esCfg.CACert = clientCA + + return nil +} + +func configureClientCertificate(tlsConfig *tls.Config, cfg *model.Config, fb filestore.FileBackend) *model.AppError { + // read the client certificate file + clientCert, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.ClientCert) + if err != nil { + return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_cert_missing", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // read the client key file + clientKey, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.ClientKey) + if err != nil { + return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_key_missing", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // load the client key and certificate + certificate, err := tls.X509KeyPair(clientCert, clientKey) + if err != nil { + return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_cert_malformed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // update the TLS config + tlsConfig.Certificates = []tls.Certificate{certificate} + + return nil +} diff --git a/server/enterprise/elasticsearch/elasticsearch/elasticsearch.go b/server/enterprise/elasticsearch/elasticsearch/elasticsearch.go new file mode 100644 index 0000000000..239be8059b --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/elasticsearch.go @@ -0,0 +1,1950 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "context" + "encoding/json" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/app/platform" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" + + elastic "github.com/elastic/go-elasticsearch/v8" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/deletebyquery" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/highlighterencoder" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/operator" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder" +) + +const elasticsearchMaxVersion = 8 + +var ( + purgeIndexListAllowedIndexes = []string{common.IndexBaseChannels} +) + +type ElasticsearchInterfaceImpl struct { + client *elastic.TypedClient + mutex sync.RWMutex + ready int32 + version int + fullVersion string + plugins []string + + bulkProcessor *Bulk + Platform *platform.PlatformService + + // This flag is for indicating if channel index's mappings + // has been verified, and if so, what was the result. + // + // value = 0 indicates it has NOT BEEN CHECKED + // value = 1 indicates index has been checked and has CORRECT mappings + // value = 2 indicates index has been checked and it has INCORRECT mappings + channelIndexVerified int32 +} + +func getJSONOrErrorStr(obj any) string { + b, err := json.Marshal(obj) + if err != nil { + return err.Error() + } + return string(b) +} + +func (*ElasticsearchInterfaceImpl) UpdateConfig(cfg *model.Config) { + // Not needed, it use the `Server` stored internally to get always the last version +} + +func (*ElasticsearchInterfaceImpl) GetName() string { + return "elasticsearch" +} + +func (es *ElasticsearchInterfaceImpl) IsEnabled() bool { + return *es.Platform.Config().ElasticsearchSettings.EnableIndexing +} + +func (es *ElasticsearchInterfaceImpl) IsActive() bool { + return *es.Platform.Config().ElasticsearchSettings.EnableIndexing && atomic.LoadInt32(&es.ready) == 1 +} + +func (es *ElasticsearchInterfaceImpl) IsIndexingEnabled() bool { + return *es.Platform.Config().ElasticsearchSettings.EnableIndexing +} + +func (es *ElasticsearchInterfaceImpl) IsSearchEnabled() bool { + return *es.Platform.Config().ElasticsearchSettings.EnableSearching +} + +func (es *ElasticsearchInterfaceImpl) IsAutocompletionEnabled() bool { + // if we encounter the index mappings haven't been checked, we check it once and store result. + // While in most cases the flag would have been set in the `Start` function, + // There's a case if you call the update config API and enable ES and autocomplete at the same time, it's not set + // so we're checking if it's unset here and trying to check the index. + if atomic.LoadInt32(&es.channelIndexVerified) == 0 { + es.Platform.Log().Debug("Elasticsearch.IsAutocompletionEnabled: channel index has not been verified yet, checking index now") + es.checkChannelIndex() + } + + return *es.Platform.Config().ElasticsearchSettings.EnableAutocomplete && atomic.LoadInt32(&es.channelIndexVerified) == 1 +} + +func (es *ElasticsearchInterfaceImpl) IsChannelsIndexVerified() bool { + if atomic.LoadInt32(&es.channelIndexVerified) == 0 { + es.Platform.Log().Debug("Elasticsearch.IsChannelsIndexVerified: channel index has not been verified yet, checking index now") + es.checkChannelIndex() + } + + return atomic.LoadInt32(&es.channelIndexVerified) == 1 +} + +func (es *ElasticsearchInterfaceImpl) IsIndexingSync() bool { + return *es.Platform.Config().ElasticsearchSettings.LiveIndexingBatchSize <= 1 +} + +func (es *ElasticsearchInterfaceImpl) Start() *model.AppError { + if license := es.Platform.License(); license == nil || !*license.Features.Elasticsearch || !*es.Platform.Config().ElasticsearchSettings.EnableIndexing { + return nil + } + + es.mutex.Lock() + defer es.mutex.Unlock() + + if atomic.LoadInt32(&es.ready) != 0 { + // Elasticsearch is already started. We don't return an error + // because "Test Connection" already re-initializes the client. So this + // can be a valid scenario. + return nil + } + + var appErr *model.AppError + if es.client, appErr = createTypedClient(es.Platform.Log(), es.Platform.Config(), es.Platform.FileBackend(), true); appErr != nil { + return appErr + } + + version, major, appErr := checkMaxVersion(es.client, es.Platform.Config()) + if appErr != nil { + return appErr + } + + // Since we are only retrieving plugins for the Support Packet generation, it doesn't make sense to kill the process if we get an error + // Instead, we will log it and move forward + resp, err := es.client.API.Cat.Plugins().Do(context.Background()) + if err != nil { + es.Platform.Log().Warn("Error retrieving elasticsearch plugins", mlog.Err(err)) + } else { + for _, p := range resp { + es.plugins = append(es.plugins, *p.Component) + } + } + + es.version = major + es.fullVersion = version + + ctx := context.Background() + + if *es.Platform.Config().ElasticsearchSettings.LiveIndexingBatchSize > 1 { + es.bulkProcessor = NewBulk(es.Platform.Config().ElasticsearchSettings, + es.Platform.Log(), + es.client) + } + + // Set up posts index template. + _, err = es.client.API.Indices.PutIndexTemplate(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts). + Request(common.GetPostTemplate(es.Platform.Config())). + Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.start", "ent.elasticsearch.create_template_posts_if_not_exists.template_create_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // Set up channels index template. + _, err = es.client.API.Indices.PutIndexTemplate(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels). + Request(common.GetChannelTemplate(es.Platform.Config())). + Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.start", "ent.elasticsearch.create_template_channels_if_not_exists.template_create_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // Set up users index template. + _, err = es.client.API.Indices.PutIndexTemplate(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers). + Request(common.GetUserTemplate(es.Platform.Config())). + Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.start", "ent.elasticsearch.create_template_users_if_not_exists.template_create_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // Set up files index template. + _, err = es.client.API.Indices.PutIndexTemplate(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles). + Request(common.GetFileInfoTemplate(es.Platform.Config())). + Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.start", "ent.elasticsearch.create_template_file_info_if_not_exists.template_create_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + if atomic.LoadInt32(&es.channelIndexVerified) == 0 { + es.checkChannelIndex() + } + + atomic.StoreInt32(&es.ready, 1) + + return nil +} + +func (es *ElasticsearchInterfaceImpl) Stop() *model.AppError { + es.mutex.Lock() + defer es.mutex.Unlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.start", "ent.elasticsearch.stop.already_stopped.app_error", nil, "", http.StatusInternalServerError) + } + + es.client = nil + // Flushing any pending requests + if es.bulkProcessor != nil { + if err := es.bulkProcessor.Stop(); err != nil { + es.Platform.Log().Warn("Error stopping bulk processor", mlog.Err(err)) + } + es.bulkProcessor = nil + } + + atomic.StoreInt32(&es.ready, 0) + + return nil +} + +func (es *ElasticsearchInterfaceImpl) GetVersion() int { + return es.version +} + +func (es *ElasticsearchInterfaceImpl) GetFullVersion() string { + return es.fullVersion +} + +func (es *ElasticsearchInterfaceImpl) GetPlugins() []string { + return es.plugins +} + +func (es *ElasticsearchInterfaceImpl) IndexPost(post *model.Post, teamId string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.IndexPost", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + indexName := common.BuildPostIndexName(*es.Platform.Config().ElasticsearchSettings.AggregatePostsAfterDays, + *es.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts, *es.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts_MONTH, time.Now(), post.CreateAt) + + searchPost, err := common.ESPostFromPost(post, teamId) + if err != nil { + return model.NewAppError("Elasticsearch.IndexPost", "ent.elasticsearch.index_post.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + if es.bulkProcessor != nil { + err = es.bulkProcessor.IndexOp(types.IndexOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(searchPost.Id), + }, searchPost) + if err != nil { + return model.NewAppError("Elasticsearch.IndexPost", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = es.client.Index(indexName). + Id(post.Id). + Document(searchPost). + Do(ctx) + } + if err != nil { + return model.NewAppError("Elasticsearch.IndexPost", "ent.elasticsearch.index_post.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + metrics := es.Platform.Metrics() + if metrics != nil { + metrics.IncrementPostIndexCounter() + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) getPostIndexNames() ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + indexes, err := es.client.API.Indices.Get("_all").Do(ctx) + if err != nil { + return nil, err + } + postIndexes := make([]string, 0) + for name := range indexes { + if strings.HasPrefix(name, *es.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts) { + postIndexes = append(postIndexes, name) + } + } + return postIndexes, nil +} + +func (es *ElasticsearchInterfaceImpl) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return []string{}, nil, model.NewAppError("Elasticsearch.SearchPosts", "ent.elasticsearch.search_posts.disabled", nil, "", http.StatusInternalServerError) + } + + var channelIds []string + for _, channel := range channels { + channelIds = append(channelIds, channel.Id) + } + + var termQueries, notTermQueries, highlightQueries []types.Query + var filters, notFilters []types.Query + for i, params := range searchParams { + newTerms := []string{} + for _, term := range strings.Split(params.Terms, " ") { + if searchengine.EmailRegex.MatchString(term) { + term = `"` + term + `"` + } + newTerms = append(newTerms, term) + } + + params.Terms = strings.Join(newTerms, " ") + + termOperator := operator.And + if searchParams[0].OrTerms { + termOperator = operator.Or + } + + // Date, channels and FromUsers filters come in all + // searchParams iteration, and as they are global to the + // query, we only need to process them once + if i == 0 { + if len(params.InChannels) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": params.InChannels}}, + }) + } + + if len(params.ExcludedChannels) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": params.ExcludedChannels}}, + }) + } + + if len(params.FromUsers) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"user_id": params.FromUsers}}, + }) + } + + if len(params.ExcludedUsers) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"user_id": params.ExcludedUsers}}, + }) + } + + if params.OnDate != "" { + before, after := params.GetOnDateMillis() + filters = append(filters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(before)), + Lte: model.NewPointer(types.Float64(after)), + }, + }, + }) + } else { + if params.AfterDate != "" || params.BeforeDate != "" { + nrQuery := types.NumberRangeQuery{} + if params.AfterDate != "" { + nrQuery.Gte = model.NewPointer(types.Float64(params.GetAfterDateMillis())) + } + + if params.BeforeDate != "" { + nrQuery.Lte = model.NewPointer(types.Float64(params.GetBeforeDateMillis())) + } + + query := types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": nrQuery, + }, + } + filters = append(filters, query) + } + + if params.ExcludedAfterDate != "" || params.ExcludedBeforeDate != "" || params.ExcludedDate != "" { + if params.ExcludedDate != "" { + before, after := params.GetExcludedDateMillis() + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(before)), + Lte: model.NewPointer(types.Float64(after)), + }, + }, + }) + } + + if params.ExcludedAfterDate != "" { + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(params.GetExcludedAfterDateMillis())), + }, + }, + }) + } + + if params.ExcludedBeforeDate != "" { + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Lte: model.NewPointer(types.Float64(params.GetExcludedBeforeDateMillis())), + }, + }, + }) + } + } + } + } + + if params.IsHashtag { + if params.Terms != "" { + query := types.Query{ + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"hashtags"}, + DefaultOperator: &termOperator, + }, + } + termQueries = append(termQueries, query) + highlightQueries = append(highlightQueries, query) + } else if params.ExcludedTerms != "" { + query := types.Query{ + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"hashtags"}, + DefaultOperator: &termOperator, + }, + } + notTermQueries = append(notTermQueries, query) + } + } else { + if params.Terms != "" { + elements := []types.Query{ + { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"message"}, + DefaultOperator: &termOperator, + }, + }, { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"attachments"}, + DefaultOperator: &termOperator, + }, + }, { + Term: map[string]types.TermQuery{ + "urls": {Value: params.Terms}, + }, + }, + } + query := types.Query{ + Bool: &types.BoolQuery{Should: append([]types.Query(nil), elements...)}, + } + + termQueries = append(termQueries, query) + + hashtagTerms := []string{} + for _, term := range strings.Split(params.Terms, " ") { + hashtagTerms = append(hashtagTerms, "#"+term) + } + + hashtagQuery := types.Query{ + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: strings.Join(hashtagTerms, " "), + Fields: []string{"hashtags"}, + DefaultOperator: &termOperator, + }, + } + highlightQuery := types.Query{ + Bool: &types.BoolQuery{Should: append(elements, hashtagQuery)}, + } + + highlightQueries = append(highlightQueries, highlightQuery) + } + + if params.ExcludedTerms != "" { + query := types.Query{ + Bool: &types.BoolQuery{Should: []types.Query{ + { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"message"}, + DefaultOperator: &termOperator, + }, + }, { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"attachments"}, + DefaultOperator: &termOperator, + }, + }, { + Term: map[string]types.TermQuery{ + "urls": {Value: params.ExcludedTerms}, + }, + }, + }}, + } + + notTermQueries = append(notTermQueries, query) + } + } + } + + allTermsQuery := &types.BoolQuery{ + MustNot: append([]types.Query(nil), notTermQueries...), + } + if searchParams[0].OrTerms { + allTermsQuery.Should = append([]types.Query(nil), termQueries...) + } else { + allTermsQuery.Must = append([]types.Query(nil), termQueries...) + } + + fullHighlightsQuery := &types.BoolQuery{ + Filter: append([]types.Query(nil), filters...), + MustNot: append([]types.Query(nil), notFilters...), + } + + if searchParams[0].OrTerms { + fullHighlightsQuery.Should = append([]types.Query(nil), highlightQueries...) + } else { + fullHighlightsQuery.Must = append([]types.Query(nil), highlightQueries...) + } + + filters = append(filters, + types.Query{ + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{"channel_id": channelIds}, + }, + }, + types.Query{ + Bool: &types.BoolQuery{ + Should: []types.Query{ + { + Term: map[string]types.TermQuery{"type": {Value: "default"}}, + }, { + Term: map[string]types.TermQuery{"type": {Value: "slack_attachment"}}, + }, + }, + }, + }, + ) + + highlight := &types.Highlight{ + HighlightQuery: &types.Query{ + Bool: fullHighlightsQuery, + }, + Fields: map[string]types.HighlightField{ + "message": {}, + "attachments": {}, + "url": {}, + "hashtag": {}, + }, + Encoder: &highlighterencoder.Html, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: append([]types.Query(nil), filters...), + Must: []types.Query{{Bool: allTermsQuery}}, + MustNot: append([]types.Query(nil), notFilters...), + }, + } + + search := es.client.Search(). + Index(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "*"). + Request(&search.Request{ + Query: query, + Highlight: highlight, + }). + Sort(types.SortOptions{SortOptions: map[string]types.FieldSort{ + "create_at": {Order: &sortorder.Desc}, + }}). + From(page * perPage). + Size(perPage) + + searchResult, err := search.Do(ctx) + if err != nil { + errorStr := "err=" + err.Error() + if *es.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return []string{}, nil, model.NewAppError("Elasticsearch.SearchPosts", "ent.elasticsearch.search_posts.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + postIds := make([]string, len(searchResult.Hits.Hits)) + matches := make(model.PostSearchMatches, len(searchResult.Hits.Hits)) + + for i, hit := range searchResult.Hits.Hits { + var post common.ESPost + err := json.Unmarshal(hit.Source_, &post) + if err != nil { + return postIds, matches, model.NewAppError("Elasticsearch.SearchPosts", "ent.elasticsearch.search_posts.unmarshall_post_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + postIds[i] = post.Id + + matchesForPost, err := common.GetMatchesForHit(hit.Highlight) + if err != nil { + return postIds, matches, model.NewAppError("Elasticsearch.SearchPosts", "ent.elasticsearch.search_posts.parse_matches_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + matches[post.Id] = matchesForPost + } + + return postIds, matches, nil +} + +func (es *ElasticsearchInterfaceImpl) DeletePost(post *model.Post) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeletePost", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + // This is racy with index aggregation, but since the posts are verified in the database when returning search + // results, there's no risk of deleted posts getting sent back to the user in response to a search query, and even + // then the race is very unlikely because it would only occur when someone deletes a post that's due to be + // aggregated but hasn't been yet, which makes the time window small and the post likelihood very low. + indexName := common.BuildPostIndexName(*es.Platform.Config().ElasticsearchSettings.AggregatePostsAfterDays, + *es.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts, *es.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts_MONTH, time.Now(), post.CreateAt) + + if err := es.deletePost(indexName, post.Id); err != nil { + return err + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) DeleteChannelPosts(rctx request.CTX, channelID string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeleteChannelPosts", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + postIndexes, err := es.getPostIndexNames() + if err != nil { + return model.NewAppError("Elasticsearch.DeleteChannelPosts", "ent.elasticsearch.delete_channel_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"channel_id": {Value: channelID}}, + }}, + }, + } + deleteQuery := es.client.DeleteByQuery(strings.Join(postIndexes, ",")). + Request(&deletebyquery.Request{ + Query: query, + }) + response, err := deleteQuery.Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.DeleteChannelPosts", "ent.elasticsearch.delete_channel_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("Posts for channel deleted", mlog.String("channel_id", channelID), mlog.Int("deleted", *response.Deleted)) + + return nil +} + +func (es *ElasticsearchInterfaceImpl) DeleteUserPosts(rctx request.CTX, userID string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeleteUserPosts", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + postIndexes, err := es.getPostIndexNames() + if err != nil { + return model.NewAppError("Elasticsearch.DeleteUserPosts", "ent.elasticsearch.delete_user_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"user_id": {Value: userID}}, + }}, + }, + } + + deleteQuery := es.client.DeleteByQuery(strings.Join(postIndexes, ",")). + Request(&deletebyquery.Request{ + Query: query, + }) + + response, err := deleteQuery.Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.DeleteUserPosts", "ent.elasticsearch.delete_user_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("Posts for user deleted", mlog.String("user_id", userID), mlog.Int("deleted", *response.Deleted)) + + return nil +} + +func (es *ElasticsearchInterfaceImpl) deletePost(indexName, postID string) *model.AppError { + var err error + if es.bulkProcessor != nil { + err = es.bulkProcessor.DeleteOp(types.DeleteOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(postID), + }) + if err != nil { + return model.NewAppError("Elasticsearch.IndexPost", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + _, err = es.client.Delete(indexName, postID).Do(ctx) + } + if err != nil { + return model.NewAppError("Elasticsearch.DeletePost", "ent.elasticsearch.delete_post.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil +} + +func (es *ElasticsearchInterfaceImpl) IndexChannel(rctx request.CTX, channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.IndexChannel", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + indexName := *es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels + + searchChannel := common.ESChannelFromChannel(channel, userIDs, teamMemberIDs) + + var err error + if es.bulkProcessor != nil { + err = es.bulkProcessor.IndexOp(types.IndexOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(searchChannel.Id), + }, searchChannel) + if err != nil { + return model.NewAppError("Elasticsearch.IndexPost", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + _, err = es.client.Index(indexName). + Id(searchChannel.Id). + Document(searchChannel). + Do(ctx) + } + if err != nil { + return model.NewAppError("Elasticsearch.IndexChannel", "ent.elasticsearch.index_channel.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + metrics := es.Platform.Metrics() + if metrics != nil { + metrics.IncrementChannelIndexCounter() + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) SearchChannels(teamId, userID string, term string, isGuest bool) ([]string, *model.AppError) { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return []string{}, model.NewAppError("Elasticsearch.SearchChannels", "ent.elasticsearch.search_channels.disabled", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + boolNotPrivate := types.Query{ + Bool: &types.BoolQuery{ + MustNot: []types.Query{{ + Term: map[string]types.TermQuery{"type": {Value: model.ChannelTypePrivate}}, + }}, + }, + } + + userQ := types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"user_ids": {Value: userID}}, + }}, + Must: []types.Query{{ + Term: map[string]types.TermQuery{"type": {Value: model.ChannelTypePrivate}}, + }}, + }, + } + + query := &types.BoolQuery{} + + if teamId != "" { + query.Filter = append(query.Filter, types.Query{Term: map[string]types.TermQuery{"team_id": {Value: teamId}}}) + } else { + query.Filter = append(query.Filter, types.Query{Term: map[string]types.TermQuery{"team_member_ids": {Value: userID}}}) + } + + if !isGuest { + query.Filter = append(query.Filter, types.Query{ + Bool: &types.BoolQuery{ + Should: []types.Query{ + boolNotPrivate, userQ, + }, + Must: []types.Query{{ + Prefix: map[string]types.PrefixQuery{ + "name_suggestions": {Value: strings.ToLower(term)}, + }, + }}, + MinimumShouldMatch: 1, + }, + }) + } else { + query.Filter = append(query.Filter, types.Query{ + Bool: &types.BoolQuery{ + Must: []types.Query{ + boolNotPrivate, { + Prefix: map[string]types.PrefixQuery{ + "name_suggestions": {Value: strings.ToLower(term)}, + }, + }}, + }, + }) + } + + search := es.client.Search(). + Index(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels). + Request(&search.Request{ + Query: &types.Query{Bool: query}, + }). + Size(model.ChannelSearchDefaultLimit) + + searchResult, err := search.Do(ctx) + + if err != nil { + errorStr := "err=" + err.Error() + if *es.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return nil, model.NewAppError("Elasticsearch.SearchChannels", "ent.elasticsearch.search_channels.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + channelIds := []string{} + for _, hit := range searchResult.Hits.Hits { + var channel common.ESChannel + err := json.Unmarshal(hit.Source_, &channel) + if err != nil { + return nil, model.NewAppError("Elasticsearch.SearchChannels", "ent.elasticsearch.search_channels.unmarshall_channel_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + channelIds = append(channelIds, channel.Id) + } + + return channelIds, nil +} + +func (es *ElasticsearchInterfaceImpl) DeleteChannel(channel *model.Channel) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeleteChannel", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + var err error + if es.bulkProcessor != nil { + err = es.bulkProcessor.DeleteOp(types.DeleteOperation{ + Index_: model.NewPointer(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels), + Id_: model.NewPointer(channel.Id), + }) + if err != nil { + return model.NewAppError("Elasticsearch.IndexPost", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = es.client.Delete(*es.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBaseChannels, channel.Id). + Do(ctx) + } + if err != nil { + return model.NewAppError("Elasticsearch.DeleteChannel", "ent.elasticsearch.delete_channel.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) IndexUser(rctx request.CTX, user *model.User, teamsIds, channelsIds []string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.IndexUser", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + indexName := *es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers + + searchUser := common.ESUserFromUserAndTeams(user, teamsIds, channelsIds) + + var err error + if es.bulkProcessor != nil { + err = es.bulkProcessor.IndexOp(types.IndexOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(searchUser.Id), + }, searchUser) + if err != nil { + return model.NewAppError("Elasticsearch.IndexPost", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = es.client.Index(indexName). + Id(searchUser.Id). + Document(searchUser). + Do(ctx) + } + if err != nil { + return model.NewAppError("Elasticsearch.IndexUser", "ent.elasticsearch.index_user.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + metrics := es.Platform.Metrics() + if metrics != nil { + metrics.IncrementUserIndexCounter() + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) autocompleteUsers(contextCategory string, categoryIds []string, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return nil, model.NewAppError("Elasticsearch.autocompleteUsers", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.BoolQuery{} + + if term != "" { + var suggestionField string + if options.AllowFullNames { + suggestionField = "suggestions_with_fullname" + } else { + suggestionField = "suggestions_without_fullname" + } + query.Must = append(query.Must, types.Query{ + Prefix: map[string]types.PrefixQuery{ + suggestionField: {Value: strings.ToLower(term)}, + }, + }) + } + + if len(categoryIds) > 0 { + var iCategoryIds []string + for _, id := range categoryIds { + if id != "" { + iCategoryIds = append(iCategoryIds, id) + } + } + if len(iCategoryIds) > 0 { + query.Filter = append(query.Filter, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{contextCategory: iCategoryIds}}, + }) + } + } + + if !options.AllowInactive { + query.Filter = append(query.Filter, types.Query{ + Bool: &types.BoolQuery{ + Should: []types.Query{ + { + Range: map[string]types.RangeQuery{ + "delete_at": types.DateRangeQuery{ + Lte: model.NewPointer("0"), + }, + }, + }, { + Bool: &types.BoolQuery{ + MustNot: []types.Query{{ + Exists: &types.ExistsQuery{Field: "delete_at"}, + }}, + }, + }, + }, + }, + }) + } + + if options.Role != "" { + query.Filter = append(query.Filter, types.Query{ + Term: map[string]types.TermQuery{ + "roles": {Value: options.Role}, + }, + }) + } + + search := es.client.Search(). + Index(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers). + Request(&search.Request{ + Query: &types.Query{Bool: query}, + }). + Size(options.Limit) + + searchResults, err := search.Do(ctx) + + if err != nil { + errorStr := "err=" + err.Error() + if *es.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return nil, model.NewAppError("Elasticsearch.autocompleteUsers", "ent.elasticsearch.search_users.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + users := []common.ESUser{} + for _, hit := range searchResults.Hits.Hits { + var user common.ESUser + err := json.Unmarshal(hit.Source_, &user) + if err != nil { + return nil, model.NewAppError("Elasticsearch.autocompleteUsers", "ent.elasticsearch.search_users.unmarshall_user_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + users = append(users, user) + } + + return users, nil +} + +func (es *ElasticsearchInterfaceImpl) autocompleteUsersInChannel(channelId, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + return es.autocompleteUsers("channel_id", []string{channelId}, term, options) +} + +func (es *ElasticsearchInterfaceImpl) autocompleteUsersInChannels(channelIds []string, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + return es.autocompleteUsers("channel_id", channelIds, term, options) +} + +func (es *ElasticsearchInterfaceImpl) autocompleteUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + return es.autocompleteUsers("team_id", []string{teamId}, term, options) +} + +func (es *ElasticsearchInterfaceImpl) autocompleteUsersNotInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return nil, model.NewAppError("Elasticsearch.autocompleteUsersNotInChannel", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + filterMust := []types.Query{{Term: map[string]types.TermQuery{ + "team_id": {Value: teamId}, + }}} + if len(restrictedToChannels) > 0 { + filterMust = append(filterMust, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": restrictedToChannels}}, + }) + } + + query := &types.BoolQuery{ + Filter: []types.Query{{ + Bool: &types.BoolQuery{ + Must: filterMust, + }, + }}, + MustNot: []types.Query{{ + Term: map[string]types.TermQuery{ + "channel_id": {Value: channelId}, + }, + }}, + } + + if term != "" { + var suggestionField string + if options.AllowFullNames { + suggestionField = "suggestions_with_fullname" + } else { + suggestionField = "suggestions_without_fullname" + } + query.Must = append(query.Must, types.Query{ + Prefix: map[string]types.PrefixQuery{ + suggestionField: {Value: strings.ToLower(term)}, + }, + }) + } + + if !options.AllowInactive { + notExistField := types.Query{ + Bool: &types.BoolQuery{ + MustNot: []types.Query{{ + Exists: &types.ExistsQuery{Field: "delete_at"}, + }}, + }, + } + deleteRangeQuery := types.Query{ + Range: map[string]types.RangeQuery{ + "delete_at": types.DateRangeQuery{ + Lte: model.NewPointer("0"), + }, + }, + } + inactiveQuery := types.Query{ + Bool: &types.BoolQuery{ + Should: []types.Query{deleteRangeQuery, notExistField}, + }, + } + query.Filter = append(query.Filter, inactiveQuery) + } + + if options.Role != "" { + query.Filter = append(query.Filter, types.Query{ + Term: map[string]types.TermQuery{ + "roles": {Value: options.Role}, + }, + }) + } + + search := es.client.Search(). + Index(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers). + Request(&search.Request{ + Query: &types.Query{Bool: query}, + }). + Size(options.Limit) + + searchResults, err := search.Do(ctx) + if err != nil { + errorStr := "err=" + err.Error() + if *es.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return nil, model.NewAppError("Elasticsearch.autocompleteUsersNotInChannel", "ent.elasticsearch.search_users.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + users := []common.ESUser{} + for _, hit := range searchResults.Hits.Hits { + var user common.ESUser + err := json.Unmarshal(hit.Source_, &user) + if err != nil { + return nil, model.NewAppError("Elasticsearch.autocompleteUsersNotInChannel", "ent.elasticsearch.search_users.unmarshall_user_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + users = append(users, user) + } + + return users, nil +} + +func (es *ElasticsearchInterfaceImpl) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) { + if restrictedToChannels != nil && len(restrictedToChannels) == 0 { + return []string{}, []string{}, nil + } + + uchan, err := es.autocompleteUsersInChannel(channelId, term, options) + if err != nil { + return nil, nil, err + } + + var nuchan []common.ESUser + nuchan, err = es.autocompleteUsersNotInChannel(teamId, channelId, restrictedToChannels, term, options) + if err != nil { + return nil, nil, err + } + + uchanIds := []string{} + for _, user := range uchan { + uchanIds = append(uchanIds, user.Id) + } + nuchanIds := []string{} + for _, user := range nuchan { + nuchanIds = append(nuchanIds, user.Id) + } + + return uchanIds, nuchanIds, nil +} + +func (es *ElasticsearchInterfaceImpl) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) { + if restrictedToChannels != nil && len(restrictedToChannels) == 0 { + return []string{}, nil + } + + var users []common.ESUser + var err *model.AppError + if restrictedToChannels == nil { + users, err = es.autocompleteUsersInTeam(teamId, term, options) + } else { + users, err = es.autocompleteUsersInChannels(restrictedToChannels, term, options) + } + if err != nil { + return nil, err + } + + usersIds := []string{} + if len(users) >= options.Limit { + users = users[:options.Limit] + } + + for _, user := range users { + usersIds = append(usersIds, user.Id) + } + + return usersIds, nil +} + +func (es *ElasticsearchInterfaceImpl) DeleteUser(user *model.User) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeleteUser", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + var err error + if es.bulkProcessor != nil { + err = es.bulkProcessor.DeleteOp(types.DeleteOperation{ + Index_: model.NewPointer(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers), + Id_: model.NewPointer(user.Id), + }) + if err != nil { + return model.NewAppError("Elasticsearch.DeleteUser", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = es.client.Delete(*es.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBaseUsers, user.Id). + Do(ctx) + } + if err != nil { + return model.NewAppError("Elasticsearch.DeleteUser", "ent.elasticsearch.delete_user.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) TestConfig(rctx request.CTX, cfg *model.Config) *model.AppError { + if license := es.Platform.License(); license == nil || !*license.Features.Elasticsearch { + return model.NewAppError("Elasticsearch.TestConfig", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) + } + + if !*cfg.ElasticsearchSettings.EnableIndexing { + return model.NewAppError("Elasticsearch.TestConfig", "ent.elasticsearch.test_config.indexing_disabled.error", nil, "", http.StatusNotImplemented) + } + + client, appErr := createTypedClient(rctx.Logger(), cfg, es.Platform.FileBackend(), true) + if appErr != nil { + return appErr + } + + _, _, appErr = checkMaxVersion(client, cfg) + if appErr != nil { + return appErr + } + + // Resetting the state. + if atomic.CompareAndSwapInt32(&es.ready, 0, 1) { + // Re-assign the client. + // This is necessary in case elasticsearch was started + // after server start. + es.mutex.Lock() + es.client = client + es.mutex.Unlock() + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) PurgeIndexes(rctx request.CTX) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if license := es.Platform.License(); license == nil || !*license.Features.Elasticsearch { + return model.NewAppError("Elasticsearch.PurgeIndexes", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) + } + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.PurgeIndexes", "ent.elasticsearch.generic.disabled", nil, "", http.StatusInternalServerError) + } + + indexPrefix := *es.Platform.Config().ElasticsearchSettings.IndexPrefix + indexesToDelete := indexPrefix + "*" + + if ignorePurgeIndexes := *es.Platform.Config().ElasticsearchSettings.IgnoredPurgeIndexes; ignorePurgeIndexes != "" { + // we are checking if provided indexes exist. If an index doesn't exist, + // elasticsearch returns an error while trying to purge it even we intend to + // ignore it. + for _, ignorePurgeIndex := range strings.Split(ignorePurgeIndexes, ",") { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err := es.client.Indices.Get(ignorePurgeIndex).Do(ctx) + if err != nil { + rctx.Logger().Warn("Elasticsearch index get error", mlog.String("index", ignorePurgeIndex), mlog.Err(err)) + continue + } + indexesToDelete += ",-" + strings.TrimSpace(ignorePurgeIndex) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err := es.client.Indices.Delete(indexesToDelete).Do(ctx) + if err != nil { + rctx.Logger().Error("Elastic Search PurgeIndexes Error", mlog.Err(err)) + return model.NewAppError("Elasticsearch.PurgeIndexes", "ent.elasticsearch.purge_indexes.delete_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return nil +} + +// PurgeIndexList purges a list of specified indexes. +// For now it only allows purging the channels index as thats all that's needed, +// but the code is written in generic fashion to allow it to purge any index. +// It needs more logic around post indexes as their name isn't the same, but rather follow a pattern +// containing the date as well. +func (es *ElasticsearchInterfaceImpl) PurgeIndexList(rctx request.CTX, indexes []string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if license := es.Platform.License(); license == nil || !*license.Features.Elasticsearch { + return model.NewAppError("Elasticsearch.PurgeIndexList", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) + } + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.PurgeIndexList", "ent.elasticsearch.generic.disabled", nil, "", http.StatusInternalServerError) + } + + indexPrefix := *es.Platform.Config().ElasticsearchSettings.IndexPrefix + indexToDeleteMap := map[string]bool{} + for _, index := range indexes { + isKnownIndex := false + for _, allowedIndex := range purgeIndexListAllowedIndexes { + if index == allowedIndex { + isKnownIndex = true + break + } + } + + if !isKnownIndex { + return model.NewAppError("Elasticsearch.PurgeIndexList", "ent.elasticsearch.purge_indexes.unknown_index", map[string]any{"unknown_index": index}, "", http.StatusBadRequest) + } + + indexToDeleteMap[indexPrefix+index] = true + } + + if ign := *es.Platform.Config().ElasticsearchSettings.IgnoredPurgeIndexes; ign != "" { + // make sure we're not purging any index configured to be ignored + for _, ix := range strings.Split(ign, ",") { + delete(indexToDeleteMap, ix) + } + } + + indexToDelete := []string{} + for key := range indexToDeleteMap { + indexToDelete = append(indexToDelete, key) + } + + if len(indexToDelete) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + _, err := es.client.Indices.Delete(strings.Join(indexToDelete, ",")).Do(ctx) + if err != nil { + elasticErr, ok := err.(*types.ElasticsearchError) + if !ok || elasticErr.Status != http.StatusNotFound { + rctx.Logger().Error("Elastic Search PurgeIndex Error", mlog.Err(err)) + return model.NewAppError("Elasticsearch.PurgeIndexList", "ent.elasticsearch.purge_index.delete_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + } + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) RefreshIndexes(rctx request.CTX) *model.AppError { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + _, err := es.client.Indices.Refresh().Do(ctx) + if err != nil { + rctx.Logger().Error("Elastic Search RefreshIndexes Error", mlog.Err(err)) + return model.NewAppError("Elasticsearch.RefreshIndexes", "ent.elasticsearch.refresh_indexes.refresh_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil +} + +func (es *ElasticsearchInterfaceImpl) DataRetentionDeleteIndexes(rctx request.CTX, cutoff time.Time) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if license := es.Platform.License(); license == nil || !*license.Features.Elasticsearch { + return model.NewAppError("Elasticsearch.DataRetentionDeleteIndexes", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) + } + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DataRetentionDeleteIndexes", "ent.elasticsearch.generic.disabled", nil, "", http.StatusInternalServerError) + } + + ctx := context.Background() + dateFormat := *es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_2006_01_02" + postIndexesResult, err := es.client.Indices.Get(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_*").Do(ctx) + if err != nil { + return model.NewAppError("ElasticSearch.DataRetentionDeleteIndexes", "ent.elasticsearch.data_retention_delete_indexes.get_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + for index := range postIndexesResult { + if indexDate, err := time.Parse(dateFormat, index); err != nil { + rctx.Logger().Warn("Failed to parse date from posts index. Ignoring index.", mlog.String("index", index)) + } else { + if indexDate.Before(cutoff) || indexDate.Equal(cutoff) { + if _, err := es.client.Indices.Delete(index).Do(ctx); err != nil { + return model.NewAppError("ElasticSearch.DataRetentionDeleteIndexes", "ent.elasticsearch.data_retention_delete_indexes.delete_index.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + } + } + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) IndexFile(file *model.FileInfo, channelId string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.IndexFile", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + indexName := *es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles + + searchFile := common.ESFileFromFileInfo(file, channelId) + + var err error + if es.bulkProcessor != nil { + err = es.bulkProcessor.IndexOp(types.IndexOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(searchFile.Id), + }, searchFile) + if err != nil { + return model.NewAppError("Elasticsearch.IndexFile", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = es.client.Index(indexName). + Id(file.Id). + Document(searchFile). + Do(ctx) + } + if err != nil { + return model.NewAppError("Elasticsearch.IndexFile", "ent.elasticsearch.index_file.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + if metrics := es.Platform.Metrics(); metrics != nil { + metrics.IncrementFileIndexCounter() + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return []string{}, model.NewAppError("Elasticsearch.SearchPosts", "ent.elasticsearch.search_files.disabled", nil, "", http.StatusInternalServerError) + } + + var channelIds []string + for _, channel := range channels { + channelIds = append(channelIds, channel.Id) + } + + var termQueries, notTermQueries []types.Query + var filters, notFilters []types.Query + for i, params := range searchParams { + newTerms := []string{} + for _, term := range strings.Split(params.Terms, " ") { + if searchengine.EmailRegex.MatchString(term) { + term = `"` + term + `"` + } + newTerms = append(newTerms, term) + } + + params.Terms = strings.Join(newTerms, " ") + + termOperator := operator.And + if searchParams[0].OrTerms { + termOperator = operator.Or + } + + // Date, channels and FromUsers filters come in all + // searchParams iteration, and as they are global to the + // query, we only need to process them once + if i == 0 { + if len(params.InChannels) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": params.InChannels}}, + }) + } + + if len(params.ExcludedChannels) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": params.ExcludedChannels}}, + }) + } + + if len(params.FromUsers) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"creator_id": params.FromUsers}}, + }) + } + + if len(params.ExcludedUsers) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"creator_id": params.ExcludedUsers}}, + }) + } + + if len(params.Extensions) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"extension": params.Extensions}}, + }) + } + + if len(params.ExcludedExtensions) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"extension": params.ExcludedExtensions}}, + }) + } + + if params.OnDate != "" { + before, after := params.GetOnDateMillis() + filters = append(filters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(before)), + Lte: model.NewPointer(types.Float64(after)), + }, + }, + }) + } else { + if params.AfterDate != "" || params.BeforeDate != "" { + nrQuery := types.NumberRangeQuery{} + if params.AfterDate != "" { + nrQuery.Gte = model.NewPointer(types.Float64(params.GetAfterDateMillis())) + } + + if params.BeforeDate != "" { + nrQuery.Lte = model.NewPointer(types.Float64(params.GetBeforeDateMillis())) + } + query := types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": nrQuery, + }, + } + filters = append(filters, query) + } + + if params.ExcludedAfterDate != "" || params.ExcludedBeforeDate != "" || params.ExcludedDate != "" { + if params.ExcludedDate != "" { + before, after := params.GetExcludedDateMillis() + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(before)), + Lte: model.NewPointer(types.Float64(after)), + }, + }, + }) + } + + if params.ExcludedAfterDate != "" { + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(params.GetExcludedAfterDateMillis())), + }, + }, + }) + } + + if params.ExcludedBeforeDate != "" { + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Lte: model.NewPointer(types.Float64(params.GetExcludedBeforeDateMillis())), + }, + }, + }) + } + } + } + } + + if params.Terms != "" { + elements := []types.Query{ + { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"content"}, + DefaultOperator: &termOperator, + }, + }, { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"name"}, + DefaultOperator: &termOperator, + }, + }, + } + query := types.Query{ + Bool: &types.BoolQuery{Should: append([]types.Query(nil), elements...)}, + } + termQueries = append(termQueries, query) + } + + if params.ExcludedTerms != "" { + elements := []types.Query{ + { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"content"}, + DefaultOperator: &termOperator, + }, + }, { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"name"}, + DefaultOperator: &termOperator, + }, + }, + } + query := types.Query{ + Bool: &types.BoolQuery{Should: append([]types.Query(nil), elements...)}, + } + notTermQueries = append(notTermQueries, query) + } + } + + allTermsQuery := &types.BoolQuery{ + MustNot: append([]types.Query(nil), notTermQueries...), + } + if searchParams[0].OrTerms { + allTermsQuery.Should = append([]types.Query(nil), termQueries...) + } else { + allTermsQuery.Must = append([]types.Query(nil), termQueries...) + } + + filters = append(filters, + types.Query{ + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{"channel_id": channelIds}, + }, + }, + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: append([]types.Query(nil), filters...), + Must: []types.Query{{Bool: allTermsQuery}}, + MustNot: append([]types.Query(nil), notFilters...), + }, + } + + search := es.client.Search(). + Index(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles). + Request(&search.Request{ + Query: query, + }). + Sort(types.SortOptions{SortOptions: map[string]types.FieldSort{ + "create_at": {Order: &sortorder.Desc}, + }}). + From(page * perPage). + Size(perPage) + + searchResult, err := search.Do(ctx) + if err != nil { + errorStr := "err=" + err.Error() + if *es.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return []string{}, model.NewAppError("Elasticsearch.SearchFiles", "ent.elasticsearch.search_files.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + fileIds := make([]string, len(searchResult.Hits.Hits)) + + for i, hit := range searchResult.Hits.Hits { + var file common.ESFile + if err := json.Unmarshal(hit.Source_, &file); err != nil { + return fileIds, model.NewAppError("Elasticsearch.SearchFiles", "ent.elasticsearch.search_files.unmarshall_file_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + fileIds[i] = file.Id + } + + return fileIds, nil +} + +func (es *ElasticsearchInterfaceImpl) DeleteFile(fileID string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeleteFile", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + var err error + if es.bulkProcessor != nil { + err = es.bulkProcessor.DeleteOp(types.DeleteOperation{ + Index_: model.NewPointer(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles), + Id_: model.NewPointer(fileID), + }) + if err != nil { + return model.NewAppError("Elasticsearch.DeleteFile", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = es.client.Delete(*es.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBaseFiles, fileID). + Do(ctx) + } + if err != nil { + return model.NewAppError("Elasticsearch.DeleteFile", "ent.elasticsearch.delete_file.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return nil +} + +func (es *ElasticsearchInterfaceImpl) DeleteUserFiles(rctx request.CTX, userID string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeleteFilesBatch", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"creator_id": {Value: userID}}, + }}, + }, + } + + deleteQuery := es.client.DeleteByQuery(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles). + Request(&deletebyquery.Request{ + Query: query, + }) + response, err := deleteQuery.Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.DeleteUserFiles", "ent.elasticsearch.delete_user_files.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("User files deleted", mlog.String("user_id", userID), mlog.Int("deleted", *response.Deleted)) + + return nil +} + +func (es *ElasticsearchInterfaceImpl) DeletePostFiles(rctx request.CTX, postID string) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeleteFilesBatch", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"post_id": {Value: postID}}, + }}, + }, + } + deleteQuery := es.client.DeleteByQuery(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles). + Request(&deletebyquery.Request{ + Query: query, + }) + response, err := deleteQuery.Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.DeletePostFiles", "ent.elasticsearch.delete_post_files.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("Post files deleted", mlog.String("post_id", postID), mlog.Int("deleted", *response.Deleted)) + + return nil +} + +func (es *ElasticsearchInterfaceImpl) DeleteFilesBatch(rctx request.CTX, endTime, limit int64) *model.AppError { + es.mutex.RLock() + defer es.mutex.RUnlock() + + if atomic.LoadInt32(&es.ready) == 0 { + return model.NewAppError("Elasticsearch.DeleteFilesBatch", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Lte: model.NewPointer(types.Float64(endTime)), + }, + }, + }}, + }, + } + + deleteQuery := es.client.DeleteByQuery(*es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles). + Request(&deletebyquery.Request{ + Query: query, + }). + // Note that max_docs is slightly different than size. + // Size will just limit the number of elements returned, which is not + // what we want. We want to limit the number of elements to be deleted. + MaxDocs(limit) + response, err := deleteQuery.Do(ctx) + if err != nil { + return model.NewAppError("Elasticsearch.DeleteUserPosts", "ent.elasticsearch.delete_user_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("Files batch deleted", mlog.Int("end_time", endTime), mlog.Int("limit", limit), mlog.Int("deleted", *response.Deleted)) + + return nil +} + +func checkMaxVersion(client *elastic.TypedClient, cfg *model.Config) (string, int, *model.AppError) { + resp, err := client.API.Core.Info().Do(context.Background()) + if err != nil { + return "", 0, model.NewAppError("Elasticsearch.checkMaxVersion", "ent.elasticsearch.start.get_server_version.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + major, _, _, esErr := common.GetVersionComponents(resp.Version.Int) + if esErr != nil { + return "", 0, model.NewAppError("Elasticsearch.checkMaxVersion", "ent.elasticsearch.start.parse_server_version.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + if major > elasticsearchMaxVersion { + return "", 0, model.NewAppError("Elasticsearch.checkMaxVersion", "ent.elasticsearch.max_version.app_error", map[string]any{"Version": major, "MaxVersion": elasticsearchMaxVersion}, "", http.StatusBadRequest) + } + return resp.Version.Int, major, nil +} + +// checkChannelIndex checks if channel index's mapping is correct. +// See Jira issue https://mattermost.atlassian.net/browse/MM-49257 +func (es *ElasticsearchInterfaceImpl) checkChannelIndex() { + es.Platform.Log().Debug("Elasticsearch.checkChannelIndex: checking if channel index field is of correct type") + isCorrect, err := es.isFieldCorrect() + if err != nil { + return + } + + if isCorrect { + es.Platform.Log().Debug("Elasticsearch.checkChannelIndex: channel index field is correct") + atomic.StoreInt32(&es.channelIndexVerified, 1) + } else { + es.Platform.Log().Debug("Elasticsearch.checkChannelIndex: channel index field is incorrect") + atomic.StoreInt32(&es.channelIndexVerified, 2) + } +} + +func (es *ElasticsearchInterfaceImpl) isFieldCorrect() (bool, error) { + // We want to check if channel index's "type" field is of type "keyword". + // If the index is in incorrect state, the field would be of type "text". + + es.Platform.Log().Debug("Elasticsearch.isFieldCorrect: querying ES to check if field is correct") + + ctx, cancel := context.WithTimeout( + context.Background(), + time.Duration(*es.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second, + ) + defer cancel() + + indexName := *es.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels + indexMappingInterface, err := es.client.Indices.GetFieldMapping("type").Index(indexName).Do(ctx) + if err != nil { + // The case of channels index not existing is fine, + // as whenever the index will be created, it will be created + // with the correct mappings. + elasticErr, ok := err.(*types.ElasticsearchError) + if ok && elasticErr.Status == http.StatusNotFound { + es.Platform.Logger().Debug("Elasticsearch isFieldCorrect: channel index doesn't exist", mlog.Err(err)) + return true, nil + } + + es.Platform.Logger().Error("Elasticsearch: Failed to fetch channels index template", mlog.Err(err)) + return false, err + } + + // this struct is declared here because + // it's not used anywhere outside this function + type channelsTypeFieldMapping struct { + Mappings struct { + Type struct { + Mapping struct { + Type struct { + Type string + } + } + } + } + } + + mappingInterface := indexMappingInterface[indexName] + mappingBytes, err := json.Marshal(mappingInterface) + if err != nil { + es.Platform.Logger().Error("Elasticsearch: Failed to marshal Elasticsearch index field mapping", mlog.Err(err)) + return false, err + } + + es.Platform.Log().Debug("Elasticsearch.isFieldCorrect: channel index type field mapping queried successfully", mlog.String("mapping", string(mappingBytes))) + + var mapping channelsTypeFieldMapping + err = json.Unmarshal(mappingBytes, &mapping) + if err != nil { + es.Platform.Logger().Error("Elasticsearch: Failed to unmarshal Elasticsearch index field mapping", mlog.Err(err)) + return false, err + } + + es.Platform.Logger().Debug("Elasticsearch: Found type of type field as", mlog.String("type", mapping.Mappings.Type.Mapping.Type.Type)) + return mapping.Mappings.Type.Mapping.Type.Type == "keyword", nil +} diff --git a/server/enterprise/elasticsearch/elasticsearch/elasticsearch_test.go b/server/enterprise/elasticsearch/elasticsearch/elasticsearch_test.go new file mode 100644 index 0000000000..62edffeea7 --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/elasticsearch_test.go @@ -0,0 +1,100 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "context" + "encoding/json" + "testing" + + elastic "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/suite" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks" +) + +type ElasticsearchInterfaceTestSuite struct { + common.CommonTestSuite + + th *api4.TestHelper + client *elastic.TypedClient + ctx context.Context + fileBackend filestore.FileBackend +} + +func TestElasticsearchInterfaceTestSuite(t *testing.T) { + testSuite := &ElasticsearchInterfaceTestSuite{ + CommonTestSuite: common.CommonTestSuite{}, + } + suite.Run(t, testSuite) +} + +func (s *ElasticsearchInterfaceTestSuite) SetupSuite() { + s.th = api4.SetupEnterprise(s.T()).InitBasic() + s.CommonTestSuite.TH = s.th + s.CommonTestSuite.GetDocumentFn = func(index, documentID string) (bool, json.RawMessage, error) { + resp, err := s.client.API.Get(index, documentID).Do(s.ctx) + if resp == nil { + return false, nil, err + } + return resp.Found, resp.Source_, err + } + s.CommonTestSuite.RefreshIndexFn = func() error { + _, err := s.client.Indices.Refresh().Do(context.Background()) + return err + } + s.CommonTestSuite.CreateIndexFn = func(index string) error { + _, err := s.client.Indices.Create(index).Do(s.ctx) + return err + } + s.CommonTestSuite.GetIndexFn = func(indexPattern string) ([]string, error) { + res, err := s.client.Indices.Get(indexPattern).Do(s.ctx) + if err != nil { + return nil, err + } + var names []string + for name := range res { + names = append(names, name) + } + return names, nil + } + + // Set up the state for the tests. + s.th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + *cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1 + *cfg.SqlSettings.DisableDatabaseSearch = true + }) + s.th.App.Srv().SetLicense(model.NewTestLicense()) + + if s.fileBackend == nil { + s.fileBackend = &mocks.FileBackend{} + } + + // Initialise other stuff for the test. + s.client = createTestClient(s.T(), s.th.Context, s.th.App.Config(), s.th.App.FileBackend()) + s.ctx = context.Background() + + // Register search engine + s.th.App.SearchEngine().RegisterElasticsearchEngine(&ElasticsearchInterfaceImpl{Platform: s.th.Server.Platform()}) +} + +func (s *ElasticsearchInterfaceTestSuite) SetupTest() { + s.CommonTestSuite.ESImpl = s.th.App.SearchEngine().ElasticsearchEngine + + if s.CommonTestSuite.ESImpl.IsActive() { + appErr := s.CommonTestSuite.ESImpl.Stop() + s.Require().Nil(appErr) + } + + s.Require().Nil(s.CommonTestSuite.ESImpl.Start()) + + s.Nil(s.CommonTestSuite.ESImpl.PurgeIndexes(s.th.Context)) +} diff --git a/server/enterprise/elasticsearch/elasticsearch/indexing_job.go b/server/enterprise/elasticsearch/elasticsearch/indexing_job.go new file mode 100644 index 0000000000..276c9f7871 --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/indexing_job.go @@ -0,0 +1,72 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "context" + "io" + "time" + + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + + "github.com/elastic/go-elasticsearch/v8/esutil" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/v8/channels/app" +) + +type ElasticsearchIndexerInterfaceImpl struct { + Server *app.Server + bulkProcessor esutil.BulkIndexer +} + +func (esi *ElasticsearchIndexerInterfaceImpl) MakeWorker() model.Worker { + const workerName = "EnterpriseElasticsearchIndexer" + + // Initializing logger + logger := esi.Server.Jobs.Logger().With(mlog.String("worker_name", workerName)) + + // Creating the client + client, appErr := createUntypedClient(logger, esi.Server.Jobs.Config(), esi.Server.Platform().FileBackend()) + if appErr != nil { + logger.Error("Worker: Failed to Create Client", mlog.Err(appErr)) + return nil + } + + return common.NewIndexerWorker(workerName, + esi.Server.Jobs, + logger, + esi.Server.Platform().FileBackend(), esi.Server.License, + func() error { + // Creating the bulk indexer from the client. + biCfg := esutil.BulkIndexerConfig{ + Client: client, + OnError: func(_ context.Context, err error) { + logger.Error("Error from elasticsearch bulk indexer", mlog.Err(err)) + }, + Timeout: time.Duration(*esi.Server.Jobs.Config().ElasticsearchSettings.RequestTimeoutSeconds) * time.Second, + NumWorkers: common.NumIndexWorkers(), + } + if *esi.Server.Jobs.Config().ElasticsearchSettings.Trace == "all" { + biCfg.DebugLogger = common.NewBulkIndexerLogger(logger, workerName) + } + var err error + esi.bulkProcessor, err = esutil.NewBulkIndexer(biCfg) + return err + }, + // Function to add an item in the bulk processor + func(indexName, indexOp, docID string, body io.ReadSeeker) error { + return esi.bulkProcessor.Add(context.Background(), esutil.BulkIndexerItem{ + Index: indexName, + Action: indexOp, + DocumentID: docID, + Body: body, + }) + }, + // Closing the bulk processor. + func() error { + return esi.bulkProcessor.Close(context.Background()) + }) +} diff --git a/server/enterprise/elasticsearch/elasticsearch/indexing_job_test.go b/server/enterprise/elasticsearch/elasticsearch/indexing_job_test.go new file mode 100644 index 0000000000..a312f29908 --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/indexing_job_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/channels/api4" +) + +func TestElasticSearchIndexerJobIsEnabled(t *testing.T) { + t.Run("ElasticSearch feature is enabled then job is enabled", func(t *testing.T) { + th := api4.SetupEnterpriseWithStoreMock(t) + defer th.TearDown() + + th.Server.SetLicense(model.NewTestLicense("elastic_search")) + + esImpl := &ElasticsearchIndexerInterfaceImpl{ + Server: th.Server, + } + worker := esImpl.MakeWorker() + + config := &model.Config{ + ElasticsearchSettings: model.ElasticsearchSettings{ + EnableIndexing: model.NewPointer(true), + }, + } + + result := worker.IsEnabled(config) + + assert.Equal(t, result, true) + }) + + t.Run("there is NO license then job is disabled", func(t *testing.T) { + th := api4.SetupEnterpriseWithStoreMock(t) + defer th.TearDown() + + th.Server.SetLicense(nil) + + esImpl := &ElasticsearchIndexerInterfaceImpl{ + Server: th.Server, + } + worker := esImpl.MakeWorker() + + config := &model.Config{ + ElasticsearchSettings: model.ElasticsearchSettings{ + EnableIndexing: model.NewPointer(true), + }, + } + + result := worker.IsEnabled(config) + + assert.Equal(t, result, false) + }) +} + +func TestElasticSearchIndexerPending(t *testing.T) { + th := api4.SetupEnterprise(t).InitBasic() + defer th.TearDown() + + // Set up the state for the tests. + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + *cfg.SqlSettings.DisableDatabaseSearch = true + }) + th.App.Srv().SetLicense(model.NewTestLicense()) + + impl := ElasticsearchIndexerInterfaceImpl{ + Server: th.App.Srv(), + } + + worker := impl.MakeWorker() + th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, worker, nil) + + go worker.Run() + + job, appErr := th.App.Srv().Jobs.CreateJob(th.Context, model.JobTypeElasticsearchPostIndexing, map[string]string{}) + require.Nil(t, appErr) + + worker.JobChannel() <- *job + + worker.Stop() + + job, err := th.App.Srv().Store().Job().Get(th.Context, job.Id) + require.NoError(t, err) + assert.Equal(t, job.Status, model.JobStatusPending) +} diff --git a/server/enterprise/elasticsearch/elasticsearch/main_test.go b/server/enterprise/elasticsearch/elasticsearch/main_test.go new file mode 100644 index 0000000000..a0d714a35c --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/main_test.go @@ -0,0 +1,21 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "testing" + + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/channels/testlib" +) + +var mainHelper *testlib.MainHelper + +func TestMain(m *testing.M) { + mainHelper = testlib.NewMainHelper() + defer mainHelper.Close() + api4.SetMainHelper(mainHelper) + + mainHelper.Main(m) +} diff --git a/server/enterprise/elasticsearch/elasticsearch/testlib.go b/server/enterprise/elasticsearch/elasticsearch/testlib.go new file mode 100644 index 0000000000..31169a994e --- /dev/null +++ b/server/enterprise/elasticsearch/elasticsearch/testlib.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "testing" + + "github.com/elastic/go-elasticsearch/v8" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks" +) + +func createTestClient(t *testing.T, rctx request.CTX, cfg *model.Config, fileStore filestore.FileBackend) *elasticsearch.TypedClient { + t.Helper() + + if fileStore == nil { + fileStore = &mocks.FileBackend{} + } + + client, err := createTypedClient(rctx.Logger(), cfg, fileStore, true) + require.Nil(t, err) + return client +} diff --git a/server/enterprise/elasticsearch/init.go b/server/enterprise/elasticsearch/init.go new file mode 100644 index 0000000000..863eac0b4c --- /dev/null +++ b/server/enterprise/elasticsearch/init.go @@ -0,0 +1,36 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package elasticsearch + +import ( + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/elasticsearch" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/opensearch" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/channels/app" + "github.com/mattermost/mattermost/server/v8/channels/app/platform" + ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs" + "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" +) + +func init() { + platform.RegisterElasticsearchInterface(func(s *platform.PlatformService) searchengine.SearchEngineInterface { + if *s.Config().ElasticsearchSettings.Backend == model.ElasticsearchSettingsESBackend { + return &elasticsearch.ElasticsearchInterfaceImpl{Platform: s} + } + return &opensearch.OpensearchInterfaceImpl{Platform: s} + }) + app.RegisterJobsElasticsearchIndexerInterface(func(s *app.Server) ejobs.IndexerJobInterface { + if *s.Config().ElasticsearchSettings.Backend == model.ElasticsearchSettingsESBackend { + return &elasticsearch.ElasticsearchIndexerInterfaceImpl{Server: s} + } + return &opensearch.OpensearchIndexerInterfaceImpl{Server: s} + }) + app.RegisterJobsElasticsearchAggregatorInterface(func(s *app.Server) ejobs.ElasticsearchAggregatorInterface { + if *s.Config().ElasticsearchSettings.Backend == model.ElasticsearchSettingsESBackend { + return &elasticsearch.ElasticsearchAggregatorInterfaceImpl{Server: s} + } + return &opensearch.OpensearchAggregatorInterfaceImpl{Server: s} + }) +} diff --git a/server/enterprise/elasticsearch/opensearch/aggregation_job.go b/server/enterprise/elasticsearch/opensearch/aggregation_job.go new file mode 100644 index 0000000000..9ec84f6a86 --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/aggregation_job.go @@ -0,0 +1,331 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "context" + "errors" + "net/http" + "strconv" + "sync" + "time" + + "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/app" + "github.com/mattermost/mattermost/server/v8/channels/jobs" + "github.com/mattermost/mattermost/server/v8/channels/store" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" +) + +const ( + aggregatorJobPollingInterval = 15 * time.Second + indexDeletionBatchSize = 20 +) + +type OpensearchAggregatorInterfaceImpl struct { + Server *app.Server +} + +type OpensearchAggregatorWorker struct { + name string + // stateMut protects stopCh and stopped and helps enforce + // ordering in case subsequent Run or Stop calls are made. + stateMut sync.Mutex + stopCh chan struct{} + stopped bool + stoppedCh chan bool + jobs chan model.Job + jobServer *jobs.JobServer + logger mlog.LoggerIFace + fileBackend filestore.FileBackend + + client *opensearchapi.Client + license func() *model.License +} + +func (esi *OpensearchAggregatorInterfaceImpl) MakeWorker() model.Worker { + const workerName = "EnterpriseOpensearchAggregator" + worker := OpensearchAggregatorWorker{ + name: workerName, + stoppedCh: make(chan bool, 1), + jobs: make(chan model.Job), + jobServer: esi.Server.Jobs, + logger: esi.Server.Jobs.Logger().With(mlog.String("worker_name", workerName)), + fileBackend: esi.Server.Platform().FileBackend(), + license: esi.Server.License, + stopped: true, + } + + return &worker +} + +func (worker *OpensearchAggregatorWorker) Run() { + worker.stateMut.Lock() + // We have to re-assign the stop channel again, because + // it might happen that the job was restarted due to a config change. + if worker.stopped { + worker.stopped = false + worker.stopCh = make(chan struct{}) + } else { + worker.stateMut.Unlock() + return + } + // Run is called from a separate goroutine and doesn't return. + // So we cannot Unlock in a defer clause. + worker.stateMut.Unlock() + + worker.logger.Debug("Worker Started") + + defer func() { + worker.logger.Debug("Worker Finished") + worker.stoppedCh <- true + }() + + client, err := createClient(worker.logger, worker.jobServer.Config(), worker.fileBackend, false) + if err != nil { + worker.logger.Error("Worker Failed to Create Client", mlog.Err(err)) + return + } + + worker.client = client + + for { + select { + case <-worker.stopCh: + worker.logger.Debug("Worker Received stop signal") + return + case job := <-worker.jobs: + worker.DoJob(&job) + } + } +} + +func (worker *OpensearchAggregatorWorker) IsEnabled(cfg *model.Config) bool { + if license := worker.license(); license == nil || !*license.Features.Elasticsearch { + return false + } + + if *cfg.ElasticsearchSettings.EnableIndexing { + return true + } + + return false +} + +func (worker *OpensearchAggregatorWorker) Stop() { + worker.stateMut.Lock() + defer worker.stateMut.Unlock() + + // Set to close, and if already closed before, then return. + if worker.stopped { + return + } + worker.stopped = true + + worker.logger.Debug("Worker Stopping") + close(worker.stopCh) + <-worker.stoppedCh +} + +func (worker *OpensearchAggregatorWorker) JobChannel() chan<- model.Job { + return worker.jobs +} + +func (worker *OpensearchAggregatorWorker) DoJob(job *model.Job) { + logger := worker.logger.With(jobs.JobLoggerFields(job)...) + logger.Debug("Worker: Received a new candidate job.") + defer worker.jobServer.HandleJobPanic(logger, job) + + claimed, appErr := worker.jobServer.ClaimJob(job) + if appErr != nil { + logger.Warn("Worker: Error occurred while trying to claim job", mlog.Err(appErr)) + return + } + + if !claimed { + return + } + + logger.Info("Worker: Aggregation job claimed by worker") + + var cancelContext request.CTX = request.EmptyContext(worker.logger) + cancelCtx, cancelCancelWatcher := context.WithCancel(context.Background()) + cancelWatcherChan := make(chan struct{}, 1) + cancelContext = cancelContext.WithContext(cancelCtx) + go worker.jobServer.CancellationWatcher(cancelContext, job.Id, cancelWatcherChan) + defer cancelCancelWatcher() + + rctx := request.EmptyContext(worker.logger) + now := time.Now() + today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.Local) + cutoff := today.AddDate(0, 0, -*worker.jobServer.Config().ElasticsearchSettings.AggregatePostsAfterDays+1) + + // Get all the daily Elasticsearch post indexes to work out which days aren't aggregated yet. + dateFormat := *worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_2006_01_02" + datedIndexes := []time.Time{} + + postIndexesResult, err := worker.client.Indices.Get(rctx.Context(), opensearchapi.IndicesGetReq{ + Indices: []string{*worker.jobServer.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_*"}, + }) + if err != nil { + appError := model.NewAppError("OpensearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.get_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err) + worker.setJobError(logger, job, appError) + return + } + + for index := range postIndexesResult.Indices { + var indexDate time.Time + indexDate, err = time.Parse(dateFormat, index) + if err != nil { + logger.Warn("Failed to parse date from posts index. Ignoring index.", mlog.String("index", index)) + } else { + datedIndexes = append(datedIndexes, indexDate) + } + } + + // Work out how far back the reindexing (and index deletion) needs to go. + var oldestDay time.Time + oldestDayFound := false + indexesToPurge := []string{} + for _, date := range datedIndexes { + if date.Before(cutoff) { + logger.Debug("Worker: Post index identified for purging", mlog.Time("date", date)) + indexesToPurge = append(indexesToPurge, date.Format(dateFormat)) + if !oldestDayFound || oldestDay.After(date) { + oldestDay = date + oldestDayFound = true + } + } else { + logger.Debug("Worker: Post index is within the range to keep", mlog.Time("date", date)) + } + } + + if !oldestDayFound { + // Nothing to purge. + logger.Info("Worker: Aggregation job completed. Nothing to aggregate.") + worker.setJobSuccess(logger, job) + return + } + + // Trigger a reindexing job with the appropriate dates. + reindexingStartDate := oldestDay + reindexingEndDate := cutoff + + logger.Info("Worker: Aggregation job reindexing", mlog.String("start_date", reindexingStartDate.Format("2006-01-02")), mlog.String("end_date", reindexingEndDate.Format("2006-01-02"))) + + var indexJob *model.Job + if indexJob, appErr = worker.jobServer.CreateJob( + rctx, + model.JobTypeElasticsearchPostIndexing, + map[string]string{ + "start_time": strconv.FormatInt(reindexingStartDate.UnixNano()/int64(time.Millisecond), 10), + "end_time": strconv.FormatInt(reindexingEndDate.UnixNano()/int64(time.Millisecond), 10), + }, + ); appErr != nil { + logger.Error("Worker: Failed to create indexing job.", mlog.Err(appErr)) + appError := model.NewAppError("OpensearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.create_index_job.error", nil, "", http.StatusInternalServerError).Wrap(appErr) + worker.setJobError(logger, job, appError) + return + } + + for { + select { + case <-cancelWatcherChan: + logger.Info("Worker: Aggregation job has been canceled via CancellationWatcher") + worker.setJobCanceled(logger, job) + return + + case <-worker.stopCh: + logger.Info("Worker: Aggregation job has been canceled via Worker Stop") + worker.setJobCanceled(logger, job) + return + + case <-time.After(aggregatorJobPollingInterval): + // Get the details of the indexing job we are waiting on. + indexJob, err = worker.jobServer.Store.Job().Get(rctx, indexJob.Id) + if err != nil { + var appErr *model.AppError + var nfErr *store.ErrNotFound + switch { + case errors.As(err, &nfErr): + appErr = model.NewAppError("DoJob", "app.job.get.app_error", nil, "", http.StatusNotFound).Wrap(nfErr) + default: + appErr = model.NewAppError("DoJob", "app.job.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + worker.setJobError(logger, job, appErr) + return + } + + // Wait for the aggregation job to finish. + // On success, we delete the old indexes. + // Otherwise, fail the job. + switch indexJob.Status { + case model.JobStatusSuccess: + // We limit the number of indexes to delete at one shot. + // A minor side-effect of this is that the aggregation job status + // will be redundantly queried multiple times, but that's not a major bottleneck. + curWindow := indexesToPurge + deleteMore := false + if len(indexesToPurge) > indexDeletionBatchSize { + curWindow = indexesToPurge[:indexDeletionBatchSize] + indexesToPurge = indexesToPurge[indexDeletionBatchSize:] + deleteMore = true + } + // Delete indexes + if _, err = worker.client.Indices.Delete(rctx.Context(), opensearchapi.IndicesDeleteReq{ + Indices: curWindow, + }); err != nil { + appError := model.NewAppError("OpensearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.delete_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err) + logger.Error("Worker: Failed to delete indexes for job", mlog.String("workername", worker.name), mlog.String("job_id", job.Id), mlog.Err(appError)) + worker.setJobError(logger, job, appError) + return + } + + if !deleteMore { + // Job done. Set the status to success. + logger.Info("Worker: Aggregation job finished successfully") + worker.setJobSuccess(logger, job) + return + } + case model.JobStatusPending, model.JobStatusInProgress: + // Indexing job is in progress or pending. Update the progress of this job. + if err := worker.jobServer.SetJobProgress(job, indexJob.Progress); err != nil { + logger.Error("Worker: Failed to set progress for job", mlog.Err(err)) + worker.setJobError(logger, job, err) + return + } + default: + // error case + appError := model.NewAppError("OpensearchAggregatorWorker", "ent.elasticsearch.aggregator_worker.index_job_failed.error", nil, "", http.StatusInternalServerError) + logger.Error("Worker: Index aggregation job failed", mlog.Err(appError)) + worker.setJobError(logger, job, appError) + return + } + } + } +} + +func (worker *OpensearchAggregatorWorker) setJobSuccess(logger mlog.LoggerIFace, job *model.Job) { + if err := worker.jobServer.SetJobSuccess(job); err != nil { + logger.Error("Worker: Failed to set success for job", mlog.Err(err)) + worker.setJobError(logger, job, err) + } +} + +func (worker *OpensearchAggregatorWorker) setJobError(logger mlog.LoggerIFace, job *model.Job, appError *model.AppError) { + if err := worker.jobServer.SetJobError(job, appError); err != nil { + logger.Error("Worker: Failed to set job error", mlog.Err(err)) + } +} + +func (worker *OpensearchAggregatorWorker) setJobCanceled(logger mlog.LoggerIFace, job *model.Job) { + if err := worker.jobServer.SetJobCanceled(job); err != nil { + logger.Error("Worker: Failed to mark job as canceled", mlog.Err(err)) + } +} diff --git a/server/enterprise/elasticsearch/opensearch/aggregation_job_test.go b/server/enterprise/elasticsearch/opensearch/aggregation_job_test.go new file mode 100644 index 0000000000..e2706dd14c --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/aggregation_job_test.go @@ -0,0 +1,223 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "bytes" + "context" + "encoding/json" + "os" + "testing" + "time" + + "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/channels/store/storetest/mocks" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" +) + +func TestElasticsearchAggregation(t *testing.T) { + if os.Getenv("IS_CI") == "true" { + os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://opensearch:9201") + os.Setenv("MM_ELASTICSEARCHSETTINGS_BACKEND", "opensearch") + } + + defer func() { + if os.Getenv("IS_CI") == "true" { + os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://elasticsearch:9201") + os.Unsetenv("MM_ELASTICSEARCHSETTINGS_BACKEND") + } + }() + + th := api4.SetupEnterpriseWithStoreMock(t) + rctx := request.TestContext(t) + + mockUserStore := mocks.UserStore{} + mockUserStore.On("Count", mock.Anything).Return(int64(10), nil) + mockUserStore.On("GetAllProfiles", mock.Anything).Return(nil, nil) + + mockPostStore := mocks.PostStore{} + mockPostStore.On("GetMaxPostSize").Return(65535, nil) + + mockSystemStore := mocks.SystemStore{} + mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil) + mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil) + mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil) + + mockJobStore := mocks.JobStore{} + mockJobStore.On("Save", mock.AnythingOfType("*model.Job")).Return(&model.Job{}, nil) + mockJobStore.On("UpdateStatus", mock.AnythingOfType("string"), model.JobStatusSuccess).Return(&model.Job{}, nil) + mockJobStore.On("Get", mock.AnythingOfType("*request.Context"), mock.AnythingOfType("string")).Return(&model.Job{ + Status: model.JobStatusSuccess, + }, nil) + mockJobStore.On("UpdateStatusOptimistically", + mock.AnythingOfType("string"), + model.JobStatusPending, + model.JobStatusInProgress).Return(true, nil) + mockJobStore.On("GetAllByType", mock.AnythingOfType("string")).Return([]*model.Job{{ + Id: "abcxyz123", + Type: "EnterpriseElasticsearchIndexer", + Status: model.JobStatusCanceled, + }}, nil) + + mockStore := th.App.Srv().Platform().Store.(*mocks.Store) + mockStore.On("User").Return(&mockUserStore) + mockStore.On("Post").Return(&mockPostStore) + mockStore.On("System").Return(&mockSystemStore) + mockStore.On("Job").Return(&mockJobStore) + mockStore.On("GetDBSchemaVersion").Return(1, nil) + + aggImpl := OpensearchAggregatorInterfaceImpl{Server: th.Server} + + // Register search engine + th.App.SearchEngine().RegisterElasticsearchEngine(&OpensearchInterfaceImpl{ + Platform: th.Server.Platform(), + }) + + // Set up the state for the tests. + th.App.UpdateConfig(func(cfg *model.Config) { + if os.Getenv("IS_CI") == "true" { + *cfg.ElasticsearchSettings.ConnectionURL = "http://opensearch:9201" + } else { + *cfg.ElasticsearchSettings.ConnectionURL = "http://localhost:9201" + } + *cfg.ElasticsearchSettings.Backend = model.ElasticsearchSettingsOSBackend + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + *cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1 + *cfg.ElasticsearchSettings.AggregatePostsAfterDays = 1 + *cfg.SqlSettings.DisableDatabaseSearch = true + }) + + esImpl := th.App.SearchEngine().ElasticsearchEngine + appErr := esImpl.Start() + if appErr != nil && appErr.Id != "ent.elasticsearch.start.already_started.app_error" { + require.Fail(t, "failed to start elasticsearch", appErr) + } + require.Nil(t, esImpl.PurgeIndexes(rctx)) + + post := &model.Post{ + Id: model.NewId(), + ChannelId: "channel", + Message: "hi", + } + for i := 0; i < indexDeletionBatchSize+1; i++ { + indexPost(t, th, esImpl.(*OpensearchInterfaceImpl), + post, + time.Now().Add(-time.Duration(4+i)*24*time.Hour)) + } + + job := &model.Job{ + Id: model.NewId(), + Type: model.JobTypeElasticsearchPostAggregation, + Status: model.JobStatusPending, + } + + _, err := th.Server.Store().Job().Save(job) + require.NoError(t, err) + + worker := aggImpl.MakeWorker().(*OpensearchAggregatorWorker) + worker.client = createTestClient(t, th.Context, th.App.Config(), th.App.FileBackend()) + worker.jobServer.Store = mockStore + + indexingImpl := OpensearchIndexerInterfaceImpl{ + Server: th.App.Srv(), + } + th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, indexingImpl.MakeWorker(), nil) + + worker.DoJob(job) + + // We assert the minimum number of calls to verify that + // batching is working correctly. Because job().Get() will happen + // in each iteration. + numCalls := 0 + for _, call := range mockJobStore.Calls { + if call.Method == "Get" { + numCalls++ + } + } + assert.GreaterOrEqual(t, numCalls, 8, "Unexpected number of Jobstore.Get calls") +} + +func TestElasticsearchAggregationSkipDuringBulkIndexing(t *testing.T) { + th := api4.SetupEnterpriseWithStoreMock(t) + + mockUserStore := mocks.UserStore{} + mockUserStore.On("Count", mock.Anything).Return(int64(10), nil) + + mockPostStore := mocks.PostStore{} + mockPostStore.On("GetMaxPostSize").Return(65535, nil) + + mockSystemStore := mocks.SystemStore{} + mockSystemStore.On("GetByName", "UpgradedFromTE").Return(&model.System{Name: "UpgradedFromTE", Value: "false"}, nil) + mockSystemStore.On("GetByName", "InstallationDate").Return(&model.System{Name: "InstallationDate", Value: "10"}, nil) + mockSystemStore.On("GetByName", "FirstServerRunTimestamp").Return(&model.System{Name: "FirstServerRunTimestamp", Value: "10"}, nil) + + mockJobStore := mocks.JobStore{} + + mockStore := th.App.Srv().Platform().Store.(*mocks.Store) + mockStore.On("User").Return(&mockUserStore) + mockStore.On("Post").Return(&mockPostStore) + mockStore.On("System").Return(&mockSystemStore) + mockStore.On("Job").Return(&mockJobStore) + mockStore.On("GetDBSchemaVersion").Return(1, nil) + + aggImpl := OpensearchAggregatorInterfaceImpl{Server: th.Server} + aggImpl.Server.Jobs.Store = mockStore + + // Register search engine + th.App.SearchEngine().RegisterElasticsearchEngine(&OpensearchInterfaceImpl{ + Platform: th.Server.Platform(), + }) + + // Set up the state for the tests. + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + *cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1 + *cfg.ElasticsearchSettings.AggregatePostsAfterDays = 1 + *cfg.SqlSettings.DisableDatabaseSearch = true + }) + + sched := aggImpl.MakeScheduler() + // Pass pending jobs as true + job, appErr := sched.ScheduleJob(th.Context, th.App.Config(), true, nil) + require.Nil(t, job) + require.Nil(t, appErr) + + mockJobStore.AssertNotCalled(t, "GetCountByStatusAndType") +} + +func indexPost(t *testing.T, th *api4.TestHelper, esImpl *OpensearchInterfaceImpl, post *model.Post, createTime time.Time) { //nolint:unused + t.Helper() + indexName := common.BuildPostIndexName(*th.Server.Config().ElasticsearchSettings.AggregatePostsAfterDays, + common.IndexBasePosts, + common.IndexBasePosts_MONTH, + createTime.Add(-1*24*time.Hour), + model.GetMillisForTime(createTime), + ) + searchPost, err := common.ESPostFromPost(post, "teamID") + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), + time.Duration(*esImpl.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + buf, err := json.Marshal(searchPost) + require.NoError(t, err) + + _, err = esImpl.client.Index(ctx, opensearchapi.IndexReq{ + Index: indexName, + DocumentID: post.Id, + Body: bytes.NewReader(buf), + }) + require.NoError(t, err) +} diff --git a/server/enterprise/elasticsearch/opensearch/aggregation_scheduler.go b/server/enterprise/elasticsearch/opensearch/aggregation_scheduler.go new file mode 100644 index 0000000000..44d858696d --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/aggregation_scheduler.go @@ -0,0 +1,73 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "net/http" + "time" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/app" + "github.com/mattermost/mattermost/server/v8/channels/jobs" + ejobs "github.com/mattermost/mattermost/server/v8/einterfaces/jobs" +) + +type OpenSearchAggregatorScheduler struct { + jobServer *jobs.JobServer + server *app.Server +} + +func (s *OpenSearchAggregatorScheduler) Enabled(cfg *model.Config) bool { + if license := s.server.License(); license == nil || !*license.Features.Elasticsearch { + return false + } + + if *cfg.ElasticsearchSettings.EnableIndexing { + return true + } + + return false +} + +func (s *OpenSearchAggregatorScheduler) NextScheduleTime(cfg *model.Config, now time.Time, pendingJobs bool, lastSuccessfulJob *model.Job) *time.Time { + parsedTime, err := time.Parse("15:04", *cfg.ElasticsearchSettings.PostsAggregatorJobStartTime) + if err != nil { + s.server.Log().Error("Cannot determine next schedule time for opensearch post aggregator. PostsAggregatorJobStartTime config value is invalid.", mlog.Err(err)) + return nil + } + + return jobs.GenerateNextStartDateTime(now, parsedTime) +} + +func (s *OpenSearchAggregatorScheduler) ScheduleJob(rctx request.CTX, _ *model.Config, pendingJobs bool, _ *model.Job) (*model.Job, *model.AppError) { + if pendingJobs { + s.server.Log().Warn("An aggregator job is already running. Skipping.") + return nil, nil + } + + // Don't schedule a job if we already have a running bulk indexing job + count, err := s.jobServer.Store.Job().GetCountByStatusAndType(model.JobStatusInProgress, model.JobTypeElasticsearchPostIndexing) + if err != nil { + return nil, model.NewAppError( + "ScheduleJob", + model.NoTranslation, + nil, + "", + http.StatusInternalServerError).Wrap(err) + } + if count > 0 { + return nil, nil + } + + return s.jobServer.CreateJob(rctx, model.JobTypeElasticsearchPostAggregation, nil) +} + +func (esi *OpensearchAggregatorInterfaceImpl) MakeScheduler() ejobs.Scheduler { + return &OpenSearchAggregatorScheduler{ + server: esi.Server, + jobServer: esi.Server.Jobs, + } +} diff --git a/server/enterprise/elasticsearch/opensearch/bulk.go b/server/enterprise/elasticsearch/opensearch/bulk.go new file mode 100644 index 0000000000..ca40793b95 --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/bulk.go @@ -0,0 +1,162 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "bytes" + "context" + "encoding/json" + "sync" + "time" + + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/opensearch-project/opensearch-go/v4/opensearchapi" +) + +type Bulk struct { + mut sync.Mutex + buf *bytes.Buffer + + logger mlog.LoggerIFace + client *opensearchapi.Client + settings model.ElasticsearchSettings + + quitFlusher chan struct{} + quitFlusherWg sync.WaitGroup + + pendingRequests int +} + +func NewBulk(settings model.ElasticsearchSettings, + logger mlog.LoggerIFace, + client *opensearchapi.Client) *Bulk { + b := &Bulk{ + settings: settings, + logger: logger, + client: client, + quitFlusher: make(chan struct{}), + buf: &bytes.Buffer{}, + } + + b.quitFlusherWg.Add(1) + go b.periodicFlusher() + + return b +} + +// IndexOp is a helper function to add an IndexOperation to the current bulk request. +// doc argument can be a []byte, json.RawMessage or a struct. +func (r *Bulk) IndexOp(op *types.IndexOperation, doc any) error { + r.mut.Lock() + defer r.mut.Unlock() + + operation := types.OperationContainer{Index: op} + header, err := json.Marshal(operation) + if err != nil { + return err + } + + r.buf.Write(header) + r.buf.Write([]byte("\n")) + + switch v := doc.(type) { + case []byte: + r.buf.Write(v) + case json.RawMessage: + r.buf.Write(v) + default: + body, err := json.Marshal(doc) + if err != nil { + return err + } + r.buf.Write(body) + } + + r.buf.Write([]byte("\n")) + + return r.flushIfNecessary() +} + +// DeleteOp is a helper function to add a DeleteOperation to the current bulk request. +func (r *Bulk) DeleteOp(op *types.DeleteOperation) error { + r.mut.Lock() + defer r.mut.Unlock() + + operation := types.OperationContainer{Delete: op} + header, err := json.Marshal(operation) + if err != nil { + return err + } + + r.buf.Write(header) + r.buf.Write([]byte("\n")) + + return r.flushIfNecessary() +} + +// flushIfNecessary flushes the pending buffer if needed. +// It MUST be called with an already acquired mutex. +func (r *Bulk) flushIfNecessary() error { + r.pendingRequests++ + + if r.pendingRequests > *r.settings.LiveIndexingBatchSize { + return r._flush() + } + + return nil +} + +func (r *Bulk) Stop() error { + r.mut.Lock() + defer r.mut.Unlock() + r.logger.Info("Stopping Bulk processor") + + if r.pendingRequests > 0 { + return r._flush() + } + + close(r.quitFlusher) + r.quitFlusherWg.Wait() + + return nil +} + +func (r *Bulk) periodicFlusher() { + defer r.quitFlusherWg.Done() + + for { + select { + case <-time.After(common.BulkFlushInterval): + r.mut.Lock() + if r.pendingRequests > 0 { + if err := r._flush(); err != nil { + r.logger.Warn("Error flushing live indexing buffer", mlog.Err(err)) + } + } + r.mut.Unlock() + case <-r.quitFlusher: + return + } + } +} + +// _flush MUST be called with an acquired lock. +func (r *Bulk) _flush() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*r.settings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err := r.client.Bulk(ctx, opensearchapi.BulkReq{ + Body: bytes.NewReader(r.buf.Bytes()), + }) + if err != nil { + return err + } + r.buf.Reset() + r.pendingRequests = 0 + + return nil +} diff --git a/server/enterprise/elasticsearch/opensearch/bulk_test.go b/server/enterprise/elasticsearch/opensearch/bulk_test.go new file mode 100644 index 0000000000..57466e9beb --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/bulk_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "os" + "testing" + + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/stretchr/testify/require" +) + +func TestBulkProcessor(t *testing.T) { + th := api4.SetupEnterprise(t) + defer th.TearDown() + + if os.Getenv("IS_CI") == "true" { + os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://opensearch:9201") + os.Setenv("MM_ELASTICSEARCHSETTINGS_BACKEND", "opensearch") + } + + defer func() { + if os.Getenv("IS_CI") == "true" { + os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://elasticsearch:9201") + os.Unsetenv("MM_ELASTICSEARCHSETTINGS_BACKEND") + } + }() + + th.App.UpdateConfig(func(cfg *model.Config) { + if os.Getenv("IS_CI") == "true" { + *cfg.ElasticsearchSettings.ConnectionURL = "http://opensearch:9201" + } else { + *cfg.ElasticsearchSettings.ConnectionURL = "http://localhost:9201" + } + *cfg.ElasticsearchSettings.Backend = model.ElasticsearchSettingsOSBackend + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + }) + + client := createTestClient(t, th.Context, th.App.Config(), th.App.FileBackend()) + bulk := NewBulk(th.App.Config().ElasticsearchSettings, + th.Server.Platform().Log(), + client) + + post, err := common.ESPostFromPost(&model.Post{ + Id: model.NewId(), + Message: "hello world", + }, "myteam") + require.NoError(t, err) + + err = bulk.IndexOp(&types.IndexOperation{ + Index_: model.NewPointer("myindex"), + Id_: model.NewPointer(post.Id), + }, post) + require.NoError(t, err) + + require.Equal(t, 1, bulk.pendingRequests) + + err = bulk.Stop() + require.NoError(t, err) + + require.Equal(t, 0, bulk.pendingRequests) +} diff --git a/server/enterprise/elasticsearch/opensearch/common.go b/server/enterprise/elasticsearch/opensearch/common.go new file mode 100644 index 0000000000..752da4842b --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/common.go @@ -0,0 +1,124 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "crypto/tls" + "net/http" + "time" + + "github.com/opensearch-project/opensearch-go/v4" + "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" +) + +func createClient(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend, debugLogging bool) (*opensearchapi.Client, *model.AppError) { + esCfg, appErr := createClientConfig(logger, cfg, fileBackend, debugLogging) + if appErr != nil { + return nil, appErr + } + + client, err := opensearchapi.NewClient(*esCfg) + if err != nil { + return nil, model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.connect_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return client, nil +} + +func createClientConfig(logger mlog.LoggerIFace, cfg *model.Config, fileBackend filestore.FileBackend, debugLogging bool) (*opensearchapi.Config, *model.AppError) { + tp := http.DefaultTransport.(*http.Transport).Clone() + tp.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: *cfg.ElasticsearchSettings.SkipTLSVerification, + } + + osCfg := &opensearchapi.Config{ + Client: opensearch.Config{ + Addresses: []string{*cfg.ElasticsearchSettings.ConnectionURL}, + RetryBackoff: func(i int) time.Duration { return time.Duration(i) * 100 * time.Millisecond }, // A minimal backoff function + RetryOnStatus: []int{502, 503, 504, 429}, // Retry on 429 TooManyRequests statuses + MaxRetries: 3, + DiscoverNodesOnStart: *cfg.ElasticsearchSettings.Sniff, + }, + } + + if osCfg.Client.DiscoverNodesOnStart { + osCfg.Client.DiscoverNodesInterval = 30 * time.Second + } + + if *cfg.ElasticsearchSettings.ClientCert != "" { + appErr := configureClientCertificate(tp.TLSClientConfig, cfg, fileBackend) + if appErr != nil { + return nil, appErr + } + } + + // custom CA + if *cfg.ElasticsearchSettings.CA != "" { + appErr := configureCA(&osCfg.Client, cfg, fileBackend) + if appErr != nil { + return nil, appErr + } + } + + osCfg.Client.Transport = tp + + if *cfg.ElasticsearchSettings.Username != "" { + osCfg.Client.Username = *cfg.ElasticsearchSettings.Username + osCfg.Client.Password = *cfg.ElasticsearchSettings.Password + } + + // This is a compatibility mode from previous config settings. + // We have to conditionally enable debug logging due to + // https://github.com/elastic/elastic-transport-go/issues/22 + // Although, this is opensearch, the issue is the same. + if *cfg.ElasticsearchSettings.Trace == "all" && debugLogging { + osCfg.Client.EnableDebugLogger = true + } + + osCfg.Client.Logger = common.NewLogger("Opensearch", logger, *cfg.ElasticsearchSettings.Trace == "all") + + return osCfg, nil +} + +func configureCA(esCfg *opensearch.Config, cfg *model.Config, fb filestore.FileBackend) *model.AppError { + // read the certificate authority (CA) file + clientCA, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.CA) + if err != nil { + return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.ca_cert_missing", nil, "", http.StatusInternalServerError).Wrap(err) + } + + esCfg.CACert = clientCA + + return nil +} + +func configureClientCertificate(tlsConfig *tls.Config, cfg *model.Config, fb filestore.FileBackend) *model.AppError { + // read the client certificate file + clientCert, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.ClientCert) + if err != nil { + return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_cert_missing", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // read the client key file + clientKey, err := common.ReadFileSafely(fb, *cfg.ElasticsearchSettings.ClientKey) + if err != nil { + return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_key_missing", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // load the client key and certificate + certificate, err := tls.X509KeyPair(clientCert, clientKey) + if err != nil { + return model.NewAppError("Elasticsearch.createClient", "ent.elasticsearch.create_client.client_cert_malformed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // update the TLS config + tlsConfig.Certificates = []tls.Certificate{certificate} + + return nil +} diff --git a/server/enterprise/elasticsearch/opensearch/indexing_job.go b/server/enterprise/elasticsearch/opensearch/indexing_job.go new file mode 100644 index 0000000000..823f81cf15 --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/indexing_job.go @@ -0,0 +1,73 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "context" + "io" + "time" + + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + + "github.com/opensearch-project/opensearch-go/v4/opensearchutil" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/v8/channels/app" +) + +type OpensearchIndexerInterfaceImpl struct { + Server *app.Server + bulkProcessor opensearchutil.BulkIndexer +} + +func (esi *OpensearchIndexerInterfaceImpl) MakeWorker() model.Worker { + const workerName = "EnterpriseOpensearchIndexer" + + // Initializing logger + logger := esi.Server.Jobs.Logger().With(mlog.String("worker_name", workerName)) + + // Creating the client + client, appErr := createClient(logger, esi.Server.Jobs.Config(), esi.Server.Platform().FileBackend(), true) + if appErr != nil { + logger.Error("Worker: Failed to Create Client", mlog.Err(appErr)) + return nil + } + + return common.NewIndexerWorker(workerName, + esi.Server.Jobs, + logger, + esi.Server.Platform().FileBackend(), + esi.Server.License, + func() error { + // Creating the bulk indexer from the client. + biCfg := opensearchutil.BulkIndexerConfig{ + Client: client, + OnError: func(_ context.Context, err error) { + logger.Error("Error from opensearch bulk indexer", mlog.Err(err)) + }, + Timeout: time.Duration(*esi.Server.Jobs.Config().ElasticsearchSettings.RequestTimeoutSeconds) * time.Second, + NumWorkers: common.NumIndexWorkers(), + } + if *esi.Server.Jobs.Config().ElasticsearchSettings.Trace == "all" { + biCfg.DebugLogger = common.NewBulkIndexerLogger(logger, workerName) + } + var err error + esi.bulkProcessor, err = opensearchutil.NewBulkIndexer(biCfg) + return err + }, + // Function to add an item in the bulk processor + func(indexName, indexOp, docID string, body io.ReadSeeker) error { + return esi.bulkProcessor.Add(context.Background(), opensearchutil.BulkIndexerItem{ + Index: indexName, + Action: indexOp, + DocumentID: docID, + Body: body, + }) + }, + // Closing the bulk processor + func() error { + return esi.bulkProcessor.Close(context.Background()) + }) +} diff --git a/server/enterprise/elasticsearch/opensearch/indexing_job_test.go b/server/enterprise/elasticsearch/opensearch/indexing_job_test.go new file mode 100644 index 0000000000..f03b212b0a --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/indexing_job_test.go @@ -0,0 +1,94 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/channels/api4" +) + +func TestOpenSearchIndexerJobIsEnabled(t *testing.T) { + t.Run("ElasticSearch feature is enabled then job is enabled", func(t *testing.T) { + th := api4.SetupEnterpriseWithStoreMock(t) + defer th.TearDown() + + th.Server.SetLicense(model.NewTestLicense("elastic_search")) + + osImpl := &OpensearchIndexerInterfaceImpl{ + Server: th.Server, + } + worker := osImpl.MakeWorker() + + config := &model.Config{ + ElasticsearchSettings: model.ElasticsearchSettings{ + EnableIndexing: model.NewPointer(true), + }, + } + + result := worker.IsEnabled(config) + + assert.Equal(t, result, true) + }) + + t.Run("there is NO license then job is disabled", func(t *testing.T) { + th := api4.SetupEnterpriseWithStoreMock(t) + defer th.TearDown() + + th.Server.SetLicense(nil) + + osImpl := &OpensearchIndexerInterfaceImpl{ + Server: th.Server, + } + worker := osImpl.MakeWorker() + + config := &model.Config{ + ElasticsearchSettings: model.ElasticsearchSettings{ + EnableIndexing: model.NewPointer(true), + }, + } + + result := worker.IsEnabled(config) + + assert.Equal(t, result, false) + }) +} + +func TestOpenSearchIndexerPending(t *testing.T) { + th := api4.SetupEnterprise(t).InitBasic() + defer th.TearDown() + + // Set up the state for the tests. + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + *cfg.SqlSettings.DisableDatabaseSearch = true + }) + th.App.Srv().SetLicense(model.NewTestLicense()) + + impl := OpensearchIndexerInterfaceImpl{ + Server: th.App.Srv(), + } + + worker := impl.MakeWorker() + th.Server.Jobs.RegisterJobType(model.JobTypeElasticsearchPostIndexing, worker, nil) + + go worker.Run() + + job, appErr := th.App.Srv().Jobs.CreateJob(th.Context, model.JobTypeElasticsearchPostIndexing, map[string]string{}) + require.Nil(t, appErr) + + worker.JobChannel() <- *job + + worker.Stop() + + job, err := th.App.Srv().Store().Job().Get(th.Context, job.Id) + require.NoError(t, err) + assert.Equal(t, job.Status, model.JobStatusPending) +} diff --git a/server/enterprise/elasticsearch/opensearch/main_test.go b/server/enterprise/elasticsearch/opensearch/main_test.go new file mode 100644 index 0000000000..f29ede3af5 --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/main_test.go @@ -0,0 +1,21 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "testing" + + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/channels/testlib" +) + +var mainHelper *testlib.MainHelper + +func TestMain(m *testing.M) { + mainHelper = testlib.NewMainHelper() + defer mainHelper.Close() + api4.SetMainHelper(mainHelper) + + mainHelper.Main(m) +} diff --git a/server/enterprise/elasticsearch/opensearch/opensearch.go b/server/enterprise/elasticsearch/opensearch/opensearch.go new file mode 100644 index 0000000000..8a4d6d5e14 --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/opensearch.go @@ -0,0 +1,2104 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/mlog" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/channels/app/platform" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/mattermost/mattermost/server/v8/platform/services/searchengine" + + "github.com/elastic/go-elasticsearch/v8/typedapi/core/deletebyquery" + "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" + "github.com/elastic/go-elasticsearch/v8/typedapi/types" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/highlighterencoder" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/operator" + "github.com/elastic/go-elasticsearch/v8/typedapi/types/enums/sortorder" + "github.com/opensearch-project/opensearch-go/v4" + "github.com/opensearch-project/opensearch-go/v4/opensearchapi" +) + +const opensearchMaxVersion = 2 + +var ( + purgeIndexListAllowedIndexes = []string{common.IndexBaseChannels} +) + +type OpensearchInterfaceImpl struct { + client *opensearchapi.Client + mutex sync.RWMutex + ready int32 + version int + fullVersion string + plugins []string + + bulkProcessor *Bulk + Platform *platform.PlatformService + + // This flag is for indicating if channel index's mappings + // has been verified, and if so, what was the result. + // + // value = 0 indicates it has NOT BEEN CHECKED + // value = 1 indicates index has been checked and has CORRECT mappings + // value = 2 indicates index has been checked and it has INCORRECT mappings + channelIndexVerified int32 +} + +func getJSONOrErrorStr(obj any) string { + b, err := json.Marshal(obj) + if err != nil { + return err.Error() + } + return string(b) +} + +func (*OpensearchInterfaceImpl) UpdateConfig(cfg *model.Config) { + // Not needed, it uses the `Server` stored internally to get always the last version +} + +func (*OpensearchInterfaceImpl) GetName() string { + return "opensearch" +} + +func (os *OpensearchInterfaceImpl) IsEnabled() bool { + return *os.Platform.Config().ElasticsearchSettings.EnableIndexing +} + +func (os *OpensearchInterfaceImpl) IsActive() bool { + return *os.Platform.Config().ElasticsearchSettings.EnableIndexing && atomic.LoadInt32(&os.ready) == 1 +} + +func (os *OpensearchInterfaceImpl) IsIndexingEnabled() bool { + return *os.Platform.Config().ElasticsearchSettings.EnableIndexing +} + +func (os *OpensearchInterfaceImpl) IsSearchEnabled() bool { + return *os.Platform.Config().ElasticsearchSettings.EnableSearching +} + +func (os *OpensearchInterfaceImpl) IsAutocompletionEnabled() bool { + // if we encounter the index mappings haven't been checked, we check it once and store result. + // While in most cases the flag would have been set in the `Start` function, + // There's a case if you call the update config API and enable ES and autocomplete at the same time, it's not set + // so we're checking if its unset here and trying to check the index. + if atomic.LoadInt32(&os.channelIndexVerified) == 0 { + os.Platform.Log().Debug("IsAutocompletionEnabled: channel index has not been verified yet, checking index now") + os.checkChannelIndex() + } + + return *os.Platform.Config().ElasticsearchSettings.EnableAutocomplete && atomic.LoadInt32(&os.channelIndexVerified) == 1 +} + +func (os *OpensearchInterfaceImpl) IsChannelsIndexVerified() bool { + if atomic.LoadInt32(&os.channelIndexVerified) == 0 { + os.Platform.Log().Debug("OpenSearch.IsChannelsIndexVerified: channel index has not been verified yet, checking index now") + os.checkChannelIndex() + } + + return atomic.LoadInt32(&os.channelIndexVerified) == 1 +} + +func (os *OpensearchInterfaceImpl) IsIndexingSync() bool { + return *os.Platform.Config().ElasticsearchSettings.LiveIndexingBatchSize <= 1 +} + +func (os *OpensearchInterfaceImpl) Start() *model.AppError { + if license := os.Platform.License(); license == nil || !*license.Features.Elasticsearch || !*os.Platform.Config().ElasticsearchSettings.EnableIndexing { + return nil + } + + os.mutex.Lock() + defer os.mutex.Unlock() + + if atomic.LoadInt32(&os.ready) != 0 { + // Elasticsearch is already started. We don't return an error + // because "Test Connection" already re-initializes the client. So this + // can be a valid scenario. + return nil + } + + var appErr *model.AppError + if os.client, appErr = createClient(os.Platform.Log(), os.Platform.Config(), os.Platform.FileBackend(), true); appErr != nil { + return appErr + } + + version, major, appErr := checkMaxVersion(os.client) + if appErr != nil { + return appErr + } + + // Since we are only retrieving plugins for the Support Packet generation, it doesn't make sense to kill the process if we get an error + // Instead, we will log it and move forward + resp, err := os.client.Cat.Plugins(context.Background(), nil) + if err != nil { + os.Platform.Log().Warn("Error retrieving opensearch plugins", mlog.Err(err)) + } else { + for _, p := range resp.Plugins { + os.plugins = append(os.plugins, p.Component) + } + } + + os.version = major + os.fullVersion = version + + ctx := context.Background() + + if *os.Platform.Config().ElasticsearchSettings.LiveIndexingBatchSize > 1 { + os.bulkProcessor = NewBulk(os.Platform.Config().ElasticsearchSettings, + os.Platform.Log(), + os.client) + } + + // Set up posts index template. + templateBuf, err := json.Marshal(common.GetPostTemplate(os.Platform.Config())) + if err != nil { + return model.NewAppError("Opensearch.start", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + _, err = os.client.IndexTemplate.Create(ctx, opensearchapi.IndexTemplateCreateReq{ + IndexTemplate: *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts, + Body: bytes.NewReader(templateBuf), + }) + if err != nil { + return model.NewAppError("Opensearch.start", "ent.elasticsearch.create_template_posts_if_not_exists.template_create_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // Set up channels index template. + templateBuf, err = json.Marshal(common.GetChannelTemplate(os.Platform.Config())) + if err != nil { + return model.NewAppError("Opensearch.start", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + _, err = os.client.IndexTemplate.Create(ctx, opensearchapi.IndexTemplateCreateReq{ + IndexTemplate: *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels, + Body: bytes.NewReader(templateBuf), + }) + if err != nil { + return model.NewAppError("Opensearch.start", "ent.elasticsearch.create_template_channels_if_not_exists.template_create_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // Set up users index template. + templateBuf, err = json.Marshal(common.GetUserTemplate(os.Platform.Config())) + if err != nil { + return model.NewAppError("Opensearch.start", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + _, err = os.client.IndexTemplate.Create(ctx, opensearchapi.IndexTemplateCreateReq{ + IndexTemplate: *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers, + Body: bytes.NewReader(templateBuf), + }) + if err != nil { + return model.NewAppError("Opensearch.start", "ent.elasticsearch.create_template_users_if_not_exists.template_create_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // Set up files index template. + templateBuf, err = json.Marshal(common.GetFileInfoTemplate(os.Platform.Config())) + if err != nil { + return model.NewAppError("Opensearch.start", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + _, err = os.client.IndexTemplate.Create(ctx, opensearchapi.IndexTemplateCreateReq{ + IndexTemplate: *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles, + Body: bytes.NewReader(templateBuf), + }) + if err != nil { + return model.NewAppError("Opensearch.start", "ent.elasticsearch.create_template_file_info_if_not_exists.template_create_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + if atomic.LoadInt32(&os.channelIndexVerified) == 0 { + os.checkChannelIndex() + } + + atomic.StoreInt32(&os.ready, 1) + + return nil +} + +func (os *OpensearchInterfaceImpl) Stop() *model.AppError { + os.mutex.Lock() + defer os.mutex.Unlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.start", "ent.elasticsearch.stop.already_stopped.app_error", nil, "", http.StatusInternalServerError) + } + + // Flushing any pending requests + if os.bulkProcessor != nil { + if err := os.bulkProcessor.Stop(); err != nil { + os.Platform.Log().Warn("Error stopping bulk processor", mlog.Err(err)) + } + os.bulkProcessor = nil + } + + os.client = nil + atomic.StoreInt32(&os.ready, 0) + + return nil +} + +func (os *OpensearchInterfaceImpl) GetVersion() int { + return os.version +} + +func (os *OpensearchInterfaceImpl) GetFullVersion() string { + return os.fullVersion +} + +func (os *OpensearchInterfaceImpl) GetPlugins() []string { + return os.plugins +} + +func (os *OpensearchInterfaceImpl) IndexPost(post *model.Post, teamId string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.IndexPost", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + indexName := common.BuildPostIndexName(*os.Platform.Config().ElasticsearchSettings.AggregatePostsAfterDays, + *os.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts, *os.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts_MONTH, time.Now(), post.CreateAt) + + searchPost, err := common.ESPostFromPost(post, teamId) + if err != nil { + return model.NewAppError("Opensearch.IndexPost", "ent.elasticsearch.index_post.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + var postBuf []byte + if os.bulkProcessor != nil { + err = os.bulkProcessor.IndexOp(&types.IndexOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(searchPost.Id), + }, searchPost) + if err != nil { + return model.NewAppError("Opensearch.IndexPost", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + postBuf, err = json.Marshal(searchPost) + if err != nil { + return model.NewAppError("Opensearch.start", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + _, err = os.client.Index(ctx, opensearchapi.IndexReq{ + Index: indexName, + DocumentID: post.Id, + Body: bytes.NewReader(postBuf), + }) + } + if err != nil { + return model.NewAppError("Opensearch.IndexPost", "ent.elasticsearch.index_post.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + metrics := os.Platform.Metrics() + if metrics != nil { + metrics.IncrementPostIndexCounter() + } + + return nil +} + +func (os *OpensearchInterfaceImpl) getPostIndexNames() ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + indexes, err := os.client.Indices.Get(ctx, opensearchapi.IndicesGetReq{ + Indices: []string{"_all"}, + }) + if err != nil { + return nil, err + } + postIndexes := make([]string, 0) + for name := range indexes.Indices { + if strings.HasPrefix(name, *os.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts) { + postIndexes = append(postIndexes, name) + } + } + return postIndexes, nil +} + +func (os *OpensearchInterfaceImpl) SearchPosts(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, model.PostSearchMatches, *model.AppError) { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return []string{}, nil, model.NewAppError("Opensearch.SearchPosts", "ent.elasticsearch.search_posts.disabled", nil, "", http.StatusInternalServerError) + } + + var channelIds []string + for _, channel := range channels { + channelIds = append(channelIds, channel.Id) + } + + var termQueries, notTermQueries, highlightQueries []types.Query + var filters, notFilters []types.Query + for i, params := range searchParams { + newTerms := []string{} + for _, term := range strings.Split(params.Terms, " ") { + if searchengine.EmailRegex.MatchString(term) { + term = `"` + term + `"` + } + newTerms = append(newTerms, term) + } + + params.Terms = strings.Join(newTerms, " ") + + termOperator := operator.And + if searchParams[0].OrTerms { + termOperator = operator.Or + } + + // Date, channels and FromUsers filters come in all + // searchParams iteration, and as they are global to the + // query, we only need to process them once + if i == 0 { + if len(params.InChannels) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": params.InChannels}}, + }) + } + + if len(params.ExcludedChannels) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": params.ExcludedChannels}}, + }) + } + + if len(params.FromUsers) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"user_id": params.FromUsers}}, + }) + } + + if len(params.ExcludedUsers) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"user_id": params.ExcludedUsers}}, + }) + } + + if params.OnDate != "" { + before, after := params.GetOnDateMillis() + filters = append(filters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(before)), + Lte: model.NewPointer(types.Float64(after)), + }, + }, + }) + } else { + if params.AfterDate != "" || params.BeforeDate != "" { + nrQuery := types.NumberRangeQuery{} + if params.AfterDate != "" { + nrQuery.Gte = model.NewPointer(types.Float64(params.GetAfterDateMillis())) + } + + if params.BeforeDate != "" { + nrQuery.Lte = model.NewPointer(types.Float64(params.GetBeforeDateMillis())) + } + + query := types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": nrQuery, + }, + } + filters = append(filters, query) + } + + if params.ExcludedAfterDate != "" || params.ExcludedBeforeDate != "" || params.ExcludedDate != "" { + if params.ExcludedDate != "" { + before, after := params.GetExcludedDateMillis() + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(before)), + Lte: model.NewPointer(types.Float64(after)), + }, + }, + }) + } + + if params.ExcludedAfterDate != "" { + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(params.GetExcludedAfterDateMillis())), + }, + }, + }) + } + + if params.ExcludedBeforeDate != "" { + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Lte: model.NewPointer(types.Float64(params.GetExcludedBeforeDateMillis())), + }, + }, + }) + } + } + } + } + + if params.IsHashtag { + if params.Terms != "" { + query := types.Query{ + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"hashtags"}, + DefaultOperator: &termOperator, + }, + } + termQueries = append(termQueries, query) + highlightQueries = append(highlightQueries, query) + } else if params.ExcludedTerms != "" { + query := types.Query{ + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"hashtags"}, + DefaultOperator: &termOperator, + }, + } + notTermQueries = append(notTermQueries, query) + } + } else { + if params.Terms != "" { + elements := []types.Query{ + { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"message"}, + DefaultOperator: &termOperator, + }, + }, { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"attachments"}, + DefaultOperator: &termOperator, + }, + }, { + Term: map[string]types.TermQuery{ + "urls": {Value: params.Terms}, + }, + }, + } + query := types.Query{ + Bool: &types.BoolQuery{Should: append([]types.Query(nil), elements...)}, + } + + termQueries = append(termQueries, query) + + hashtagTerms := []string{} + for _, term := range strings.Split(params.Terms, " ") { + hashtagTerms = append(hashtagTerms, "#"+term) + } + + hashtagQuery := types.Query{ + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: strings.Join(hashtagTerms, " "), + Fields: []string{"hashtags"}, + DefaultOperator: &termOperator, + }, + } + highlightQuery := types.Query{ + Bool: &types.BoolQuery{Should: append(elements, hashtagQuery)}, + } + + highlightQueries = append(highlightQueries, highlightQuery) + } + + if params.ExcludedTerms != "" { + query := types.Query{ + Bool: &types.BoolQuery{Should: []types.Query{ + { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"message"}, + DefaultOperator: &termOperator, + }, + }, { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"attachments"}, + DefaultOperator: &termOperator, + }, + }, { + Term: map[string]types.TermQuery{ + "urls": {Value: params.ExcludedTerms}, + }, + }, + }}, + } + + notTermQueries = append(notTermQueries, query) + } + } + } + + allTermsQuery := &types.BoolQuery{ + MustNot: append([]types.Query(nil), notTermQueries...), + } + if searchParams[0].OrTerms { + allTermsQuery.Should = append([]types.Query(nil), termQueries...) + } else { + allTermsQuery.Must = append([]types.Query(nil), termQueries...) + } + + fullHighlightsQuery := &types.BoolQuery{ + Filter: append([]types.Query(nil), filters...), + MustNot: append([]types.Query(nil), notFilters...), + } + + if searchParams[0].OrTerms { + fullHighlightsQuery.Should = append([]types.Query(nil), highlightQueries...) + } else { + fullHighlightsQuery.Must = append([]types.Query(nil), highlightQueries...) + } + + filters = append(filters, + types.Query{ + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{"channel_id": channelIds}, + }, + }, + types.Query{ + Bool: &types.BoolQuery{ + Should: []types.Query{ + { + Term: map[string]types.TermQuery{"type": {Value: "default"}}, + }, { + Term: map[string]types.TermQuery{"type": {Value: "slack_attachment"}}, + }, + }, + }, + }, + ) + + highlight := &types.Highlight{ + HighlightQuery: &types.Query{ + Bool: fullHighlightsQuery, + }, + Fields: map[string]types.HighlightField{ + "message": {}, + "attachments": {}, + "url": {}, + "hashtag": {}, + }, + Encoder: &highlighterencoder.Html, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: append([]types.Query(nil), filters...), + Must: []types.Query{{Bool: allTermsQuery}}, + MustNot: append([]types.Query(nil), notFilters...), + }, + } + + searchBuf, err := json.Marshal(search.Request{ + Query: query, + Highlight: highlight, + Sort: []types.SortCombinations{types.SortOptions{ + SortOptions: map[string]types.FieldSort{"create_at": {Order: &sortorder.Desc}}, + }}, + }) + if err != nil { + return []string{}, nil, model.NewAppError("Opensearch.SearchPosts", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + // We need to declare the response structs because + // OS client doesn't have the highlight field in the struct. + type SearchHit struct { + Index string `json:"_index"` + ID string `json:"_id"` + Score float32 `json:"_score"` + Source json.RawMessage `json:"_source"` + Fields json.RawMessage `json:"fields"` + Type string `json:"_type"` // Deprecated field + Sort []any `json:"sort"` + Highlight map[string][]string `json:"highlight,omitempty"` + } + + type searchResp struct { + Took int `json:"took"` + Timeout bool `json:"timed_out"` + Hits struct { + Total struct { + Value int `json:"value"` + Relation string `json:"relation"` + } `json:"total"` + MaxScore float32 `json:"max_score"` + Hits []SearchHit `json:"hits"` + } `json:"hits"` + Errors bool `json:"errors"` + } + + var searchResult searchResp + _, err = os.client.Client.Do(ctx, &opensearchapi.SearchReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "*"}, + Body: bytes.NewReader(searchBuf), + Params: opensearchapi.SearchParams{ + From: model.NewPointer(page * perPage), + Size: model.NewPointer(perPage), + }, + }, &searchResult) + if err != nil { + errorStr := "err=" + err.Error() + if *os.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return []string{}, nil, model.NewAppError("Opensearch.SearchPosts", "ent.elasticsearch.search_posts.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + postIds := make([]string, len(searchResult.Hits.Hits)) + matches := make(model.PostSearchMatches, len(searchResult.Hits.Hits)) + + for i, hit := range searchResult.Hits.Hits { + var post common.ESPost + err := json.Unmarshal(hit.Source, &post) + if err != nil { + return postIds, matches, model.NewAppError("Opensearch.SearchPosts", "ent.elasticsearch.search_posts.unmarshall_post_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + postIds[i] = post.Id + + matchesForPost, err := common.GetMatchesForHit(hit.Highlight) + if err != nil { + return postIds, matches, model.NewAppError("Opensearch.SearchPosts", "ent.elasticsearch.search_posts.parse_matches_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + matches[post.Id] = matchesForPost + } + + return postIds, matches, nil +} + +func (os *OpensearchInterfaceImpl) DeletePost(post *model.Post) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeletePost", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + // This is racy with index aggregation, but since the posts are verified in the database when returning search + // results, there's no risk of deleted posts getting sent back to the user in response to a search query, and even + // then the race is very unlikely because it would only occur when someone deletes a post that's due to be + // aggregated but hasn't been yet, which makes the time window small and the post likelihood very low. + indexName := common.BuildPostIndexName(*os.Platform.Config().ElasticsearchSettings.AggregatePostsAfterDays, + *os.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts, *os.Platform.Config().ElasticsearchSettings.IndexPrefix+common.IndexBasePosts_MONTH, time.Now(), post.CreateAt) + + if err := os.deletePost(indexName, post.Id); err != nil { + return err + } + + return nil +} + +func (os *OpensearchInterfaceImpl) DeleteChannelPosts(rctx request.CTX, channelID string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeleteChannelPosts", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + postIndexes, err := os.getPostIndexNames() + if err != nil { + return model.NewAppError("Opensearch.DeleteChannelPosts", "ent.elasticsearch.delete_channel_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"channel_id": {Value: channelID}}, + }}, + }, + } + queryBuf, err := json.Marshal(deletebyquery.Request{ + Query: query, + }) + if err != nil { + return model.NewAppError("Opensearch.SearchPosts", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + response, err := os.client.Document.DeleteByQuery(ctx, opensearchapi.DocumentDeleteByQueryReq{ + Indices: postIndexes, + Body: bytes.NewReader(queryBuf), + }) + if err != nil { + return model.NewAppError("Opensearch.DeleteChannelPosts", "ent.elasticsearch.delete_channel_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("Posts for channel deleted", mlog.String("channel_id", channelID), mlog.Int("deleted", response.Deleted)) + + return nil +} + +func (os *OpensearchInterfaceImpl) DeleteUserPosts(rctx request.CTX, userID string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeleteUserPosts", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + postIndexes, err := os.getPostIndexNames() + if err != nil { + return model.NewAppError("Opensearch.DeleteUserPosts", "ent.elasticsearch.delete_user_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"user_id": {Value: userID}}, + }}, + }, + } + + queryBuf, err := json.Marshal(deletebyquery.Request{ + Query: query, + }) + if err != nil { + return model.NewAppError("Opensearch.SearchPosts", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + response, err := os.client.Document.DeleteByQuery(ctx, opensearchapi.DocumentDeleteByQueryReq{ + Indices: postIndexes, + Body: bytes.NewReader(queryBuf), + }) + if err != nil { + return model.NewAppError("Opensearch.DeleteUserPosts", "ent.elasticsearch.delete_user_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("Posts for user deleted", mlog.String("user_id", userID), mlog.Int("deleted", response.Deleted)) + + return nil +} + +func (os *OpensearchInterfaceImpl) deletePost(indexName, postID string) *model.AppError { + var err error + if os.bulkProcessor != nil { + err = os.bulkProcessor.DeleteOp(&types.DeleteOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(postID), + }) + if err != nil { + return model.NewAppError("Opensearch.IndexPost", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + _, err = os.client.Document.Delete(ctx, opensearchapi.DocumentDeleteReq{ + Index: indexName, + DocumentID: postID, + }) + } + if err != nil { + return model.NewAppError("Opensearch.DeletePost", "ent.elasticsearch.delete_post.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil +} + +func (os *OpensearchInterfaceImpl) IndexChannel(rctx request.CTX, channel *model.Channel, userIDs, teamMemberIDs []string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.IndexChannel", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + indexName := *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels + + searchChannel := common.ESChannelFromChannel(channel, userIDs, teamMemberIDs) + + var err error + var buf []byte + if os.bulkProcessor != nil { + err = os.bulkProcessor.IndexOp(&types.IndexOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(searchChannel.Id), + }, searchChannel) + if err != nil { + return model.NewAppError("Opensearch.IndexChannel", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + buf, err = json.Marshal(searchChannel) + if err != nil { + return model.NewAppError("Opensearch.IndexChannel", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + _, err = os.client.Index(ctx, opensearchapi.IndexReq{ + Index: indexName, + DocumentID: searchChannel.Id, + Body: bytes.NewReader(buf), + }) + } + if err != nil { + return model.NewAppError("Opensearch.IndexChannel", "ent.elasticsearch.index_channel.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + metrics := os.Platform.Metrics() + if metrics != nil { + metrics.IncrementChannelIndexCounter() + } + + return nil +} + +func (os *OpensearchInterfaceImpl) SearchChannels(teamId, userID string, term string, isGuest bool) ([]string, *model.AppError) { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return []string{}, model.NewAppError("Opensearch.SearchChannels", "ent.elasticsearch.search_channels.disabled", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + boolNotPrivate := types.Query{ + Bool: &types.BoolQuery{ + MustNot: []types.Query{{ + Term: map[string]types.TermQuery{"type": {Value: model.ChannelTypePrivate}}, + }}, + }, + } + + userQ := types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"user_ids": {Value: userID}}, + }}, + Must: []types.Query{{ + Term: map[string]types.TermQuery{"type": {Value: model.ChannelTypePrivate}}, + }}, + }, + } + + query := &types.BoolQuery{} + + if teamId != "" { + query.Filter = append(query.Filter, types.Query{Term: map[string]types.TermQuery{"team_id": {Value: teamId}}}) + } else { + query.Filter = append(query.Filter, types.Query{Term: map[string]types.TermQuery{"team_member_ids": {Value: userID}}}) + } + + if !isGuest { + query.Filter = append(query.Filter, types.Query{ + Bool: &types.BoolQuery{ + Should: []types.Query{ + boolNotPrivate, userQ, + }, + Must: []types.Query{{ + Prefix: map[string]types.PrefixQuery{ + "name_suggestions": {Value: strings.ToLower(term)}, + }, + }}, + MinimumShouldMatch: 1, + }, + }) + } else { + query.Filter = append(query.Filter, types.Query{ + Bool: &types.BoolQuery{ + Must: []types.Query{ + boolNotPrivate, { + Prefix: map[string]types.PrefixQuery{ + "name_suggestions": {Value: strings.ToLower(term)}, + }, + }}, + }, + }) + } + + buf, err := json.Marshal(search.Request{ + Query: &types.Query{Bool: query}, + }) + if err != nil { + return []string{}, model.NewAppError("Opensearch.SearchChannels", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + searchResult, err := os.client.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels}, + Body: bytes.NewReader(buf), + Params: opensearchapi.SearchParams{ + Size: model.NewPointer(model.ChannelSearchDefaultLimit), + }, + }) + if err != nil { + errorStr := "err=" + err.Error() + if *os.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return nil, model.NewAppError("Opensearch.SearchChannels", "ent.elasticsearch.search_channels.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + channelIds := []string{} + for _, hit := range searchResult.Hits.Hits { + var channel common.ESChannel + err := json.Unmarshal(hit.Source, &channel) + if err != nil { + return nil, model.NewAppError("Opensearch.SearchChannels", "ent.elasticsearch.search_channels.unmarshall_channel_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + channelIds = append(channelIds, channel.Id) + } + + return channelIds, nil +} + +func (os *OpensearchInterfaceImpl) DeleteChannel(channel *model.Channel) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeleteChannel", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + var err error + if os.bulkProcessor != nil { + err = os.bulkProcessor.DeleteOp(&types.DeleteOperation{ + Index_: model.NewPointer(*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels), + Id_: model.NewPointer(channel.Id), + }) + if err != nil { + return model.NewAppError("Opensearch.IndexPost", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = os.client.Document.Delete(ctx, opensearchapi.DocumentDeleteReq{ + Index: *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels, + DocumentID: channel.Id, + }) + } + if err != nil { + return model.NewAppError("Opensearch.DeleteChannel", "ent.elasticsearch.delete_channel.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return nil +} + +func (os *OpensearchInterfaceImpl) IndexUser(rctx request.CTX, user *model.User, teamsIds, channelsIds []string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.IndexUser", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + indexName := *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers + + searchUser := common.ESUserFromUserAndTeams(user, teamsIds, channelsIds) + + var err error + var buf []byte + if os.bulkProcessor != nil { + err = os.bulkProcessor.IndexOp(&types.IndexOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(searchUser.Id), + }, searchUser) + if err != nil { + return model.NewAppError("Opensearch.IndexUser", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + buf, err = json.Marshal(searchUser) + if err != nil { + return model.NewAppError("Opensearch.IndexUser", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + _, err = os.client.Index(ctx, opensearchapi.IndexReq{ + Index: indexName, + DocumentID: searchUser.Id, + Body: bytes.NewReader(buf), + }) + } + if err != nil { + return model.NewAppError("Opensearch.IndexUser", "ent.elasticsearch.index_user.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + metrics := os.Platform.Metrics() + if metrics != nil { + metrics.IncrementUserIndexCounter() + } + + return nil +} + +func (os *OpensearchInterfaceImpl) autocompleteUsers(contextCategory string, categoryIds []string, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return nil, model.NewAppError("Opensearch.autocompleteUsers", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.BoolQuery{} + + if term != "" { + var suggestionField string + if options.AllowFullNames { + suggestionField = "suggestions_with_fullname" + } else { + suggestionField = "suggestions_without_fullname" + } + query.Must = append(query.Must, types.Query{ + Prefix: map[string]types.PrefixQuery{ + suggestionField: {Value: strings.ToLower(term)}, + }, + }) + } + + if len(categoryIds) > 0 { + var iCategoryIds []string + for _, id := range categoryIds { + if id != "" { + iCategoryIds = append(iCategoryIds, id) + } + } + if len(iCategoryIds) > 0 { + query.Filter = append(query.Filter, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{contextCategory: iCategoryIds}}, + }) + } + } + + if !options.AllowInactive { + query.Filter = append(query.Filter, types.Query{ + Bool: &types.BoolQuery{ + Should: []types.Query{ + { + Range: map[string]types.RangeQuery{ + "delete_at": types.DateRangeQuery{ + Lte: model.NewPointer("0"), + }, + }, + }, { + Bool: &types.BoolQuery{ + MustNot: []types.Query{{ + Exists: &types.ExistsQuery{Field: "delete_at"}, + }}, + }, + }, + }, + }, + }) + } + + if options.Role != "" { + query.Filter = append(query.Filter, types.Query{ + Term: map[string]types.TermQuery{ + "roles": {Value: options.Role}, + }, + }) + } + + buf, err := json.Marshal(search.Request{ + Query: &types.Query{Bool: query}, + }) + if err != nil { + return nil, model.NewAppError("Opensearch.autocompleteUsers", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + searchResults, err := os.client.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers}, + Body: bytes.NewReader(buf), + Params: opensearchapi.SearchParams{ + Size: model.NewPointer(options.Limit), + }, + }) + + if err != nil { + errorStr := "err=" + err.Error() + if *os.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return nil, model.NewAppError("Opensearch.autocompleteUsers", "ent.elasticsearch.search_users.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + users := []common.ESUser{} + for _, hit := range searchResults.Hits.Hits { + var user common.ESUser + err := json.Unmarshal(hit.Source, &user) + if err != nil { + return nil, model.NewAppError("Opensearch.autocompleteUsers", "ent.elasticsearch.search_users.unmarshall_user_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + users = append(users, user) + } + + return users, nil +} + +func (os *OpensearchInterfaceImpl) autocompleteUsersInChannel(channelId, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + return os.autocompleteUsers("channel_id", []string{channelId}, term, options) +} + +func (os *OpensearchInterfaceImpl) autocompleteUsersInChannels(channelIds []string, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + return os.autocompleteUsers("channel_id", channelIds, term, options) +} + +func (os *OpensearchInterfaceImpl) autocompleteUsersInTeam(teamId, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + return os.autocompleteUsers("team_id", []string{teamId}, term, options) +} + +func (os *OpensearchInterfaceImpl) autocompleteUsersNotInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]common.ESUser, *model.AppError) { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return nil, model.NewAppError("Opensearch.autocompleteUsersNotInChannel", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + filterMust := []types.Query{{Term: map[string]types.TermQuery{ + "team_id": {Value: teamId}, + }}} + if len(restrictedToChannels) > 0 { + filterMust = append(filterMust, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": restrictedToChannels}}, + }) + } + + query := &types.BoolQuery{ + Filter: []types.Query{{ + Bool: &types.BoolQuery{ + Must: filterMust, + }, + }}, + MustNot: []types.Query{{ + Term: map[string]types.TermQuery{ + "channel_id": {Value: channelId}, + }, + }}, + } + + if term != "" { + var suggestionField string + if options.AllowFullNames { + suggestionField = "suggestions_with_fullname" + } else { + suggestionField = "suggestions_without_fullname" + } + query.Must = append(query.Must, types.Query{ + Prefix: map[string]types.PrefixQuery{ + suggestionField: {Value: strings.ToLower(term)}, + }, + }) + } + + if !options.AllowInactive { + notExistField := types.Query{ + Bool: &types.BoolQuery{ + MustNot: []types.Query{{ + Exists: &types.ExistsQuery{Field: "delete_at"}, + }}, + }, + } + deleteRangeQuery := types.Query{ + Range: map[string]types.RangeQuery{ + "delete_at": types.DateRangeQuery{ + Lte: model.NewPointer("0"), + }, + }, + } + inactiveQuery := types.Query{ + Bool: &types.BoolQuery{ + Should: []types.Query{deleteRangeQuery, notExistField}, + }, + } + query.Filter = append(query.Filter, inactiveQuery) + } + + if options.Role != "" { + query.Filter = append(query.Filter, types.Query{ + Term: map[string]types.TermQuery{ + "roles": {Value: options.Role}, + }, + }) + } + + buf, err := json.Marshal(search.Request{ + Query: &types.Query{Bool: query}, + }) + if err != nil { + return nil, model.NewAppError("Opensearch.autocompleteUsersNotInChannel", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + searchResults, err := os.client.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers}, + Body: bytes.NewReader(buf), + Params: opensearchapi.SearchParams{ + Size: model.NewPointer(options.Limit), + }, + }) + if err != nil { + errorStr := "err=" + err.Error() + if *os.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return nil, model.NewAppError("Opensearch.autocompleteUsersNotInChannel", "ent.elasticsearch.search_users.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + users := []common.ESUser{} + for _, hit := range searchResults.Hits.Hits { + var user common.ESUser + err := json.Unmarshal(hit.Source, &user) + if err != nil { + return nil, model.NewAppError("Opensearch.autocompleteUsersNotInChannel", "ent.elasticsearch.search_users.unmarshall_user_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + users = append(users, user) + } + + return users, nil +} + +func (os *OpensearchInterfaceImpl) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) { + if restrictedToChannels != nil && len(restrictedToChannels) == 0 { + return []string{}, []string{}, nil + } + + uchan, err := os.autocompleteUsersInChannel(channelId, term, options) + if err != nil { + return nil, nil, err + } + + var nuchan []common.ESUser + nuchan, err = os.autocompleteUsersNotInChannel(teamId, channelId, restrictedToChannels, term, options) + if err != nil { + return nil, nil, err + } + + uchanIds := []string{} + for _, user := range uchan { + uchanIds = append(uchanIds, user.Id) + } + nuchanIds := []string{} + for _, user := range nuchan { + nuchanIds = append(nuchanIds, user.Id) + } + + return uchanIds, nuchanIds, nil +} + +func (os *OpensearchInterfaceImpl) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) { + if restrictedToChannels != nil && len(restrictedToChannels) == 0 { + return []string{}, nil + } + + var users []common.ESUser + var err *model.AppError + if restrictedToChannels == nil { + users, err = os.autocompleteUsersInTeam(teamId, term, options) + } else { + users, err = os.autocompleteUsersInChannels(restrictedToChannels, term, options) + } + if err != nil { + return nil, err + } + + usersIds := []string{} + if len(users) >= options.Limit { + users = users[:options.Limit] + } + + for _, user := range users { + usersIds = append(usersIds, user.Id) + } + + return usersIds, nil +} + +func (os *OpensearchInterfaceImpl) DeleteUser(user *model.User) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeleteUser", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + var err error + if os.bulkProcessor != nil { + err = os.bulkProcessor.DeleteOp(&types.DeleteOperation{ + Index_: model.NewPointer(*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers), + Id_: model.NewPointer(user.Id), + }) + if err != nil { + return model.NewAppError("Opensearch.DeleteUser", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = os.client.Document.Delete(ctx, opensearchapi.DocumentDeleteReq{ + Index: *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseUsers, + DocumentID: user.Id, + }) + } + if err != nil { + return model.NewAppError("Opensearch.DeleteUser", "ent.elasticsearch.delete_user.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return nil +} + +func (os *OpensearchInterfaceImpl) TestConfig(rctx request.CTX, cfg *model.Config) *model.AppError { + if license := os.Platform.License(); license == nil || !*license.Features.Elasticsearch { + return model.NewAppError("Opensearch.TestConfig", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) + } + + if !*cfg.ElasticsearchSettings.EnableIndexing { + return model.NewAppError("Opensearch.TestConfig", "ent.elasticsearch.test_config.indexing_disabled.error", nil, "", http.StatusNotImplemented) + } + + client, appErr := createClient(rctx.Logger(), cfg, os.Platform.FileBackend(), true) + if appErr != nil { + return appErr + } + + _, _, appErr = checkMaxVersion(client) + if appErr != nil { + return appErr + } + + // Resetting the state. + if atomic.CompareAndSwapInt32(&os.ready, 0, 1) { + // Re-assign the client. + // This is necessary in case opensearch was started + // after server start. + os.mutex.Lock() + os.client = client + os.mutex.Unlock() + } + + return nil +} + +func (os *OpensearchInterfaceImpl) PurgeIndexes(rctx request.CTX) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if license := os.Platform.License(); license == nil || !*license.Features.Elasticsearch { + return model.NewAppError("Opensearch.PurgeIndexes", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) + } + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.PurgeIndexes", "ent.elasticsearch.generic.disabled", nil, "", http.StatusInternalServerError) + } + + indexPrefix := *os.Platform.Config().ElasticsearchSettings.IndexPrefix + indexesToDelete := []string{indexPrefix + "*"} + + if ignorePurgeIndexes := *os.Platform.Config().ElasticsearchSettings.IgnoredPurgeIndexes; ignorePurgeIndexes != "" { + // we are checking if provided indexes exist. If an index doesn't exist, + // opensearch returns an error while trying to purge it even we intend to + // ignore it. + for _, ignorePurgeIndex := range strings.Split(ignorePurgeIndexes, ",") { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err := os.client.Indices.Get(ctx, opensearchapi.IndicesGetReq{ + Indices: []string{ignorePurgeIndex}, + }) + if err != nil { + rctx.Logger().Warn("Opensearch index get error", mlog.String("index", ignorePurgeIndex), mlog.Err(err)) + continue + } + indexesToDelete = append(indexesToDelete, "-"+strings.TrimSpace(ignorePurgeIndex)) + } + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err := os.client.Indices.Delete(ctx, opensearchapi.IndicesDeleteReq{ + Indices: indexesToDelete, + }) + if err != nil { + rctx.Logger().Error("Opensearch PurgeIndexes Error", mlog.Err(err)) + return model.NewAppError("Opensearch.PurgeIndexes", "ent.elasticsearch.purge_indexes.delete_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return nil +} + +// PurgeIndexList purges a list of specified indexes. +// For now it only allows purging the channels index as thats all that's needed, +// but the code is written in generic fashion to allow it to purge any index. +// It needs more logic around post indexes as their name isn't the same, but rather follow a pattern +// containing the date as well. +func (os *OpensearchInterfaceImpl) PurgeIndexList(rctx request.CTX, indexes []string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if license := os.Platform.License(); license == nil || !*license.Features.Elasticsearch { + return model.NewAppError("Opensearch.PurgeIndexList", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) + } + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.PurgeIndexList", "ent.elasticsearch.generic.disabled", nil, "", http.StatusInternalServerError) + } + + indexPrefix := *os.Platform.Config().ElasticsearchSettings.IndexPrefix + indexToDeleteMap := map[string]bool{} + for _, index := range indexes { + isKnownIndex := false + for _, allowedIndex := range purgeIndexListAllowedIndexes { + if index == allowedIndex { + isKnownIndex = true + break + } + } + + if !isKnownIndex { + return model.NewAppError("Opensearch.PurgeIndexList", "ent.elasticsearch.purge_indexes.unknown_index", map[string]any{"unknown_index": index}, "", http.StatusBadRequest) + } + + indexToDeleteMap[indexPrefix+index] = true + } + + if ign := *os.Platform.Config().ElasticsearchSettings.IgnoredPurgeIndexes; ign != "" { + // make sure we're not purging any index configured to be ignored + for _, ix := range strings.Split(ign, ",") { + delete(indexToDeleteMap, ix) + } + } + + indexToDelete := []string{} + for key := range indexToDeleteMap { + indexToDelete = append(indexToDelete, key) + } + + if len(indexToDelete) > 0 { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + _, err := os.client.Indices.Delete(ctx, opensearchapi.IndicesDeleteReq{ + Indices: indexToDelete, + }) + if err != nil { + openErr, ok := err.(*opensearch.StructError) + if !ok || openErr.Status != http.StatusNotFound { + rctx.Logger().Error("Elastic Search PurgeIndex Error", mlog.Err(err)) + return model.NewAppError("Opensearch.PurgeIndexList", "ent.elasticsearch.purge_index.delete_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + } + } + + return nil +} + +func (os *OpensearchInterfaceImpl) RefreshIndexes(rctx request.CTX) *model.AppError { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + _, err := os.client.Indices.Refresh(ctx, nil) + if err != nil { + rctx.Logger().Error("Elastic Search RefreshIndexes Error", mlog.Err(err)) + return model.NewAppError("Opensearch.RefreshIndexes", "ent.elasticsearch.refresh_indexes.refresh_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + return nil +} + +func (os *OpensearchInterfaceImpl) DataRetentionDeleteIndexes(rctx request.CTX, cutoff time.Time) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if license := os.Platform.License(); license == nil || !*license.Features.Elasticsearch { + return model.NewAppError("Opensearch.DataRetentionDeleteIndexes", "ent.elasticsearch.test_config.license.error", nil, "", http.StatusNotImplemented) + } + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DataRetentionDeleteIndexes", "ent.elasticsearch.generic.disabled", nil, "", http.StatusInternalServerError) + } + + ctx := context.Background() + dateFormat := *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_2006_01_02" + postIndexesResult, err := os.client.Indices.Get(ctx, opensearchapi.IndicesGetReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBasePosts + "_*"}, + }) + if err != nil { + return model.NewAppError("Opensearch.DataRetentionDeleteIndexes", "ent.elasticsearch.data_retention_delete_indexes.get_indexes.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + for index := range postIndexesResult.Indices { + if indexDate, err := time.Parse(dateFormat, index); err != nil { + rctx.Logger().Warn("Failed to parse date from posts index. Ignoring index.", mlog.String("index", index)) + } else { + if indexDate.Before(cutoff) || indexDate.Equal(cutoff) { + if _, err := os.client.Indices.Delete(ctx, opensearchapi.IndicesDeleteReq{ + Indices: []string{index}, + }); err != nil { + return model.NewAppError("Opensearch.DataRetentionDeleteIndexes", "ent.elasticsearch.data_retention_delete_indexes.delete_index.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + } + } + } + + return nil +} + +func (os *OpensearchInterfaceImpl) IndexFile(file *model.FileInfo, channelId string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.IndexFile", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + indexName := *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles + + searchFile := common.ESFileFromFileInfo(file, channelId) + + var err error + var fileBuf []byte + if os.bulkProcessor != nil { + err = os.bulkProcessor.IndexOp(&types.IndexOperation{ + Index_: model.NewPointer(indexName), + Id_: model.NewPointer(searchFile.Id), + }, searchFile) + if err != nil { + return model.NewAppError("Opensearch.IndexFile", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + fileBuf, err = json.Marshal(searchFile) + if err != nil { + return model.NewAppError("Opensearch.SearchPosts", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + _, err = os.client.Index(ctx, opensearchapi.IndexReq{ + Index: indexName, + DocumentID: file.Id, + Body: bytes.NewReader(fileBuf), + }) + } + if err != nil { + return model.NewAppError("Opensearch.IndexFile", "ent.elasticsearch.index_file.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + if metrics := os.Platform.Metrics(); metrics != nil { + metrics.IncrementFileIndexCounter() + } + + return nil +} + +func (os *OpensearchInterfaceImpl) SearchFiles(channels model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return []string{}, model.NewAppError("Opensearch.SearchPosts", "ent.elasticsearch.search_files.disabled", nil, "", http.StatusInternalServerError) + } + + var channelIds []string + for _, channel := range channels { + channelIds = append(channelIds, channel.Id) + } + + var termQueries, notTermQueries []types.Query + var filters, notFilters []types.Query + for i, params := range searchParams { + newTerms := []string{} + for _, term := range strings.Split(params.Terms, " ") { + if searchengine.EmailRegex.MatchString(term) { + term = `"` + term + `"` + } + newTerms = append(newTerms, term) + } + + params.Terms = strings.Join(newTerms, " ") + + termOperator := operator.And + if searchParams[0].OrTerms { + termOperator = operator.Or + } + + // Date, channels and FromUsers filters come in all + // searchParams iteration, and as they are global to the + // query, we only need to process them once + if i == 0 { + if len(params.InChannels) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": params.InChannels}}, + }) + } + + if len(params.ExcludedChannels) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"channel_id": params.ExcludedChannels}}, + }) + } + + if len(params.FromUsers) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"creator_id": params.FromUsers}}, + }) + } + + if len(params.ExcludedUsers) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"creator_id": params.ExcludedUsers}}, + }) + } + + if len(params.Extensions) > 0 { + filters = append(filters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"extension": params.Extensions}}, + }) + } + + if len(params.ExcludedExtensions) > 0 { + notFilters = append(notFilters, types.Query{ + Terms: &types.TermsQuery{TermsQuery: map[string]types.TermsQueryField{"extension": params.ExcludedExtensions}}, + }) + } + + if params.OnDate != "" { + before, after := params.GetOnDateMillis() + filters = append(filters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(before)), + Lte: model.NewPointer(types.Float64(after)), + }, + }, + }) + } else { + if params.AfterDate != "" || params.BeforeDate != "" { + nrQuery := types.NumberRangeQuery{} + if params.AfterDate != "" { + nrQuery.Gte = model.NewPointer(types.Float64(params.GetAfterDateMillis())) + } + + if params.BeforeDate != "" { + nrQuery.Lte = model.NewPointer(types.Float64(params.GetBeforeDateMillis())) + } + query := types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": nrQuery, + }, + } + filters = append(filters, query) + } + + if params.ExcludedAfterDate != "" || params.ExcludedBeforeDate != "" || params.ExcludedDate != "" { + if params.ExcludedDate != "" { + before, after := params.GetExcludedDateMillis() + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(before)), + Lte: model.NewPointer(types.Float64(after)), + }, + }, + }) + } + + if params.ExcludedAfterDate != "" { + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Gte: model.NewPointer(types.Float64(params.GetExcludedAfterDateMillis())), + }, + }, + }) + } + + if params.ExcludedBeforeDate != "" { + notFilters = append(notFilters, types.Query{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Lte: model.NewPointer(types.Float64(params.GetExcludedBeforeDateMillis())), + }, + }, + }) + } + } + } + } + + if params.Terms != "" { + elements := []types.Query{ + { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"content"}, + DefaultOperator: &termOperator, + }, + }, { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.Terms, + Fields: []string{"name"}, + DefaultOperator: &termOperator, + }, + }, + } + query := types.Query{ + Bool: &types.BoolQuery{Should: append([]types.Query(nil), elements...)}, + } + termQueries = append(termQueries, query) + } + + if params.ExcludedTerms != "" { + elements := []types.Query{ + { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"content"}, + DefaultOperator: &termOperator, + }, + }, { + SimpleQueryString: &types.SimpleQueryStringQuery{ + Query: params.ExcludedTerms, + Fields: []string{"name"}, + DefaultOperator: &termOperator, + }, + }, + } + query := types.Query{ + Bool: &types.BoolQuery{Should: append([]types.Query(nil), elements...)}, + } + notTermQueries = append(notTermQueries, query) + } + } + + allTermsQuery := &types.BoolQuery{ + MustNot: append([]types.Query(nil), notTermQueries...), + } + if searchParams[0].OrTerms { + allTermsQuery.Should = append([]types.Query(nil), termQueries...) + } else { + allTermsQuery.Must = append([]types.Query(nil), termQueries...) + } + + filters = append(filters, + types.Query{ + Terms: &types.TermsQuery{ + TermsQuery: map[string]types.TermsQueryField{"channel_id": channelIds}, + }, + }, + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: append([]types.Query(nil), filters...), + Must: []types.Query{{Bool: allTermsQuery}}, + MustNot: append([]types.Query(nil), notFilters...), + }, + } + + searchBuf, err := json.Marshal(search.Request{ + Query: query, + Sort: []types.SortCombinations{types.SortOptions{ + SortOptions: map[string]types.FieldSort{"create_at": {Order: &sortorder.Desc}}, + }}, + }) + if err != nil { + return []string{}, model.NewAppError("Opensearch.SearchFiles", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + searchResult, err := os.client.Search(ctx, &opensearchapi.SearchReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles}, + Body: bytes.NewReader(searchBuf), + Params: opensearchapi.SearchParams{ + From: model.NewPointer(page * perPage), + Size: model.NewPointer(perPage), + }, + }) + if err != nil { + errorStr := "err=" + err.Error() + if *os.Platform.Config().ElasticsearchSettings.Trace == "error" { + errorStr = "Query=" + getJSONOrErrorStr(query) + ", " + errorStr + } + return []string{}, model.NewAppError("Opensearch.SearchFiles", "ent.elasticsearch.search_files.search_failed", nil, errorStr, http.StatusInternalServerError) + } + + fileIds := make([]string, len(searchResult.Hits.Hits)) + + for i, hit := range searchResult.Hits.Hits { + var file common.ESFile + if err := json.Unmarshal(hit.Source, &file); err != nil { + return fileIds, model.NewAppError("Opensearch.SearchFiles", "ent.elasticsearch.search_files.unmarshall_file_failed", nil, "", http.StatusInternalServerError).Wrap(err) + } + fileIds[i] = file.Id + } + + return fileIds, nil +} + +func (os *OpensearchInterfaceImpl) DeleteFile(fileID string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeleteFile", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + var err error + if os.bulkProcessor != nil { + err = os.bulkProcessor.DeleteOp(&types.DeleteOperation{ + Index_: model.NewPointer(*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles), + Id_: model.NewPointer(fileID), + }) + if err != nil { + return model.NewAppError("Opensearch.DeleteFile", model.NoTranslation, nil, "", http.StatusInternalServerError).Wrap(err) + } + } else { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + _, err = os.client.Document.Delete(ctx, opensearchapi.DocumentDeleteReq{ + Index: *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles, + DocumentID: fileID, + }) + } + if err != nil { + return model.NewAppError("Opensearch.DeleteFile", "ent.elasticsearch.delete_file.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + return nil +} + +func (os *OpensearchInterfaceImpl) DeleteUserFiles(rctx request.CTX, userID string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeleteFilesBatch", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"creator_id": {Value: userID}}, + }}, + }, + } + + queryBuf, err := json.Marshal(deletebyquery.Request{ + Query: query, + }) + if err != nil { + return model.NewAppError("Opensearch.DeleteUserFiles", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + response, err := os.client.Document.DeleteByQuery(ctx, opensearchapi.DocumentDeleteByQueryReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles}, + Body: bytes.NewReader(queryBuf), + }) + if err != nil { + return model.NewAppError("Opensearch.DeleteUserFiles", "ent.elasticsearch.delete_user_files.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("User files deleted", mlog.String("user_id", userID), mlog.Int("deleted", response.Deleted)) + + return nil +} + +func (os *OpensearchInterfaceImpl) DeletePostFiles(rctx request.CTX, postID string) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeleteFilesBatch", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Term: map[string]types.TermQuery{"post_id": {Value: postID}}, + }}, + }, + } + queryBuf, err := json.Marshal(deletebyquery.Request{ + Query: query, + }) + if err != nil { + return model.NewAppError("Opensearch.DeletePostFiles", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + response, err := os.client.Document.DeleteByQuery(ctx, opensearchapi.DocumentDeleteByQueryReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles}, + Body: bytes.NewReader(queryBuf), + }) + if err != nil { + return model.NewAppError("Opensearch.DeletePostFiles", "ent.elasticsearch.delete_post_files.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("Post files deleted", mlog.String("post_id", postID), mlog.Int("deleted", response.Deleted)) + + return nil +} + +func (os *OpensearchInterfaceImpl) DeleteFilesBatch(rctx request.CTX, endTime, limit int64) *model.AppError { + os.mutex.RLock() + defer os.mutex.RUnlock() + + if atomic.LoadInt32(&os.ready) == 0 { + return model.NewAppError("Opensearch.DeleteFilesBatch", "ent.elasticsearch.not_started.error", nil, "", http.StatusInternalServerError) + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second) + defer cancel() + + query := &types.Query{ + Bool: &types.BoolQuery{ + Filter: []types.Query{{ + Range: map[string]types.RangeQuery{ + "create_at": types.NumberRangeQuery{ + Lte: model.NewPointer(types.Float64(endTime)), + }, + }, + }}, + }, + } + + queryBuf, err := json.Marshal(deletebyquery.Request{ + Query: query, + }) + if err != nil { + return model.NewAppError("Opensearch.DeleteUserFiles", "api.marshal_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + response, err := os.client.Document.DeleteByQuery(ctx, opensearchapi.DocumentDeleteByQueryReq{ + Indices: []string{*os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseFiles}, + Body: bytes.NewReader(queryBuf), + Params: opensearchapi.DocumentDeleteByQueryParams{ + // Note that max_docs is slightly different than size. + // Size will just limit the number of elements returned, which is not + // what we want. We want to limit the number of elements to be deleted. + MaxDocs: model.NewPointer(int(limit)), + }, + }) + if err != nil { + return model.NewAppError("Opensearch.DeleteUserPosts", "ent.elasticsearch.delete_user_posts.error", nil, "", http.StatusInternalServerError).Wrap(err) + } + rctx.Logger().Info("Files batch deleted", mlog.Int("end_time", endTime), mlog.Int("limit", limit), mlog.Int("deleted", response.Deleted)) + + return nil +} + +func checkMaxVersion(client *opensearchapi.Client) (string, int, *model.AppError) { + resp, err := client.Info(context.Background(), nil) + if err != nil { + return "", 0, model.NewAppError("Opensearch.checkMaxVersion", "ent.elasticsearch.start.get_server_version.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + major, _, _, esErr := common.GetVersionComponents(resp.Version.Number) + if esErr != nil { + return "", 0, model.NewAppError("Opensearch.checkMaxVersion", "ent.elasticsearch.start.parse_server_version.app_error", nil, "", http.StatusInternalServerError).Wrap(err) + } + + if major > opensearchMaxVersion { + return "", 0, model.NewAppError("Opensearch.checkMaxVersion", "ent.elasticsearch.max_version.app_error", map[string]any{"Version": major, "MaxVersion": opensearchMaxVersion}, "", http.StatusBadRequest) + } + return resp.Version.Number, major, nil +} + +// checkChannelIndex checks if channel index's mapping is correct. +// See Jira issue https://mattermost.atlassian.net/browse/MM-49257 +func (os *OpensearchInterfaceImpl) checkChannelIndex() { + os.Platform.Log().Debug("Opensearch.checkChannelIndex: checking if channel index field is of correct type") + isCorrect, err := os.isFieldCorrect() + if err != nil { + return + } + + if isCorrect { + os.Platform.Log().Debug("Opensearch.checkChannelIndex: channel index field is correct") + atomic.StoreInt32(&os.channelIndexVerified, 1) + } else { + os.Platform.Log().Debug("Opensearch.checkChannelIndex: channel index field is incorrect") + atomic.StoreInt32(&os.channelIndexVerified, 2) + } +} + +func (os *OpensearchInterfaceImpl) isFieldCorrect() (bool, error) { + // We want to check if channel index's "type" field is of type "keyword". + // If the index is in incorrect state, the field would be of type "text". + + os.Platform.Log().Debug("Opensearch.isFieldCorrect: querying ES to check if field is correct") + + ctx, cancel := context.WithTimeout( + context.Background(), + time.Duration(*os.Platform.Config().ElasticsearchSettings.RequestTimeoutSeconds)*time.Second, + ) + defer cancel() + + var mappingFieldResp map[string]struct { + Mappings json.RawMessage `json:"mappings"` + } + + indexName := *os.Platform.Config().ElasticsearchSettings.IndexPrefix + common.IndexBaseChannels + httpResp, err := os.client.Client.Do(ctx, &opensearchapi.MappingFieldReq{ + Fields: []string{"type"}, + Indices: []string{indexName}, + }, &mappingFieldResp) + if err != nil { + os.Platform.Logger().Error("Opensearch: Failed to fetch channels index template", mlog.Err(err)) + return false, err + } + // The case of channels index not existing is fine, + // as whenever the index will be created, it will be created + // with the correct mappings. + if httpResp != nil && httpResp.StatusCode == http.StatusNotFound { + os.Platform.Logger().Debug("Opensearch isFieldCorrect: channel index doesn't exist", mlog.Err(err)) + return true, nil + } + + // this struct is declared here because + // it's not used anywhere outside this function + type channelsTypeFieldMapping struct { + Mappings struct { + Type struct { + Mapping struct { + Type struct { + Type string + } + } + } + } + } + + mappingInterface := mappingFieldResp[indexName] + mappingBytes, err := json.Marshal(mappingInterface) + if err != nil { + os.Platform.Logger().Error("Opensearch: Failed to marshal Opensearch index field mapping", mlog.Err(err)) + return false, err + } + + os.Platform.Log().Debug("Opensearch.isFieldCorrect: channel index type field mapping queried successfully", mlog.String("mapping", string(mappingBytes))) + + var mapping channelsTypeFieldMapping + err = json.Unmarshal(mappingBytes, &mapping) + if err != nil { + os.Platform.Logger().Error("Opensearch: Failed to unmarshal Opensearch index field mapping", mlog.Err(err)) + return false, err + } + + os.Platform.Logger().Debug("Opensearch: Found type of type field as", mlog.String("type", mapping.Mappings.Type.Mapping.Type.Type)) + return mapping.Mappings.Type.Mapping.Type.Type == "keyword", nil +} diff --git a/server/enterprise/elasticsearch/opensearch/opensearch_test.go b/server/enterprise/elasticsearch/opensearch/opensearch_test.go new file mode 100644 index 0000000000..5cfada6612 --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/opensearch_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "context" + "encoding/json" + "os" + "testing" + + "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + "github.com/stretchr/testify/suite" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/v8/channels/api4" + "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch/common" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks" +) + +type OpensearchInterfaceTestSuite struct { + common.CommonTestSuite + + th *api4.TestHelper + client *opensearchapi.Client + ctx context.Context + fileBackend filestore.FileBackend +} + +func TestOpensearchInterfaceTestSuite(t *testing.T) { + testSuite := &OpensearchInterfaceTestSuite{ + CommonTestSuite: common.CommonTestSuite{}, + } + suite.Run(t, testSuite) +} + +func (s *OpensearchInterfaceTestSuite) SetupSuite() { + if os.Getenv("IS_CI") == "true" { + os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://opensearch:9201") + os.Setenv("MM_ELASTICSEARCHSETTINGS_BACKEND", "opensearch") + } + + s.th = api4.SetupEnterprise(s.T()).InitBasic() + s.CommonTestSuite.TH = s.th + s.CommonTestSuite.GetDocumentFn = func(index, documentID string) (bool, json.RawMessage, error) { + resp, err := s.client.Document.Get(s.ctx, opensearchapi.DocumentGetReq{ + Index: index, + DocumentID: documentID, + }) + if resp == nil { + return false, nil, err + } + return resp.Found, resp.Source, err + } + s.CommonTestSuite.RefreshIndexFn = func() error { + _, err := s.client.Indices.Refresh(context.Background(), nil) + return err + } + s.CommonTestSuite.CreateIndexFn = func(index string) error { + _, err := s.client.Indices.Create(s.ctx, opensearchapi.IndicesCreateReq{ + Index: index, + }) + return err + } + s.CommonTestSuite.GetIndexFn = func(indexPattern string) ([]string, error) { + res, err := s.client.Indices.Get(s.ctx, opensearchapi.IndicesGetReq{ + Indices: []string{indexPattern}, + }) + if err != nil { + return nil, err + } + var names []string + for name := range res.Indices { + names = append(names, name) + } + return names, nil + } + + // Set up the state for the tests. + s.th.App.UpdateConfig(func(cfg *model.Config) { + if os.Getenv("IS_CI") == "true" { + *cfg.ElasticsearchSettings.ConnectionURL = "http://opensearch:9201" + } else { + *cfg.ElasticsearchSettings.ConnectionURL = "http://localhost:9201" + } + *cfg.ElasticsearchSettings.Backend = model.ElasticsearchSettingsOSBackend + *cfg.ElasticsearchSettings.EnableIndexing = true + *cfg.ElasticsearchSettings.EnableSearching = true + *cfg.ElasticsearchSettings.EnableAutocomplete = true + *cfg.ElasticsearchSettings.LiveIndexingBatchSize = 1 + *cfg.SqlSettings.DisableDatabaseSearch = true + }) + s.th.App.Srv().SetLicense(model.NewTestLicense()) + + if s.fileBackend == nil { + s.fileBackend = &mocks.FileBackend{} + } + + // Initialise other stuff for the test. + s.client = createTestClient(s.T(), s.th.Context, s.th.App.Config(), s.th.App.FileBackend()) + s.ctx = context.Background() + + // Register search engine + s.th.App.SearchEngine().RegisterElasticsearchEngine(&OpensearchInterfaceImpl{Platform: s.th.Server.Platform()}) +} + +func (s *OpensearchInterfaceTestSuite) TearDownSuite() { + if os.Getenv("IS_CI") == "true" { + os.Setenv("MM_ELASTICSEARCHSETTINGS_CONNECTIONURL", "http://elasticsearch:9201") + os.Unsetenv("MM_ELASTICSEARCHSETTINGS_BACKEND") + } +} + +func (s *OpensearchInterfaceTestSuite) SetupTest() { + s.CommonTestSuite.ESImpl = s.th.App.SearchEngine().ElasticsearchEngine + + if s.CommonTestSuite.ESImpl.IsActive() { + appErr := s.CommonTestSuite.ESImpl.Stop() + s.Require().Nil(appErr) + } + + s.Require().Nil(s.CommonTestSuite.ESImpl.Start()) + + s.Nil(s.CommonTestSuite.ESImpl.PurgeIndexes(s.th.Context)) +} diff --git a/server/enterprise/elasticsearch/opensearch/testlib.go b/server/enterprise/elasticsearch/opensearch/testlib.go new file mode 100644 index 0000000000..8ed9353477 --- /dev/null +++ b/server/enterprise/elasticsearch/opensearch/testlib.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.enterprise for license information. + +package opensearch + +import ( + "testing" + + "github.com/opensearch-project/opensearch-go/v4/opensearchapi" + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost/server/public/model" + "github.com/mattermost/mattermost/server/public/shared/request" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore" + "github.com/mattermost/mattermost/server/v8/platform/shared/filestore/mocks" +) + +func createTestClient(t *testing.T, rctx request.CTX, cfg *model.Config, fileStore filestore.FileBackend) *opensearchapi.Client { + t.Helper() + + if fileStore == nil { + fileStore = &mocks.FileBackend{} + } + + client, err := createClient(rctx.Logger(), cfg, fileStore, true) + require.Nil(t, err) + return client +} diff --git a/server/enterprise/external_imports.go b/server/enterprise/external_imports.go index 4ad0bf051c..c6272585c3 100644 --- a/server/enterprise/external_imports.go +++ b/server/enterprise/external_imports.go @@ -15,8 +15,6 @@ import ( // Needed to ensure the init() method in the EE gets run _ "github.com/mattermost/enterprise/data_retention" // Needed to ensure the init() method in the EE gets run - _ "github.com/mattermost/enterprise/elasticsearch" - // Needed to ensure the init() method in the EE gets run _ "github.com/mattermost/enterprise/ldap" // Needed to ensure the init() method in the EE gets run _ "github.com/mattermost/enterprise/cloud" diff --git a/server/enterprise/local_imports.go b/server/enterprise/local_imports.go index adeee7463e..449f0a03a5 100644 --- a/server/enterprise/local_imports.go +++ b/server/enterprise/local_imports.go @@ -16,4 +16,6 @@ import ( _ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/csv_export" // Needed to ensure the init() method in the EE gets run _ "github.com/mattermost/mattermost/server/v8/enterprise/message_export/global_relay_export" + // Needed to ensure the init() method in the EE gets run + _ "github.com/mattermost/mattermost/server/v8/enterprise/elasticsearch" ) diff --git a/server/go.mod b/server/go.mod index 0a79ea7d9f..397c705f3d 100644 --- a/server/go.mod +++ b/server/go.mod @@ -8,7 +8,7 @@ require ( code.sajari.com/docconv/v2 v2.0.0-pre.4 github.com/Masterminds/semver/v3 v3.2.1 github.com/avct/uasurfer v0.0.0-20240501094946-ca0c4d1e541b - github.com/aws/aws-sdk-go v1.55.0 + github.com/aws/aws-sdk-go v1.55.5 github.com/blang/semver/v4 v4.0.0 github.com/blevesearch/bleve/v2 v2.4.1 github.com/cespare/xxhash/v2 v2.3.0 @@ -51,6 +51,7 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 github.com/minio/minio-go/v7 v7.0.74 github.com/oov/psd v0.0.0-20220121172623-5db5eafcecbb + github.com/opensearch-project/opensearch-go/v4 v4.3.0 github.com/opentracing/opentracing-go v1.2.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.19.1 diff --git a/server/go.sum b/server/go.sum index e46f4e45a1..c273c70a84 100644 --- a/server/go.sum +++ b/server/go.sum @@ -47,8 +47,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/avct/uasurfer v0.0.0-20240501094946-ca0c4d1e541b h1:F1IDheTR2BqSIznXwfgxursfutFj5pNezhneejTPUYQ= github.com/avct/uasurfer v0.0.0-20240501094946-ca0c4d1e541b/go.mod h1:s+GCtuP4kZNxh1WGoqdWI1+PbluBcycrMMWuKQ9e5Nk= -github.com/aws/aws-sdk-go v1.55.0 h1:hVALKPjXz33kP1R9nTyJpUK7qF59dO2mleQxUW9mCVE= -github.com/aws/aws-sdk-go v1.55.0/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= @@ -440,6 +440,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/oov/psd v0.0.0-20220121172623-5db5eafcecbb h1:JF9kOhBBk4WPF7luXFu5yR+WgaFm9L/KiHJHhU9vDwA= github.com/oov/psd v0.0.0-20220121172623-5db5eafcecbb/go.mod h1:GHI1bnmAcbp96z6LNfBJvtrjxhaXGkbsk967utPlvL8= +github.com/opensearch-project/opensearch-go/v4 v4.3.0 h1:gmQ+ILFJW6AJimivf+lHGVqCS2SCr/PBBf2Qr1xOCgE= +github.com/opensearch-project/opensearch-go/v4 v4.3.0/go.mod h1:+w6KAvEX3S0fVVmZciNLN0CkXhxxem26+F6Y7DoPp04= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= @@ -616,6 +618,8 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/tinylib/msgp v1.2.0 h1:0uKB/662twsVBpYUPbokj4sTSKhWFKB7LopO2kWK8lY= github.com/tinylib/msgp v1.2.0/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= @@ -637,6 +641,8 @@ github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IU github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/wI2L/jsondiff v0.6.0 h1:zrsH3FbfVa3JO9llxrcDy/XLkYPLgoMX6Mz3T2PP2AI= +github.com/wI2L/jsondiff v0.6.0/go.mod h1:D6aQ5gKgPF9g17j+E9N7aasmU1O+XvfmWm1y8UMmNpw= github.com/wiggin77/merror v1.0.5 h1:P+lzicsn4vPMycAf2mFf7Zk6G9eco5N+jB1qJ2XW3ME= github.com/wiggin77/merror v1.0.5/go.mod h1:H2ETSu7/bPE0Ymf4bEwdUoo73OOEkdClnoRisfw0Nm0= github.com/wiggin77/srslog v1.0.1 h1:gA2XjSMy3DrRdX9UqLuDtuVAAshb8bE1NhX1YK0Qe+8=