From 2a63b5552a7bac3a719e73827ba91cf185808ef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Mon, 11 Jan 2021 15:14:16 +0100 Subject: [PATCH] Add search engine support for files (#16190) * Add search engine support for files * Fixing i18n * Fix golangci-lint * Fix consistency problem in the Search receiver functio of the SqlFileStore * Fixing some tests * Fixing test * Apply suggestions from code review Co-authored-by: Mario de Frutos Dieguez * Addressing PR review comments * Removing some empty lines * Address PR review comments * Fixing problem after merge master * Fixing spelling problem * Add missed translations * Fixing certain global variable usages after merge master * Fixing some constants usage * Fixing goimports order Co-authored-by: Mario de Frutos Dieguez --- einterfaces/metrics.go | 3 + einterfaces/mocks/MetricsInterface.go | 15 + i18n/en.json | 68 + model/file_info_list.go | 128 ++ model/file_info_search_results.go | 37 + model/post.go | 6 + model/search_params.go | 19 +- model/search_params_test.go | 600 +++--- services/searchengine/bleveengine/bleve.go | 28 + .../searchengine/bleveengine/bleve_test.go | 4 + services/searchengine/bleveengine/common.go | 34 + .../bleveengine/indexer/indexing_job.go | 24 + services/searchengine/bleveengine/search.go | 318 ++++ services/searchengine/interface.go | 6 + .../mocks/SearchEngineInterface.go | 105 ++ store/opentracinglayer/opentracinglayer.go | 36 + store/retrylayer/retrylayer.go | 40 + store/searchlayer/channel_layer.go | 2 + store/searchlayer/file_info_layer.go | 197 ++ store/searchlayer/layer.go | 6 + store/searchlayer/post_layer.go | 4 + store/searchtest/file_info_layer.go | 1646 +++++++++++++++++ store/searchtest/helper.go | 46 + store/sqlstore/file_info_store.go | 200 ++ store/sqlstore/file_info_store_test.go | 5 + store/store.go | 2 + store/storetest/mocks/FileInfoStore.go | 46 + store/timerlayer/timerlayer.go | 32 + 28 files changed, 3438 insertions(+), 219 deletions(-) create mode 100644 model/file_info_list.go create mode 100644 model/file_info_search_results.go create mode 100644 store/searchlayer/file_info_layer.go create mode 100644 store/searchtest/file_info_layer.go diff --git a/einterfaces/metrics.go b/einterfaces/metrics.go index 5c0f91ded9..a2ae546fb5 100644 --- a/einterfaces/metrics.go +++ b/einterfaces/metrics.go @@ -50,9 +50,12 @@ type MetricsInterface interface { IncrementPostsSearchCounter() ObservePostsSearchDuration(elapsed float64) + IncrementFilesSearchCounter() + ObserveFilesSearchDuration(elapsed float64) ObserveStoreMethodDuration(method, success string, elapsed float64) ObserveApiEndpointDuration(endpoint, method, statusCode string, elapsed float64) IncrementPostIndexCounter() + IncrementFileIndexCounter() IncrementUserIndexCounter() IncrementChannelIndexCounter() diff --git a/einterfaces/mocks/MetricsInterface.go b/einterfaces/mocks/MetricsInterface.go index 69560fcebb..7fb2606153 100644 --- a/einterfaces/mocks/MetricsInterface.go +++ b/einterfaces/mocks/MetricsInterface.go @@ -75,6 +75,16 @@ func (_m *MetricsInterface) IncrementEtagMissCounter(route string) { _m.Called(route) } +// IncrementFileIndexCounter provides a mock function with given fields: +func (_m *MetricsInterface) IncrementFileIndexCounter() { + _m.Called() +} + +// IncrementFilesSearchCounter provides a mock function with given fields: +func (_m *MetricsInterface) IncrementFilesSearchCounter() { + _m.Called() +} + // IncrementHttpError provides a mock function with given fields: func (_m *MetricsInterface) IncrementHttpError() { _m.Called() @@ -205,6 +215,11 @@ func (_m *MetricsInterface) ObserveEnabledUsers(users int64) { _m.Called(users) } +// ObserveFilesSearchDuration provides a mock function with given fields: elapsed +func (_m *MetricsInterface) ObserveFilesSearchDuration(elapsed float64) { + _m.Called(elapsed) +} + // ObservePluginApiDuration provides a mock function with given fields: pluginID, apiName, success, elapsed func (_m *MetricsInterface) ObservePluginApiDuration(pluginID string, apiName string, success bool, elapsed float64) { _m.Called(pluginID, apiName, success, elapsed) diff --git a/i18n/en.json b/i18n/en.json index 633e771565..cd43b2d0d2 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -5822,6 +5822,10 @@ "id": "bleveengine.create_channel_index.error", "translation": "Error creating the bleve channel index." }, + { + "id": "bleveengine.create_file_index.error", + "translation": "Error creating the bleve file index." + }, { "id": "bleveengine.create_post_index.error", "translation": "Error creating the bleve post index." @@ -5838,14 +5842,30 @@ "id": "bleveengine.delete_channel_posts.error", "translation": "Failed to delete channel posts" }, + { + "id": "bleveengine.delete_file.error", + "translation": "Failed to delete the file." + }, + { + "id": "bleveengine.delete_files_batch.error", + "translation": "Failed to delete the files." + }, { "id": "bleveengine.delete_post.error", "translation": "Failed to delete the post." }, + { + "id": "bleveengine.delete_post_files.error", + "translation": "Fiiled to delete the post files." + }, { "id": "bleveengine.delete_user.error", "translation": "Failed to delete the user." }, + { + "id": "bleveengine.delete_user_files.error", + "translation": "Failed to delete the user files." + }, { "id": "bleveengine.delete_user_posts.error", "translation": "Failed to delete user posts" @@ -5854,6 +5874,10 @@ "id": "bleveengine.index_channel.error", "translation": "Failed to index the channel." }, + { + "id": "bleveengine.index_file.error", + "translation": "Failed to index the file." + }, { "id": "bleveengine.index_post.error", "translation": "Failed to index the post." @@ -5866,6 +5890,10 @@ "id": "bleveengine.indexer.do_job.bulk_index_channels.batch_error", "translation": "Failed to index channel batch." }, + { + "id": "bleveengine.indexer.do_job.bulk_index_files.batch_error", + "translation": "Failed to index file batch." + }, { "id": "bleveengine.indexer.do_job.bulk_index_posts.batch_error", "translation": "Failed to index post batch." @@ -5910,6 +5938,10 @@ "id": "bleveengine.search_channels.error", "translation": "Channel search failed to complete." }, + { + "id": "bleveengine.search_files.error", + "translation": "File search failed to complete." + }, { "id": "bleveengine.search_posts.error", "translation": "Post search failed to complete." @@ -5930,6 +5962,10 @@ "id": "bleveengine.stop_channel_index.error", "translation": "Failed to close channel index." }, + { + "id": "bleveengine.stop_file_index.error", + "translation": "Failed to close file index." + }, { "id": "bleveengine.stop_post_index.error", "translation": "Failed to close post index." @@ -6150,6 +6186,10 @@ "id": "ent.elasticsearch.create_template_channels_if_not_exists.template_create_failed", "translation": "Failed to create Elasticsearch template for channels" }, + { + "id": "ent.elasticsearch.create_template_file_info_if_not_exists.template_create_failed", + "translation": "Failed to create Elasticsearch template for files" + }, { "id": "ent.elasticsearch.create_template_posts_if_not_exists.template_create_failed", "translation": "Failed to create Elasticsearch template for posts" @@ -6174,14 +6214,26 @@ "id": "ent.elasticsearch.delete_channel_posts.error", "translation": "Failed to delete channel posts" }, + { + "id": "ent.elasticsearch.delete_file.error", + "translation": "Failed to delete file" + }, { "id": "ent.elasticsearch.delete_post.error", "translation": "Failed to delete the post" }, + { + "id": "ent.elasticsearch.delete_post_files.error", + "translation": "Failed to delete post files" + }, { "id": "ent.elasticsearch.delete_user.error", "translation": "Failed to delete the user" }, + { + "id": "ent.elasticsearch.delete_user_files.error", + "translation": "Failed to delete user files" + }, { "id": "ent.elasticsearch.delete_user_posts.error", "translation": "Failed to delete user posts" @@ -6198,6 +6250,10 @@ "id": "ent.elasticsearch.index_channels_batch.error", "translation": "Unable to get the channels batch for indexing." }, + { + "id": "ent.elasticsearch.index_file.error", + "translation": "Failed to index the file" + }, { "id": "ent.elasticsearch.index_post.error", "translation": "Failed to index the post" @@ -6250,6 +6306,18 @@ "id": "ent.elasticsearch.search_channels.unmarshall_channel_failed", "translation": "Failed to decode search results" }, + { + "id": "ent.elasticsearch.search_files.disabled", + "translation": "Elasticsearch files searching is disabled on this server" + }, + { + "id": "ent.elasticsearch.search_files.search_failed", + "translation": "Search failed to complete" + }, + { + "id": "ent.elasticsearch.search_files.unmarshall_file_failed", + "translation": "Failed to decode search results" + }, { "id": "ent.elasticsearch.search_posts.disabled", "translation": "Elasticsearch searching is disabled on this server" diff --git a/model/file_info_list.go b/model/file_info_list.go new file mode 100644 index 0000000000..cd9694f511 --- /dev/null +++ b/model/file_info_list.go @@ -0,0 +1,128 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package model + +import ( + "encoding/json" + "io" + "sort" +) + +type FileInfoList struct { + Order []string `json:"order"` + FileInfos map[string]*FileInfo `json:"file_infos"` + NextFileInfoId string `json:"next_file_info_id"` + PrevFileInfoId string `json:"prev_file_info_id"` +} + +func NewFileInfoList() *FileInfoList { + return &FileInfoList{ + Order: make([]string, 0), + FileInfos: make(map[string]*FileInfo), + NextFileInfoId: "", + PrevFileInfoId: "", + } +} + +func (o *FileInfoList) ToSlice() []*FileInfo { + var fileInfos []*FileInfo + for _, id := range o.Order { + fileInfos = append(fileInfos, o.FileInfos[id]) + } + return fileInfos +} + +func (o *FileInfoList) ToJson() string { + b, err := json.Marshal(o) + if err != nil { + return "" + } else { + return string(b) + } +} + +func (o *FileInfoList) MakeNonNil() { + if o.Order == nil { + o.Order = make([]string, 0) + } + + if o.FileInfos == nil { + o.FileInfos = make(map[string]*FileInfo) + } +} + +func (o *FileInfoList) AddOrder(id string) { + if o.Order == nil { + o.Order = make([]string, 0, 128) + } + + o.Order = append(o.Order, id) +} + +func (o *FileInfoList) AddFileInfo(fileInfo *FileInfo) { + if o.FileInfos == nil { + o.FileInfos = make(map[string]*FileInfo) + } + + o.FileInfos[fileInfo.Id] = fileInfo +} + +func (o *FileInfoList) UniqueOrder() { + keys := make(map[string]bool) + order := []string{} + for _, fileInfoId := range o.Order { + if _, value := keys[fileInfoId]; !value { + keys[fileInfoId] = true + order = append(order, fileInfoId) + } + } + + o.Order = order +} + +func (o *FileInfoList) Extend(other *FileInfoList) { + for fileInfoId := range other.FileInfos { + o.AddFileInfo(other.FileInfos[fileInfoId]) + } + + for _, fileInfoId := range other.Order { + o.AddOrder(fileInfoId) + } + + o.UniqueOrder() +} + +func (o *FileInfoList) SortByCreateAt() { + sort.Slice(o.Order, func(i, j int) bool { + return o.FileInfos[o.Order[i]].CreateAt > o.FileInfos[o.Order[j]].CreateAt + }) +} + +func (o *FileInfoList) Etag() string { + id := "0" + var t int64 = 0 + + for _, v := range o.FileInfos { + if v.UpdateAt > t { + t = v.UpdateAt + id = v.Id + } else if v.UpdateAt == t && v.Id > id { + t = v.UpdateAt + id = v.Id + } + } + + orderId := "" + if len(o.Order) > 0 { + orderId = o.Order[0] + } + + return Etag(orderId, id, t) +} + +func FileInfoListFromJson(data io.Reader) *FileInfoList { + var o *FileInfoList + json.NewDecoder(data).Decode(&o) + return o +} diff --git a/model/file_info_search_results.go b/model/file_info_search_results.go new file mode 100644 index 0000000000..90f2922b9c --- /dev/null +++ b/model/file_info_search_results.go @@ -0,0 +1,37 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package model + +import ( + "encoding/json" + "io" +) + +type FileInfoSearchMatches map[string][]string + +type FileInfoSearchResults struct { + *FileInfoList + Matches FileInfoSearchMatches `json:"matches"` +} + +func MakeFileInfoSearchResults(fileInfos *FileInfoList, matches FileInfoSearchMatches) *FileInfoSearchResults { + return &FileInfoSearchResults{ + fileInfos, + matches, + } +} + +func (o *FileInfoSearchResults) ToJson() string { + b, err := json.Marshal(o) + if err != nil { + return "" + } + return string(b) +} + +func FileInfoSearchResultsFromJson(data io.Reader) *FileInfoSearchResults { + var o *FileInfoSearchResults + json.NewDecoder(data).Decode(&o) + return o +} diff --git a/model/post.go b/model/post.go index 0e0795bb39..8133e6ae25 100644 --- a/model/post.go +++ b/model/post.go @@ -164,6 +164,12 @@ type PostForIndexing struct { ParentCreateAt *int64 `json:"parent_create_at"` } +type FileForIndexing struct { + FileInfo + ChannelId string `json:"channel_id"` + Content string `json:"content"` +} + // ShallowCopy is an utility function to shallow copy a Post to the given // destination without touching the internal RWMutex. func (o *Post) ShallowCopy(dst *Post) error { diff --git a/model/search_params.go b/model/search_params.go index bf562a6bed..3a5bf8416c 100644 --- a/model/search_params.go +++ b/model/search_params.go @@ -25,6 +25,8 @@ type SearchParams struct { ExcludedAfterDate string BeforeDate string ExcludedBeforeDate string + Extensions []string + ExcludedExtensions []string OnDate string ExcludedDate string OrTerms bool @@ -106,7 +108,7 @@ func (p *SearchParams) GetExcludedDateMillis() (int64, int64) { return GetStartOfDayMillis(date, p.TimeZoneOffset), GetEndOfDayMillis(date, p.TimeZoneOffset) } -var searchFlags = [...]string{"from", "channel", "in", "before", "after", "on"} +var searchFlags = [...]string{"from", "channel", "in", "before", "after", "on", "ext"} type flag struct { name string @@ -265,6 +267,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams { excludedBeforeDate := "" onDate := "" excludedDate := "" + excludedExtensions := []string{} + extensions := []string{} for _, flag := range flags { if flag.name == "in" || flag.name == "channel" { @@ -297,6 +301,12 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams { } else { onDate = flag.value } + } else if flag.name == "ext" { + if flag.exclude { + excludedExtensions = append(excludedExtensions, flag.value) + } else { + extensions = append(extensions, flag.value) + } } } @@ -315,6 +325,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams { ExcludedAfterDate: excludedAfterDate, BeforeDate: beforeDate, ExcludedBeforeDate: excludedBeforeDate, + Extensions: extensions, + ExcludedExtensions: excludedExtensions, OnDate: onDate, ExcludedDate: excludedDate, TimeZoneOffset: timeZoneOffset, @@ -334,6 +346,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams { ExcludedAfterDate: excludedAfterDate, BeforeDate: beforeDate, ExcludedBeforeDate: excludedBeforeDate, + Extensions: extensions, + ExcludedExtensions: excludedExtensions, OnDate: onDate, ExcludedDate: excludedDate, TimeZoneOffset: timeZoneOffset, @@ -345,6 +359,7 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams { len(excludedPlainTerms) == 0 && len(excludedHashtagTerms) == 0 && (len(inChannels) != 0 || len(fromUsers) != 0 || len(excludedChannels) != 0 || len(excludedUsers) != 0 || + len(extensions) != 0 || len(excludedExtensions) != 0 || afterDate != "" || excludedAfterDate != "" || beforeDate != "" || excludedBeforeDate != "" || onDate != "" || excludedDate != "") { @@ -360,6 +375,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams { ExcludedAfterDate: excludedAfterDate, BeforeDate: beforeDate, ExcludedBeforeDate: excludedBeforeDate, + Extensions: extensions, + ExcludedExtensions: excludedExtensions, OnDate: onDate, ExcludedDate: excludedDate, TimeZoneOffset: timeZoneOffset, diff --git a/model/search_params_test.go b/model/search_params_test.go index 0000dad900..e36e8d0301 100644 --- a/model/search_params_test.go +++ b/model/search_params_test.go @@ -1043,13 +1043,15 @@ func TestParseSearchParams(t *testing.T) { Input: "words words", Output: []*SearchParams{ { - Terms: "words words", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "words words", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1058,13 +1060,15 @@ func TestParseSearchParams(t *testing.T) { Input: "-word1 -word2", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "word1 word2", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "word1 word2", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1073,13 +1077,15 @@ func TestParseSearchParams(t *testing.T) { Input: "\"my stuff\"", Output: []*SearchParams{ { - Terms: "\"my stuff\"", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "\"my stuff\"", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1088,13 +1094,15 @@ func TestParseSearchParams(t *testing.T) { Input: "-\"my stuff\"", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "\"my stuff\"", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "\"my stuff\"", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1103,13 +1111,15 @@ func TestParseSearchParams(t *testing.T) { Input: "#words #words", Output: []*SearchParams{ { - Terms: "#words #words", - ExcludedTerms: "", - IsHashtag: true, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "#words #words", + ExcludedTerms: "", + IsHashtag: true, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1118,22 +1128,26 @@ func TestParseSearchParams(t *testing.T) { Input: "#words words", Output: []*SearchParams{ { - Terms: "words", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "words", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, { - Terms: "#words", - ExcludedTerms: "", - IsHashtag: true, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "#words", + ExcludedTerms: "", + IsHashtag: true, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1142,13 +1156,15 @@ func TestParseSearchParams(t *testing.T) { Input: "-#hashtag", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "#hashtag", - IsHashtag: true, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "#hashtag", + IsHashtag: true, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1157,13 +1173,15 @@ func TestParseSearchParams(t *testing.T) { Input: "-#hashtag1 -#hashtag2", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "#hashtag1 #hashtag2", - IsHashtag: true, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "#hashtag1 #hashtag2", + IsHashtag: true, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1172,13 +1190,15 @@ func TestParseSearchParams(t *testing.T) { Input: "#hashtag1 -#hashtag2", Output: []*SearchParams{ { - Terms: "#hashtag1", - ExcludedTerms: "#hashtag2", - IsHashtag: true, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "#hashtag1", + ExcludedTerms: "#hashtag2", + IsHashtag: true, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1187,22 +1207,26 @@ func TestParseSearchParams(t *testing.T) { Input: "word1 #hashtag1 -#hashtag2 -word2", Output: []*SearchParams{ { - Terms: "word1", - ExcludedTerms: "word2", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "word1", + ExcludedTerms: "word2", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, { - Terms: "#hashtag1", - ExcludedTerms: "#hashtag2", - IsHashtag: true, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "#hashtag1", + ExcludedTerms: "#hashtag2", + IsHashtag: true, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1211,13 +1235,15 @@ func TestParseSearchParams(t *testing.T) { Input: "in:channel", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{"channel"}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{"channel"}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1226,13 +1252,15 @@ func TestParseSearchParams(t *testing.T) { Input: "-in:channel", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{"channel"}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{"channel"}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1241,13 +1269,15 @@ func TestParseSearchParams(t *testing.T) { Input: "testing in:channel", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{"channel"}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{"channel"}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1256,13 +1286,15 @@ func TestParseSearchParams(t *testing.T) { Input: "testing -in:channel", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{"channel"}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{"channel"}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1271,13 +1303,15 @@ func TestParseSearchParams(t *testing.T) { Input: "in:channel testing", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{"channel"}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{"channel"}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1286,13 +1320,15 @@ func TestParseSearchParams(t *testing.T) { Input: "in:channel in:otherchannel", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{"channel", "otherchannel"}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{"channel", "otherchannel"}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1301,13 +1337,15 @@ func TestParseSearchParams(t *testing.T) { Input: "testing in:channel in:otherchannel", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{"channel", "otherchannel"}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{"channel", "otherchannel"}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1316,13 +1354,15 @@ func TestParseSearchParams(t *testing.T) { Input: "testing in:channel from:someone", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{"channel"}, - ExcludedChannels: []string{}, - FromUsers: []string{"someone"}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{"channel"}, + ExcludedChannels: []string{}, + FromUsers: []string{"someone"}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1331,13 +1371,15 @@ func TestParseSearchParams(t *testing.T) { Input: "testing in:channel -from:someone", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{"channel"}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{"someone"}, + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{"channel"}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{"someone"}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1346,13 +1388,15 @@ func TestParseSearchParams(t *testing.T) { Input: "testing in:channel from:someone -from:someoneelse", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{"channel"}, - ExcludedChannels: []string{}, - FromUsers: []string{"someone"}, - ExcludedUsers: []string{"someoneelse"}, + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{"channel"}, + ExcludedChannels: []string{}, + FromUsers: []string{"someone"}, + ExcludedUsers: []string{"someoneelse"}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1361,13 +1405,15 @@ func TestParseSearchParams(t *testing.T) { Input: "##hashtag +#plus+", Output: []*SearchParams{ { - Terms: "#hashtag #plus", - ExcludedTerms: "", - IsHashtag: true, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "#hashtag #plus", + ExcludedTerms: "", + IsHashtag: true, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1376,13 +1422,15 @@ func TestParseSearchParams(t *testing.T) { Input: "wildcar*", Output: []*SearchParams{ { - Terms: "wildcar*", - ExcludedTerms: "", - IsHashtag: false, - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "wildcar*", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1391,14 +1439,16 @@ func TestParseSearchParams(t *testing.T) { Input: "after:2018-8-1 testing", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - AfterDate: "2018-8-1", - ExcludedAfterDate: "", - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + AfterDate: "2018-8-1", + ExcludedAfterDate: "", + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1407,14 +1457,16 @@ func TestParseSearchParams(t *testing.T) { Input: "-after:2018-8-1 testing", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - AfterDate: "", - ExcludedAfterDate: "2018-8-1", - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + AfterDate: "", + ExcludedAfterDate: "2018-8-1", + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1423,15 +1475,17 @@ func TestParseSearchParams(t *testing.T) { Input: "on:2018-8-1 testing", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - OnDate: "2018-8-1", - AfterDate: "", - ExcludedAfterDate: "", - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + OnDate: "2018-8-1", + AfterDate: "", + ExcludedAfterDate: "", + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1440,14 +1494,16 @@ func TestParseSearchParams(t *testing.T) { Input: "-on:2018-8-1 testing", Output: []*SearchParams{ { - Terms: "testing", - ExcludedTerms: "", - AfterDate: "", - ExcludedDate: "2018-8-1", - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "testing", + ExcludedTerms: "", + AfterDate: "", + ExcludedDate: "2018-8-1", + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1456,14 +1512,16 @@ func TestParseSearchParams(t *testing.T) { Input: "after:2018-8-1", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "", - AfterDate: "2018-8-1", - ExcludedDate: "", - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "", + AfterDate: "2018-8-1", + ExcludedDate: "", + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1472,15 +1530,17 @@ func TestParseSearchParams(t *testing.T) { Input: "before:2018-8-1", Output: []*SearchParams{ { - Terms: "", - ExcludedTerms: "", - BeforeDate: "2018-8-1", - AfterDate: "", - ExcludedDate: "", - InChannels: []string{}, - ExcludedChannels: []string{}, - FromUsers: []string{}, - ExcludedUsers: []string{}, + Terms: "", + ExcludedTerms: "", + BeforeDate: "2018-8-1", + AfterDate: "", + ExcludedDate: "", + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, }, }, }, @@ -1498,6 +1558,110 @@ func TestParseSearchParams(t *testing.T) { ExcludedChannels: []string{}, FromUsers: []string{}, ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{}, + }, + }, + }, + { + Name: "input is two words separated with : and should result in a single Extension", + Input: "ext:png", + Output: []*SearchParams{ + { + Terms: "", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{"png"}, + ExcludedExtensions: []string{}, + }, + }, + }, + { + Name: "input is two words separated with :, prefied with - and should result in a single ExcludedExtensions", + Input: "-ext:png", + Output: []*SearchParams{ + { + Terms: "", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{"png"}, + }, + }, + }, + { + Name: "input is two words separated with : with a prefixed word should result in a single Extension and a term", + Input: "testing ext:png", + Output: []*SearchParams{ + { + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{"png"}, + ExcludedExtensions: []string{}, + }, + }, + }, + { + Name: "input is two words separated with : with a prefixed word should result in a single ExcludedExtension and a term", + Input: "testing -ext:png", + Output: []*SearchParams{ + { + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{}, + ExcludedExtensions: []string{"png"}, + }, + }, + }, + { + Name: "input is two words separated with : with a postfix word should result in a single Extension and a term", + Input: "ext:png testing", + Output: []*SearchParams{ + { + Terms: "testing", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{"png"}, + ExcludedExtensions: []string{}, + }, + }, + }, + { + Name: "input is four words separated with : should result in a two Extensions", + Input: "ext:png ext:jpg", + Output: []*SearchParams{ + { + Terms: "", + ExcludedTerms: "", + IsHashtag: false, + InChannels: []string{}, + ExcludedChannels: []string{}, + FromUsers: []string{}, + ExcludedUsers: []string{}, + Extensions: []string{"png", "jpg"}, + ExcludedExtensions: []string{}, }, }, }, diff --git a/services/searchengine/bleveengine/bleve.go b/services/searchengine/bleveengine/bleve.go index 8c4b70760f..f5a3d2c080 100644 --- a/services/searchengine/bleveengine/bleve.go +++ b/services/searchengine/bleveengine/bleve.go @@ -24,12 +24,14 @@ import ( const ( EngineName = "bleve" PostIndex = "posts" + FileIndex = "files" UserIndex = "users" ChannelIndex = "channels" ) type BleveEngine struct { PostIndex bleve.Index + FileIndex bleve.Index UserIndex bleve.Index ChannelIndex bleve.Index Mutex sync.RWMutex @@ -83,6 +85,23 @@ func getPostIndexMapping() *mapping.IndexMappingImpl { return indexMapping } +func getFileIndexMapping() *mapping.IndexMappingImpl { + fileMapping := bleve.NewDocumentMapping() + fileMapping.AddFieldMappingsAt("Id", keywordMapping) + fileMapping.AddFieldMappingsAt("CreatorId", keywordMapping) + fileMapping.AddFieldMappingsAt("ChannelId", keywordMapping) + fileMapping.AddFieldMappingsAt("CreateAt", dateMapping) + fileMapping.AddFieldMappingsAt("Name", standardMapping) + fileMapping.AddFieldMappingsAt("Content", standardMapping) + fileMapping.AddFieldMappingsAt("Extension", keywordMapping) + fileMapping.AddFieldMappingsAt("Content", standardMapping) + + indexMapping := bleve.NewIndexMapping() + indexMapping.AddDocumentMapping("_default", fileMapping) + + return indexMapping +} + func getUserIndexMapping() *mapping.IndexMappingImpl { userMapping := bleve.NewDocumentMapping() userMapping.AddFieldMappingsAt("Id", keywordMapping) @@ -132,6 +151,11 @@ func (b *BleveEngine) openIndexes() *model.AppError { return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, err.Error(), http.StatusInternalServerError) } + b.FileIndex, err = b.createOrOpenIndex(FileIndex, getFileIndexMapping()) + if err != nil { + return model.NewAppError("Bleveengine.Start", "bleveengine.create_file_index.error", nil, err.Error(), http.StatusInternalServerError) + } + b.UserIndex, err = b.createOrOpenIndex(UserIndex, getUserIndexMapping()) if err != nil { return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, err.Error(), http.StatusInternalServerError) @@ -165,6 +189,10 @@ func (b *BleveEngine) closeIndexes() *model.AppError { return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, err.Error(), http.StatusInternalServerError) } + if err := b.FileIndex.Close(); err != nil { + return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_file_index.error", nil, err.Error(), http.StatusInternalServerError) + } + if err := b.UserIndex.Close(); err != nil { return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, err.Error(), http.StatusInternalServerError) } diff --git a/services/searchengine/bleveengine/bleve_test.go b/services/searchengine/bleveengine/bleve_test.go index 36f6b2d12f..5748f109d4 100644 --- a/services/searchengine/bleveengine/bleve_test.go +++ b/services/searchengine/bleveengine/bleve_test.go @@ -98,6 +98,10 @@ func (s *BleveEngineTestSuite) TestBleveSearchStoreTests() { s.Run("TestSearchPostStore", func() { searchtest.TestSearchPostStore(s.T(), s.Store, searchTestEngine) }) + + s.Run("TestSearchFileInfoStore", func() { + searchtest.TestSearchFileInfoStore(s.T(), s.Store, searchTestEngine) + }) } func (s *BleveEngineTestSuite) TestDeleteChannelPosts() { diff --git a/services/searchengine/bleveengine/common.go b/services/searchengine/bleveengine/common.go index 9a834292ef..97bb0624d2 100644 --- a/services/searchengine/bleveengine/common.go +++ b/services/searchengine/bleveengine/common.go @@ -36,6 +36,16 @@ type BLVPost struct { Attachments string } +type BLVFile struct { + Id string + CreatorId string + ChannelId string + CreateAt int64 + Name string + Content string + Extension string +} + func BLVChannelFromChannel(channel *model.Channel) *BLVChannel { displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ") nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"}) @@ -114,3 +124,27 @@ func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost { Hashtags: strings.Fields(post.Hashtags), } } + +func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile { + return &BLVFile{ + Id: fileInfo.Id, + ChannelId: channelId, + CreatorId: fileInfo.CreatorId, + CreateAt: fileInfo.CreateAt, + Content: fileInfo.Content, + Extension: fileInfo.Extension, + Name: fileInfo.Name, + } +} + +func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile { + return &BLVFile{ + Id: file.Id, + ChannelId: file.ChannelId, + CreatorId: file.CreatorId, + CreateAt: file.CreateAt, + Content: file.Content, + Extension: file.Extension, + Name: file.Name, + } +} diff --git a/services/searchengine/bleveengine/indexer/indexing_job.go b/services/searchengine/bleveengine/indexer/indexing_job.go index 7e378f22e2..bfdd835872 100644 --- a/services/searchengine/bleveengine/indexer/indexing_job.go +++ b/services/searchengine/bleveengine/indexer/indexing_job.go @@ -354,6 +354,30 @@ func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing, return lastCreateAt, nil } +func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (int64, *model.AppError) { + lastCreateAt := int64(0) + batch := worker.engine.FileIndex.NewBatch() + + for _, file := range files { + if file.DeleteAt == 0 { + searchFile := bleveengine.BLVFileFromFileForIndexing(file) + batch.Index(searchFile.Id, searchFile) + } else { + batch.Delete(file.Id) + } + + lastCreateAt = file.CreateAt + } + + worker.engine.Mutex.RLock() + defer worker.engine.Mutex.RUnlock() + + if err := worker.engine.FileIndex.Batch(batch); err != nil { + return 0, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_files.batch_error", nil, err.Error(), http.StatusInternalServerError) + } + return lastCreateAt, nil +} + func (worker *BleveIndexerWorker) IndexChannelsBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) { endTime := progress.LastEntityTime + int64(*worker.jobServer.Config().BleveSettings.BulkIndexingTimeWindowSeconds*1000) diff --git a/services/searchengine/bleveengine/search.go b/services/searchengine/bleveengine/search.go index a6360a971e..a0ff5bb357 100644 --- a/services/searchengine/bleveengine/search.go +++ b/services/searchengine/bleveengine/search.go @@ -15,6 +15,7 @@ import ( ) const DeletePostsBatchSize = 500 +const DeleteFilesBatchSize = 500 func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError { b.Mutex.RLock() @@ -507,3 +508,320 @@ func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError { } return nil } + +func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError { + b.Mutex.RLock() + defer b.Mutex.RUnlock() + + blvFile := BLVFileFromFileInfo(file, channelId) + if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil { + return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *BleveEngine) SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) { + channelQueries := []query.Query{} + for _, channel := range *channels { + channelIdQ := bleve.NewTermQuery(channel.Id) + channelIdQ.SetField("ChannelId") + channelQueries = append(channelQueries, channelIdQ) + } + channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...) + + var termQueries []query.Query + var notTermQueries []query.Query + var filters []query.Query + var notFilters []query.Query + + for i, params := range searchParams { + var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd + if searchParams[0].OrTerms { + termOperator = query.MatchQueryOperatorOr + } + + // Date, channels and FromUsers filters come in all + // searchParams iteration, and as they are global to the + // query, we only need to process them once + if i == 0 { + if len(params.InChannels) > 0 { + inChannels := []query.Query{} + for _, channelId := range params.InChannels { + channelQ := bleve.NewTermQuery(channelId) + channelQ.SetField("ChannelId") + inChannels = append(inChannels, channelQ) + } + filters = append(filters, bleve.NewDisjunctionQuery(inChannels...)) + } + + if len(params.ExcludedChannels) > 0 { + excludedChannels := []query.Query{} + for _, channelId := range params.ExcludedChannels { + channelQ := bleve.NewTermQuery(channelId) + channelQ.SetField("ChannelId") + excludedChannels = append(excludedChannels, channelQ) + } + notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...)) + } + + if len(params.FromUsers) > 0 { + fromUsers := []query.Query{} + for _, userId := range params.FromUsers { + userQ := bleve.NewTermQuery(userId) + userQ.SetField("CreatorId") + fromUsers = append(fromUsers, userQ) + } + filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...)) + } + + if len(params.ExcludedUsers) > 0 { + excludedUsers := []query.Query{} + for _, userId := range params.ExcludedUsers { + userQ := bleve.NewTermQuery(userId) + userQ.SetField("CreatorId") + excludedUsers = append(excludedUsers, userQ) + } + notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...)) + } + + if len(params.Extensions) > 0 { + extensions := []query.Query{} + for _, extension := range params.Extensions { + extensionQ := bleve.NewTermQuery(extension) + extensionQ.SetField("Extension") + extensions = append(extensions, extensionQ) + } + filters = append(filters, bleve.NewDisjunctionQuery(extensions...)) + } + + if len(params.ExcludedExtensions) > 0 { + excludedExtensions := []query.Query{} + for _, extension := range params.ExcludedExtensions { + extensionQ := bleve.NewTermQuery(extension) + extensionQ.SetField("Extension") + excludedExtensions = append(excludedExtensions, extensionQ) + } + notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...)) + } + + if params.OnDate != "" { + before, after := params.GetOnDateMillis() + beforeFloat64 := float64(before) + afterFloat64 := float64(after) + onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64) + onDateQ.SetField("CreateAt") + filters = append(filters, onDateQ) + } else { + if params.AfterDate != "" || params.BeforeDate != "" { + var min, max *float64 + if params.AfterDate != "" { + minf := float64(params.GetAfterDateMillis()) + min = &minf + } + + if params.BeforeDate != "" { + maxf := float64(params.GetBeforeDateMillis()) + max = &maxf + } + + dateQ := bleve.NewNumericRangeQuery(min, max) + dateQ.SetField("CreateAt") + filters = append(filters, dateQ) + } + + if params.ExcludedAfterDate != "" { + minf := float64(params.GetExcludedAfterDateMillis()) + dateQ := bleve.NewNumericRangeQuery(&minf, nil) + dateQ.SetField("CreateAt") + notFilters = append(notFilters, dateQ) + } + + if params.ExcludedBeforeDate != "" { + maxf := float64(params.GetExcludedBeforeDateMillis()) + dateQ := bleve.NewNumericRangeQuery(nil, &maxf) + dateQ.SetField("CreateAt") + notFilters = append(notFilters, dateQ) + } + + if params.ExcludedDate != "" { + before, after := params.GetExcludedDateMillis() + beforef := float64(before) + afterf := float64(after) + onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf) + onDateQ.SetField("CreateAt") + notFilters = append(notFilters, onDateQ) + } + } + } + + if len(params.Terms) > 0 { + terms := []string{} + for _, term := range strings.Split(params.Terms, " ") { + if strings.HasSuffix(term, "*") { + nameQ := bleve.NewWildcardQuery(term) + nameQ.SetField("Name") + contentQ := bleve.NewWildcardQuery(term) + contentQ.SetField("Content") + termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) + } else { + terms = append(terms, term) + } + } + + if len(terms) > 0 { + nameQ := bleve.NewMatchQuery(strings.Join(terms, " ")) + nameQ.SetField("Name") + nameQ.SetOperator(termOperator) + contentQ := bleve.NewMatchQuery(strings.Join(terms, " ")) + contentQ.SetField("Content") + contentQ.SetOperator(termOperator) + termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) + } + } + + if len(params.ExcludedTerms) > 0 { + nameQ := bleve.NewMatchQuery(params.ExcludedTerms) + nameQ.SetField("Name") + nameQ.SetOperator(termOperator) + contentQ := bleve.NewMatchQuery(params.ExcludedTerms) + contentQ.SetField("Content") + contentQ.SetOperator(termOperator) + notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ)) + } + } + + allTermsQ := bleve.NewBooleanQuery() + allTermsQ.AddMustNot(notTermQueries...) + if searchParams[0].OrTerms { + allTermsQ.AddShould(termQueries...) + } else { + allTermsQ.AddMust(termQueries...) + } + + query := bleve.NewBooleanQuery() + query.AddMust(channelDisjunctionQ) + + if len(termQueries) > 0 || len(notTermQueries) > 0 { + query.AddMust(allTermsQ) + } + + if len(filters) > 0 { + query.AddMust(bleve.NewConjunctionQuery(filters...)) + } + if len(notFilters) > 0 { + query.AddMustNot(notFilters...) + } + + search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false) + search.SortBy([]string{"-CreateAt"}) + results, err := b.FileIndex.Search(search) + if err != nil { + return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, err.Error(), http.StatusInternalServerError) + } + + fileIds := []string{} + + for _, r := range results.Hits { + fileIds = append(fileIds, r.ID) + } + + return fileIds, nil +} + +func (b *BleveEngine) DeleteFile(fileID string) *model.AppError { + b.Mutex.RLock() + defer b.Mutex.RUnlock() + + if err := b.FileIndex.Delete(fileID); err != nil { + return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, err.Error(), http.StatusInternalServerError) + } + return nil +} + +func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) { + resultsCount := int64(0) + + for { + // As we are deleting the files after fetching them, we need to keep + // From fixed always to 0 + searchRequest.From = 0 + searchRequest.Size = batchSize + results, err := b.FileIndex.Search(searchRequest) + if err != nil { + return -1, err + } + batch := b.FileIndex.NewBatch() + for _, file := range results.Hits { + batch.Delete(file.ID) + } + if err := b.FileIndex.Batch(batch); err != nil { + return -1, err + } + resultsCount += int64(results.Hits.Len()) + if results.Hits.Len() < batchSize { + break + } + } + + return resultsCount, nil +} + +func (b *BleveEngine) DeleteUserFiles(userID string) *model.AppError { + b.Mutex.RLock() + defer b.Mutex.RUnlock() + + query := bleve.NewTermQuery(userID) + query.SetField("CreatorId") + search := bleve.NewSearchRequest(query) + deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) + if err != nil { + return model.NewAppError("Bleveengine.DeleteUserFiles", + "bleveengine.delete_user_files.error", nil, + err.Error(), http.StatusInternalServerError) + } + + mlog.Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted)) + + return nil +} + +func (b *BleveEngine) DeletePostFiles(postID string) *model.AppError { + b.Mutex.RLock() + defer b.Mutex.RUnlock() + + query := bleve.NewTermQuery(postID) + query.SetField("PostId") + search := bleve.NewSearchRequest(query) + deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) + if err != nil { + return model.NewAppError("Bleveengine.DeletePostFiles", + "bleveengine.delete_post_files.error", nil, + err.Error(), http.StatusInternalServerError) + } + + mlog.Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int64("deleted", deleted)) + + return nil +} + +func (b *BleveEngine) DeleteFilesBatch(endTime, limit int64) *model.AppError { + b.Mutex.RLock() + defer b.Mutex.RUnlock() + + endTimeFloat := float64(endTime) + query := bleve.NewNumericRangeQuery(nil, &endTimeFloat) + query.SetField("CreateAt") + search := bleve.NewSearchRequestOptions(query, int(limit), 0, false) + search.SortBy([]string{"-CreateAt"}) + + deleted, err := b.deleteFiles(search, DeleteFilesBatchSize) + if err != nil { + return model.NewAppError("Bleveengine.DeleteFilesBatch", + "bleveengine.delete_files_batch.error", nil, + err.Error(), http.StatusInternalServerError) + } + + mlog.Info("Files in batch deleted", mlog.Int64("endTime", endTime), mlog.Int64("limit", limit), mlog.Int64("deleted", deleted)) + + return nil +} diff --git a/services/searchengine/interface.go b/services/searchengine/interface.go index f8e35fc792..da0f78154a 100644 --- a/services/searchengine/interface.go +++ b/services/searchengine/interface.go @@ -32,6 +32,12 @@ type SearchEngineInterface interface { SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) DeleteUser(user *model.User) *model.AppError + IndexFile(file *model.FileInfo, channelId string) *model.AppError + SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) + DeleteFile(fileID string) *model.AppError + DeletePostFiles(postID string) *model.AppError + DeleteUserFiles(userID string) *model.AppError + DeleteFilesBatch(endTime, limit int64) *model.AppError TestConfig(cfg *model.Config) *model.AppError PurgeIndexes() *model.AppError RefreshIndexes() *model.AppError diff --git a/services/searchengine/mocks/SearchEngineInterface.go b/services/searchengine/mocks/SearchEngineInterface.go index f36128bc2a..19918f4d6c 100644 --- a/services/searchengine/mocks/SearchEngineInterface.go +++ b/services/searchengine/mocks/SearchEngineInterface.go @@ -64,6 +64,38 @@ func (_m *SearchEngineInterface) DeleteChannelPosts(channelID string) *model.App return r0 } +// DeleteFile provides a mock function with given fields: fileID +func (_m *SearchEngineInterface) DeleteFile(fileID string) *model.AppError { + ret := _m.Called(fileID) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string) *model.AppError); ok { + r0 = rf(fileID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + +// DeleteFilesBatch provides a mock function with given fields: endTime, limit +func (_m *SearchEngineInterface) DeleteFilesBatch(endTime int64, limit int64) *model.AppError { + ret := _m.Called(endTime, limit) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(int64, int64) *model.AppError); ok { + r0 = rf(endTime, limit) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + // DeletePost provides a mock function with given fields: post func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError { ret := _m.Called(post) @@ -80,6 +112,22 @@ func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError { return r0 } +// DeletePostFiles provides a mock function with given fields: postID +func (_m *SearchEngineInterface) DeletePostFiles(postID string) *model.AppError { + ret := _m.Called(postID) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string) *model.AppError); ok { + r0 = rf(postID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + // DeleteUser provides a mock function with given fields: user func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError { ret := _m.Called(user) @@ -96,6 +144,22 @@ func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError { return r0 } +// DeleteUserFiles provides a mock function with given fields: userID +func (_m *SearchEngineInterface) DeleteUserFiles(userID string) *model.AppError { + ret := _m.Called(userID) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(string) *model.AppError); ok { + r0 = rf(userID) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + // DeleteUserPosts provides a mock function with given fields: userID func (_m *SearchEngineInterface) DeleteUserPosts(userID string) *model.AppError { ret := _m.Called(userID) @@ -156,6 +220,22 @@ func (_m *SearchEngineInterface) IndexChannel(channel *model.Channel) *model.App return r0 } +// IndexFile provides a mock function with given fields: file, channelId +func (_m *SearchEngineInterface) IndexFile(file *model.FileInfo, channelId string) *model.AppError { + ret := _m.Called(file, channelId) + + var r0 *model.AppError + if rf, ok := ret.Get(0).(func(*model.FileInfo, string) *model.AppError); ok { + r0 = rf(file, channelId) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.AppError) + } + } + + return r0 +} + // IndexPost provides a mock function with given fields: post, teamId func (_m *SearchEngineInterface) IndexPost(post *model.Post, teamId string) *model.AppError { ret := _m.Called(post, teamId) @@ -315,6 +395,31 @@ func (_m *SearchEngineInterface) SearchChannels(teamId string, term string) ([]s return r0, r1 } +// SearchFiles provides a mock function with given fields: channels, searchParams, page, perPage +func (_m *SearchEngineInterface) SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, *model.AppError) { + ret := _m.Called(channels, searchParams, page, perPage) + + var r0 []string + if rf, ok := ret.Get(0).(func(*model.ChannelList, []*model.SearchParams, int, int) []string); ok { + r0 = rf(channels, searchParams, page, perPage) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } + + var r1 *model.AppError + if rf, ok := ret.Get(1).(func(*model.ChannelList, []*model.SearchParams, int, int) *model.AppError); ok { + r1 = rf(channels, searchParams, page, perPage) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(*model.AppError) + } + } + + return r0, r1 +} + // SearchPosts provides a mock function with given fields: channels, searchParams, page, perPage func (_m *SearchEngineInterface) SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) { ret := _m.Called(channels, searchParams, page, perPage) diff --git a/store/opentracinglayer/opentracinglayer.go b/store/opentracinglayer/opentracinglayer.go index b22596721a..bbd8d78ff2 100644 --- a/store/opentracinglayer/opentracinglayer.go +++ b/store/opentracinglayer/opentracinglayer.go @@ -2982,6 +2982,24 @@ func (s *OpenTracingLayerFileInfoStore) Get(id string) (*model.FileInfo, error) return result, err } +func (s *OpenTracingLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetByIds") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.FileInfoStore.GetByIds(ids) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetByPath") @@ -3139,6 +3157,24 @@ func (s *OpenTracingLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileI return result, err } +func (s *OpenTracingLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) { + origCtx := s.Root.Store.Context() + span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.Search") + s.Root.Store.SetContext(newCtx) + defer func() { + s.Root.Store.SetContext(origCtx) + }() + + defer span.Finish() + result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage) + if err != nil { + span.LogFields(spanlog.Error(err)) + ext.Error.Set(span, true) + } + + return result, err +} + func (s *OpenTracingLayerFileInfoStore) SetContent(fileId string, content string) error { origCtx := s.Root.Store.Context() span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.SetContent") diff --git a/store/retrylayer/retrylayer.go b/store/retrylayer/retrylayer.go index 2e22ad4d9e..8a096f62e8 100644 --- a/store/retrylayer/retrylayer.go +++ b/store/retrylayer/retrylayer.go @@ -3190,6 +3190,26 @@ func (s *RetryLayerFileInfoStore) Get(id string) (*model.FileInfo, error) { } +func (s *RetryLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) { + + tries := 0 + for { + result, err := s.FileInfoStore.GetByIds(ids) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + } + +} + func (s *RetryLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) { tries := 0 @@ -3356,6 +3376,26 @@ func (s *RetryLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, e } +func (s *RetryLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) { + + tries := 0 + for { + result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage) + if err == nil { + return result, nil + } + if !isRepeatableError(err) { + return result, err + } + tries++ + if tries >= 3 { + err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures") + return result, err + } + } + +} + func (s *RetryLayerFileInfoStore) SetContent(fileId string, content string) error { tries := 0 diff --git a/store/searchlayer/channel_layer.go b/store/searchlayer/channel_layer.go index e8e71a7278..f4e4d78072 100644 --- a/store/searchlayer/channel_layer.go +++ b/store/searchlayer/channel_layer.go @@ -24,6 +24,7 @@ func (c *SearchChannelStore) deleteChannelIndex(channel *model.Channel) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { if err := engineCopy.DeleteChannel(channel); err != nil { mlog.Warn("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return } mlog.Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id)) }) @@ -39,6 +40,7 @@ func (c *SearchChannelStore) indexChannel(channel *model.Channel) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { if err := engineCopy.IndexChannel(channel); err != nil { mlog.Warn("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return } mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id)) }) diff --git a/store/searchlayer/file_info_layer.go b/store/searchlayer/file_info_layer.go new file mode 100644 index 0000000000..e281b1e573 --- /dev/null +++ b/store/searchlayer/file_info_layer.go @@ -0,0 +1,197 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchlayer + +import ( + "github.com/mattermost/mattermost-server/v5/mlog" + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/services/searchengine" + "github.com/mattermost/mattermost-server/v5/store" +) + +type SearchFileInfoStore struct { + store.FileInfoStore + rootStore *SearchStore +} + +func (s SearchFileInfoStore) indexFile(file *model.FileInfo) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { + if file.PostId == "" { + return + } + post, postErr := s.rootStore.Post().GetSingle(file.PostId) + if postErr != nil { + mlog.Error("Couldn't get post for file for SearchEngine indexing.", mlog.String("post_id", file.PostId), mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id), mlog.Err(postErr)) + return + } + + if err := engineCopy.IndexFile(file, post.ChannelId); err != nil { + mlog.Error("Encountered error indexing file", mlog.String("file_info_id", file.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + mlog.Debug("Indexed file in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id)) + }) + } + } +} + +func (s SearchFileInfoStore) deleteFileIndex(fileID string) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { + if err := engineCopy.DeleteFile(fileID); err != nil { + mlog.Error("Encountered error deleting file", mlog.String("file_info_id", fileID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + mlog.Debug("Removed file from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", fileID)) + }) + } + } +} + +func (s SearchFileInfoStore) deleteFileIndexForUser(userID string) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { + if err := engineCopy.DeleteUserFiles(userID); err != nil { + mlog.Error("Encountered error deleting files for user", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + mlog.Debug("Removed user's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", userID)) + }) + } + } +} + +func (s SearchFileInfoStore) deleteFileIndexForPost(postID string) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { + if err := engineCopy.DeletePostFiles(postID); err != nil { + mlog.Error("Encountered error deleting files for post", mlog.String("post_id", postID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + mlog.Debug("Removed post's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", postID)) + }) + } + } +} + +func (s SearchFileInfoStore) deleteFileIndexBatch(endTime, limit int64) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsIndexingEnabled() { + runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { + if err := engineCopy.DeleteFilesBatch(endTime, limit); err != nil { + mlog.Error("Encountered error deleting a batch of files", mlog.Int64("limit", limit), mlog.Int64("end_time", endTime), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return + } + mlog.Debug("Removed batch of files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.Int64("end_time", endTime), mlog.Int64("limit", limit)) + }) + } + } +} + +func (s SearchFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) { + nfile, err := s.FileInfoStore.Save(info) + if err == nil { + s.indexFile(nfile) + } + return nfile, err +} + +func (s SearchFileInfoStore) SetContent(fileID, content string) error { + err := s.FileInfoStore.SetContent(fileID, content) + if err == nil { + nfile, err2 := s.FileInfoStore.Get(fileID) + if err2 == nil { + nfile.Content = content + s.indexFile(nfile) + } + } + return err +} + +func (s SearchFileInfoStore) AttachToPost(fileId, postId, creatorId string) error { + err := s.FileInfoStore.AttachToPost(fileId, postId, creatorId) + if err == nil { + nFileInfo, err2 := s.FileInfoStore.Get(fileId) + if err2 == nil { + s.indexFile(nFileInfo) + } + } + return err +} + +func (s SearchFileInfoStore) DeleteForPost(postId string) (string, error) { + result, err := s.FileInfoStore.DeleteForPost(postId) + if err == nil { + s.deleteFileIndexForPost(postId) + } + return result, err +} + +func (s SearchFileInfoStore) PermanentDelete(fileId string) error { + err := s.FileInfoStore.PermanentDelete(fileId) + if err == nil { + s.deleteFileIndex(fileId) + } + return err +} + +func (s SearchFileInfoStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) { + result, err := s.FileInfoStore.PermanentDeleteBatch(endTime, limit) + if err == nil { + s.deleteFileIndexBatch(endTime, limit) + } + return result, err +} + +func (s SearchFileInfoStore) PermanentDeleteByUser(userId string) (int64, error) { + result, err := s.FileInfoStore.PermanentDeleteByUser(userId) + if err == nil { + s.deleteFileIndexForUser(userId) + } + return result, err +} + +func (s SearchFileInfoStore) Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) { + for _, engine := range s.rootStore.searchEngine.GetActiveEngines() { + if engine.IsSearchEnabled() { + userChannels, nErr := s.rootStore.Channel().GetChannels(teamId, userId, paramsList[0].IncludeDeletedChannels, 0) + if nErr != nil { + return nil, nErr + } + fileIds, appErr := engine.SearchFiles(userChannels, paramsList, page, perPage) + if appErr != nil { + mlog.Error("Encountered error on Search.", mlog.String("search_engine", engine.GetName()), mlog.Err(appErr)) + continue + } + mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName())) + + // Get the files + filesList := model.NewFileInfoList() + if len(fileIds) > 0 { + files, nErr := s.FileInfoStore.GetByIds(fileIds) + if nErr != nil { + return nil, nErr + } + for _, f := range files { + filesList.AddFileInfo(f) + filesList.AddOrder(f.Id) + } + } + return filesList, nil + } + } + + if *s.rootStore.getConfig().SqlSettings.DisableDatabaseSearch { + mlog.Debug("Returning empty results for file Search as the database search is disabled") + return model.NewFileInfoList(), nil + } + + mlog.Debug("Using database search because no other search engine is available") + return s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage) +} diff --git a/store/searchlayer/layer.go b/store/searchlayer/layer.go index 8544e6e464..e15f50380e 100644 --- a/store/searchlayer/layer.go +++ b/store/searchlayer/layer.go @@ -19,6 +19,7 @@ type SearchStore struct { team *SearchTeamStore channel *SearchChannelStore post *SearchPostStore + fileInfo *SearchFileInfoStore configValue atomic.Value } @@ -32,6 +33,7 @@ func NewSearchLayer(baseStore store.Store, searchEngine *searchengine.Broker, cf searchStore.post = &SearchPostStore{PostStore: baseStore.Post(), rootStore: searchStore} searchStore.team = &SearchTeamStore{TeamStore: baseStore.Team(), rootStore: searchStore} searchStore.user = &SearchUserStore{UserStore: baseStore.User(), rootStore: searchStore} + searchStore.fileInfo = &SearchFileInfoStore{FileInfoStore: baseStore.FileInfo(), rootStore: searchStore} return searchStore } @@ -52,6 +54,10 @@ func (s *SearchStore) Post() store.PostStore { return s.post } +func (s *SearchStore) FileInfo() store.FileInfoStore { + return s.fileInfo +} + func (s *SearchStore) Team() store.TeamStore { return s.team } diff --git a/store/searchlayer/post_layer.go b/store/searchlayer/post_layer.go index b077e11c6d..f6207bec54 100644 --- a/store/searchlayer/post_layer.go +++ b/store/searchlayer/post_layer.go @@ -28,6 +28,7 @@ func (s SearchPostStore) indexPost(post *model.Post) { } if err := engineCopy.IndexPost(post, channel.TeamId); err != nil { mlog.Warn("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return } mlog.Debug("Indexed post in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id)) }) @@ -41,6 +42,7 @@ func (s SearchPostStore) deletePostIndex(post *model.Post) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { if err := engineCopy.DeletePost(post); err != nil { mlog.Warn("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return } mlog.Debug("Removed post from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id)) }) @@ -54,6 +56,7 @@ func (s SearchPostStore) deleteChannelPostsIndex(channelID string) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { if err := engineCopy.DeleteChannelPosts(channelID); err != nil { mlog.Warn("Encountered error deleting channel posts", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return } mlog.Debug("Removed all channel posts from the index in search engine", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName())) }) @@ -67,6 +70,7 @@ func (s SearchPostStore) deleteUserPostsIndex(userID string) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { if err := engineCopy.DeleteUserPosts(userID); err != nil { mlog.Warn("Encountered error deleting user posts", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) + return } mlog.Debug("Removed all user posts from the index in search engine", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName())) }) diff --git a/store/searchtest/file_info_layer.go b/store/searchtest/file_info_layer.go new file mode 100644 index 0000000000..f3659711da --- /dev/null +++ b/store/searchtest/file_info_layer.go @@ -0,0 +1,1646 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See LICENSE.txt for license information. + +package searchtest + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/mattermost/mattermost-server/v5/model" + "github.com/mattermost/mattermost-server/v5/store" +) + +var searchFileInfoStoreTests = []searchTest{ + { + Name: "Should be able to search posts including results from DMs", + Fn: testFileInfoSearchFileInfosIncludingDMs, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to search posts using pagination", + Fn: testFileInfoSearchFileInfosWithPagination, + Tags: []string{EngineElasticSearch, EngineBleve}, + }, + { + Name: "Should be able to search for exact phrases in quotes", + Fn: testFileInfoSearchExactPhraseInQuotes, + Tags: []string{EnginePostgres, EngineMySql, EngineElasticSearch}, + }, + { + Name: "Should be able to search for email addresses with or without quotes", + Fn: testFileInfoSearchEmailAddresses, + Tags: []string{EngineElasticSearch}, + }, + { + Name: "Should be able to search when markdown underscores are applied", + Fn: testFileInfoSearchMarkdownUnderscores, + Tags: []string{EnginePostgres, EngineElasticSearch}, + }, + { + Name: "Should be able to search for non-latin words", + Fn: testFileInfoSearchNonLatinWords, + Tags: []string{EngineElasticSearch}, + }, + { + Name: "Should be able to search for alternative spellings of words", + Fn: testFileInfoSearchAlternativeSpellings, + Tags: []string{EngineElasticSearch}, + }, + { + Name: "Should be able to search for alternative spellings of words with and without accents", + Fn: testFileInfoSearchAlternativeSpellingsAccents, + Tags: []string{EngineElasticSearch}, + }, + { + Name: "Should be able to search or exclude messages written by a specific user", + Fn: testFileInfoSearchOrExcludeFileInfosBySpecificUser, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to search or exclude messages written in a specific channel", + Fn: testFileInfoSearchOrExcludeFileInfosInChannel, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to search or exclude messages written in a DM or GM", + Fn: testFileInfoSearchOrExcludeFileInfosInDMGM, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to search or exclude files by extensions", + Fn: testFileInfoSearchOrExcludeByExtensions, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to filter messages written after a specific date", + Fn: testFileInfoFilterFilesAfterSpecificDate, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to filter messages written before a specific date", + Fn: testFileInfoFilterFilesBeforeSpecificDate, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to filter messages written on a specific date", + Fn: testFileInfoFilterFilesInSpecificDate, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to exclude messages that contain a serch term", + Fn: testFileInfoFilterFilesWithATerm, + Tags: []string{EngineMySql, EnginePostgres}, + }, + { + Name: "Should be able to search using boolean operators", + Fn: testFileInfoSearchUsingBooleanOperators, + Tags: []string{EngineMySql, EnginePostgres, EngineElasticSearch}, + }, + { + Name: "Should be able to search with combined filters", + Fn: testFileInfoSearchUsingCombinedFilters, + Tags: []string{EngineAll}, + }, + { + Name: "Should be able to ignore stop words", + Fn: testFileInfoSearchIgnoringStopWords, + Tags: []string{EngineMySql, EngineElasticSearch}, + }, + { + Name: "Should support search stemming", + Fn: testFileInfoSupportStemming, + Tags: []string{EnginePostgres, EngineElasticSearch}, + }, + { + Name: "Should support search with wildcards", + Fn: testFileInfoSupportWildcards, + Tags: []string{EngineAll}, + }, + { + Name: "Should not support search with preceding wildcards", + Fn: testFileInfoNotSupportPrecedingWildcards, + Tags: []string{EngineAll}, + }, + { + Name: "Should discard a wildcard if it's not placed immediately by text", + Fn: testFileInfoSearchDiscardWildcardAlone, + Tags: []string{EngineAll}, + }, + { + Name: "Should support terms with dash", + Fn: testFileInfoSupportTermsWithDash, + Tags: []string{EngineAll}, + Skip: true, + }, + { + Name: "Should support terms with underscore", + Fn: testFileInfoSupportTermsWithUnderscore, + Tags: []string{EngineMySql, EngineElasticSearch}, + }, + { + Name: "Should be able to search in deleted/archived channels", + Fn: testFileInfoSearchInDeletedOrArchivedChannels, + Tags: []string{EngineMySql, EnginePostgres}, + }, + { + Name: "Should be able to search terms with dashes", + Fn: testFileInfoSearchTermsWithDashes, + Tags: []string{EngineAll}, + Skip: true, + SkipMessage: "Not working", + }, + { + Name: "Should be able to search terms with dots", + Fn: testFileInfoSearchTermsWithDots, + Tags: []string{EnginePostgres, EngineElasticSearch}, + }, + { + Name: "Should be able to search terms with underscores", + Fn: testFileInfoSearchTermsWithUnderscores, + Tags: []string{EngineMySql, EngineElasticSearch}, + }, + { + Name: "Should be able to combine stemming and wildcards", + Fn: testFileInfoSupportStemmingAndWildcards, + Tags: []string{EngineElasticSearch}, + }, + { + Name: "Should support wildcard outside quotes", + Fn: testFileInfoSupportWildcardOutsideQuotes, + Tags: []string{EngineElasticSearch}, + }, + { + Name: "Should not support slash as character separator", + Fn: testFileInfoSlashShouldNotBeCharSeparator, + Tags: []string{EngineMySql, EngineElasticSearch}, + }, + { + Name: "Should be able to search emails without quoting them", + Fn: testFileInfoSearchEmailsWithoutQuotes, + Tags: []string{EngineElasticSearch}, + }, +} + +func TestSearchFileInfoStore(t *testing.T, s store.Store, testEngine *SearchTestEngine) { + th := &SearchTestHelper{ + Store: s, + } + err := th.SetupBasicFixtures() + require.Nil(t, err) + defer th.CleanFixtures() + + runTestSearch(t, testEngine, searchFileInfoStoreTests, th) +} + +func testFileInfoSearchFileInfosIncludingDMs(t *testing.T, th *SearchTestHelper) { + direct, err := th.createDirectChannel(th.Team.Id, "direct", "direct", []*model.User{th.User, th.User2}) + require.Nil(t, err) + defer th.deleteChannel(direct) + + post, err := th.createPost(th.User.Id, direct.Id, "dm test", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + post2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "dm test", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "dm test filename", "dm contenttest filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "dm other filename", "dm other filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "channel test filename", "channel contenttest filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("by-name", func(t *testing.T) { + params := &model.SearchParams{Terms: "test"} + results, err := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, err) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + + t.Run("by-content", func(t *testing.T) { + params := &model.SearchParams{Terms: "contenttest"} + results, err := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, err) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) +} + +func testFileInfoSearchFileInfosWithPagination(t *testing.T, th *SearchTestHelper) { + direct, err := th.createDirectChannel(th.Team.Id, "direct", "direct", []*model.User{th.User, th.User2}) + require.Nil(t, err) + defer th.deleteChannel(direct) + + post, err := th.createPost(th.User.Id, direct.Id, "dm test", "", model.POST_DEFAULT, 10000, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + post2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "dm test", "", model.POST_DEFAULT, 20000, false) + require.Nil(t, err) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "dm test filename", "dm contenttest filename", "jpg", "image/jpeg", 10000, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "dm other filename", "dm other filename", "jpg", "image/jpeg", 20000, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "channel test filename", "channel contenttest filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("by-name", func(t *testing.T) { + params := &model.SearchParams{Terms: "test"} + results, err := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 1) + require.Nil(t, err) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + + results, err = th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 1, 1) + require.Nil(t, err) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("by-content", func(t *testing.T) { + params := &model.SearchParams{Terms: "contenttest"} + results, err := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 1) + require.Nil(t, err) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + + results, err = th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 1, 1) + require.Nil(t, err) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) +} + +func testFileInfoSearchExactPhraseInQuotes(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "channel test 1 2 3 filename", "channel content test 1 2 3 filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "channel test 123 filename", "channel content test 123 filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("by-name", func(t *testing.T) { + params := &model.SearchParams{Terms: "\"channel test 1 2 3\""} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("by-content", func(t *testing.T) { + params := &model.SearchParams{Terms: "\"channel content test 1 2 3\""} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) +} + +func testFileInfoSearchEmailAddresses(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "test email test@test.com", "test email test@content.com", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "test email test2@test.com", "test email test2@content.com", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("by-name", func(t *testing.T) { + t.Run("Should search email addresses enclosed by quotes", func(t *testing.T) { + params := &model.SearchParams{Terms: "\"test@test.com\""} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should search email addresses without quotes", func(t *testing.T) { + params := &model.SearchParams{Terms: "test@test.com"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + }) + t.Run("by-content", func(t *testing.T) { + t.Run("Should search email addresses enclosed by quotes", func(t *testing.T) { + params := &model.SearchParams{Terms: "\"test@content.com\""} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should search email addresses without quotes", func(t *testing.T) { + params := &model.SearchParams{Terms: "test@content.com"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + }) +} + +func testFileInfoSearchMarkdownUnderscores(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "_start middle end_ _another_", "_start middle end_ _another_", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should search the start inside the markdown underscore", func(t *testing.T) { + params := &model.SearchParams{Terms: "start"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should search a word in the middle of the markdown underscore", func(t *testing.T) { + params := &model.SearchParams{Terms: "middle"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should search in the end of the markdown underscore", func(t *testing.T) { + params := &model.SearchParams{Terms: "end"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should search inside markdown underscore", func(t *testing.T) { + params := &model.SearchParams{Terms: "another"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) +} + +func testFileInfoSearchNonLatinWords(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + t.Run("Should be able to search chinese words", func(t *testing.T) { + p1, err := th.createFileInfo(th.User.Id, post.Id, "你好", "你好", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "你", "你", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should search one word", func(t *testing.T) { + params := &model.SearchParams{Terms: "你"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + t.Run("Should search two words", func(t *testing.T) { + params := &model.SearchParams{Terms: "你好"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + t.Run("Should search with wildcard", func(t *testing.T) { + params := &model.SearchParams{Terms: "你*"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + }) + t.Run("Should be able to search cyrillic words", func(t *testing.T) { + p1, err := th.createFileInfo(th.User.Id, post.Id, "слово test", "слово test", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should search one word", func(t *testing.T) { + params := &model.SearchParams{Terms: "слово"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + t.Run("Should search using wildcard", func(t *testing.T) { + params := &model.SearchParams{Terms: "слов*"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + }) + + t.Run("Should be able to search japanese words", func(t *testing.T) { + p1, err := th.createFileInfo(th.User.Id, post.Id, "本", "本", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "本木", "本木", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should search one word", func(t *testing.T) { + params := &model.SearchParams{Terms: "本"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + t.Run("Should search two words", func(t *testing.T) { + params := &model.SearchParams{Terms: "本木"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + t.Run("Should search with wildcard", func(t *testing.T) { + params := &model.SearchParams{Terms: "本*"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + }) + + t.Run("Should be able to search korean words", func(t *testing.T) { + p1, err := th.createFileInfo(th.User.Id, post.Id, "불", "불", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "불다", "불다", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should search one word", func(t *testing.T) { + params := &model.SearchParams{Terms: "불"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + t.Run("Should search two words", func(t *testing.T) { + params := &model.SearchParams{Terms: "불다"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + t.Run("Should search with wildcard", func(t *testing.T) { + params := &model.SearchParams{Terms: "불*"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + }) +} + +func testFileInfoSearchAlternativeSpellings(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "Straße test", "Straße test", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "Strasse test", "Strasse test", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + params := &model.SearchParams{Terms: "Straße"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + + params = &model.SearchParams{Terms: "Strasse"} + results, apperr = th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) +} + +func testFileInfoSearchAlternativeSpellingsAccents(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "café", "café", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "café", "café", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + params := &model.SearchParams{Terms: "café"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + + params = &model.SearchParams{Terms: "café"} + results, apperr = th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + + params = &model.SearchParams{Terms: "cafe"} + results, apperr = th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 0) +} + +func testFileInfoSearchOrExcludeFileInfosBySpecificUser(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "test fromuser filename", "test fromuser filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User2.Id, post.Id, "test fromuser filename", "test fromuser filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + defer th.deleteUserFileInfos(th.User2.Id) + + params := &model.SearchParams{Terms: "fromuser", FromUsers: []string{th.User.Id}} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) +} + +func testFileInfoSearchOrExcludeFileInfosInChannel(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + post2, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "test fromuser filename", "test fromuser filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post2.Id, "test fromuser filename", "test fromuser filename", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + defer th.deleteUserFileInfos(th.User2.Id) + + params := &model.SearchParams{Terms: "fromuser", InChannels: []string{th.ChannelBasic.Id}} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) +} + +func testFileInfoSearchOrExcludeFileInfosInDMGM(t *testing.T, th *SearchTestHelper) { + direct, err := th.createDirectChannel(th.Team.Id, "direct", "direct", []*model.User{th.User, th.User2}) + require.Nil(t, err) + defer th.deleteChannel(direct) + + group, err := th.createGroupChannel(th.Team.Id, "test group", []*model.User{th.User, th.User2}) + require.Nil(t, err) + defer th.deleteChannel(group) + + post1, err := th.createPost(th.User.Id, direct.Id, "test fromuser", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + post2, err := th.createPost(th.User2.Id, group.Id, "test fromuser 2", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + defer th.deleteUserPosts(th.User2.Id) + + p1, err := th.createFileInfo(th.User.Id, post1.Id, "test fromuser", "test fromuser", "jpg", "image/jpg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User2.Id, post2.Id, "test fromuser 2", "test fromuser 2", "jpg", "image/jpg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + defer th.deleteUserFileInfos(th.User2.Id) + + t.Run("Should be able to search in both DM and GM channels", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "fromuser", + InChannels: []string{direct.Id, group.Id}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + + t.Run("Should be able to search only in DM channel", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "fromuser", + InChannels: []string{direct.Id}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should be able to search only in GM channel", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "fromuser", + InChannels: []string{group.Id}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) +} + +func testFileInfoSearchOrExcludeByExtensions(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "test", "test", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "test", "test", "png", "image/png", 0, 0) + require.Nil(t, err) + p3, err := th.createFileInfo(th.User.Id, post.Id, "test", "test", "bmp", "image/bmp", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Search by one extension", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + InChannels: []string{th.ChannelBasic.Id}, + Extensions: []string{"jpg"}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search by multiple extensions", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + InChannels: []string{th.ChannelBasic.Id}, + Extensions: []string{"jpg", "bmp"}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("Search excluding one extension", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + InChannels: []string{th.ChannelBasic.Id}, + ExcludedExtensions: []string{"jpg"}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("Search excluding multiple extensions", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + InChannels: []string{th.ChannelBasic.Id}, + ExcludedExtensions: []string{"jpg", "bmp"}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) +} + +func testFileInfoFilterFilesInSpecificDate(t *testing.T, th *SearchTestHelper) { + post1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + post2, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + creationDate := model.GetMillisForTime(time.Date(2020, 03, 22, 12, 0, 0, 0, time.UTC)) + p1, err := th.createFileInfo(th.User.Id, post1.Id, "test in specific date", "test in specific date", "jpg", "image/jpeg", creationDate, 0) + require.Nil(t, err) + creationDate2 := model.GetMillisForTime(time.Date(2020, 03, 23, 0, 0, 0, 0, time.UTC)) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "test in the present", "test in the present", "jpg", "image/jpeg", creationDate2, 0) + require.Nil(t, err) + creationDate3 := model.GetMillisForTime(time.Date(2020, 03, 21, 23, 59, 59, 0, time.UTC)) + p3, err := th.createFileInfo(th.User.Id, post1.Id, "test in the present", "test in the present", "jpg", "image/jpeg", creationDate3, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should be able to search posts on date", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + OnDate: "2020-03-22", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + t.Run("Should be able to exclude posts on date", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + ExcludedDate: "2020-03-22", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) +} + +func testFileInfoFilterFilesBeforeSpecificDate(t *testing.T, th *SearchTestHelper) { + post1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + post2, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + creationDate := model.GetMillisForTime(time.Date(2020, 03, 01, 12, 0, 0, 0, time.UTC)) + p1, err := th.createFileInfo(th.User.Id, post1.Id, "test in specific date", "test in specific date", "jpg", "image/jpeg", creationDate, 0) + require.Nil(t, err) + creationDate2 := model.GetMillisForTime(time.Date(2020, 03, 22, 23, 59, 59, 0, time.UTC)) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "test in specific date 2", "test in specific date 2", "jpg", "image/jpeg", creationDate2, 0) + require.Nil(t, err) + creationDate3 := model.GetMillisForTime(time.Date(2020, 03, 26, 16, 55, 0, 0, time.UTC)) + p3, err := th.createFileInfo(th.User.Id, post1.Id, "test in the present", "test in the present", "jpg", "image/jpeg", creationDate3, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should be able to search posts before a date", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + BeforeDate: "2020-03-23", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + + t.Run("Should be able to exclude posts before a date", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + ExcludedBeforeDate: "2020-03-23", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) +} + +func testFileInfoFilterFilesAfterSpecificDate(t *testing.T, th *SearchTestHelper) { + post1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + post2, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + creationDate := model.GetMillisForTime(time.Date(2020, 03, 01, 12, 0, 0, 0, time.UTC)) + p1, err := th.createFileInfo(th.User.Id, post1.Id, "test in specific date", "test in specific date", "jpg", "image/jpeg", creationDate, 0) + require.Nil(t, err) + creationDate2 := model.GetMillisForTime(time.Date(2020, 03, 22, 23, 59, 59, 0, time.UTC)) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "test in specific date 2", "test in specific date 2", "jpg", "image/jpeg", creationDate2, 0) + require.Nil(t, err) + creationDate3 := model.GetMillisForTime(time.Date(2020, 03, 26, 16, 55, 0, 0, time.UTC)) + p3, err := th.createFileInfo(th.User.Id, post1.Id, "test in the present", "test in the present", "jpg", "image/jpeg", creationDate3, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should be able to search posts after a date", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + AfterDate: "2020-03-23", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("Should be able to exclude posts after a date", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "test", + ExcludedAfterDate: "2020-03-23", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) +} + +func testFileInfoFilterFilesWithATerm(t *testing.T, th *SearchTestHelper) { + post1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + post2, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post1.Id, "one two three", "one two three", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "one four five six", "one four five six", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post1.Id, "one seven eight nine", "one seven eight nine", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should exclude terms", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "one", + ExcludedTerms: "five eight", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should exclude quoted terms", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "one", + ExcludedTerms: "\"eight nine\"", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) +} + +func testFileInfoSearchUsingBooleanOperators(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "one two three message", "one two three message", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "two messages", "two messages", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "another message", "another message", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should search posts using OR operator", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "one two", + OrTerms: true, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + + t.Run("Should search posts using AND operator", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "one two", + OrTerms: false, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) +} + +func testFileInfoSearchUsingCombinedFilters(t *testing.T, th *SearchTestHelper) { + post1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + post2, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + creationDate := model.GetMillisForTime(time.Date(2020, 03, 01, 12, 0, 0, 0, time.UTC)) + p1, err := th.createFileInfo(th.User.Id, post2.Id, "one two three message", "one two three message", "jpg", "image/jpeg", creationDate, 0) + require.Nil(t, err) + creationDate2 := model.GetMillisForTime(time.Date(2020, 03, 10, 12, 0, 0, 0, time.UTC)) + p2, err := th.createFileInfo(th.User2.Id, post2.Id, "two messages", "two messages", "jpg", "image/jpeg", creationDate2, 0) + require.Nil(t, err) + creationDate3 := model.GetMillisForTime(time.Date(2020, 03, 20, 12, 0, 0, 0, time.UTC)) + p3, err := th.createFileInfo(th.User.Id, post1.Id, "two another message", "two another message", "jpg", "image/jpeg", creationDate3, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + defer th.deleteUserFileInfos(th.User2.Id) + + t.Run("Should search combining from user and in channel filters", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "two", + FromUsers: []string{th.User2.Id}, + InChannels: []string{th.ChannelPrivate.Id}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + + t.Run("Should search combining excluding users and in channel filters", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "two", + ExcludedUsers: []string{th.User2.Id}, + InChannels: []string{th.ChannelPrivate.Id}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should search combining excluding dates and in channel filters", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "two", + ExcludedBeforeDate: "2020-03-09", + ExcludedAfterDate: "2020-03-11", + InChannels: []string{th.ChannelPrivate.Id}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + t.Run("Should search combining excluding dates and in channel filters", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "two", + AfterDate: "2020-03-11", + ExcludedChannels: []string{th.ChannelPrivate.Id}, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) +} + +func testFileInfoSearchIgnoringStopWords(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "the search for a bunch of stop words", "the search for a bunch of stop words", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "the objective is to avoid a bunch of stop words", "the objective is to avoid a bunch of stop words", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p3, err := th.createFileInfo(th.User.Id, post.Id, "in the a on to where you", "in the a on to where you", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p4, err := th.createFileInfo(th.User.Id, post.Id, "where is the car?", "where is the car?", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should avoid stop word 'the'", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "the search", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should avoid stop word 'a'", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "a avoid", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + + t.Run("Should avoid stop word 'in'", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "in where you", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("Should avoid stop words 'where', 'is' and 'the'", func(t *testing.T) { + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{{Terms: "is the car"}}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p4.Id, results.FileInfos) + }) + + t.Run("Should remove all terms and return empty list", func(t *testing.T) { + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{{Terms: "is the"}}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + require.Empty(t, results.FileInfos) + }) +} + +func testFileInfoSupportStemming(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "search post", "search post", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "searching post", "searching post", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "another post", "another post", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + params := &model.SearchParams{ + Terms: "search", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) +} + +func testFileInfoSupportWildcards(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "search post", "search post", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "searching", "searching", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "another post", "another post", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Simple wildcard-only search", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "search*", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + + t.Run("Wildcard search with another term placed after", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "sear* post", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) +} + +func testFileInfoNotSupportPrecedingWildcards(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + _, err = th.createFileInfo(th.User.Id, post.Id, "search post", "search post", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "searching post", "searching post", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "another post", "another post", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + params := &model.SearchParams{ + Terms: "*earch", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 0) +} + +func testFileInfoSearchDiscardWildcardAlone(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "qwerty", "qwerty", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "qwertyjkl", "qwertyjkl", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + params := &model.SearchParams{ + Terms: "qwerty *", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) +} + +func testFileInfoSupportTermsWithDash(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "search term-with-dash", "search term-with-dash", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "searching term with dash", "searching term with dash", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should search terms with dash", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "term-with-dash", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should search terms with dash using quotes", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "\"term-with-dash\"", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) +} + +func testFileInfoSupportTermsWithUnderscore(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "search term_with_underscore", "search term_with_underscore", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "searching term with underscore", "searching term with underscore", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should search terms with underscore", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "term_with_underscore", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Should search terms with underscore using quotes", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "\"term_with_underscore\"", + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) +} + +func testFileInfoSearchInDeletedOrArchivedChannels(t *testing.T, th *SearchTestHelper) { + post1, err := th.createPost(th.User.Id, th.ChannelDeleted.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + post2, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + post3, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post1.Id, "message in deleted channel", "message in deleted channel", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "message in regular channel", "message in regular channel", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p3, err := th.createFileInfo(th.User.Id, post3.Id, "message in private channel", "message in private channel", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Doesn't include posts in deleted channels", func(t *testing.T) { + params := &model.SearchParams{Terms: "message", IncludeDeletedChannels: false} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("Include posts in deleted channels", func(t *testing.T) { + params := &model.SearchParams{Terms: "message", IncludeDeletedChannels: true} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 3) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("Include posts in deleted channels using multiple terms", func(t *testing.T) { + params := &model.SearchParams{Terms: "message channel", IncludeDeletedChannels: true} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 3) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("Include posts in deleted channels using multiple OR terms", func(t *testing.T) { + params := &model.SearchParams{ + Terms: "message channel", + IncludeDeletedChannels: true, + OrTerms: true, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 3) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("All IncludeDeletedChannels params should have same value if multiple SearchParams provided", func(t *testing.T) { + params1 := &model.SearchParams{ + Terms: "message channel", + IncludeDeletedChannels: true, + } + params2 := &model.SearchParams{ + Terms: "#hashtag", + IncludeDeletedChannels: false, + } + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params1, params2}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, results) + require.NotNil(t, apperr) + }) +} + +func testFileInfoSearchTermsWithDashes(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "message with-dash-term", "message with-dash-term", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "message with dash term", "message with dash term", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Search for terms with dash", func(t *testing.T) { + params := &model.SearchParams{Terms: "with-dash-term"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for terms with quoted dash", func(t *testing.T) { + params := &model.SearchParams{Terms: "\"with-dash-term\""} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for multiple terms with one having dash", func(t *testing.T) { + params := &model.SearchParams{Terms: "with-dash-term message"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for multiple OR terms with one having dash", func(t *testing.T) { + params := &model.SearchParams{Terms: "with-dash-term message", OrTerms: true} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) +} + +func testFileInfoSearchTermsWithDots(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "message with.dots.term", "message with.dots.term", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "message with dots term", "message with dots term", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Search for terms with dots", func(t *testing.T) { + params := &model.SearchParams{Terms: "with.dots.term"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for terms with quoted dots", func(t *testing.T) { + params := &model.SearchParams{Terms: "\"with.dots.term\""} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for multiple terms with one having dots", func(t *testing.T) { + params := &model.SearchParams{Terms: "with.dots.term message"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for multiple OR terms with one having dots", func(t *testing.T) { + params := &model.SearchParams{Terms: "with.dots.term message", OrTerms: true} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) +} + +func testFileInfoSearchTermsWithUnderscores(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "message with_underscores_term", "message with_underscores_term", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post.Id, "message with underscores term", "message with underscores term", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Search for terms with underscores", func(t *testing.T) { + params := &model.SearchParams{Terms: "with_underscores_term"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for terms with quoted underscores", func(t *testing.T) { + params := &model.SearchParams{Terms: "\"with_underscores_term\""} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for multiple terms with one having underscores", func(t *testing.T) { + params := &model.SearchParams{Terms: "with_underscores_term message"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + }) + + t.Run("Search for multiple OR terms with one having underscores", func(t *testing.T) { + params := &model.SearchParams{Terms: "with_underscores_term message", OrTerms: true} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) +} + +func testFileInfoSupportStemmingAndWildcards(t *testing.T, th *SearchTestHelper) { + post1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + post2, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + + defer th.deleteUserPosts(th.User.Id) + p1, err := th.createFileInfo(th.User.Id, post1.Id, "approve", "approve", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "approved", "approved", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p3, err := th.createFileInfo(th.User.Id, post2.Id, "approvedz", "approvedz", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should stem appr", func(t *testing.T) { + params := &model.SearchParams{Terms: "appr*"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 3) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) + + t.Run("Should stem approve", func(t *testing.T) { + params := &model.SearchParams{Terms: "approve*"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p3.Id, results.FileInfos) + }) +} + +func testFileInfoSupportWildcardOutsideQuotes(t *testing.T, th *SearchTestHelper) { + post1, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + post2, err := th.createPost(th.User.Id, th.ChannelPrivate.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + + p1, err := th.createFileInfo(th.User.Id, post1.Id, "hello world", "hello world", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + p2, err := th.createFileInfo(th.User.Id, post2.Id, "hell or heaven", "hell or heaven", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + t.Run("Should return results without quotes", func(t *testing.T) { + params := &model.SearchParams{Terms: "hell*"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 2) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + + t.Run("Should return just one result with quotes", func(t *testing.T) { + params := &model.SearchParams{Terms: "\"hell\"*"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p2.Id, results.FileInfos) + }) + +} + +func testFileInfoSlashShouldNotBeCharSeparator(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "alpha/beta gamma, theta", "alpha/beta gamma, theta", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + params := &model.SearchParams{Terms: "gamma"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + + params = &model.SearchParams{Terms: "beta"} + results, apperr = th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) + + params = &model.SearchParams{Terms: "alpha"} + results, apperr = th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) +} + +func testFileInfoSearchEmailsWithoutQuotes(t *testing.T, th *SearchTestHelper) { + post, err := th.createPost(th.User.Id, th.ChannelBasic.Id, "testmessage", "", model.POST_DEFAULT, 0, false) + require.Nil(t, err) + defer th.deleteUserPosts(th.User.Id) + + p1, err := th.createFileInfo(th.User.Id, post.Id, "message test@test.com", "message test@test.com", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + _, err = th.createFileInfo(th.User.Id, post.Id, "message test2@test.com", "message test2@test.com", "jpg", "image/jpeg", 0, 0) + require.Nil(t, err) + defer th.deleteUserFileInfos(th.User.Id) + + params := &model.SearchParams{Terms: "test@test.com"} + results, apperr := th.Store.FileInfo().Search([]*model.SearchParams{params}, th.User.Id, th.Team.Id, 0, 20) + require.Nil(t, apperr) + + require.Len(t, results.FileInfos, 1) + th.checkFileInfoInSearchResults(t, p1.Id, results.FileInfos) +} diff --git a/store/searchtest/helper.go b/store/searchtest/helper.go index b7b75c6d7f..3939150d32 100644 --- a/store/searchtest/helper.go +++ b/store/searchtest/helper.go @@ -392,6 +392,36 @@ func (th *SearchTestHelper) createPost(userID, channelID, message, hashtags, pos return post, nil } +func (th *SearchTestHelper) createFileInfoModel(creatorID, postID, name, content, extension, mimeType string, createAt, size int64) *model.FileInfo { + return &model.FileInfo{ + CreatorId: creatorID, + PostId: postID, + CreateAt: createAt, + UpdateAt: createAt, + DeleteAt: 0, + Name: name, + Content: content, + Path: name, + Extension: extension, + Size: size, + MimeType: mimeType, + } +} + +func (th *SearchTestHelper) createFileInfo(creatorID, postID, name, content, extension, mimeType string, createAt, size int64) (*model.FileInfo, error) { + var creationTime int64 = 1000000 + if createAt > 0 { + creationTime = createAt + } + fileInfoModel := th.createFileInfoModel(creatorID, postID, name, content, extension, mimeType, creationTime, size) + fileInfo, appError := th.Store.FileInfo().Save(fileInfoModel) + if appError != nil { + return nil, errors.New(appError.Error()) + } + + return fileInfo, nil +} + func (th *SearchTestHelper) createReply(userID, message, hashtags string, parent *model.Post, createAt int64, pinned bool) (*model.Post, error) { replyModel := th.createPostModel(userID, parent.ChannelId, message, hashtags, parent.Type, createAt, pinned) replyModel.ParentId = parent.Id @@ -411,6 +441,13 @@ func (th *SearchTestHelper) deleteUserPosts(userID string) error { return nil } +func (th *SearchTestHelper) deleteUserFileInfos(userID string) error { + if _, err := th.Store.FileInfo().PermanentDeleteByUser(userID); err != nil { + return errors.New(err.Error()) + } + return nil +} + func (th *SearchTestHelper) addUserToTeams(user *model.User, teamIDS []string) error { for _, teamID := range teamIDS { _, err := th.Store.Team().SaveMember(&model.TeamMember{TeamId: teamID, UserId: user.Id}, -1) @@ -467,6 +504,15 @@ func (th *SearchTestHelper) checkPostInSearchResults(t *testing.T, postID string assert.Contains(t, postIDS, postID, "Did not find expected post in search results.") } +func (th *SearchTestHelper) checkFileInfoInSearchResults(t *testing.T, fileID string, searchResults map[string]*model.FileInfo) { + t.Helper() + fileIDS := make([]string, len(searchResults)) + for ID := range searchResults { + fileIDS = append(fileIDS, ID) + } + assert.Contains(t, fileIDS, fileID, "Did not find expected file in search results.") +} + func (th *SearchTestHelper) checkChannelIdsMatch(t *testing.T, expected []string, results *model.ChannelList) { t.Helper() channelIds := make([]string, len(*results)) diff --git a/store/sqlstore/file_info_store.go b/store/sqlstore/file_info_store.go index bf8e0e32e4..07e9c2b4e5 100644 --- a/store/sqlstore/file_info_store.go +++ b/store/sqlstore/file_info_store.go @@ -6,11 +6,15 @@ package sqlstore import ( "database/sql" "fmt" + "regexp" + "strconv" + "strings" sq "github.com/Masterminds/squirrel" "github.com/pkg/errors" "github.com/mattermost/mattermost-server/v5/einterfaces" + "github.com/mattermost/mattermost-server/v5/mlog" "github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/store" ) @@ -73,6 +77,9 @@ func (fs SqlFileInfoStore) createIndexesIfNotExists() { fs.CreateIndexIfNotExists("idx_fileinfo_create_at", "FileInfo", "CreateAt") fs.CreateIndexIfNotExists("idx_fileinfo_delete_at", "FileInfo", "DeleteAt") fs.CreateIndexIfNotExists("idx_fileinfo_postid_at", "FileInfo", "PostId") + fs.CreateIndexIfNotExists("idx_fileinfo_extension_at", "FileInfo", "Extension") + fs.CreateFullTextIndexIfNotExists("idx_fileinfo_name_txt", "FileInfo", "Name") + fs.CreateFullTextIndexIfNotExists("idx_fileinfo_content_txt", "FileInfo", "Content") } func (fs SqlFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) { @@ -87,6 +94,26 @@ func (fs SqlFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) { return info, nil } +func (fs SqlFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) { + query := fs.getQueryBuilder(). + Select("*"). + From("FileInfo"). + Where(sq.Eq{"Id": ids}). + Where(sq.Eq{"DeleteAt": 0}). + OrderBy("CreateAt DESC") + + queryString, args, err := query.ToSql() + if err != nil { + return nil, errors.Wrap(err, "file_info_tosql") + } + + var infos []*model.FileInfo + if _, err := fs.GetReplica().Select(&infos, queryString, args...); err != nil { + return nil, errors.Wrap(err, "failed to find FileInfos") + } + return infos, nil +} + func (fs SqlFileInfoStore) Upsert(info *model.FileInfo) (*model.FileInfo, error) { info.PreSave() if err := info.IsValid(); err != nil { @@ -393,3 +420,176 @@ func (fs SqlFileInfoStore) PermanentDeleteByUser(userId string) (int64, error) { return rowsAffected, nil } + +func (fs SqlFileInfoStore) Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) { + // Since we don't support paging for DB search, we just return nothing for later pages + if page > 0 { + return model.NewFileInfoList(), nil + } + if err := model.IsSearchParamsListValid(paramsList); err != nil { + return nil, err + } + query := fs.getQueryBuilder(). + Select("FI.*"). + From("FileInfo AS FI"). + LeftJoin("Posts as P ON FI.PostId=P.Id"). + LeftJoin("Channels as C ON C.Id=P.ChannelId"). + LeftJoin("ChannelMembers as CM ON C.Id=CM.ChannelId"). + Where(sq.Or{sq.Eq{"C.TeamId": teamId}, sq.Eq{"C.TeamId": ""}}). + Where(sq.Eq{"FI.DeleteAt": 0}). + OrderBy("FI.CreateAt DESC"). + Limit(100) + + for _, params := range paramsList { + params.Terms = removeNonAlphaNumericUnquotedTerms(params.Terms, " ") + + if !params.IncludeDeletedChannels { + query = query.Where(sq.Eq{"C.DeleteAt": 0}) + } + + if !params.SearchWithoutUserId { + query = query.Where(sq.Eq{"CM.UserId": userId}) + } + + if len(params.InChannels) != 0 { + query = query.Where(sq.Eq{"C.Id": params.InChannels}) + } + + if len(params.Extensions) != 0 { + query = query.Where(sq.Eq{"FI.Extension": params.Extensions}) + } + + if len(params.ExcludedExtensions) != 0 { + query = query.Where(sq.NotEq{"FI.Extension": params.ExcludedExtensions}) + } + + if len(params.ExcludedChannels) != 0 { + query = query.Where(sq.NotEq{"C.Id": params.ExcludedChannels}) + } + + if len(params.FromUsers) != 0 { + query = query.Where(sq.Eq{"FI.CreatorId": params.FromUsers}) + } + + if len(params.ExcludedUsers) != 0 { + query = query.Where(sq.NotEq{"FI.CreatorId": params.ExcludedUsers}) + } + + // handle after: before: on: filters + if len(params.OnDate) > 0 { + onDateStart, onDateEnd := params.GetOnDateMillis() + query = query.Where(sq.Expr("FI.CreateAt BETWEEN ? AND ?", strconv.FormatInt(onDateStart, 10), strconv.FormatInt(onDateEnd, 10))) + } else { + if len(params.ExcludedDate) > 0 { + excludedDateStart, excludedDateEnd := params.GetExcludedDateMillis() + query = query.Where(sq.Expr("FI.CreateAt NOT BETWEEN ? AND ?", strconv.FormatInt(excludedDateStart, 10), strconv.FormatInt(excludedDateEnd, 10))) + } + + if len(params.AfterDate) > 0 { + afterDate := params.GetAfterDateMillis() + query = query.Where(sq.GtOrEq{"FI.CreateAt": strconv.FormatInt(afterDate, 10)}) + } + + if len(params.BeforeDate) > 0 { + beforeDate := params.GetBeforeDateMillis() + query = query.Where(sq.LtOrEq{"FI.CreateAt": strconv.FormatInt(beforeDate, 10)}) + } + + if len(params.ExcludedAfterDate) > 0 { + afterDate := params.GetExcludedAfterDateMillis() + query = query.Where(sq.Lt{"FI.CreateAt": strconv.FormatInt(afterDate, 10)}) + } + + if len(params.ExcludedBeforeDate) > 0 { + beforeDate := params.GetExcludedBeforeDateMillis() + query = query.Where(sq.Gt{"FI.CreateAt": strconv.FormatInt(beforeDate, 10)}) + } + } + + terms := params.Terms + excludedTerms := params.ExcludedTerms + + // these chars have special meaning and can be treated as spaces + for _, c := range specialSearchChar { + terms = strings.Replace(terms, c, " ", -1) + excludedTerms = strings.Replace(excludedTerms, c, " ", -1) + } + + if terms == "" && excludedTerms == "" { + // we've already confirmed that we have a channel or user to search for + } else if fs.DriverName() == model.DATABASE_DRIVER_POSTGRES { + // Parse text for wildcards + if wildcard, err := regexp.Compile(`\*($| )`); err == nil { + terms = wildcard.ReplaceAllLiteralString(terms, ":* ") + excludedTerms = wildcard.ReplaceAllLiteralString(excludedTerms, ":* ") + } + + excludeClause := "" + if excludedTerms != "" { + excludeClause = " & !(" + strings.Join(strings.Fields(excludedTerms), " | ") + ")" + } + + queryTerms := "" + if params.OrTerms { + queryTerms = "(" + strings.Join(strings.Fields(terms), " | ") + ")" + excludeClause + } else { + queryTerms = "(" + strings.Join(strings.Fields(terms), " & ") + ")" + excludeClause + } + + query = query.Where(sq.Or{ + sq.Expr("to_tsvector('english', FI.Name) @@ to_tsquery('english', ?)", queryTerms), + sq.Expr("to_tsvector('english', FI.Content) @@ to_tsquery('english', ?)", queryTerms), + }) + } else if fs.DriverName() == model.DATABASE_DRIVER_MYSQL { + var err error + terms, err = removeMysqlStopWordsFromTerms(terms) + if err != nil { + return nil, errors.Wrap(err, "failed to remove Mysql stop-words from terms") + } + + if terms == "" { + return model.NewFileInfoList(), nil + } + + excludeClause := "" + if excludedTerms != "" { + excludeClause = " -(" + excludedTerms + ")" + } + + queryTerms := "" + if params.OrTerms { + queryTerms = terms + excludeClause + } else { + splitTerms := []string{} + for _, t := range strings.Fields(terms) { + splitTerms = append(splitTerms, "+"+t) + } + queryTerms = strings.Join(splitTerms, " ") + excludeClause + } + query = query.Where(sq.Or{ + sq.Expr("MATCH (FI.Name) AGAINST (? IN BOOLEAN MODE)", queryTerms), + sq.Expr("MATCH (FI.Content) AGAINST (? IN BOOLEAN MODE)", queryTerms), + }) + } + } + + queryString, args, err := query.ToSql() + if err != nil { + return nil, errors.Wrap(err, "file_info_tosql") + } + + list := model.NewFileInfoList() + fileInfos := []*model.FileInfo{} + _, err = fs.GetSearchReplica().Select(&fileInfos, queryString, args...) + if err != nil { + mlog.Warn("Query error searching files.", mlog.Err(err)) + // Don't return the error to the caller as it is of no use to the user. Instead return an empty set of search results. + } else { + for _, f := range fileInfos { + list.AddFileInfo(f) + list.AddOrder(f.Id) + } + } + list.MakeNonNil() + return list, nil +} diff --git a/store/sqlstore/file_info_store_test.go b/store/sqlstore/file_info_store_test.go index e20b028f76..549d15138b 100644 --- a/store/sqlstore/file_info_store_test.go +++ b/store/sqlstore/file_info_store_test.go @@ -6,9 +6,14 @@ package sqlstore import ( "testing" + "github.com/mattermost/mattermost-server/v5/store/searchtest" "github.com/mattermost/mattermost-server/v5/store/storetest" ) func TestFileInfoStore(t *testing.T) { StoreTest(t, storetest.TestFileInfoStore) } + +func TestSearchFileInfoStore(t *testing.T) { + StoreTestWithSearchTestEngine(t, searchtest.TestSearchFileInfoStore) +} diff --git a/store/store.go b/store/store.go index 5bc62b2814..d9826a43d1 100644 --- a/store/store.go +++ b/store/store.go @@ -564,6 +564,7 @@ type FileInfoStore interface { Save(info *model.FileInfo) (*model.FileInfo, error) Upsert(info *model.FileInfo) (*model.FileInfo, error) Get(id string) (*model.FileInfo, error) + GetByIds(ids []string) ([]*model.FileInfo, error) GetByPath(path string) (*model.FileInfo, error) GetForPost(postId string, readFromMaster, includeDeleted, allowFromCache bool) ([]*model.FileInfo, error) GetForUser(userId string) ([]*model.FileInfo, error) @@ -575,6 +576,7 @@ type FileInfoStore interface { PermanentDeleteBatch(endTime int64, limit int64) (int64, error) PermanentDeleteByUser(userId string) (int64, error) SetContent(fileId, content string) error + Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) ClearCaches() } diff --git a/store/storetest/mocks/FileInfoStore.go b/store/storetest/mocks/FileInfoStore.go index 9cbf13aeac..6e566007ca 100644 --- a/store/storetest/mocks/FileInfoStore.go +++ b/store/storetest/mocks/FileInfoStore.go @@ -77,6 +77,29 @@ func (_m *FileInfoStore) Get(id string) (*model.FileInfo, error) { return r0, r1 } +// GetByIds provides a mock function with given fields: ids +func (_m *FileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) { + ret := _m.Called(ids) + + var r0 []*model.FileInfo + if rf, ok := ret.Get(0).(func([]string) []*model.FileInfo); ok { + r0 = rf(ids) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*model.FileInfo) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func([]string) error); ok { + r1 = rf(ids) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetByPath provides a mock function with given fields: path func (_m *FileInfoStore) GetByPath(path string) (*model.FileInfo, error) { ret := _m.Called(path) @@ -253,6 +276,29 @@ func (_m *FileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) { return r0, r1 } +// Search provides a mock function with given fields: paramsList, userId, teamId, page, perPage +func (_m *FileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) { + ret := _m.Called(paramsList, userId, teamId, page, perPage) + + var r0 *model.FileInfoList + if rf, ok := ret.Get(0).(func([]*model.SearchParams, string, string, int, int) *model.FileInfoList); ok { + r0 = rf(paramsList, userId, teamId, page, perPage) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*model.FileInfoList) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func([]*model.SearchParams, string, string, int, int) error); ok { + r1 = rf(paramsList, userId, teamId, page, perPage) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // SetContent provides a mock function with given fields: fileId, content func (_m *FileInfoStore) SetContent(fileId string, content string) error { ret := _m.Called(fileId, content) diff --git a/store/timerlayer/timerlayer.go b/store/timerlayer/timerlayer.go index 1e95f4e441..01e812c05c 100644 --- a/store/timerlayer/timerlayer.go +++ b/store/timerlayer/timerlayer.go @@ -2734,6 +2734,22 @@ func (s *TimerLayerFileInfoStore) Get(id string) (*model.FileInfo, error) { return result, err } +func (s *TimerLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) { + start := timemodule.Now() + + result, err := s.FileInfoStore.GetByIds(ids) + + elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.GetByIds", success, elapsed) + } + return result, err +} + func (s *TimerLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) { start := timemodule.Now() @@ -2877,6 +2893,22 @@ func (s *TimerLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, e return result, err } +func (s *TimerLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) { + start := timemodule.Now() + + result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage) + + elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second) + if s.Root.Metrics != nil { + success := "false" + if err == nil { + success = "true" + } + s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.Search", success, elapsed) + } + return result, err +} + func (s *TimerLayerFileInfoStore) SetContent(fileId string, content string) error { start := timemodule.Now()