Add search engine support for files (#16190)

* Add search engine support for files

* Fixing i18n

* Fix golangci-lint

* Fix consistency problem in the Search receiver functio of the SqlFileStore

* Fixing some tests

* Fixing test

* Apply suggestions from code review

Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>

* Addressing PR review comments

* Removing some empty lines

* Address PR review comments

* Fixing problem after merge master

* Fixing spelling problem

* Add missed translations

* Fixing certain global variable usages after merge master

* Fixing some constants usage

* Fixing goimports order

Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>
Этот коммит содержится в:
Jesús Espino
2021-01-11 15:14:16 +01:00
коммит произвёл GitHub
родитель bcacc78f77
Коммит 2a63b5552a
28 изменённых файлов: 3438 добавлений и 219 удалений

Просмотреть файл

@@ -50,9 +50,12 @@ type MetricsInterface interface {
IncrementPostsSearchCounter()
ObservePostsSearchDuration(elapsed float64)
IncrementFilesSearchCounter()
ObserveFilesSearchDuration(elapsed float64)
ObserveStoreMethodDuration(method, success string, elapsed float64)
ObserveApiEndpointDuration(endpoint, method, statusCode string, elapsed float64)
IncrementPostIndexCounter()
IncrementFileIndexCounter()
IncrementUserIndexCounter()
IncrementChannelIndexCounter()

Просмотреть файл

@@ -75,6 +75,16 @@ func (_m *MetricsInterface) IncrementEtagMissCounter(route string) {
_m.Called(route)
}
// IncrementFileIndexCounter provides a mock function with given fields:
func (_m *MetricsInterface) IncrementFileIndexCounter() {
_m.Called()
}
// IncrementFilesSearchCounter provides a mock function with given fields:
func (_m *MetricsInterface) IncrementFilesSearchCounter() {
_m.Called()
}
// IncrementHttpError provides a mock function with given fields:
func (_m *MetricsInterface) IncrementHttpError() {
_m.Called()
@@ -205,6 +215,11 @@ func (_m *MetricsInterface) ObserveEnabledUsers(users int64) {
_m.Called(users)
}
// ObserveFilesSearchDuration provides a mock function with given fields: elapsed
func (_m *MetricsInterface) ObserveFilesSearchDuration(elapsed float64) {
_m.Called(elapsed)
}
// ObservePluginApiDuration provides a mock function with given fields: pluginID, apiName, success, elapsed
func (_m *MetricsInterface) ObservePluginApiDuration(pluginID string, apiName string, success bool, elapsed float64) {
_m.Called(pluginID, apiName, success, elapsed)

Просмотреть файл

@@ -5822,6 +5822,10 @@
"id": "bleveengine.create_channel_index.error",
"translation": "Error creating the bleve channel index."
},
{
"id": "bleveengine.create_file_index.error",
"translation": "Error creating the bleve file index."
},
{
"id": "bleveengine.create_post_index.error",
"translation": "Error creating the bleve post index."
@@ -5838,14 +5842,30 @@
"id": "bleveengine.delete_channel_posts.error",
"translation": "Failed to delete channel posts"
},
{
"id": "bleveengine.delete_file.error",
"translation": "Failed to delete the file."
},
{
"id": "bleveengine.delete_files_batch.error",
"translation": "Failed to delete the files."
},
{
"id": "bleveengine.delete_post.error",
"translation": "Failed to delete the post."
},
{
"id": "bleveengine.delete_post_files.error",
"translation": "Fiiled to delete the post files."
},
{
"id": "bleveengine.delete_user.error",
"translation": "Failed to delete the user."
},
{
"id": "bleveengine.delete_user_files.error",
"translation": "Failed to delete the user files."
},
{
"id": "bleveengine.delete_user_posts.error",
"translation": "Failed to delete user posts"
@@ -5854,6 +5874,10 @@
"id": "bleveengine.index_channel.error",
"translation": "Failed to index the channel."
},
{
"id": "bleveengine.index_file.error",
"translation": "Failed to index the file."
},
{
"id": "bleveengine.index_post.error",
"translation": "Failed to index the post."
@@ -5866,6 +5890,10 @@
"id": "bleveengine.indexer.do_job.bulk_index_channels.batch_error",
"translation": "Failed to index channel batch."
},
{
"id": "bleveengine.indexer.do_job.bulk_index_files.batch_error",
"translation": "Failed to index file batch."
},
{
"id": "bleveengine.indexer.do_job.bulk_index_posts.batch_error",
"translation": "Failed to index post batch."
@@ -5910,6 +5938,10 @@
"id": "bleveengine.search_channels.error",
"translation": "Channel search failed to complete."
},
{
"id": "bleveengine.search_files.error",
"translation": "File search failed to complete."
},
{
"id": "bleveengine.search_posts.error",
"translation": "Post search failed to complete."
@@ -5930,6 +5962,10 @@
"id": "bleveengine.stop_channel_index.error",
"translation": "Failed to close channel index."
},
{
"id": "bleveengine.stop_file_index.error",
"translation": "Failed to close file index."
},
{
"id": "bleveengine.stop_post_index.error",
"translation": "Failed to close post index."
@@ -6150,6 +6186,10 @@
"id": "ent.elasticsearch.create_template_channels_if_not_exists.template_create_failed",
"translation": "Failed to create Elasticsearch template for channels"
},
{
"id": "ent.elasticsearch.create_template_file_info_if_not_exists.template_create_failed",
"translation": "Failed to create Elasticsearch template for files"
},
{
"id": "ent.elasticsearch.create_template_posts_if_not_exists.template_create_failed",
"translation": "Failed to create Elasticsearch template for posts"
@@ -6174,14 +6214,26 @@
"id": "ent.elasticsearch.delete_channel_posts.error",
"translation": "Failed to delete channel posts"
},
{
"id": "ent.elasticsearch.delete_file.error",
"translation": "Failed to delete file"
},
{
"id": "ent.elasticsearch.delete_post.error",
"translation": "Failed to delete the post"
},
{
"id": "ent.elasticsearch.delete_post_files.error",
"translation": "Failed to delete post files"
},
{
"id": "ent.elasticsearch.delete_user.error",
"translation": "Failed to delete the user"
},
{
"id": "ent.elasticsearch.delete_user_files.error",
"translation": "Failed to delete user files"
},
{
"id": "ent.elasticsearch.delete_user_posts.error",
"translation": "Failed to delete user posts"
@@ -6198,6 +6250,10 @@
"id": "ent.elasticsearch.index_channels_batch.error",
"translation": "Unable to get the channels batch for indexing."
},
{
"id": "ent.elasticsearch.index_file.error",
"translation": "Failed to index the file"
},
{
"id": "ent.elasticsearch.index_post.error",
"translation": "Failed to index the post"
@@ -6250,6 +6306,18 @@
"id": "ent.elasticsearch.search_channels.unmarshall_channel_failed",
"translation": "Failed to decode search results"
},
{
"id": "ent.elasticsearch.search_files.disabled",
"translation": "Elasticsearch files searching is disabled on this server"
},
{
"id": "ent.elasticsearch.search_files.search_failed",
"translation": "Search failed to complete"
},
{
"id": "ent.elasticsearch.search_files.unmarshall_file_failed",
"translation": "Failed to decode search results"
},
{
"id": "ent.elasticsearch.search_posts.disabled",
"translation": "Elasticsearch searching is disabled on this server"

128
model/file_info_list.go Обычный файл
Просмотреть файл

@@ -0,0 +1,128 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package model
import (
"encoding/json"
"io"
"sort"
)
type FileInfoList struct {
Order []string `json:"order"`
FileInfos map[string]*FileInfo `json:"file_infos"`
NextFileInfoId string `json:"next_file_info_id"`
PrevFileInfoId string `json:"prev_file_info_id"`
}
func NewFileInfoList() *FileInfoList {
return &FileInfoList{
Order: make([]string, 0),
FileInfos: make(map[string]*FileInfo),
NextFileInfoId: "",
PrevFileInfoId: "",
}
}
func (o *FileInfoList) ToSlice() []*FileInfo {
var fileInfos []*FileInfo
for _, id := range o.Order {
fileInfos = append(fileInfos, o.FileInfos[id])
}
return fileInfos
}
func (o *FileInfoList) ToJson() string {
b, err := json.Marshal(o)
if err != nil {
return ""
} else {
return string(b)
}
}
func (o *FileInfoList) MakeNonNil() {
if o.Order == nil {
o.Order = make([]string, 0)
}
if o.FileInfos == nil {
o.FileInfos = make(map[string]*FileInfo)
}
}
func (o *FileInfoList) AddOrder(id string) {
if o.Order == nil {
o.Order = make([]string, 0, 128)
}
o.Order = append(o.Order, id)
}
func (o *FileInfoList) AddFileInfo(fileInfo *FileInfo) {
if o.FileInfos == nil {
o.FileInfos = make(map[string]*FileInfo)
}
o.FileInfos[fileInfo.Id] = fileInfo
}
func (o *FileInfoList) UniqueOrder() {
keys := make(map[string]bool)
order := []string{}
for _, fileInfoId := range o.Order {
if _, value := keys[fileInfoId]; !value {
keys[fileInfoId] = true
order = append(order, fileInfoId)
}
}
o.Order = order
}
func (o *FileInfoList) Extend(other *FileInfoList) {
for fileInfoId := range other.FileInfos {
o.AddFileInfo(other.FileInfos[fileInfoId])
}
for _, fileInfoId := range other.Order {
o.AddOrder(fileInfoId)
}
o.UniqueOrder()
}
func (o *FileInfoList) SortByCreateAt() {
sort.Slice(o.Order, func(i, j int) bool {
return o.FileInfos[o.Order[i]].CreateAt > o.FileInfos[o.Order[j]].CreateAt
})
}
func (o *FileInfoList) Etag() string {
id := "0"
var t int64 = 0
for _, v := range o.FileInfos {
if v.UpdateAt > t {
t = v.UpdateAt
id = v.Id
} else if v.UpdateAt == t && v.Id > id {
t = v.UpdateAt
id = v.Id
}
}
orderId := ""
if len(o.Order) > 0 {
orderId = o.Order[0]
}
return Etag(orderId, id, t)
}
func FileInfoListFromJson(data io.Reader) *FileInfoList {
var o *FileInfoList
json.NewDecoder(data).Decode(&o)
return o
}

37
model/file_info_search_results.go Обычный файл
Просмотреть файл

@@ -0,0 +1,37 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package model
import (
"encoding/json"
"io"
)
type FileInfoSearchMatches map[string][]string
type FileInfoSearchResults struct {
*FileInfoList
Matches FileInfoSearchMatches `json:"matches"`
}
func MakeFileInfoSearchResults(fileInfos *FileInfoList, matches FileInfoSearchMatches) *FileInfoSearchResults {
return &FileInfoSearchResults{
fileInfos,
matches,
}
}
func (o *FileInfoSearchResults) ToJson() string {
b, err := json.Marshal(o)
if err != nil {
return ""
}
return string(b)
}
func FileInfoSearchResultsFromJson(data io.Reader) *FileInfoSearchResults {
var o *FileInfoSearchResults
json.NewDecoder(data).Decode(&o)
return o
}

Просмотреть файл

@@ -164,6 +164,12 @@ type PostForIndexing struct {
ParentCreateAt *int64 `json:"parent_create_at"`
}
type FileForIndexing struct {
FileInfo
ChannelId string `json:"channel_id"`
Content string `json:"content"`
}
// ShallowCopy is an utility function to shallow copy a Post to the given
// destination without touching the internal RWMutex.
func (o *Post) ShallowCopy(dst *Post) error {

Просмотреть файл

@@ -25,6 +25,8 @@ type SearchParams struct {
ExcludedAfterDate string
BeforeDate string
ExcludedBeforeDate string
Extensions []string
ExcludedExtensions []string
OnDate string
ExcludedDate string
OrTerms bool
@@ -106,7 +108,7 @@ func (p *SearchParams) GetExcludedDateMillis() (int64, int64) {
return GetStartOfDayMillis(date, p.TimeZoneOffset), GetEndOfDayMillis(date, p.TimeZoneOffset)
}
var searchFlags = [...]string{"from", "channel", "in", "before", "after", "on"}
var searchFlags = [...]string{"from", "channel", "in", "before", "after", "on", "ext"}
type flag struct {
name string
@@ -265,6 +267,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
excludedBeforeDate := ""
onDate := ""
excludedDate := ""
excludedExtensions := []string{}
extensions := []string{}
for _, flag := range flags {
if flag.name == "in" || flag.name == "channel" {
@@ -297,6 +301,12 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
} else {
onDate = flag.value
}
} else if flag.name == "ext" {
if flag.exclude {
excludedExtensions = append(excludedExtensions, flag.value)
} else {
extensions = append(extensions, flag.value)
}
}
}
@@ -315,6 +325,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
ExcludedAfterDate: excludedAfterDate,
BeforeDate: beforeDate,
ExcludedBeforeDate: excludedBeforeDate,
Extensions: extensions,
ExcludedExtensions: excludedExtensions,
OnDate: onDate,
ExcludedDate: excludedDate,
TimeZoneOffset: timeZoneOffset,
@@ -334,6 +346,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
ExcludedAfterDate: excludedAfterDate,
BeforeDate: beforeDate,
ExcludedBeforeDate: excludedBeforeDate,
Extensions: extensions,
ExcludedExtensions: excludedExtensions,
OnDate: onDate,
ExcludedDate: excludedDate,
TimeZoneOffset: timeZoneOffset,
@@ -345,6 +359,7 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
len(excludedPlainTerms) == 0 && len(excludedHashtagTerms) == 0 &&
(len(inChannels) != 0 || len(fromUsers) != 0 ||
len(excludedChannels) != 0 || len(excludedUsers) != 0 ||
len(extensions) != 0 || len(excludedExtensions) != 0 ||
afterDate != "" || excludedAfterDate != "" ||
beforeDate != "" || excludedBeforeDate != "" ||
onDate != "" || excludedDate != "") {
@@ -360,6 +375,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
ExcludedAfterDate: excludedAfterDate,
BeforeDate: beforeDate,
ExcludedBeforeDate: excludedBeforeDate,
Extensions: extensions,
ExcludedExtensions: excludedExtensions,
OnDate: onDate,
ExcludedDate: excludedDate,
TimeZoneOffset: timeZoneOffset,

Просмотреть файл

@@ -1043,13 +1043,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "words words",
Output: []*SearchParams{
{
Terms: "words words",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "words words",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1058,13 +1060,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-word1 -word2",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "word1 word2",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "word1 word2",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1073,13 +1077,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "\"my stuff\"",
Output: []*SearchParams{
{
Terms: "\"my stuff\"",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "\"my stuff\"",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1088,13 +1094,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-\"my stuff\"",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "\"my stuff\"",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "\"my stuff\"",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1103,13 +1111,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "#words #words",
Output: []*SearchParams{
{
Terms: "#words #words",
ExcludedTerms: "",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "#words #words",
ExcludedTerms: "",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1118,22 +1128,26 @@ func TestParseSearchParams(t *testing.T) {
Input: "#words words",
Output: []*SearchParams{
{
Terms: "words",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "words",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
{
Terms: "#words",
ExcludedTerms: "",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "#words",
ExcludedTerms: "",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1142,13 +1156,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-#hashtag",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "#hashtag",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "#hashtag",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1157,13 +1173,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-#hashtag1 -#hashtag2",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "#hashtag1 #hashtag2",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "#hashtag1 #hashtag2",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1172,13 +1190,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "#hashtag1 -#hashtag2",
Output: []*SearchParams{
{
Terms: "#hashtag1",
ExcludedTerms: "#hashtag2",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "#hashtag1",
ExcludedTerms: "#hashtag2",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1187,22 +1207,26 @@ func TestParseSearchParams(t *testing.T) {
Input: "word1 #hashtag1 -#hashtag2 -word2",
Output: []*SearchParams{
{
Terms: "word1",
ExcludedTerms: "word2",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "word1",
ExcludedTerms: "word2",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
{
Terms: "#hashtag1",
ExcludedTerms: "#hashtag2",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "#hashtag1",
ExcludedTerms: "#hashtag2",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1211,13 +1235,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "in:channel",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1226,13 +1252,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-in:channel",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{"channel"},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{"channel"},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1241,13 +1269,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1256,13 +1286,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing -in:channel",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{"channel"},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{"channel"},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1271,13 +1303,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "in:channel testing",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1286,13 +1320,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "in:channel in:otherchannel",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel", "otherchannel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel", "otherchannel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1301,13 +1337,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel in:otherchannel",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel", "otherchannel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel", "otherchannel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1316,13 +1354,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel from:someone",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{"someone"},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{"someone"},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1331,13 +1371,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel -from:someone",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{"someone"},
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{"someone"},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1346,13 +1388,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel from:someone -from:someoneelse",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{"someone"},
ExcludedUsers: []string{"someoneelse"},
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{"channel"},
ExcludedChannels: []string{},
FromUsers: []string{"someone"},
ExcludedUsers: []string{"someoneelse"},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1361,13 +1405,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "##hashtag +#plus+",
Output: []*SearchParams{
{
Terms: "#hashtag #plus",
ExcludedTerms: "",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "#hashtag #plus",
ExcludedTerms: "",
IsHashtag: true,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1376,13 +1422,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "wildcar*",
Output: []*SearchParams{
{
Terms: "wildcar*",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "wildcar*",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1391,14 +1439,16 @@ func TestParseSearchParams(t *testing.T) {
Input: "after:2018-8-1 testing",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
AfterDate: "2018-8-1",
ExcludedAfterDate: "",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
AfterDate: "2018-8-1",
ExcludedAfterDate: "",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1407,14 +1457,16 @@ func TestParseSearchParams(t *testing.T) {
Input: "-after:2018-8-1 testing",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
AfterDate: "",
ExcludedAfterDate: "2018-8-1",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
AfterDate: "",
ExcludedAfterDate: "2018-8-1",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1423,15 +1475,17 @@ func TestParseSearchParams(t *testing.T) {
Input: "on:2018-8-1 testing",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
OnDate: "2018-8-1",
AfterDate: "",
ExcludedAfterDate: "",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
OnDate: "2018-8-1",
AfterDate: "",
ExcludedAfterDate: "",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1440,14 +1494,16 @@ func TestParseSearchParams(t *testing.T) {
Input: "-on:2018-8-1 testing",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
AfterDate: "",
ExcludedDate: "2018-8-1",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "testing",
ExcludedTerms: "",
AfterDate: "",
ExcludedDate: "2018-8-1",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1456,14 +1512,16 @@ func TestParseSearchParams(t *testing.T) {
Input: "after:2018-8-1",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
AfterDate: "2018-8-1",
ExcludedDate: "",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "",
AfterDate: "2018-8-1",
ExcludedDate: "",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1472,15 +1530,17 @@ func TestParseSearchParams(t *testing.T) {
Input: "before:2018-8-1",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
BeforeDate: "2018-8-1",
AfterDate: "",
ExcludedDate: "",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Terms: "",
ExcludedTerms: "",
BeforeDate: "2018-8-1",
AfterDate: "",
ExcludedDate: "",
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
@@ -1498,6 +1558,110 @@ func TestParseSearchParams(t *testing.T) {
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
{
Name: "input is two words separated with : and should result in a single Extension",
Input: "ext:png",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{"png"},
ExcludedExtensions: []string{},
},
},
},
{
Name: "input is two words separated with :, prefied with - and should result in a single ExcludedExtensions",
Input: "-ext:png",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{"png"},
},
},
},
{
Name: "input is two words separated with : with a prefixed word should result in a single Extension and a term",
Input: "testing ext:png",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{"png"},
ExcludedExtensions: []string{},
},
},
},
{
Name: "input is two words separated with : with a prefixed word should result in a single ExcludedExtension and a term",
Input: "testing -ext:png",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{"png"},
},
},
},
{
Name: "input is two words separated with : with a postfix word should result in a single Extension and a term",
Input: "ext:png testing",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{"png"},
ExcludedExtensions: []string{},
},
},
},
{
Name: "input is four words separated with : should result in a two Extensions",
Input: "ext:png ext:jpg",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{"png", "jpg"},
ExcludedExtensions: []string{},
},
},
},

Просмотреть файл

@@ -24,12 +24,14 @@ import (
const (
EngineName = "bleve"
PostIndex = "posts"
FileIndex = "files"
UserIndex = "users"
ChannelIndex = "channels"
)
type BleveEngine struct {
PostIndex bleve.Index
FileIndex bleve.Index
UserIndex bleve.Index
ChannelIndex bleve.Index
Mutex sync.RWMutex
@@ -83,6 +85,23 @@ func getPostIndexMapping() *mapping.IndexMappingImpl {
return indexMapping
}
func getFileIndexMapping() *mapping.IndexMappingImpl {
fileMapping := bleve.NewDocumentMapping()
fileMapping.AddFieldMappingsAt("Id", keywordMapping)
fileMapping.AddFieldMappingsAt("CreatorId", keywordMapping)
fileMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
fileMapping.AddFieldMappingsAt("CreateAt", dateMapping)
fileMapping.AddFieldMappingsAt("Name", standardMapping)
fileMapping.AddFieldMappingsAt("Content", standardMapping)
fileMapping.AddFieldMappingsAt("Extension", keywordMapping)
fileMapping.AddFieldMappingsAt("Content", standardMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.AddDocumentMapping("_default", fileMapping)
return indexMapping
}
func getUserIndexMapping() *mapping.IndexMappingImpl {
userMapping := bleve.NewDocumentMapping()
userMapping.AddFieldMappingsAt("Id", keywordMapping)
@@ -132,6 +151,11 @@ func (b *BleveEngine) openIndexes() *model.AppError {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, err.Error(), http.StatusInternalServerError)
}
b.FileIndex, err = b.createOrOpenIndex(FileIndex, getFileIndexMapping())
if err != nil {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_file_index.error", nil, err.Error(), http.StatusInternalServerError)
}
b.UserIndex, err = b.createOrOpenIndex(UserIndex, getUserIndexMapping())
if err != nil {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, err.Error(), http.StatusInternalServerError)
@@ -165,6 +189,10 @@ func (b *BleveEngine) closeIndexes() *model.AppError {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, err.Error(), http.StatusInternalServerError)
}
if err := b.FileIndex.Close(); err != nil {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_file_index.error", nil, err.Error(), http.StatusInternalServerError)
}
if err := b.UserIndex.Close(); err != nil {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, err.Error(), http.StatusInternalServerError)
}

Просмотреть файл

@@ -98,6 +98,10 @@ func (s *BleveEngineTestSuite) TestBleveSearchStoreTests() {
s.Run("TestSearchPostStore", func() {
searchtest.TestSearchPostStore(s.T(), s.Store, searchTestEngine)
})
s.Run("TestSearchFileInfoStore", func() {
searchtest.TestSearchFileInfoStore(s.T(), s.Store, searchTestEngine)
})
}
func (s *BleveEngineTestSuite) TestDeleteChannelPosts() {

Просмотреть файл

@@ -36,6 +36,16 @@ type BLVPost struct {
Attachments string
}
type BLVFile struct {
Id string
CreatorId string
ChannelId string
CreateAt int64
Name string
Content string
Extension string
}
func BLVChannelFromChannel(channel *model.Channel) *BLVChannel {
displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ")
nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"})
@@ -114,3 +124,27 @@ func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost {
Hashtags: strings.Fields(post.Hashtags),
}
}
func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile {
return &BLVFile{
Id: fileInfo.Id,
ChannelId: channelId,
CreatorId: fileInfo.CreatorId,
CreateAt: fileInfo.CreateAt,
Content: fileInfo.Content,
Extension: fileInfo.Extension,
Name: fileInfo.Name,
}
}
func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile {
return &BLVFile{
Id: file.Id,
ChannelId: file.ChannelId,
CreatorId: file.CreatorId,
CreateAt: file.CreateAt,
Content: file.Content,
Extension: file.Extension,
Name: file.Name,
}
}

Просмотреть файл

@@ -354,6 +354,30 @@ func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing,
return lastCreateAt, nil
}
func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (int64, *model.AppError) {
lastCreateAt := int64(0)
batch := worker.engine.FileIndex.NewBatch()
for _, file := range files {
if file.DeleteAt == 0 {
searchFile := bleveengine.BLVFileFromFileForIndexing(file)
batch.Index(searchFile.Id, searchFile)
} else {
batch.Delete(file.Id)
}
lastCreateAt = file.CreateAt
}
worker.engine.Mutex.RLock()
defer worker.engine.Mutex.RUnlock()
if err := worker.engine.FileIndex.Batch(batch); err != nil {
return 0, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_files.batch_error", nil, err.Error(), http.StatusInternalServerError)
}
return lastCreateAt, nil
}
func (worker *BleveIndexerWorker) IndexChannelsBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
endTime := progress.LastEntityTime + int64(*worker.jobServer.Config().BleveSettings.BulkIndexingTimeWindowSeconds*1000)

Просмотреть файл

@@ -15,6 +15,7 @@ import (
)
const DeletePostsBatchSize = 500
const DeleteFilesBatchSize = 500
func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError {
b.Mutex.RLock()
@@ -507,3 +508,320 @@ func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError {
}
return nil
}
func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
blvFile := BLVFileFromFileInfo(file, channelId)
if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil {
return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *BleveEngine) SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) {
channelQueries := []query.Query{}
for _, channel := range *channels {
channelIdQ := bleve.NewTermQuery(channel.Id)
channelIdQ.SetField("ChannelId")
channelQueries = append(channelQueries, channelIdQ)
}
channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
var termQueries []query.Query
var notTermQueries []query.Query
var filters []query.Query
var notFilters []query.Query
for i, params := range searchParams {
var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
if searchParams[0].OrTerms {
termOperator = query.MatchQueryOperatorOr
}
// Date, channels and FromUsers filters come in all
// searchParams iteration, and as they are global to the
// query, we only need to process them once
if i == 0 {
if len(params.InChannels) > 0 {
inChannels := []query.Query{}
for _, channelId := range params.InChannels {
channelQ := bleve.NewTermQuery(channelId)
channelQ.SetField("ChannelId")
inChannels = append(inChannels, channelQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
}
if len(params.ExcludedChannels) > 0 {
excludedChannels := []query.Query{}
for _, channelId := range params.ExcludedChannels {
channelQ := bleve.NewTermQuery(channelId)
channelQ.SetField("ChannelId")
excludedChannels = append(excludedChannels, channelQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
}
if len(params.FromUsers) > 0 {
fromUsers := []query.Query{}
for _, userId := range params.FromUsers {
userQ := bleve.NewTermQuery(userId)
userQ.SetField("CreatorId")
fromUsers = append(fromUsers, userQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
}
if len(params.ExcludedUsers) > 0 {
excludedUsers := []query.Query{}
for _, userId := range params.ExcludedUsers {
userQ := bleve.NewTermQuery(userId)
userQ.SetField("CreatorId")
excludedUsers = append(excludedUsers, userQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
}
if len(params.Extensions) > 0 {
extensions := []query.Query{}
for _, extension := range params.Extensions {
extensionQ := bleve.NewTermQuery(extension)
extensionQ.SetField("Extension")
extensions = append(extensions, extensionQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(extensions...))
}
if len(params.ExcludedExtensions) > 0 {
excludedExtensions := []query.Query{}
for _, extension := range params.ExcludedExtensions {
extensionQ := bleve.NewTermQuery(extension)
extensionQ.SetField("Extension")
excludedExtensions = append(excludedExtensions, extensionQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...))
}
if params.OnDate != "" {
before, after := params.GetOnDateMillis()
beforeFloat64 := float64(before)
afterFloat64 := float64(after)
onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
onDateQ.SetField("CreateAt")
filters = append(filters, onDateQ)
} else {
if params.AfterDate != "" || params.BeforeDate != "" {
var min, max *float64
if params.AfterDate != "" {
minf := float64(params.GetAfterDateMillis())
min = &minf
}
if params.BeforeDate != "" {
maxf := float64(params.GetBeforeDateMillis())
max = &maxf
}
dateQ := bleve.NewNumericRangeQuery(min, max)
dateQ.SetField("CreateAt")
filters = append(filters, dateQ)
}
if params.ExcludedAfterDate != "" {
minf := float64(params.GetExcludedAfterDateMillis())
dateQ := bleve.NewNumericRangeQuery(&minf, nil)
dateQ.SetField("CreateAt")
notFilters = append(notFilters, dateQ)
}
if params.ExcludedBeforeDate != "" {
maxf := float64(params.GetExcludedBeforeDateMillis())
dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
dateQ.SetField("CreateAt")
notFilters = append(notFilters, dateQ)
}
if params.ExcludedDate != "" {
before, after := params.GetExcludedDateMillis()
beforef := float64(before)
afterf := float64(after)
onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
onDateQ.SetField("CreateAt")
notFilters = append(notFilters, onDateQ)
}
}
}
if len(params.Terms) > 0 {
terms := []string{}
for _, term := range strings.Split(params.Terms, " ") {
if strings.HasSuffix(term, "*") {
nameQ := bleve.NewWildcardQuery(term)
nameQ.SetField("Name")
contentQ := bleve.NewWildcardQuery(term)
contentQ.SetField("Content")
termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
} else {
terms = append(terms, term)
}
}
if len(terms) > 0 {
nameQ := bleve.NewMatchQuery(strings.Join(terms, " "))
nameQ.SetField("Name")
nameQ.SetOperator(termOperator)
contentQ := bleve.NewMatchQuery(strings.Join(terms, " "))
contentQ.SetField("Content")
contentQ.SetOperator(termOperator)
termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
}
}
if len(params.ExcludedTerms) > 0 {
nameQ := bleve.NewMatchQuery(params.ExcludedTerms)
nameQ.SetField("Name")
nameQ.SetOperator(termOperator)
contentQ := bleve.NewMatchQuery(params.ExcludedTerms)
contentQ.SetField("Content")
contentQ.SetOperator(termOperator)
notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
}
}
allTermsQ := bleve.NewBooleanQuery()
allTermsQ.AddMustNot(notTermQueries...)
if searchParams[0].OrTerms {
allTermsQ.AddShould(termQueries...)
} else {
allTermsQ.AddMust(termQueries...)
}
query := bleve.NewBooleanQuery()
query.AddMust(channelDisjunctionQ)
if len(termQueries) > 0 || len(notTermQueries) > 0 {
query.AddMust(allTermsQ)
}
if len(filters) > 0 {
query.AddMust(bleve.NewConjunctionQuery(filters...))
}
if len(notFilters) > 0 {
query.AddMustNot(notFilters...)
}
search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
search.SortBy([]string{"-CreateAt"})
results, err := b.FileIndex.Search(search)
if err != nil {
return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, err.Error(), http.StatusInternalServerError)
}
fileIds := []string{}
for _, r := range results.Hits {
fileIds = append(fileIds, r.ID)
}
return fileIds, nil
}
func (b *BleveEngine) DeleteFile(fileID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
if err := b.FileIndex.Delete(fileID); err != nil {
return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
resultsCount := int64(0)
for {
// As we are deleting the files after fetching them, we need to keep
// From fixed always to 0
searchRequest.From = 0
searchRequest.Size = batchSize
results, err := b.FileIndex.Search(searchRequest)
if err != nil {
return -1, err
}
batch := b.FileIndex.NewBatch()
for _, file := range results.Hits {
batch.Delete(file.ID)
}
if err := b.FileIndex.Batch(batch); err != nil {
return -1, err
}
resultsCount += int64(results.Hits.Len())
if results.Hits.Len() < batchSize {
break
}
}
return resultsCount, nil
}
func (b *BleveEngine) DeleteUserFiles(userID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
query := bleve.NewTermQuery(userID)
query.SetField("CreatorId")
search := bleve.NewSearchRequest(query)
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeleteUserFiles",
"bleveengine.delete_user_files.error", nil,
err.Error(), http.StatusInternalServerError)
}
mlog.Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted))
return nil
}
func (b *BleveEngine) DeletePostFiles(postID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
query := bleve.NewTermQuery(postID)
query.SetField("PostId")
search := bleve.NewSearchRequest(query)
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeletePostFiles",
"bleveengine.delete_post_files.error", nil,
err.Error(), http.StatusInternalServerError)
}
mlog.Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int64("deleted", deleted))
return nil
}
func (b *BleveEngine) DeleteFilesBatch(endTime, limit int64) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
endTimeFloat := float64(endTime)
query := bleve.NewNumericRangeQuery(nil, &endTimeFloat)
query.SetField("CreateAt")
search := bleve.NewSearchRequestOptions(query, int(limit), 0, false)
search.SortBy([]string{"-CreateAt"})
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeleteFilesBatch",
"bleveengine.delete_files_batch.error", nil,
err.Error(), http.StatusInternalServerError)
}
mlog.Info("Files in batch deleted", mlog.Int64("endTime", endTime), mlog.Int64("limit", limit), mlog.Int64("deleted", deleted))
return nil
}

Просмотреть файл

@@ -32,6 +32,12 @@ type SearchEngineInterface interface {
SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError)
SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError)
DeleteUser(user *model.User) *model.AppError
IndexFile(file *model.FileInfo, channelId string) *model.AppError
SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError)
DeleteFile(fileID string) *model.AppError
DeletePostFiles(postID string) *model.AppError
DeleteUserFiles(userID string) *model.AppError
DeleteFilesBatch(endTime, limit int64) *model.AppError
TestConfig(cfg *model.Config) *model.AppError
PurgeIndexes() *model.AppError
RefreshIndexes() *model.AppError

Просмотреть файл

@@ -64,6 +64,38 @@ func (_m *SearchEngineInterface) DeleteChannelPosts(channelID string) *model.App
return r0
}
// DeleteFile provides a mock function with given fields: fileID
func (_m *SearchEngineInterface) DeleteFile(fileID string) *model.AppError {
ret := _m.Called(fileID)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
r0 = rf(fileID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// DeleteFilesBatch provides a mock function with given fields: endTime, limit
func (_m *SearchEngineInterface) DeleteFilesBatch(endTime int64, limit int64) *model.AppError {
ret := _m.Called(endTime, limit)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(int64, int64) *model.AppError); ok {
r0 = rf(endTime, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// DeletePost provides a mock function with given fields: post
func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError {
ret := _m.Called(post)
@@ -80,6 +112,22 @@ func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError {
return r0
}
// DeletePostFiles provides a mock function with given fields: postID
func (_m *SearchEngineInterface) DeletePostFiles(postID string) *model.AppError {
ret := _m.Called(postID)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
r0 = rf(postID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// DeleteUser provides a mock function with given fields: user
func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError {
ret := _m.Called(user)
@@ -96,6 +144,22 @@ func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError {
return r0
}
// DeleteUserFiles provides a mock function with given fields: userID
func (_m *SearchEngineInterface) DeleteUserFiles(userID string) *model.AppError {
ret := _m.Called(userID)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
r0 = rf(userID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// DeleteUserPosts provides a mock function with given fields: userID
func (_m *SearchEngineInterface) DeleteUserPosts(userID string) *model.AppError {
ret := _m.Called(userID)
@@ -156,6 +220,22 @@ func (_m *SearchEngineInterface) IndexChannel(channel *model.Channel) *model.App
return r0
}
// IndexFile provides a mock function with given fields: file, channelId
func (_m *SearchEngineInterface) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
ret := _m.Called(file, channelId)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(*model.FileInfo, string) *model.AppError); ok {
r0 = rf(file, channelId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// IndexPost provides a mock function with given fields: post, teamId
func (_m *SearchEngineInterface) IndexPost(post *model.Post, teamId string) *model.AppError {
ret := _m.Called(post, teamId)
@@ -315,6 +395,31 @@ func (_m *SearchEngineInterface) SearchChannels(teamId string, term string) ([]s
return r0, r1
}
// SearchFiles provides a mock function with given fields: channels, searchParams, page, perPage
func (_m *SearchEngineInterface) SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, *model.AppError) {
ret := _m.Called(channels, searchParams, page, perPage)
var r0 []string
if rf, ok := ret.Get(0).(func(*model.ChannelList, []*model.SearchParams, int, int) []string); ok {
r0 = rf(channels, searchParams, page, perPage)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 *model.AppError
if rf, ok := ret.Get(1).(func(*model.ChannelList, []*model.SearchParams, int, int) *model.AppError); ok {
r1 = rf(channels, searchParams, page, perPage)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*model.AppError)
}
}
return r0, r1
}
// SearchPosts provides a mock function with given fields: channels, searchParams, page, perPage
func (_m *SearchEngineInterface) SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
ret := _m.Called(channels, searchParams, page, perPage)

Просмотреть файл

@@ -2982,6 +2982,24 @@ func (s *OpenTracingLayerFileInfoStore) Get(id string) (*model.FileInfo, error)
return result, err
}
func (s *OpenTracingLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetByIds")
s.Root.Store.SetContext(newCtx)
defer func() {
s.Root.Store.SetContext(origCtx)
}()
defer span.Finish()
result, err := s.FileInfoStore.GetByIds(ids)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
}
return result, err
}
func (s *OpenTracingLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetByPath")
@@ -3139,6 +3157,24 @@ func (s *OpenTracingLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileI
return result, err
}
func (s *OpenTracingLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.Search")
s.Root.Store.SetContext(newCtx)
defer func() {
s.Root.Store.SetContext(origCtx)
}()
defer span.Finish()
result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
}
return result, err
}
func (s *OpenTracingLayerFileInfoStore) SetContent(fileId string, content string) error {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.SetContent")

Просмотреть файл

@@ -3190,6 +3190,26 @@ func (s *RetryLayerFileInfoStore) Get(id string) (*model.FileInfo, error) {
}
func (s *RetryLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
tries := 0
for {
result, err := s.FileInfoStore.GetByIds(ids)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
}
}
func (s *RetryLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
tries := 0
@@ -3356,6 +3376,26 @@ func (s *RetryLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, e
}
func (s *RetryLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) {
tries := 0
for {
result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
}
}
func (s *RetryLayerFileInfoStore) SetContent(fileId string, content string) error {
tries := 0

Просмотреть файл

@@ -24,6 +24,7 @@ func (c *SearchChannelStore) deleteChannelIndex(channel *model.Channel) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteChannel(channel); err != nil {
mlog.Warn("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
})
@@ -39,6 +40,7 @@ func (c *SearchChannelStore) indexChannel(channel *model.Channel) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.IndexChannel(channel); err != nil {
mlog.Warn("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
})

197
store/searchlayer/file_info_layer.go Обычный файл
Просмотреть файл

@@ -0,0 +1,197 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
"github.com/mattermost/mattermost-server/v5/mlog"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/services/searchengine"
"github.com/mattermost/mattermost-server/v5/store"
)
type SearchFileInfoStore struct {
store.FileInfoStore
rootStore *SearchStore
}
func (s SearchFileInfoStore) indexFile(file *model.FileInfo) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if file.PostId == "" {
return
}
post, postErr := s.rootStore.Post().GetSingle(file.PostId)
if postErr != nil {
mlog.Error("Couldn't get post for file for SearchEngine indexing.", mlog.String("post_id", file.PostId), mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id), mlog.Err(postErr))
return
}
if err := engineCopy.IndexFile(file, post.ChannelId); err != nil {
mlog.Error("Encountered error indexing file", mlog.String("file_info_id", file.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Indexed file in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndex(fileID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteFile(fileID); err != nil {
mlog.Error("Encountered error deleting file", mlog.String("file_info_id", fileID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed file from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", fileID))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndexForUser(userID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteUserFiles(userID); err != nil {
mlog.Error("Encountered error deleting files for user", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed user's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", userID))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndexForPost(postID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeletePostFiles(postID); err != nil {
mlog.Error("Encountered error deleting files for post", mlog.String("post_id", postID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed post's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", postID))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndexBatch(endTime, limit int64) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteFilesBatch(endTime, limit); err != nil {
mlog.Error("Encountered error deleting a batch of files", mlog.Int64("limit", limit), mlog.Int64("end_time", endTime), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed batch of files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.Int64("end_time", endTime), mlog.Int64("limit", limit))
})
}
}
}
func (s SearchFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
nfile, err := s.FileInfoStore.Save(info)
if err == nil {
s.indexFile(nfile)
}
return nfile, err
}
func (s SearchFileInfoStore) SetContent(fileID, content string) error {
err := s.FileInfoStore.SetContent(fileID, content)
if err == nil {
nfile, err2 := s.FileInfoStore.Get(fileID)
if err2 == nil {
nfile.Content = content
s.indexFile(nfile)
}
}
return err
}
func (s SearchFileInfoStore) AttachToPost(fileId, postId, creatorId string) error {
err := s.FileInfoStore.AttachToPost(fileId, postId, creatorId)
if err == nil {
nFileInfo, err2 := s.FileInfoStore.Get(fileId)
if err2 == nil {
s.indexFile(nFileInfo)
}
}
return err
}
func (s SearchFileInfoStore) DeleteForPost(postId string) (string, error) {
result, err := s.FileInfoStore.DeleteForPost(postId)
if err == nil {
s.deleteFileIndexForPost(postId)
}
return result, err
}
func (s SearchFileInfoStore) PermanentDelete(fileId string) error {
err := s.FileInfoStore.PermanentDelete(fileId)
if err == nil {
s.deleteFileIndex(fileId)
}
return err
}
func (s SearchFileInfoStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
result, err := s.FileInfoStore.PermanentDeleteBatch(endTime, limit)
if err == nil {
s.deleteFileIndexBatch(endTime, limit)
}
return result, err
}
func (s SearchFileInfoStore) PermanentDeleteByUser(userId string) (int64, error) {
result, err := s.FileInfoStore.PermanentDeleteByUser(userId)
if err == nil {
s.deleteFileIndexForUser(userId)
}
return result, err
}
func (s SearchFileInfoStore) Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsSearchEnabled() {
userChannels, nErr := s.rootStore.Channel().GetChannels(teamId, userId, paramsList[0].IncludeDeletedChannels, 0)
if nErr != nil {
return nil, nErr
}
fileIds, appErr := engine.SearchFiles(userChannels, paramsList, page, perPage)
if appErr != nil {
mlog.Error("Encountered error on Search.", mlog.String("search_engine", engine.GetName()), mlog.Err(appErr))
continue
}
mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
// Get the files
filesList := model.NewFileInfoList()
if len(fileIds) > 0 {
files, nErr := s.FileInfoStore.GetByIds(fileIds)
if nErr != nil {
return nil, nErr
}
for _, f := range files {
filesList.AddFileInfo(f)
filesList.AddOrder(f.Id)
}
}
return filesList, nil
}
}
if *s.rootStore.getConfig().SqlSettings.DisableDatabaseSearch {
mlog.Debug("Returning empty results for file Search as the database search is disabled")
return model.NewFileInfoList(), nil
}
mlog.Debug("Using database search because no other search engine is available")
return s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
}

Просмотреть файл

@@ -19,6 +19,7 @@ type SearchStore struct {
team *SearchTeamStore
channel *SearchChannelStore
post *SearchPostStore
fileInfo *SearchFileInfoStore
configValue atomic.Value
}
@@ -32,6 +33,7 @@ func NewSearchLayer(baseStore store.Store, searchEngine *searchengine.Broker, cf
searchStore.post = &SearchPostStore{PostStore: baseStore.Post(), rootStore: searchStore}
searchStore.team = &SearchTeamStore{TeamStore: baseStore.Team(), rootStore: searchStore}
searchStore.user = &SearchUserStore{UserStore: baseStore.User(), rootStore: searchStore}
searchStore.fileInfo = &SearchFileInfoStore{FileInfoStore: baseStore.FileInfo(), rootStore: searchStore}
return searchStore
}
@@ -52,6 +54,10 @@ func (s *SearchStore) Post() store.PostStore {
return s.post
}
func (s *SearchStore) FileInfo() store.FileInfoStore {
return s.fileInfo
}
func (s *SearchStore) Team() store.TeamStore {
return s.team
}

Просмотреть файл

@@ -28,6 +28,7 @@ func (s SearchPostStore) indexPost(post *model.Post) {
}
if err := engineCopy.IndexPost(post, channel.TeamId); err != nil {
mlog.Warn("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Indexed post in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
})
@@ -41,6 +42,7 @@ func (s SearchPostStore) deletePostIndex(post *model.Post) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeletePost(post); err != nil {
mlog.Warn("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed post from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
})
@@ -54,6 +56,7 @@ func (s SearchPostStore) deleteChannelPostsIndex(channelID string) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteChannelPosts(channelID); err != nil {
mlog.Warn("Encountered error deleting channel posts", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed all channel posts from the index in search engine", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()))
})
@@ -67,6 +70,7 @@ func (s SearchPostStore) deleteUserPostsIndex(userID string) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteUserPosts(userID); err != nil {
mlog.Warn("Encountered error deleting user posts", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed all user posts from the index in search engine", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()))
})

1646
store/searchtest/file_info_layer.go Обычный файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@@ -392,6 +392,36 @@ func (th *SearchTestHelper) createPost(userID, channelID, message, hashtags, pos
return post, nil
}
func (th *SearchTestHelper) createFileInfoModel(creatorID, postID, name, content, extension, mimeType string, createAt, size int64) *model.FileInfo {
return &model.FileInfo{
CreatorId: creatorID,
PostId: postID,
CreateAt: createAt,
UpdateAt: createAt,
DeleteAt: 0,
Name: name,
Content: content,
Path: name,
Extension: extension,
Size: size,
MimeType: mimeType,
}
}
func (th *SearchTestHelper) createFileInfo(creatorID, postID, name, content, extension, mimeType string, createAt, size int64) (*model.FileInfo, error) {
var creationTime int64 = 1000000
if createAt > 0 {
creationTime = createAt
}
fileInfoModel := th.createFileInfoModel(creatorID, postID, name, content, extension, mimeType, creationTime, size)
fileInfo, appError := th.Store.FileInfo().Save(fileInfoModel)
if appError != nil {
return nil, errors.New(appError.Error())
}
return fileInfo, nil
}
func (th *SearchTestHelper) createReply(userID, message, hashtags string, parent *model.Post, createAt int64, pinned bool) (*model.Post, error) {
replyModel := th.createPostModel(userID, parent.ChannelId, message, hashtags, parent.Type, createAt, pinned)
replyModel.ParentId = parent.Id
@@ -411,6 +441,13 @@ func (th *SearchTestHelper) deleteUserPosts(userID string) error {
return nil
}
func (th *SearchTestHelper) deleteUserFileInfos(userID string) error {
if _, err := th.Store.FileInfo().PermanentDeleteByUser(userID); err != nil {
return errors.New(err.Error())
}
return nil
}
func (th *SearchTestHelper) addUserToTeams(user *model.User, teamIDS []string) error {
for _, teamID := range teamIDS {
_, err := th.Store.Team().SaveMember(&model.TeamMember{TeamId: teamID, UserId: user.Id}, -1)
@@ -467,6 +504,15 @@ func (th *SearchTestHelper) checkPostInSearchResults(t *testing.T, postID string
assert.Contains(t, postIDS, postID, "Did not find expected post in search results.")
}
func (th *SearchTestHelper) checkFileInfoInSearchResults(t *testing.T, fileID string, searchResults map[string]*model.FileInfo) {
t.Helper()
fileIDS := make([]string, len(searchResults))
for ID := range searchResults {
fileIDS = append(fileIDS, ID)
}
assert.Contains(t, fileIDS, fileID, "Did not find expected file in search results.")
}
func (th *SearchTestHelper) checkChannelIdsMatch(t *testing.T, expected []string, results *model.ChannelList) {
t.Helper()
channelIds := make([]string, len(*results))

Просмотреть файл

@@ -6,11 +6,15 @@ package sqlstore
import (
"database/sql"
"fmt"
"regexp"
"strconv"
"strings"
sq "github.com/Masterminds/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost-server/v5/einterfaces"
"github.com/mattermost/mattermost-server/v5/mlog"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/store"
)
@@ -73,6 +77,9 @@ func (fs SqlFileInfoStore) createIndexesIfNotExists() {
fs.CreateIndexIfNotExists("idx_fileinfo_create_at", "FileInfo", "CreateAt")
fs.CreateIndexIfNotExists("idx_fileinfo_delete_at", "FileInfo", "DeleteAt")
fs.CreateIndexIfNotExists("idx_fileinfo_postid_at", "FileInfo", "PostId")
fs.CreateIndexIfNotExists("idx_fileinfo_extension_at", "FileInfo", "Extension")
fs.CreateFullTextIndexIfNotExists("idx_fileinfo_name_txt", "FileInfo", "Name")
fs.CreateFullTextIndexIfNotExists("idx_fileinfo_content_txt", "FileInfo", "Content")
}
func (fs SqlFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
@@ -87,6 +94,26 @@ func (fs SqlFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
return info, nil
}
func (fs SqlFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
query := fs.getQueryBuilder().
Select("*").
From("FileInfo").
Where(sq.Eq{"Id": ids}).
Where(sq.Eq{"DeleteAt": 0}).
OrderBy("CreateAt DESC")
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "file_info_tosql")
}
var infos []*model.FileInfo
if _, err := fs.GetReplica().Select(&infos, queryString, args...); err != nil {
return nil, errors.Wrap(err, "failed to find FileInfos")
}
return infos, nil
}
func (fs SqlFileInfoStore) Upsert(info *model.FileInfo) (*model.FileInfo, error) {
info.PreSave()
if err := info.IsValid(); err != nil {
@@ -393,3 +420,176 @@ func (fs SqlFileInfoStore) PermanentDeleteByUser(userId string) (int64, error) {
return rowsAffected, nil
}
func (fs SqlFileInfoStore) Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) {
// Since we don't support paging for DB search, we just return nothing for later pages
if page > 0 {
return model.NewFileInfoList(), nil
}
if err := model.IsSearchParamsListValid(paramsList); err != nil {
return nil, err
}
query := fs.getQueryBuilder().
Select("FI.*").
From("FileInfo AS FI").
LeftJoin("Posts as P ON FI.PostId=P.Id").
LeftJoin("Channels as C ON C.Id=P.ChannelId").
LeftJoin("ChannelMembers as CM ON C.Id=CM.ChannelId").
Where(sq.Or{sq.Eq{"C.TeamId": teamId}, sq.Eq{"C.TeamId": ""}}).
Where(sq.Eq{"FI.DeleteAt": 0}).
OrderBy("FI.CreateAt DESC").
Limit(100)
for _, params := range paramsList {
params.Terms = removeNonAlphaNumericUnquotedTerms(params.Terms, " ")
if !params.IncludeDeletedChannels {
query = query.Where(sq.Eq{"C.DeleteAt": 0})
}
if !params.SearchWithoutUserId {
query = query.Where(sq.Eq{"CM.UserId": userId})
}
if len(params.InChannels) != 0 {
query = query.Where(sq.Eq{"C.Id": params.InChannels})
}
if len(params.Extensions) != 0 {
query = query.Where(sq.Eq{"FI.Extension": params.Extensions})
}
if len(params.ExcludedExtensions) != 0 {
query = query.Where(sq.NotEq{"FI.Extension": params.ExcludedExtensions})
}
if len(params.ExcludedChannels) != 0 {
query = query.Where(sq.NotEq{"C.Id": params.ExcludedChannels})
}
if len(params.FromUsers) != 0 {
query = query.Where(sq.Eq{"FI.CreatorId": params.FromUsers})
}
if len(params.ExcludedUsers) != 0 {
query = query.Where(sq.NotEq{"FI.CreatorId": params.ExcludedUsers})
}
// handle after: before: on: filters
if len(params.OnDate) > 0 {
onDateStart, onDateEnd := params.GetOnDateMillis()
query = query.Where(sq.Expr("FI.CreateAt BETWEEN ? AND ?", strconv.FormatInt(onDateStart, 10), strconv.FormatInt(onDateEnd, 10)))
} else {
if len(params.ExcludedDate) > 0 {
excludedDateStart, excludedDateEnd := params.GetExcludedDateMillis()
query = query.Where(sq.Expr("FI.CreateAt NOT BETWEEN ? AND ?", strconv.FormatInt(excludedDateStart, 10), strconv.FormatInt(excludedDateEnd, 10)))
}
if len(params.AfterDate) > 0 {
afterDate := params.GetAfterDateMillis()
query = query.Where(sq.GtOrEq{"FI.CreateAt": strconv.FormatInt(afterDate, 10)})
}
if len(params.BeforeDate) > 0 {
beforeDate := params.GetBeforeDateMillis()
query = query.Where(sq.LtOrEq{"FI.CreateAt": strconv.FormatInt(beforeDate, 10)})
}
if len(params.ExcludedAfterDate) > 0 {
afterDate := params.GetExcludedAfterDateMillis()
query = query.Where(sq.Lt{"FI.CreateAt": strconv.FormatInt(afterDate, 10)})
}
if len(params.ExcludedBeforeDate) > 0 {
beforeDate := params.GetExcludedBeforeDateMillis()
query = query.Where(sq.Gt{"FI.CreateAt": strconv.FormatInt(beforeDate, 10)})
}
}
terms := params.Terms
excludedTerms := params.ExcludedTerms
// these chars have special meaning and can be treated as spaces
for _, c := range specialSearchChar {
terms = strings.Replace(terms, c, " ", -1)
excludedTerms = strings.Replace(excludedTerms, c, " ", -1)
}
if terms == "" && excludedTerms == "" {
// we've already confirmed that we have a channel or user to search for
} else if fs.DriverName() == model.DATABASE_DRIVER_POSTGRES {
// Parse text for wildcards
if wildcard, err := regexp.Compile(`\*($| )`); err == nil {
terms = wildcard.ReplaceAllLiteralString(terms, ":* ")
excludedTerms = wildcard.ReplaceAllLiteralString(excludedTerms, ":* ")
}
excludeClause := ""
if excludedTerms != "" {
excludeClause = " & !(" + strings.Join(strings.Fields(excludedTerms), " | ") + ")"
}
queryTerms := ""
if params.OrTerms {
queryTerms = "(" + strings.Join(strings.Fields(terms), " | ") + ")" + excludeClause
} else {
queryTerms = "(" + strings.Join(strings.Fields(terms), " & ") + ")" + excludeClause
}
query = query.Where(sq.Or{
sq.Expr("to_tsvector('english', FI.Name) @@ to_tsquery('english', ?)", queryTerms),
sq.Expr("to_tsvector('english', FI.Content) @@ to_tsquery('english', ?)", queryTerms),
})
} else if fs.DriverName() == model.DATABASE_DRIVER_MYSQL {
var err error
terms, err = removeMysqlStopWordsFromTerms(terms)
if err != nil {
return nil, errors.Wrap(err, "failed to remove Mysql stop-words from terms")
}
if terms == "" {
return model.NewFileInfoList(), nil
}
excludeClause := ""
if excludedTerms != "" {
excludeClause = " -(" + excludedTerms + ")"
}
queryTerms := ""
if params.OrTerms {
queryTerms = terms + excludeClause
} else {
splitTerms := []string{}
for _, t := range strings.Fields(terms) {
splitTerms = append(splitTerms, "+"+t)
}
queryTerms = strings.Join(splitTerms, " ") + excludeClause
}
query = query.Where(sq.Or{
sq.Expr("MATCH (FI.Name) AGAINST (? IN BOOLEAN MODE)", queryTerms),
sq.Expr("MATCH (FI.Content) AGAINST (? IN BOOLEAN MODE)", queryTerms),
})
}
}
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "file_info_tosql")
}
list := model.NewFileInfoList()
fileInfos := []*model.FileInfo{}
_, err = fs.GetSearchReplica().Select(&fileInfos, queryString, args...)
if err != nil {
mlog.Warn("Query error searching files.", mlog.Err(err))
// Don't return the error to the caller as it is of no use to the user. Instead return an empty set of search results.
} else {
for _, f := range fileInfos {
list.AddFileInfo(f)
list.AddOrder(f.Id)
}
}
list.MakeNonNil()
return list, nil
}

Просмотреть файл

@@ -6,9 +6,14 @@ package sqlstore
import (
"testing"
"github.com/mattermost/mattermost-server/v5/store/searchtest"
"github.com/mattermost/mattermost-server/v5/store/storetest"
)
func TestFileInfoStore(t *testing.T) {
StoreTest(t, storetest.TestFileInfoStore)
}
func TestSearchFileInfoStore(t *testing.T) {
StoreTestWithSearchTestEngine(t, searchtest.TestSearchFileInfoStore)
}

Просмотреть файл

@@ -564,6 +564,7 @@ type FileInfoStore interface {
Save(info *model.FileInfo) (*model.FileInfo, error)
Upsert(info *model.FileInfo) (*model.FileInfo, error)
Get(id string) (*model.FileInfo, error)
GetByIds(ids []string) ([]*model.FileInfo, error)
GetByPath(path string) (*model.FileInfo, error)
GetForPost(postId string, readFromMaster, includeDeleted, allowFromCache bool) ([]*model.FileInfo, error)
GetForUser(userId string) ([]*model.FileInfo, error)
@@ -575,6 +576,7 @@ type FileInfoStore interface {
PermanentDeleteBatch(endTime int64, limit int64) (int64, error)
PermanentDeleteByUser(userId string) (int64, error)
SetContent(fileId, content string) error
Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error)
ClearCaches()
}

Просмотреть файл

@@ -77,6 +77,29 @@ func (_m *FileInfoStore) Get(id string) (*model.FileInfo, error) {
return r0, r1
}
// GetByIds provides a mock function with given fields: ids
func (_m *FileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
ret := _m.Called(ids)
var r0 []*model.FileInfo
if rf, ok := ret.Get(0).(func([]string) []*model.FileInfo); ok {
r0 = rf(ids)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.FileInfo)
}
}
var r1 error
if rf, ok := ret.Get(1).(func([]string) error); ok {
r1 = rf(ids)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetByPath provides a mock function with given fields: path
func (_m *FileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
ret := _m.Called(path)
@@ -253,6 +276,29 @@ func (_m *FileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
return r0, r1
}
// Search provides a mock function with given fields: paramsList, userId, teamId, page, perPage
func (_m *FileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) {
ret := _m.Called(paramsList, userId, teamId, page, perPage)
var r0 *model.FileInfoList
if rf, ok := ret.Get(0).(func([]*model.SearchParams, string, string, int, int) *model.FileInfoList); ok {
r0 = rf(paramsList, userId, teamId, page, perPage)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.FileInfoList)
}
}
var r1 error
if rf, ok := ret.Get(1).(func([]*model.SearchParams, string, string, int, int) error); ok {
r1 = rf(paramsList, userId, teamId, page, perPage)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// SetContent provides a mock function with given fields: fileId, content
func (_m *FileInfoStore) SetContent(fileId string, content string) error {
ret := _m.Called(fileId, content)

Просмотреть файл

@@ -2734,6 +2734,22 @@ func (s *TimerLayerFileInfoStore) Get(id string) (*model.FileInfo, error) {
return result, err
}
func (s *TimerLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
start := timemodule.Now()
result, err := s.FileInfoStore.GetByIds(ids)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.GetByIds", success, elapsed)
}
return result, err
}
func (s *TimerLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
start := timemodule.Now()
@@ -2877,6 +2893,22 @@ func (s *TimerLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, e
return result, err
}
func (s *TimerLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) {
start := timemodule.Now()
result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.Search", success, elapsed)
}
return result, err
}
func (s *TimerLayerFileInfoStore) SetContent(fileId string, content string) error {
start := timemodule.Now()