Add search engine support for files (#16190)

* Add search engine support for files

* Fixing i18n

* Fix golangci-lint

* Fix consistency problem in the Search receiver functio of the SqlFileStore

* Fixing some tests

* Fixing test

* Apply suggestions from code review

Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>

* Addressing PR review comments

* Removing some empty lines

* Address PR review comments

* Fixing problem after merge master

* Fixing spelling problem

* Add missed translations

* Fixing certain global variable usages after merge master

* Fixing some constants usage

* Fixing goimports order

Co-authored-by: Mario de Frutos Dieguez <mario@defrutos.org>
Этот коммит содержится в:
Jesús Espino
2021-01-11 15:14:16 +01:00
коммит произвёл GitHub
родитель bcacc78f77
Коммит 2a63b5552a
28 изменённых файлов: 3438 добавлений и 219 удалений

Просмотреть файл

@@ -50,9 +50,12 @@ type MetricsInterface interface {
IncrementPostsSearchCounter() IncrementPostsSearchCounter()
ObservePostsSearchDuration(elapsed float64) ObservePostsSearchDuration(elapsed float64)
IncrementFilesSearchCounter()
ObserveFilesSearchDuration(elapsed float64)
ObserveStoreMethodDuration(method, success string, elapsed float64) ObserveStoreMethodDuration(method, success string, elapsed float64)
ObserveApiEndpointDuration(endpoint, method, statusCode string, elapsed float64) ObserveApiEndpointDuration(endpoint, method, statusCode string, elapsed float64)
IncrementPostIndexCounter() IncrementPostIndexCounter()
IncrementFileIndexCounter()
IncrementUserIndexCounter() IncrementUserIndexCounter()
IncrementChannelIndexCounter() IncrementChannelIndexCounter()

Просмотреть файл

@@ -75,6 +75,16 @@ func (_m *MetricsInterface) IncrementEtagMissCounter(route string) {
_m.Called(route) _m.Called(route)
} }
// IncrementFileIndexCounter provides a mock function with given fields:
func (_m *MetricsInterface) IncrementFileIndexCounter() {
_m.Called()
}
// IncrementFilesSearchCounter provides a mock function with given fields:
func (_m *MetricsInterface) IncrementFilesSearchCounter() {
_m.Called()
}
// IncrementHttpError provides a mock function with given fields: // IncrementHttpError provides a mock function with given fields:
func (_m *MetricsInterface) IncrementHttpError() { func (_m *MetricsInterface) IncrementHttpError() {
_m.Called() _m.Called()
@@ -205,6 +215,11 @@ func (_m *MetricsInterface) ObserveEnabledUsers(users int64) {
_m.Called(users) _m.Called(users)
} }
// ObserveFilesSearchDuration provides a mock function with given fields: elapsed
func (_m *MetricsInterface) ObserveFilesSearchDuration(elapsed float64) {
_m.Called(elapsed)
}
// ObservePluginApiDuration provides a mock function with given fields: pluginID, apiName, success, elapsed // ObservePluginApiDuration provides a mock function with given fields: pluginID, apiName, success, elapsed
func (_m *MetricsInterface) ObservePluginApiDuration(pluginID string, apiName string, success bool, elapsed float64) { func (_m *MetricsInterface) ObservePluginApiDuration(pluginID string, apiName string, success bool, elapsed float64) {
_m.Called(pluginID, apiName, success, elapsed) _m.Called(pluginID, apiName, success, elapsed)

Просмотреть файл

@@ -5822,6 +5822,10 @@
"id": "bleveengine.create_channel_index.error", "id": "bleveengine.create_channel_index.error",
"translation": "Error creating the bleve channel index." "translation": "Error creating the bleve channel index."
}, },
{
"id": "bleveengine.create_file_index.error",
"translation": "Error creating the bleve file index."
},
{ {
"id": "bleveengine.create_post_index.error", "id": "bleveengine.create_post_index.error",
"translation": "Error creating the bleve post index." "translation": "Error creating the bleve post index."
@@ -5838,14 +5842,30 @@
"id": "bleveengine.delete_channel_posts.error", "id": "bleveengine.delete_channel_posts.error",
"translation": "Failed to delete channel posts" "translation": "Failed to delete channel posts"
}, },
{
"id": "bleveengine.delete_file.error",
"translation": "Failed to delete the file."
},
{
"id": "bleveengine.delete_files_batch.error",
"translation": "Failed to delete the files."
},
{ {
"id": "bleveengine.delete_post.error", "id": "bleveengine.delete_post.error",
"translation": "Failed to delete the post." "translation": "Failed to delete the post."
}, },
{
"id": "bleveengine.delete_post_files.error",
"translation": "Fiiled to delete the post files."
},
{ {
"id": "bleveengine.delete_user.error", "id": "bleveengine.delete_user.error",
"translation": "Failed to delete the user." "translation": "Failed to delete the user."
}, },
{
"id": "bleveengine.delete_user_files.error",
"translation": "Failed to delete the user files."
},
{ {
"id": "bleveengine.delete_user_posts.error", "id": "bleveengine.delete_user_posts.error",
"translation": "Failed to delete user posts" "translation": "Failed to delete user posts"
@@ -5854,6 +5874,10 @@
"id": "bleveengine.index_channel.error", "id": "bleveengine.index_channel.error",
"translation": "Failed to index the channel." "translation": "Failed to index the channel."
}, },
{
"id": "bleveengine.index_file.error",
"translation": "Failed to index the file."
},
{ {
"id": "bleveengine.index_post.error", "id": "bleveengine.index_post.error",
"translation": "Failed to index the post." "translation": "Failed to index the post."
@@ -5866,6 +5890,10 @@
"id": "bleveengine.indexer.do_job.bulk_index_channels.batch_error", "id": "bleveengine.indexer.do_job.bulk_index_channels.batch_error",
"translation": "Failed to index channel batch." "translation": "Failed to index channel batch."
}, },
{
"id": "bleveengine.indexer.do_job.bulk_index_files.batch_error",
"translation": "Failed to index file batch."
},
{ {
"id": "bleveengine.indexer.do_job.bulk_index_posts.batch_error", "id": "bleveengine.indexer.do_job.bulk_index_posts.batch_error",
"translation": "Failed to index post batch." "translation": "Failed to index post batch."
@@ -5910,6 +5938,10 @@
"id": "bleveengine.search_channels.error", "id": "bleveengine.search_channels.error",
"translation": "Channel search failed to complete." "translation": "Channel search failed to complete."
}, },
{
"id": "bleveengine.search_files.error",
"translation": "File search failed to complete."
},
{ {
"id": "bleveengine.search_posts.error", "id": "bleveengine.search_posts.error",
"translation": "Post search failed to complete." "translation": "Post search failed to complete."
@@ -5930,6 +5962,10 @@
"id": "bleveengine.stop_channel_index.error", "id": "bleveengine.stop_channel_index.error",
"translation": "Failed to close channel index." "translation": "Failed to close channel index."
}, },
{
"id": "bleveengine.stop_file_index.error",
"translation": "Failed to close file index."
},
{ {
"id": "bleveengine.stop_post_index.error", "id": "bleveengine.stop_post_index.error",
"translation": "Failed to close post index." "translation": "Failed to close post index."
@@ -6150,6 +6186,10 @@
"id": "ent.elasticsearch.create_template_channels_if_not_exists.template_create_failed", "id": "ent.elasticsearch.create_template_channels_if_not_exists.template_create_failed",
"translation": "Failed to create Elasticsearch template for channels" "translation": "Failed to create Elasticsearch template for channels"
}, },
{
"id": "ent.elasticsearch.create_template_file_info_if_not_exists.template_create_failed",
"translation": "Failed to create Elasticsearch template for files"
},
{ {
"id": "ent.elasticsearch.create_template_posts_if_not_exists.template_create_failed", "id": "ent.elasticsearch.create_template_posts_if_not_exists.template_create_failed",
"translation": "Failed to create Elasticsearch template for posts" "translation": "Failed to create Elasticsearch template for posts"
@@ -6174,14 +6214,26 @@
"id": "ent.elasticsearch.delete_channel_posts.error", "id": "ent.elasticsearch.delete_channel_posts.error",
"translation": "Failed to delete channel posts" "translation": "Failed to delete channel posts"
}, },
{
"id": "ent.elasticsearch.delete_file.error",
"translation": "Failed to delete file"
},
{ {
"id": "ent.elasticsearch.delete_post.error", "id": "ent.elasticsearch.delete_post.error",
"translation": "Failed to delete the post" "translation": "Failed to delete the post"
}, },
{
"id": "ent.elasticsearch.delete_post_files.error",
"translation": "Failed to delete post files"
},
{ {
"id": "ent.elasticsearch.delete_user.error", "id": "ent.elasticsearch.delete_user.error",
"translation": "Failed to delete the user" "translation": "Failed to delete the user"
}, },
{
"id": "ent.elasticsearch.delete_user_files.error",
"translation": "Failed to delete user files"
},
{ {
"id": "ent.elasticsearch.delete_user_posts.error", "id": "ent.elasticsearch.delete_user_posts.error",
"translation": "Failed to delete user posts" "translation": "Failed to delete user posts"
@@ -6198,6 +6250,10 @@
"id": "ent.elasticsearch.index_channels_batch.error", "id": "ent.elasticsearch.index_channels_batch.error",
"translation": "Unable to get the channels batch for indexing." "translation": "Unable to get the channels batch for indexing."
}, },
{
"id": "ent.elasticsearch.index_file.error",
"translation": "Failed to index the file"
},
{ {
"id": "ent.elasticsearch.index_post.error", "id": "ent.elasticsearch.index_post.error",
"translation": "Failed to index the post" "translation": "Failed to index the post"
@@ -6250,6 +6306,18 @@
"id": "ent.elasticsearch.search_channels.unmarshall_channel_failed", "id": "ent.elasticsearch.search_channels.unmarshall_channel_failed",
"translation": "Failed to decode search results" "translation": "Failed to decode search results"
}, },
{
"id": "ent.elasticsearch.search_files.disabled",
"translation": "Elasticsearch files searching is disabled on this server"
},
{
"id": "ent.elasticsearch.search_files.search_failed",
"translation": "Search failed to complete"
},
{
"id": "ent.elasticsearch.search_files.unmarshall_file_failed",
"translation": "Failed to decode search results"
},
{ {
"id": "ent.elasticsearch.search_posts.disabled", "id": "ent.elasticsearch.search_posts.disabled",
"translation": "Elasticsearch searching is disabled on this server" "translation": "Elasticsearch searching is disabled on this server"

128
model/file_info_list.go Обычный файл
Просмотреть файл

@@ -0,0 +1,128 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package model
import (
"encoding/json"
"io"
"sort"
)
type FileInfoList struct {
Order []string `json:"order"`
FileInfos map[string]*FileInfo `json:"file_infos"`
NextFileInfoId string `json:"next_file_info_id"`
PrevFileInfoId string `json:"prev_file_info_id"`
}
func NewFileInfoList() *FileInfoList {
return &FileInfoList{
Order: make([]string, 0),
FileInfos: make(map[string]*FileInfo),
NextFileInfoId: "",
PrevFileInfoId: "",
}
}
func (o *FileInfoList) ToSlice() []*FileInfo {
var fileInfos []*FileInfo
for _, id := range o.Order {
fileInfos = append(fileInfos, o.FileInfos[id])
}
return fileInfos
}
func (o *FileInfoList) ToJson() string {
b, err := json.Marshal(o)
if err != nil {
return ""
} else {
return string(b)
}
}
func (o *FileInfoList) MakeNonNil() {
if o.Order == nil {
o.Order = make([]string, 0)
}
if o.FileInfos == nil {
o.FileInfos = make(map[string]*FileInfo)
}
}
func (o *FileInfoList) AddOrder(id string) {
if o.Order == nil {
o.Order = make([]string, 0, 128)
}
o.Order = append(o.Order, id)
}
func (o *FileInfoList) AddFileInfo(fileInfo *FileInfo) {
if o.FileInfos == nil {
o.FileInfos = make(map[string]*FileInfo)
}
o.FileInfos[fileInfo.Id] = fileInfo
}
func (o *FileInfoList) UniqueOrder() {
keys := make(map[string]bool)
order := []string{}
for _, fileInfoId := range o.Order {
if _, value := keys[fileInfoId]; !value {
keys[fileInfoId] = true
order = append(order, fileInfoId)
}
}
o.Order = order
}
func (o *FileInfoList) Extend(other *FileInfoList) {
for fileInfoId := range other.FileInfos {
o.AddFileInfo(other.FileInfos[fileInfoId])
}
for _, fileInfoId := range other.Order {
o.AddOrder(fileInfoId)
}
o.UniqueOrder()
}
func (o *FileInfoList) SortByCreateAt() {
sort.Slice(o.Order, func(i, j int) bool {
return o.FileInfos[o.Order[i]].CreateAt > o.FileInfos[o.Order[j]].CreateAt
})
}
func (o *FileInfoList) Etag() string {
id := "0"
var t int64 = 0
for _, v := range o.FileInfos {
if v.UpdateAt > t {
t = v.UpdateAt
id = v.Id
} else if v.UpdateAt == t && v.Id > id {
t = v.UpdateAt
id = v.Id
}
}
orderId := ""
if len(o.Order) > 0 {
orderId = o.Order[0]
}
return Etag(orderId, id, t)
}
func FileInfoListFromJson(data io.Reader) *FileInfoList {
var o *FileInfoList
json.NewDecoder(data).Decode(&o)
return o
}

37
model/file_info_search_results.go Обычный файл
Просмотреть файл

@@ -0,0 +1,37 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package model
import (
"encoding/json"
"io"
)
type FileInfoSearchMatches map[string][]string
type FileInfoSearchResults struct {
*FileInfoList
Matches FileInfoSearchMatches `json:"matches"`
}
func MakeFileInfoSearchResults(fileInfos *FileInfoList, matches FileInfoSearchMatches) *FileInfoSearchResults {
return &FileInfoSearchResults{
fileInfos,
matches,
}
}
func (o *FileInfoSearchResults) ToJson() string {
b, err := json.Marshal(o)
if err != nil {
return ""
}
return string(b)
}
func FileInfoSearchResultsFromJson(data io.Reader) *FileInfoSearchResults {
var o *FileInfoSearchResults
json.NewDecoder(data).Decode(&o)
return o
}

Просмотреть файл

@@ -164,6 +164,12 @@ type PostForIndexing struct {
ParentCreateAt *int64 `json:"parent_create_at"` ParentCreateAt *int64 `json:"parent_create_at"`
} }
type FileForIndexing struct {
FileInfo
ChannelId string `json:"channel_id"`
Content string `json:"content"`
}
// ShallowCopy is an utility function to shallow copy a Post to the given // ShallowCopy is an utility function to shallow copy a Post to the given
// destination without touching the internal RWMutex. // destination without touching the internal RWMutex.
func (o *Post) ShallowCopy(dst *Post) error { func (o *Post) ShallowCopy(dst *Post) error {

Просмотреть файл

@@ -25,6 +25,8 @@ type SearchParams struct {
ExcludedAfterDate string ExcludedAfterDate string
BeforeDate string BeforeDate string
ExcludedBeforeDate string ExcludedBeforeDate string
Extensions []string
ExcludedExtensions []string
OnDate string OnDate string
ExcludedDate string ExcludedDate string
OrTerms bool OrTerms bool
@@ -106,7 +108,7 @@ func (p *SearchParams) GetExcludedDateMillis() (int64, int64) {
return GetStartOfDayMillis(date, p.TimeZoneOffset), GetEndOfDayMillis(date, p.TimeZoneOffset) return GetStartOfDayMillis(date, p.TimeZoneOffset), GetEndOfDayMillis(date, p.TimeZoneOffset)
} }
var searchFlags = [...]string{"from", "channel", "in", "before", "after", "on"} var searchFlags = [...]string{"from", "channel", "in", "before", "after", "on", "ext"}
type flag struct { type flag struct {
name string name string
@@ -265,6 +267,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
excludedBeforeDate := "" excludedBeforeDate := ""
onDate := "" onDate := ""
excludedDate := "" excludedDate := ""
excludedExtensions := []string{}
extensions := []string{}
for _, flag := range flags { for _, flag := range flags {
if flag.name == "in" || flag.name == "channel" { if flag.name == "in" || flag.name == "channel" {
@@ -297,6 +301,12 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
} else { } else {
onDate = flag.value onDate = flag.value
} }
} else if flag.name == "ext" {
if flag.exclude {
excludedExtensions = append(excludedExtensions, flag.value)
} else {
extensions = append(extensions, flag.value)
}
} }
} }
@@ -315,6 +325,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
ExcludedAfterDate: excludedAfterDate, ExcludedAfterDate: excludedAfterDate,
BeforeDate: beforeDate, BeforeDate: beforeDate,
ExcludedBeforeDate: excludedBeforeDate, ExcludedBeforeDate: excludedBeforeDate,
Extensions: extensions,
ExcludedExtensions: excludedExtensions,
OnDate: onDate, OnDate: onDate,
ExcludedDate: excludedDate, ExcludedDate: excludedDate,
TimeZoneOffset: timeZoneOffset, TimeZoneOffset: timeZoneOffset,
@@ -334,6 +346,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
ExcludedAfterDate: excludedAfterDate, ExcludedAfterDate: excludedAfterDate,
BeforeDate: beforeDate, BeforeDate: beforeDate,
ExcludedBeforeDate: excludedBeforeDate, ExcludedBeforeDate: excludedBeforeDate,
Extensions: extensions,
ExcludedExtensions: excludedExtensions,
OnDate: onDate, OnDate: onDate,
ExcludedDate: excludedDate, ExcludedDate: excludedDate,
TimeZoneOffset: timeZoneOffset, TimeZoneOffset: timeZoneOffset,
@@ -345,6 +359,7 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
len(excludedPlainTerms) == 0 && len(excludedHashtagTerms) == 0 && len(excludedPlainTerms) == 0 && len(excludedHashtagTerms) == 0 &&
(len(inChannels) != 0 || len(fromUsers) != 0 || (len(inChannels) != 0 || len(fromUsers) != 0 ||
len(excludedChannels) != 0 || len(excludedUsers) != 0 || len(excludedChannels) != 0 || len(excludedUsers) != 0 ||
len(extensions) != 0 || len(excludedExtensions) != 0 ||
afterDate != "" || excludedAfterDate != "" || afterDate != "" || excludedAfterDate != "" ||
beforeDate != "" || excludedBeforeDate != "" || beforeDate != "" || excludedBeforeDate != "" ||
onDate != "" || excludedDate != "") { onDate != "" || excludedDate != "") {
@@ -360,6 +375,8 @@ func ParseSearchParams(text string, timeZoneOffset int) []*SearchParams {
ExcludedAfterDate: excludedAfterDate, ExcludedAfterDate: excludedAfterDate,
BeforeDate: beforeDate, BeforeDate: beforeDate,
ExcludedBeforeDate: excludedBeforeDate, ExcludedBeforeDate: excludedBeforeDate,
Extensions: extensions,
ExcludedExtensions: excludedExtensions,
OnDate: onDate, OnDate: onDate,
ExcludedDate: excludedDate, ExcludedDate: excludedDate,
TimeZoneOffset: timeZoneOffset, TimeZoneOffset: timeZoneOffset,

Просмотреть файл

@@ -1043,13 +1043,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "words words", Input: "words words",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "words words", Terms: "words words",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1058,13 +1060,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-word1 -word2", Input: "-word1 -word2",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "word1 word2", ExcludedTerms: "word1 word2",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1073,13 +1077,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "\"my stuff\"", Input: "\"my stuff\"",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "\"my stuff\"", Terms: "\"my stuff\"",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1088,13 +1094,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-\"my stuff\"", Input: "-\"my stuff\"",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "\"my stuff\"", ExcludedTerms: "\"my stuff\"",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1103,13 +1111,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "#words #words", Input: "#words #words",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "#words #words", Terms: "#words #words",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: true, IsHashtag: true,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1118,22 +1128,26 @@ func TestParseSearchParams(t *testing.T) {
Input: "#words words", Input: "#words words",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "words", Terms: "words",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
{ {
Terms: "#words", Terms: "#words",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: true, IsHashtag: true,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1142,13 +1156,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-#hashtag", Input: "-#hashtag",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "#hashtag", ExcludedTerms: "#hashtag",
IsHashtag: true, IsHashtag: true,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1157,13 +1173,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-#hashtag1 -#hashtag2", Input: "-#hashtag1 -#hashtag2",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "#hashtag1 #hashtag2", ExcludedTerms: "#hashtag1 #hashtag2",
IsHashtag: true, IsHashtag: true,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1172,13 +1190,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "#hashtag1 -#hashtag2", Input: "#hashtag1 -#hashtag2",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "#hashtag1", Terms: "#hashtag1",
ExcludedTerms: "#hashtag2", ExcludedTerms: "#hashtag2",
IsHashtag: true, IsHashtag: true,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1187,22 +1207,26 @@ func TestParseSearchParams(t *testing.T) {
Input: "word1 #hashtag1 -#hashtag2 -word2", Input: "word1 #hashtag1 -#hashtag2 -word2",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "word1", Terms: "word1",
ExcludedTerms: "word2", ExcludedTerms: "word2",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
{ {
Terms: "#hashtag1", Terms: "#hashtag1",
ExcludedTerms: "#hashtag2", ExcludedTerms: "#hashtag2",
IsHashtag: true, IsHashtag: true,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1211,13 +1235,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "in:channel", Input: "in:channel",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{"channel"}, InChannels: []string{"channel"},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1226,13 +1252,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "-in:channel", Input: "-in:channel",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{"channel"}, ExcludedChannels: []string{"channel"},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1241,13 +1269,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel", Input: "testing in:channel",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{"channel"}, InChannels: []string{"channel"},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1256,13 +1286,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing -in:channel", Input: "testing -in:channel",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{"channel"}, ExcludedChannels: []string{"channel"},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1271,13 +1303,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "in:channel testing", Input: "in:channel testing",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{"channel"}, InChannels: []string{"channel"},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1286,13 +1320,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "in:channel in:otherchannel", Input: "in:channel in:otherchannel",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{"channel", "otherchannel"}, InChannels: []string{"channel", "otherchannel"},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1301,13 +1337,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel in:otherchannel", Input: "testing in:channel in:otherchannel",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{"channel", "otherchannel"}, InChannels: []string{"channel", "otherchannel"},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1316,13 +1354,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel from:someone", Input: "testing in:channel from:someone",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{"channel"}, InChannels: []string{"channel"},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{"someone"}, FromUsers: []string{"someone"},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1331,13 +1371,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel -from:someone", Input: "testing in:channel -from:someone",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{"channel"}, InChannels: []string{"channel"},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{"someone"}, ExcludedUsers: []string{"someone"},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1346,13 +1388,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "testing in:channel from:someone -from:someoneelse", Input: "testing in:channel from:someone -from:someoneelse",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{"channel"}, InChannels: []string{"channel"},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{"someone"}, FromUsers: []string{"someone"},
ExcludedUsers: []string{"someoneelse"}, ExcludedUsers: []string{"someoneelse"},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1361,13 +1405,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "##hashtag +#plus+", Input: "##hashtag +#plus+",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "#hashtag #plus", Terms: "#hashtag #plus",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: true, IsHashtag: true,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1376,13 +1422,15 @@ func TestParseSearchParams(t *testing.T) {
Input: "wildcar*", Input: "wildcar*",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "wildcar*", Terms: "wildcar*",
ExcludedTerms: "", ExcludedTerms: "",
IsHashtag: false, IsHashtag: false,
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1391,14 +1439,16 @@ func TestParseSearchParams(t *testing.T) {
Input: "after:2018-8-1 testing", Input: "after:2018-8-1 testing",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
AfterDate: "2018-8-1", AfterDate: "2018-8-1",
ExcludedAfterDate: "", ExcludedAfterDate: "",
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1407,14 +1457,16 @@ func TestParseSearchParams(t *testing.T) {
Input: "-after:2018-8-1 testing", Input: "-after:2018-8-1 testing",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
AfterDate: "", AfterDate: "",
ExcludedAfterDate: "2018-8-1", ExcludedAfterDate: "2018-8-1",
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1423,15 +1475,17 @@ func TestParseSearchParams(t *testing.T) {
Input: "on:2018-8-1 testing", Input: "on:2018-8-1 testing",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
OnDate: "2018-8-1", OnDate: "2018-8-1",
AfterDate: "", AfterDate: "",
ExcludedAfterDate: "", ExcludedAfterDate: "",
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1440,14 +1494,16 @@ func TestParseSearchParams(t *testing.T) {
Input: "-on:2018-8-1 testing", Input: "-on:2018-8-1 testing",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "testing", Terms: "testing",
ExcludedTerms: "", ExcludedTerms: "",
AfterDate: "", AfterDate: "",
ExcludedDate: "2018-8-1", ExcludedDate: "2018-8-1",
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1456,14 +1512,16 @@ func TestParseSearchParams(t *testing.T) {
Input: "after:2018-8-1", Input: "after:2018-8-1",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "", ExcludedTerms: "",
AfterDate: "2018-8-1", AfterDate: "2018-8-1",
ExcludedDate: "", ExcludedDate: "",
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1472,15 +1530,17 @@ func TestParseSearchParams(t *testing.T) {
Input: "before:2018-8-1", Input: "before:2018-8-1",
Output: []*SearchParams{ Output: []*SearchParams{
{ {
Terms: "", Terms: "",
ExcludedTerms: "", ExcludedTerms: "",
BeforeDate: "2018-8-1", BeforeDate: "2018-8-1",
AfterDate: "", AfterDate: "",
ExcludedDate: "", ExcludedDate: "",
InChannels: []string{}, InChannels: []string{},
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
}, },
}, },
}, },
@@ -1498,6 +1558,110 @@ func TestParseSearchParams(t *testing.T) {
ExcludedChannels: []string{}, ExcludedChannels: []string{},
FromUsers: []string{}, FromUsers: []string{},
ExcludedUsers: []string{}, ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{},
},
},
},
{
Name: "input is two words separated with : and should result in a single Extension",
Input: "ext:png",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{"png"},
ExcludedExtensions: []string{},
},
},
},
{
Name: "input is two words separated with :, prefied with - and should result in a single ExcludedExtensions",
Input: "-ext:png",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{"png"},
},
},
},
{
Name: "input is two words separated with : with a prefixed word should result in a single Extension and a term",
Input: "testing ext:png",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{"png"},
ExcludedExtensions: []string{},
},
},
},
{
Name: "input is two words separated with : with a prefixed word should result in a single ExcludedExtension and a term",
Input: "testing -ext:png",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{},
ExcludedExtensions: []string{"png"},
},
},
},
{
Name: "input is two words separated with : with a postfix word should result in a single Extension and a term",
Input: "ext:png testing",
Output: []*SearchParams{
{
Terms: "testing",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{"png"},
ExcludedExtensions: []string{},
},
},
},
{
Name: "input is four words separated with : should result in a two Extensions",
Input: "ext:png ext:jpg",
Output: []*SearchParams{
{
Terms: "",
ExcludedTerms: "",
IsHashtag: false,
InChannels: []string{},
ExcludedChannels: []string{},
FromUsers: []string{},
ExcludedUsers: []string{},
Extensions: []string{"png", "jpg"},
ExcludedExtensions: []string{},
}, },
}, },
}, },

Просмотреть файл

@@ -24,12 +24,14 @@ import (
const ( const (
EngineName = "bleve" EngineName = "bleve"
PostIndex = "posts" PostIndex = "posts"
FileIndex = "files"
UserIndex = "users" UserIndex = "users"
ChannelIndex = "channels" ChannelIndex = "channels"
) )
type BleveEngine struct { type BleveEngine struct {
PostIndex bleve.Index PostIndex bleve.Index
FileIndex bleve.Index
UserIndex bleve.Index UserIndex bleve.Index
ChannelIndex bleve.Index ChannelIndex bleve.Index
Mutex sync.RWMutex Mutex sync.RWMutex
@@ -83,6 +85,23 @@ func getPostIndexMapping() *mapping.IndexMappingImpl {
return indexMapping return indexMapping
} }
func getFileIndexMapping() *mapping.IndexMappingImpl {
fileMapping := bleve.NewDocumentMapping()
fileMapping.AddFieldMappingsAt("Id", keywordMapping)
fileMapping.AddFieldMappingsAt("CreatorId", keywordMapping)
fileMapping.AddFieldMappingsAt("ChannelId", keywordMapping)
fileMapping.AddFieldMappingsAt("CreateAt", dateMapping)
fileMapping.AddFieldMappingsAt("Name", standardMapping)
fileMapping.AddFieldMappingsAt("Content", standardMapping)
fileMapping.AddFieldMappingsAt("Extension", keywordMapping)
fileMapping.AddFieldMappingsAt("Content", standardMapping)
indexMapping := bleve.NewIndexMapping()
indexMapping.AddDocumentMapping("_default", fileMapping)
return indexMapping
}
func getUserIndexMapping() *mapping.IndexMappingImpl { func getUserIndexMapping() *mapping.IndexMappingImpl {
userMapping := bleve.NewDocumentMapping() userMapping := bleve.NewDocumentMapping()
userMapping.AddFieldMappingsAt("Id", keywordMapping) userMapping.AddFieldMappingsAt("Id", keywordMapping)
@@ -132,6 +151,11 @@ func (b *BleveEngine) openIndexes() *model.AppError {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, err.Error(), http.StatusInternalServerError) return model.NewAppError("Bleveengine.Start", "bleveengine.create_post_index.error", nil, err.Error(), http.StatusInternalServerError)
} }
b.FileIndex, err = b.createOrOpenIndex(FileIndex, getFileIndexMapping())
if err != nil {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_file_index.error", nil, err.Error(), http.StatusInternalServerError)
}
b.UserIndex, err = b.createOrOpenIndex(UserIndex, getUserIndexMapping()) b.UserIndex, err = b.createOrOpenIndex(UserIndex, getUserIndexMapping())
if err != nil { if err != nil {
return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, err.Error(), http.StatusInternalServerError) return model.NewAppError("Bleveengine.Start", "bleveengine.create_user_index.error", nil, err.Error(), http.StatusInternalServerError)
@@ -165,6 +189,10 @@ func (b *BleveEngine) closeIndexes() *model.AppError {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, err.Error(), http.StatusInternalServerError) return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_post_index.error", nil, err.Error(), http.StatusInternalServerError)
} }
if err := b.FileIndex.Close(); err != nil {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_file_index.error", nil, err.Error(), http.StatusInternalServerError)
}
if err := b.UserIndex.Close(); err != nil { if err := b.UserIndex.Close(); err != nil {
return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, err.Error(), http.StatusInternalServerError) return model.NewAppError("Bleveengine.Stop", "bleveengine.stop_user_index.error", nil, err.Error(), http.StatusInternalServerError)
} }

Просмотреть файл

@@ -98,6 +98,10 @@ func (s *BleveEngineTestSuite) TestBleveSearchStoreTests() {
s.Run("TestSearchPostStore", func() { s.Run("TestSearchPostStore", func() {
searchtest.TestSearchPostStore(s.T(), s.Store, searchTestEngine) searchtest.TestSearchPostStore(s.T(), s.Store, searchTestEngine)
}) })
s.Run("TestSearchFileInfoStore", func() {
searchtest.TestSearchFileInfoStore(s.T(), s.Store, searchTestEngine)
})
} }
func (s *BleveEngineTestSuite) TestDeleteChannelPosts() { func (s *BleveEngineTestSuite) TestDeleteChannelPosts() {

Просмотреть файл

@@ -36,6 +36,16 @@ type BLVPost struct {
Attachments string Attachments string
} }
type BLVFile struct {
Id string
CreatorId string
ChannelId string
CreateAt int64
Name string
Content string
Extension string
}
func BLVChannelFromChannel(channel *model.Channel) *BLVChannel { func BLVChannelFromChannel(channel *model.Channel) *BLVChannel {
displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ") displayNameInputs := searchengine.GetSuggestionInputsSplitBy(channel.DisplayName, " ")
nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"}) nameInputs := searchengine.GetSuggestionInputsSplitByMultiple(channel.Name, []string{"-", "_"})
@@ -114,3 +124,27 @@ func BLVPostFromPostForIndexing(post *model.PostForIndexing) *BLVPost {
Hashtags: strings.Fields(post.Hashtags), Hashtags: strings.Fields(post.Hashtags),
} }
} }
func BLVFileFromFileInfo(fileInfo *model.FileInfo, channelId string) *BLVFile {
return &BLVFile{
Id: fileInfo.Id,
ChannelId: channelId,
CreatorId: fileInfo.CreatorId,
CreateAt: fileInfo.CreateAt,
Content: fileInfo.Content,
Extension: fileInfo.Extension,
Name: fileInfo.Name,
}
}
func BLVFileFromFileForIndexing(file *model.FileForIndexing) *BLVFile {
return &BLVFile{
Id: file.Id,
ChannelId: file.ChannelId,
CreatorId: file.CreatorId,
CreateAt: file.CreateAt,
Content: file.Content,
Extension: file.Extension,
Name: file.Name,
}
}

Просмотреть файл

@@ -354,6 +354,30 @@ func (worker *BleveIndexerWorker) BulkIndexPosts(posts []*model.PostForIndexing,
return lastCreateAt, nil return lastCreateAt, nil
} }
func (worker *BleveIndexerWorker) BulkIndexFiles(files []*model.FileForIndexing, progress IndexingProgress) (int64, *model.AppError) {
lastCreateAt := int64(0)
batch := worker.engine.FileIndex.NewBatch()
for _, file := range files {
if file.DeleteAt == 0 {
searchFile := bleveengine.BLVFileFromFileForIndexing(file)
batch.Index(searchFile.Id, searchFile)
} else {
batch.Delete(file.Id)
}
lastCreateAt = file.CreateAt
}
worker.engine.Mutex.RLock()
defer worker.engine.Mutex.RUnlock()
if err := worker.engine.FileIndex.Batch(batch); err != nil {
return 0, model.NewAppError("BleveIndexerWorker.BulkIndexPosts", "bleveengine.indexer.do_job.bulk_index_files.batch_error", nil, err.Error(), http.StatusInternalServerError)
}
return lastCreateAt, nil
}
func (worker *BleveIndexerWorker) IndexChannelsBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) { func (worker *BleveIndexerWorker) IndexChannelsBatch(progress IndexingProgress) (IndexingProgress, *model.AppError) {
endTime := progress.LastEntityTime + int64(*worker.jobServer.Config().BleveSettings.BulkIndexingTimeWindowSeconds*1000) endTime := progress.LastEntityTime + int64(*worker.jobServer.Config().BleveSettings.BulkIndexingTimeWindowSeconds*1000)

Просмотреть файл

@@ -15,6 +15,7 @@ import (
) )
const DeletePostsBatchSize = 500 const DeletePostsBatchSize = 500
const DeleteFilesBatchSize = 500
func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError { func (b *BleveEngine) IndexPost(post *model.Post, teamId string) *model.AppError {
b.Mutex.RLock() b.Mutex.RLock()
@@ -507,3 +508,320 @@ func (b *BleveEngine) DeleteUser(user *model.User) *model.AppError {
} }
return nil return nil
} }
func (b *BleveEngine) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
blvFile := BLVFileFromFileInfo(file, channelId)
if err := b.FileIndex.Index(blvFile.Id, blvFile); err != nil {
return model.NewAppError("Bleveengine.IndexFile", "bleveengine.index_file.error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *BleveEngine) SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError) {
channelQueries := []query.Query{}
for _, channel := range *channels {
channelIdQ := bleve.NewTermQuery(channel.Id)
channelIdQ.SetField("ChannelId")
channelQueries = append(channelQueries, channelIdQ)
}
channelDisjunctionQ := bleve.NewDisjunctionQuery(channelQueries...)
var termQueries []query.Query
var notTermQueries []query.Query
var filters []query.Query
var notFilters []query.Query
for i, params := range searchParams {
var termOperator query.MatchQueryOperator = query.MatchQueryOperatorAnd
if searchParams[0].OrTerms {
termOperator = query.MatchQueryOperatorOr
}
// Date, channels and FromUsers filters come in all
// searchParams iteration, and as they are global to the
// query, we only need to process them once
if i == 0 {
if len(params.InChannels) > 0 {
inChannels := []query.Query{}
for _, channelId := range params.InChannels {
channelQ := bleve.NewTermQuery(channelId)
channelQ.SetField("ChannelId")
inChannels = append(inChannels, channelQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(inChannels...))
}
if len(params.ExcludedChannels) > 0 {
excludedChannels := []query.Query{}
for _, channelId := range params.ExcludedChannels {
channelQ := bleve.NewTermQuery(channelId)
channelQ.SetField("ChannelId")
excludedChannels = append(excludedChannels, channelQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedChannels...))
}
if len(params.FromUsers) > 0 {
fromUsers := []query.Query{}
for _, userId := range params.FromUsers {
userQ := bleve.NewTermQuery(userId)
userQ.SetField("CreatorId")
fromUsers = append(fromUsers, userQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(fromUsers...))
}
if len(params.ExcludedUsers) > 0 {
excludedUsers := []query.Query{}
for _, userId := range params.ExcludedUsers {
userQ := bleve.NewTermQuery(userId)
userQ.SetField("CreatorId")
excludedUsers = append(excludedUsers, userQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedUsers...))
}
if len(params.Extensions) > 0 {
extensions := []query.Query{}
for _, extension := range params.Extensions {
extensionQ := bleve.NewTermQuery(extension)
extensionQ.SetField("Extension")
extensions = append(extensions, extensionQ)
}
filters = append(filters, bleve.NewDisjunctionQuery(extensions...))
}
if len(params.ExcludedExtensions) > 0 {
excludedExtensions := []query.Query{}
for _, extension := range params.ExcludedExtensions {
extensionQ := bleve.NewTermQuery(extension)
extensionQ.SetField("Extension")
excludedExtensions = append(excludedExtensions, extensionQ)
}
notFilters = append(notFilters, bleve.NewDisjunctionQuery(excludedExtensions...))
}
if params.OnDate != "" {
before, after := params.GetOnDateMillis()
beforeFloat64 := float64(before)
afterFloat64 := float64(after)
onDateQ := bleve.NewNumericRangeQuery(&beforeFloat64, &afterFloat64)
onDateQ.SetField("CreateAt")
filters = append(filters, onDateQ)
} else {
if params.AfterDate != "" || params.BeforeDate != "" {
var min, max *float64
if params.AfterDate != "" {
minf := float64(params.GetAfterDateMillis())
min = &minf
}
if params.BeforeDate != "" {
maxf := float64(params.GetBeforeDateMillis())
max = &maxf
}
dateQ := bleve.NewNumericRangeQuery(min, max)
dateQ.SetField("CreateAt")
filters = append(filters, dateQ)
}
if params.ExcludedAfterDate != "" {
minf := float64(params.GetExcludedAfterDateMillis())
dateQ := bleve.NewNumericRangeQuery(&minf, nil)
dateQ.SetField("CreateAt")
notFilters = append(notFilters, dateQ)
}
if params.ExcludedBeforeDate != "" {
maxf := float64(params.GetExcludedBeforeDateMillis())
dateQ := bleve.NewNumericRangeQuery(nil, &maxf)
dateQ.SetField("CreateAt")
notFilters = append(notFilters, dateQ)
}
if params.ExcludedDate != "" {
before, after := params.GetExcludedDateMillis()
beforef := float64(before)
afterf := float64(after)
onDateQ := bleve.NewNumericRangeQuery(&beforef, &afterf)
onDateQ.SetField("CreateAt")
notFilters = append(notFilters, onDateQ)
}
}
}
if len(params.Terms) > 0 {
terms := []string{}
for _, term := range strings.Split(params.Terms, " ") {
if strings.HasSuffix(term, "*") {
nameQ := bleve.NewWildcardQuery(term)
nameQ.SetField("Name")
contentQ := bleve.NewWildcardQuery(term)
contentQ.SetField("Content")
termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
} else {
terms = append(terms, term)
}
}
if len(terms) > 0 {
nameQ := bleve.NewMatchQuery(strings.Join(terms, " "))
nameQ.SetField("Name")
nameQ.SetOperator(termOperator)
contentQ := bleve.NewMatchQuery(strings.Join(terms, " "))
contentQ.SetField("Content")
contentQ.SetOperator(termOperator)
termQueries = append(termQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
}
}
if len(params.ExcludedTerms) > 0 {
nameQ := bleve.NewMatchQuery(params.ExcludedTerms)
nameQ.SetField("Name")
nameQ.SetOperator(termOperator)
contentQ := bleve.NewMatchQuery(params.ExcludedTerms)
contentQ.SetField("Content")
contentQ.SetOperator(termOperator)
notTermQueries = append(notTermQueries, bleve.NewDisjunctionQuery(nameQ, contentQ))
}
}
allTermsQ := bleve.NewBooleanQuery()
allTermsQ.AddMustNot(notTermQueries...)
if searchParams[0].OrTerms {
allTermsQ.AddShould(termQueries...)
} else {
allTermsQ.AddMust(termQueries...)
}
query := bleve.NewBooleanQuery()
query.AddMust(channelDisjunctionQ)
if len(termQueries) > 0 || len(notTermQueries) > 0 {
query.AddMust(allTermsQ)
}
if len(filters) > 0 {
query.AddMust(bleve.NewConjunctionQuery(filters...))
}
if len(notFilters) > 0 {
query.AddMustNot(notFilters...)
}
search := bleve.NewSearchRequestOptions(query, perPage, page*perPage, false)
search.SortBy([]string{"-CreateAt"})
results, err := b.FileIndex.Search(search)
if err != nil {
return nil, model.NewAppError("Bleveengine.SearchFiles", "bleveengine.search_files.error", nil, err.Error(), http.StatusInternalServerError)
}
fileIds := []string{}
for _, r := range results.Hits {
fileIds = append(fileIds, r.ID)
}
return fileIds, nil
}
func (b *BleveEngine) DeleteFile(fileID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
if err := b.FileIndex.Delete(fileID); err != nil {
return model.NewAppError("Bleveengine.DeleteFile", "bleveengine.delete_file.error", nil, err.Error(), http.StatusInternalServerError)
}
return nil
}
func (b *BleveEngine) deleteFiles(searchRequest *bleve.SearchRequest, batchSize int) (int64, error) {
resultsCount := int64(0)
for {
// As we are deleting the files after fetching them, we need to keep
// From fixed always to 0
searchRequest.From = 0
searchRequest.Size = batchSize
results, err := b.FileIndex.Search(searchRequest)
if err != nil {
return -1, err
}
batch := b.FileIndex.NewBatch()
for _, file := range results.Hits {
batch.Delete(file.ID)
}
if err := b.FileIndex.Batch(batch); err != nil {
return -1, err
}
resultsCount += int64(results.Hits.Len())
if results.Hits.Len() < batchSize {
break
}
}
return resultsCount, nil
}
func (b *BleveEngine) DeleteUserFiles(userID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
query := bleve.NewTermQuery(userID)
query.SetField("CreatorId")
search := bleve.NewSearchRequest(query)
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeleteUserFiles",
"bleveengine.delete_user_files.error", nil,
err.Error(), http.StatusInternalServerError)
}
mlog.Info("Files for user deleted", mlog.String("user_id", userID), mlog.Int64("deleted", deleted))
return nil
}
func (b *BleveEngine) DeletePostFiles(postID string) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
query := bleve.NewTermQuery(postID)
query.SetField("PostId")
search := bleve.NewSearchRequest(query)
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeletePostFiles",
"bleveengine.delete_post_files.error", nil,
err.Error(), http.StatusInternalServerError)
}
mlog.Info("Files for post deleted", mlog.String("post_id", postID), mlog.Int64("deleted", deleted))
return nil
}
func (b *BleveEngine) DeleteFilesBatch(endTime, limit int64) *model.AppError {
b.Mutex.RLock()
defer b.Mutex.RUnlock()
endTimeFloat := float64(endTime)
query := bleve.NewNumericRangeQuery(nil, &endTimeFloat)
query.SetField("CreateAt")
search := bleve.NewSearchRequestOptions(query, int(limit), 0, false)
search.SortBy([]string{"-CreateAt"})
deleted, err := b.deleteFiles(search, DeleteFilesBatchSize)
if err != nil {
return model.NewAppError("Bleveengine.DeleteFilesBatch",
"bleveengine.delete_files_batch.error", nil,
err.Error(), http.StatusInternalServerError)
}
mlog.Info("Files in batch deleted", mlog.Int64("endTime", endTime), mlog.Int64("limit", limit), mlog.Int64("deleted", deleted))
return nil
}

Просмотреть файл

@@ -32,6 +32,12 @@ type SearchEngineInterface interface {
SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError) SearchUsersInChannel(teamId, channelId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, []string, *model.AppError)
SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError) SearchUsersInTeam(teamId string, restrictedToChannels []string, term string, options *model.UserSearchOptions) ([]string, *model.AppError)
DeleteUser(user *model.User) *model.AppError DeleteUser(user *model.User) *model.AppError
IndexFile(file *model.FileInfo, channelId string) *model.AppError
SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page, perPage int) ([]string, *model.AppError)
DeleteFile(fileID string) *model.AppError
DeletePostFiles(postID string) *model.AppError
DeleteUserFiles(userID string) *model.AppError
DeleteFilesBatch(endTime, limit int64) *model.AppError
TestConfig(cfg *model.Config) *model.AppError TestConfig(cfg *model.Config) *model.AppError
PurgeIndexes() *model.AppError PurgeIndexes() *model.AppError
RefreshIndexes() *model.AppError RefreshIndexes() *model.AppError

Просмотреть файл

@@ -64,6 +64,38 @@ func (_m *SearchEngineInterface) DeleteChannelPosts(channelID string) *model.App
return r0 return r0
} }
// DeleteFile provides a mock function with given fields: fileID
func (_m *SearchEngineInterface) DeleteFile(fileID string) *model.AppError {
ret := _m.Called(fileID)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
r0 = rf(fileID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// DeleteFilesBatch provides a mock function with given fields: endTime, limit
func (_m *SearchEngineInterface) DeleteFilesBatch(endTime int64, limit int64) *model.AppError {
ret := _m.Called(endTime, limit)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(int64, int64) *model.AppError); ok {
r0 = rf(endTime, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// DeletePost provides a mock function with given fields: post // DeletePost provides a mock function with given fields: post
func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError { func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError {
ret := _m.Called(post) ret := _m.Called(post)
@@ -80,6 +112,22 @@ func (_m *SearchEngineInterface) DeletePost(post *model.Post) *model.AppError {
return r0 return r0
} }
// DeletePostFiles provides a mock function with given fields: postID
func (_m *SearchEngineInterface) DeletePostFiles(postID string) *model.AppError {
ret := _m.Called(postID)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
r0 = rf(postID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// DeleteUser provides a mock function with given fields: user // DeleteUser provides a mock function with given fields: user
func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError { func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError {
ret := _m.Called(user) ret := _m.Called(user)
@@ -96,6 +144,22 @@ func (_m *SearchEngineInterface) DeleteUser(user *model.User) *model.AppError {
return r0 return r0
} }
// DeleteUserFiles provides a mock function with given fields: userID
func (_m *SearchEngineInterface) DeleteUserFiles(userID string) *model.AppError {
ret := _m.Called(userID)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(string) *model.AppError); ok {
r0 = rf(userID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// DeleteUserPosts provides a mock function with given fields: userID // DeleteUserPosts provides a mock function with given fields: userID
func (_m *SearchEngineInterface) DeleteUserPosts(userID string) *model.AppError { func (_m *SearchEngineInterface) DeleteUserPosts(userID string) *model.AppError {
ret := _m.Called(userID) ret := _m.Called(userID)
@@ -156,6 +220,22 @@ func (_m *SearchEngineInterface) IndexChannel(channel *model.Channel) *model.App
return r0 return r0
} }
// IndexFile provides a mock function with given fields: file, channelId
func (_m *SearchEngineInterface) IndexFile(file *model.FileInfo, channelId string) *model.AppError {
ret := _m.Called(file, channelId)
var r0 *model.AppError
if rf, ok := ret.Get(0).(func(*model.FileInfo, string) *model.AppError); ok {
r0 = rf(file, channelId)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.AppError)
}
}
return r0
}
// IndexPost provides a mock function with given fields: post, teamId // IndexPost provides a mock function with given fields: post, teamId
func (_m *SearchEngineInterface) IndexPost(post *model.Post, teamId string) *model.AppError { func (_m *SearchEngineInterface) IndexPost(post *model.Post, teamId string) *model.AppError {
ret := _m.Called(post, teamId) ret := _m.Called(post, teamId)
@@ -315,6 +395,31 @@ func (_m *SearchEngineInterface) SearchChannels(teamId string, term string) ([]s
return r0, r1 return r0, r1
} }
// SearchFiles provides a mock function with given fields: channels, searchParams, page, perPage
func (_m *SearchEngineInterface) SearchFiles(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, *model.AppError) {
ret := _m.Called(channels, searchParams, page, perPage)
var r0 []string
if rf, ok := ret.Get(0).(func(*model.ChannelList, []*model.SearchParams, int, int) []string); ok {
r0 = rf(channels, searchParams, page, perPage)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 *model.AppError
if rf, ok := ret.Get(1).(func(*model.ChannelList, []*model.SearchParams, int, int) *model.AppError); ok {
r1 = rf(channels, searchParams, page, perPage)
} else {
if ret.Get(1) != nil {
r1 = ret.Get(1).(*model.AppError)
}
}
return r0, r1
}
// SearchPosts provides a mock function with given fields: channels, searchParams, page, perPage // SearchPosts provides a mock function with given fields: channels, searchParams, page, perPage
func (_m *SearchEngineInterface) SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) { func (_m *SearchEngineInterface) SearchPosts(channels *model.ChannelList, searchParams []*model.SearchParams, page int, perPage int) ([]string, model.PostSearchMatches, *model.AppError) {
ret := _m.Called(channels, searchParams, page, perPage) ret := _m.Called(channels, searchParams, page, perPage)

Просмотреть файл

@@ -2982,6 +2982,24 @@ func (s *OpenTracingLayerFileInfoStore) Get(id string) (*model.FileInfo, error)
return result, err return result, err
} }
func (s *OpenTracingLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetByIds")
s.Root.Store.SetContext(newCtx)
defer func() {
s.Root.Store.SetContext(origCtx)
}()
defer span.Finish()
result, err := s.FileInfoStore.GetByIds(ids)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
}
return result, err
}
func (s *OpenTracingLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) { func (s *OpenTracingLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
origCtx := s.Root.Store.Context() origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetByPath") span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetByPath")
@@ -3139,6 +3157,24 @@ func (s *OpenTracingLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileI
return result, err return result, err
} }
func (s *OpenTracingLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.Search")
s.Root.Store.SetContext(newCtx)
defer func() {
s.Root.Store.SetContext(origCtx)
}()
defer span.Finish()
result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
}
return result, err
}
func (s *OpenTracingLayerFileInfoStore) SetContent(fileId string, content string) error { func (s *OpenTracingLayerFileInfoStore) SetContent(fileId string, content string) error {
origCtx := s.Root.Store.Context() origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.SetContent") span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.SetContent")

Просмотреть файл

@@ -3190,6 +3190,26 @@ func (s *RetryLayerFileInfoStore) Get(id string) (*model.FileInfo, error) {
} }
func (s *RetryLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
tries := 0
for {
result, err := s.FileInfoStore.GetByIds(ids)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
}
}
func (s *RetryLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) { func (s *RetryLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
tries := 0 tries := 0
@@ -3356,6 +3376,26 @@ func (s *RetryLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, e
} }
func (s *RetryLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) {
tries := 0
for {
result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
if err == nil {
return result, nil
}
if !isRepeatableError(err) {
return result, err
}
tries++
if tries >= 3 {
err = errors.Wrap(err, "giving up after 3 consecutive repeatable transaction failures")
return result, err
}
}
}
func (s *RetryLayerFileInfoStore) SetContent(fileId string, content string) error { func (s *RetryLayerFileInfoStore) SetContent(fileId string, content string) error {
tries := 0 tries := 0

Просмотреть файл

@@ -24,6 +24,7 @@ func (c *SearchChannelStore) deleteChannelIndex(channel *model.Channel) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteChannel(channel); err != nil { if err := engineCopy.DeleteChannel(channel); err != nil {
mlog.Warn("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) mlog.Warn("Encountered error deleting channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
} }
mlog.Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id)) mlog.Debug("Removed channel from index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
}) })
@@ -39,6 +40,7 @@ func (c *SearchChannelStore) indexChannel(channel *model.Channel) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.IndexChannel(channel); err != nil { if err := engineCopy.IndexChannel(channel); err != nil {
mlog.Warn("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) mlog.Warn("Encountered error indexing channel", mlog.String("channel_id", channel.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
} }
mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id)) mlog.Debug("Indexed channel in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("channel_id", channel.Id))
}) })

197
store/searchlayer/file_info_layer.go Обычный файл
Просмотреть файл

@@ -0,0 +1,197 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package searchlayer
import (
"github.com/mattermost/mattermost-server/v5/mlog"
"github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/services/searchengine"
"github.com/mattermost/mattermost-server/v5/store"
)
type SearchFileInfoStore struct {
store.FileInfoStore
rootStore *SearchStore
}
func (s SearchFileInfoStore) indexFile(file *model.FileInfo) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if file.PostId == "" {
return
}
post, postErr := s.rootStore.Post().GetSingle(file.PostId)
if postErr != nil {
mlog.Error("Couldn't get post for file for SearchEngine indexing.", mlog.String("post_id", file.PostId), mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id), mlog.Err(postErr))
return
}
if err := engineCopy.IndexFile(file, post.ChannelId); err != nil {
mlog.Error("Encountered error indexing file", mlog.String("file_info_id", file.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Indexed file in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", file.Id))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndex(fileID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteFile(fileID); err != nil {
mlog.Error("Encountered error deleting file", mlog.String("file_info_id", fileID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed file from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("file_info_id", fileID))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndexForUser(userID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteUserFiles(userID); err != nil {
mlog.Error("Encountered error deleting files for user", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed user's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("user_id", userID))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndexForPost(postID string) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeletePostFiles(postID); err != nil {
mlog.Error("Encountered error deleting files for post", mlog.String("post_id", postID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed post's files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", postID))
})
}
}
}
func (s SearchFileInfoStore) deleteFileIndexBatch(endTime, limit int64) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsIndexingEnabled() {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteFilesBatch(endTime, limit); err != nil {
mlog.Error("Encountered error deleting a batch of files", mlog.Int64("limit", limit), mlog.Int64("end_time", endTime), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
}
mlog.Debug("Removed batch of files from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.Int64("end_time", endTime), mlog.Int64("limit", limit))
})
}
}
}
func (s SearchFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
nfile, err := s.FileInfoStore.Save(info)
if err == nil {
s.indexFile(nfile)
}
return nfile, err
}
func (s SearchFileInfoStore) SetContent(fileID, content string) error {
err := s.FileInfoStore.SetContent(fileID, content)
if err == nil {
nfile, err2 := s.FileInfoStore.Get(fileID)
if err2 == nil {
nfile.Content = content
s.indexFile(nfile)
}
}
return err
}
func (s SearchFileInfoStore) AttachToPost(fileId, postId, creatorId string) error {
err := s.FileInfoStore.AttachToPost(fileId, postId, creatorId)
if err == nil {
nFileInfo, err2 := s.FileInfoStore.Get(fileId)
if err2 == nil {
s.indexFile(nFileInfo)
}
}
return err
}
func (s SearchFileInfoStore) DeleteForPost(postId string) (string, error) {
result, err := s.FileInfoStore.DeleteForPost(postId)
if err == nil {
s.deleteFileIndexForPost(postId)
}
return result, err
}
func (s SearchFileInfoStore) PermanentDelete(fileId string) error {
err := s.FileInfoStore.PermanentDelete(fileId)
if err == nil {
s.deleteFileIndex(fileId)
}
return err
}
func (s SearchFileInfoStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
result, err := s.FileInfoStore.PermanentDeleteBatch(endTime, limit)
if err == nil {
s.deleteFileIndexBatch(endTime, limit)
}
return result, err
}
func (s SearchFileInfoStore) PermanentDeleteByUser(userId string) (int64, error) {
result, err := s.FileInfoStore.PermanentDeleteByUser(userId)
if err == nil {
s.deleteFileIndexForUser(userId)
}
return result, err
}
func (s SearchFileInfoStore) Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) {
for _, engine := range s.rootStore.searchEngine.GetActiveEngines() {
if engine.IsSearchEnabled() {
userChannels, nErr := s.rootStore.Channel().GetChannels(teamId, userId, paramsList[0].IncludeDeletedChannels, 0)
if nErr != nil {
return nil, nErr
}
fileIds, appErr := engine.SearchFiles(userChannels, paramsList, page, perPage)
if appErr != nil {
mlog.Error("Encountered error on Search.", mlog.String("search_engine", engine.GetName()), mlog.Err(appErr))
continue
}
mlog.Debug("Using the first available search engine", mlog.String("search_engine", engine.GetName()))
// Get the files
filesList := model.NewFileInfoList()
if len(fileIds) > 0 {
files, nErr := s.FileInfoStore.GetByIds(fileIds)
if nErr != nil {
return nil, nErr
}
for _, f := range files {
filesList.AddFileInfo(f)
filesList.AddOrder(f.Id)
}
}
return filesList, nil
}
}
if *s.rootStore.getConfig().SqlSettings.DisableDatabaseSearch {
mlog.Debug("Returning empty results for file Search as the database search is disabled")
return model.NewFileInfoList(), nil
}
mlog.Debug("Using database search because no other search engine is available")
return s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
}

Просмотреть файл

@@ -19,6 +19,7 @@ type SearchStore struct {
team *SearchTeamStore team *SearchTeamStore
channel *SearchChannelStore channel *SearchChannelStore
post *SearchPostStore post *SearchPostStore
fileInfo *SearchFileInfoStore
configValue atomic.Value configValue atomic.Value
} }
@@ -32,6 +33,7 @@ func NewSearchLayer(baseStore store.Store, searchEngine *searchengine.Broker, cf
searchStore.post = &SearchPostStore{PostStore: baseStore.Post(), rootStore: searchStore} searchStore.post = &SearchPostStore{PostStore: baseStore.Post(), rootStore: searchStore}
searchStore.team = &SearchTeamStore{TeamStore: baseStore.Team(), rootStore: searchStore} searchStore.team = &SearchTeamStore{TeamStore: baseStore.Team(), rootStore: searchStore}
searchStore.user = &SearchUserStore{UserStore: baseStore.User(), rootStore: searchStore} searchStore.user = &SearchUserStore{UserStore: baseStore.User(), rootStore: searchStore}
searchStore.fileInfo = &SearchFileInfoStore{FileInfoStore: baseStore.FileInfo(), rootStore: searchStore}
return searchStore return searchStore
} }
@@ -52,6 +54,10 @@ func (s *SearchStore) Post() store.PostStore {
return s.post return s.post
} }
func (s *SearchStore) FileInfo() store.FileInfoStore {
return s.fileInfo
}
func (s *SearchStore) Team() store.TeamStore { func (s *SearchStore) Team() store.TeamStore {
return s.team return s.team
} }

Просмотреть файл

@@ -28,6 +28,7 @@ func (s SearchPostStore) indexPost(post *model.Post) {
} }
if err := engineCopy.IndexPost(post, channel.TeamId); err != nil { if err := engineCopy.IndexPost(post, channel.TeamId); err != nil {
mlog.Warn("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) mlog.Warn("Encountered error indexing post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
} }
mlog.Debug("Indexed post in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id)) mlog.Debug("Indexed post in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
}) })
@@ -41,6 +42,7 @@ func (s SearchPostStore) deletePostIndex(post *model.Post) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeletePost(post); err != nil { if err := engineCopy.DeletePost(post); err != nil {
mlog.Warn("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) mlog.Warn("Encountered error deleting post", mlog.String("post_id", post.Id), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
} }
mlog.Debug("Removed post from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id)) mlog.Debug("Removed post from the index in search engine", mlog.String("search_engine", engineCopy.GetName()), mlog.String("post_id", post.Id))
}) })
@@ -54,6 +56,7 @@ func (s SearchPostStore) deleteChannelPostsIndex(channelID string) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteChannelPosts(channelID); err != nil { if err := engineCopy.DeleteChannelPosts(channelID); err != nil {
mlog.Warn("Encountered error deleting channel posts", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) mlog.Warn("Encountered error deleting channel posts", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
} }
mlog.Debug("Removed all channel posts from the index in search engine", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName())) mlog.Debug("Removed all channel posts from the index in search engine", mlog.String("channel_id", channelID), mlog.String("search_engine", engineCopy.GetName()))
}) })
@@ -67,6 +70,7 @@ func (s SearchPostStore) deleteUserPostsIndex(userID string) {
runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) { runIndexFn(engine, func(engineCopy searchengine.SearchEngineInterface) {
if err := engineCopy.DeleteUserPosts(userID); err != nil { if err := engineCopy.DeleteUserPosts(userID); err != nil {
mlog.Warn("Encountered error deleting user posts", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err)) mlog.Warn("Encountered error deleting user posts", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()), mlog.Err(err))
return
} }
mlog.Debug("Removed all user posts from the index in search engine", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName())) mlog.Debug("Removed all user posts from the index in search engine", mlog.String("user_id", userID), mlog.String("search_engine", engineCopy.GetName()))
}) })

1646
store/searchtest/file_info_layer.go Обычный файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@@ -392,6 +392,36 @@ func (th *SearchTestHelper) createPost(userID, channelID, message, hashtags, pos
return post, nil return post, nil
} }
func (th *SearchTestHelper) createFileInfoModel(creatorID, postID, name, content, extension, mimeType string, createAt, size int64) *model.FileInfo {
return &model.FileInfo{
CreatorId: creatorID,
PostId: postID,
CreateAt: createAt,
UpdateAt: createAt,
DeleteAt: 0,
Name: name,
Content: content,
Path: name,
Extension: extension,
Size: size,
MimeType: mimeType,
}
}
func (th *SearchTestHelper) createFileInfo(creatorID, postID, name, content, extension, mimeType string, createAt, size int64) (*model.FileInfo, error) {
var creationTime int64 = 1000000
if createAt > 0 {
creationTime = createAt
}
fileInfoModel := th.createFileInfoModel(creatorID, postID, name, content, extension, mimeType, creationTime, size)
fileInfo, appError := th.Store.FileInfo().Save(fileInfoModel)
if appError != nil {
return nil, errors.New(appError.Error())
}
return fileInfo, nil
}
func (th *SearchTestHelper) createReply(userID, message, hashtags string, parent *model.Post, createAt int64, pinned bool) (*model.Post, error) { func (th *SearchTestHelper) createReply(userID, message, hashtags string, parent *model.Post, createAt int64, pinned bool) (*model.Post, error) {
replyModel := th.createPostModel(userID, parent.ChannelId, message, hashtags, parent.Type, createAt, pinned) replyModel := th.createPostModel(userID, parent.ChannelId, message, hashtags, parent.Type, createAt, pinned)
replyModel.ParentId = parent.Id replyModel.ParentId = parent.Id
@@ -411,6 +441,13 @@ func (th *SearchTestHelper) deleteUserPosts(userID string) error {
return nil return nil
} }
func (th *SearchTestHelper) deleteUserFileInfos(userID string) error {
if _, err := th.Store.FileInfo().PermanentDeleteByUser(userID); err != nil {
return errors.New(err.Error())
}
return nil
}
func (th *SearchTestHelper) addUserToTeams(user *model.User, teamIDS []string) error { func (th *SearchTestHelper) addUserToTeams(user *model.User, teamIDS []string) error {
for _, teamID := range teamIDS { for _, teamID := range teamIDS {
_, err := th.Store.Team().SaveMember(&model.TeamMember{TeamId: teamID, UserId: user.Id}, -1) _, err := th.Store.Team().SaveMember(&model.TeamMember{TeamId: teamID, UserId: user.Id}, -1)
@@ -467,6 +504,15 @@ func (th *SearchTestHelper) checkPostInSearchResults(t *testing.T, postID string
assert.Contains(t, postIDS, postID, "Did not find expected post in search results.") assert.Contains(t, postIDS, postID, "Did not find expected post in search results.")
} }
func (th *SearchTestHelper) checkFileInfoInSearchResults(t *testing.T, fileID string, searchResults map[string]*model.FileInfo) {
t.Helper()
fileIDS := make([]string, len(searchResults))
for ID := range searchResults {
fileIDS = append(fileIDS, ID)
}
assert.Contains(t, fileIDS, fileID, "Did not find expected file in search results.")
}
func (th *SearchTestHelper) checkChannelIdsMatch(t *testing.T, expected []string, results *model.ChannelList) { func (th *SearchTestHelper) checkChannelIdsMatch(t *testing.T, expected []string, results *model.ChannelList) {
t.Helper() t.Helper()
channelIds := make([]string, len(*results)) channelIds := make([]string, len(*results))

Просмотреть файл

@@ -6,11 +6,15 @@ package sqlstore
import ( import (
"database/sql" "database/sql"
"fmt" "fmt"
"regexp"
"strconv"
"strings"
sq "github.com/Masterminds/squirrel" sq "github.com/Masterminds/squirrel"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/mattermost/mattermost-server/v5/einterfaces" "github.com/mattermost/mattermost-server/v5/einterfaces"
"github.com/mattermost/mattermost-server/v5/mlog"
"github.com/mattermost/mattermost-server/v5/model" "github.com/mattermost/mattermost-server/v5/model"
"github.com/mattermost/mattermost-server/v5/store" "github.com/mattermost/mattermost-server/v5/store"
) )
@@ -73,6 +77,9 @@ func (fs SqlFileInfoStore) createIndexesIfNotExists() {
fs.CreateIndexIfNotExists("idx_fileinfo_create_at", "FileInfo", "CreateAt") fs.CreateIndexIfNotExists("idx_fileinfo_create_at", "FileInfo", "CreateAt")
fs.CreateIndexIfNotExists("idx_fileinfo_delete_at", "FileInfo", "DeleteAt") fs.CreateIndexIfNotExists("idx_fileinfo_delete_at", "FileInfo", "DeleteAt")
fs.CreateIndexIfNotExists("idx_fileinfo_postid_at", "FileInfo", "PostId") fs.CreateIndexIfNotExists("idx_fileinfo_postid_at", "FileInfo", "PostId")
fs.CreateIndexIfNotExists("idx_fileinfo_extension_at", "FileInfo", "Extension")
fs.CreateFullTextIndexIfNotExists("idx_fileinfo_name_txt", "FileInfo", "Name")
fs.CreateFullTextIndexIfNotExists("idx_fileinfo_content_txt", "FileInfo", "Content")
} }
func (fs SqlFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) { func (fs SqlFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
@@ -87,6 +94,26 @@ func (fs SqlFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
return info, nil return info, nil
} }
func (fs SqlFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
query := fs.getQueryBuilder().
Select("*").
From("FileInfo").
Where(sq.Eq{"Id": ids}).
Where(sq.Eq{"DeleteAt": 0}).
OrderBy("CreateAt DESC")
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "file_info_tosql")
}
var infos []*model.FileInfo
if _, err := fs.GetReplica().Select(&infos, queryString, args...); err != nil {
return nil, errors.Wrap(err, "failed to find FileInfos")
}
return infos, nil
}
func (fs SqlFileInfoStore) Upsert(info *model.FileInfo) (*model.FileInfo, error) { func (fs SqlFileInfoStore) Upsert(info *model.FileInfo) (*model.FileInfo, error) {
info.PreSave() info.PreSave()
if err := info.IsValid(); err != nil { if err := info.IsValid(); err != nil {
@@ -393,3 +420,176 @@ func (fs SqlFileInfoStore) PermanentDeleteByUser(userId string) (int64, error) {
return rowsAffected, nil return rowsAffected, nil
} }
func (fs SqlFileInfoStore) Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error) {
// Since we don't support paging for DB search, we just return nothing for later pages
if page > 0 {
return model.NewFileInfoList(), nil
}
if err := model.IsSearchParamsListValid(paramsList); err != nil {
return nil, err
}
query := fs.getQueryBuilder().
Select("FI.*").
From("FileInfo AS FI").
LeftJoin("Posts as P ON FI.PostId=P.Id").
LeftJoin("Channels as C ON C.Id=P.ChannelId").
LeftJoin("ChannelMembers as CM ON C.Id=CM.ChannelId").
Where(sq.Or{sq.Eq{"C.TeamId": teamId}, sq.Eq{"C.TeamId": ""}}).
Where(sq.Eq{"FI.DeleteAt": 0}).
OrderBy("FI.CreateAt DESC").
Limit(100)
for _, params := range paramsList {
params.Terms = removeNonAlphaNumericUnquotedTerms(params.Terms, " ")
if !params.IncludeDeletedChannels {
query = query.Where(sq.Eq{"C.DeleteAt": 0})
}
if !params.SearchWithoutUserId {
query = query.Where(sq.Eq{"CM.UserId": userId})
}
if len(params.InChannels) != 0 {
query = query.Where(sq.Eq{"C.Id": params.InChannels})
}
if len(params.Extensions) != 0 {
query = query.Where(sq.Eq{"FI.Extension": params.Extensions})
}
if len(params.ExcludedExtensions) != 0 {
query = query.Where(sq.NotEq{"FI.Extension": params.ExcludedExtensions})
}
if len(params.ExcludedChannels) != 0 {
query = query.Where(sq.NotEq{"C.Id": params.ExcludedChannels})
}
if len(params.FromUsers) != 0 {
query = query.Where(sq.Eq{"FI.CreatorId": params.FromUsers})
}
if len(params.ExcludedUsers) != 0 {
query = query.Where(sq.NotEq{"FI.CreatorId": params.ExcludedUsers})
}
// handle after: before: on: filters
if len(params.OnDate) > 0 {
onDateStart, onDateEnd := params.GetOnDateMillis()
query = query.Where(sq.Expr("FI.CreateAt BETWEEN ? AND ?", strconv.FormatInt(onDateStart, 10), strconv.FormatInt(onDateEnd, 10)))
} else {
if len(params.ExcludedDate) > 0 {
excludedDateStart, excludedDateEnd := params.GetExcludedDateMillis()
query = query.Where(sq.Expr("FI.CreateAt NOT BETWEEN ? AND ?", strconv.FormatInt(excludedDateStart, 10), strconv.FormatInt(excludedDateEnd, 10)))
}
if len(params.AfterDate) > 0 {
afterDate := params.GetAfterDateMillis()
query = query.Where(sq.GtOrEq{"FI.CreateAt": strconv.FormatInt(afterDate, 10)})
}
if len(params.BeforeDate) > 0 {
beforeDate := params.GetBeforeDateMillis()
query = query.Where(sq.LtOrEq{"FI.CreateAt": strconv.FormatInt(beforeDate, 10)})
}
if len(params.ExcludedAfterDate) > 0 {
afterDate := params.GetExcludedAfterDateMillis()
query = query.Where(sq.Lt{"FI.CreateAt": strconv.FormatInt(afterDate, 10)})
}
if len(params.ExcludedBeforeDate) > 0 {
beforeDate := params.GetExcludedBeforeDateMillis()
query = query.Where(sq.Gt{"FI.CreateAt": strconv.FormatInt(beforeDate, 10)})
}
}
terms := params.Terms
excludedTerms := params.ExcludedTerms
// these chars have special meaning and can be treated as spaces
for _, c := range specialSearchChar {
terms = strings.Replace(terms, c, " ", -1)
excludedTerms = strings.Replace(excludedTerms, c, " ", -1)
}
if terms == "" && excludedTerms == "" {
// we've already confirmed that we have a channel or user to search for
} else if fs.DriverName() == model.DATABASE_DRIVER_POSTGRES {
// Parse text for wildcards
if wildcard, err := regexp.Compile(`\*($| )`); err == nil {
terms = wildcard.ReplaceAllLiteralString(terms, ":* ")
excludedTerms = wildcard.ReplaceAllLiteralString(excludedTerms, ":* ")
}
excludeClause := ""
if excludedTerms != "" {
excludeClause = " & !(" + strings.Join(strings.Fields(excludedTerms), " | ") + ")"
}
queryTerms := ""
if params.OrTerms {
queryTerms = "(" + strings.Join(strings.Fields(terms), " | ") + ")" + excludeClause
} else {
queryTerms = "(" + strings.Join(strings.Fields(terms), " & ") + ")" + excludeClause
}
query = query.Where(sq.Or{
sq.Expr("to_tsvector('english', FI.Name) @@ to_tsquery('english', ?)", queryTerms),
sq.Expr("to_tsvector('english', FI.Content) @@ to_tsquery('english', ?)", queryTerms),
})
} else if fs.DriverName() == model.DATABASE_DRIVER_MYSQL {
var err error
terms, err = removeMysqlStopWordsFromTerms(terms)
if err != nil {
return nil, errors.Wrap(err, "failed to remove Mysql stop-words from terms")
}
if terms == "" {
return model.NewFileInfoList(), nil
}
excludeClause := ""
if excludedTerms != "" {
excludeClause = " -(" + excludedTerms + ")"
}
queryTerms := ""
if params.OrTerms {
queryTerms = terms + excludeClause
} else {
splitTerms := []string{}
for _, t := range strings.Fields(terms) {
splitTerms = append(splitTerms, "+"+t)
}
queryTerms = strings.Join(splitTerms, " ") + excludeClause
}
query = query.Where(sq.Or{
sq.Expr("MATCH (FI.Name) AGAINST (? IN BOOLEAN MODE)", queryTerms),
sq.Expr("MATCH (FI.Content) AGAINST (? IN BOOLEAN MODE)", queryTerms),
})
}
}
queryString, args, err := query.ToSql()
if err != nil {
return nil, errors.Wrap(err, "file_info_tosql")
}
list := model.NewFileInfoList()
fileInfos := []*model.FileInfo{}
_, err = fs.GetSearchReplica().Select(&fileInfos, queryString, args...)
if err != nil {
mlog.Warn("Query error searching files.", mlog.Err(err))
// Don't return the error to the caller as it is of no use to the user. Instead return an empty set of search results.
} else {
for _, f := range fileInfos {
list.AddFileInfo(f)
list.AddOrder(f.Id)
}
}
list.MakeNonNil()
return list, nil
}

Просмотреть файл

@@ -6,9 +6,14 @@ package sqlstore
import ( import (
"testing" "testing"
"github.com/mattermost/mattermost-server/v5/store/searchtest"
"github.com/mattermost/mattermost-server/v5/store/storetest" "github.com/mattermost/mattermost-server/v5/store/storetest"
) )
func TestFileInfoStore(t *testing.T) { func TestFileInfoStore(t *testing.T) {
StoreTest(t, storetest.TestFileInfoStore) StoreTest(t, storetest.TestFileInfoStore)
} }
func TestSearchFileInfoStore(t *testing.T) {
StoreTestWithSearchTestEngine(t, searchtest.TestSearchFileInfoStore)
}

Просмотреть файл

@@ -564,6 +564,7 @@ type FileInfoStore interface {
Save(info *model.FileInfo) (*model.FileInfo, error) Save(info *model.FileInfo) (*model.FileInfo, error)
Upsert(info *model.FileInfo) (*model.FileInfo, error) Upsert(info *model.FileInfo) (*model.FileInfo, error)
Get(id string) (*model.FileInfo, error) Get(id string) (*model.FileInfo, error)
GetByIds(ids []string) ([]*model.FileInfo, error)
GetByPath(path string) (*model.FileInfo, error) GetByPath(path string) (*model.FileInfo, error)
GetForPost(postId string, readFromMaster, includeDeleted, allowFromCache bool) ([]*model.FileInfo, error) GetForPost(postId string, readFromMaster, includeDeleted, allowFromCache bool) ([]*model.FileInfo, error)
GetForUser(userId string) ([]*model.FileInfo, error) GetForUser(userId string) ([]*model.FileInfo, error)
@@ -575,6 +576,7 @@ type FileInfoStore interface {
PermanentDeleteBatch(endTime int64, limit int64) (int64, error) PermanentDeleteBatch(endTime int64, limit int64) (int64, error)
PermanentDeleteByUser(userId string) (int64, error) PermanentDeleteByUser(userId string) (int64, error)
SetContent(fileId, content string) error SetContent(fileId, content string) error
Search(paramsList []*model.SearchParams, userId, teamId string, page, perPage int) (*model.FileInfoList, error)
ClearCaches() ClearCaches()
} }

Просмотреть файл

@@ -77,6 +77,29 @@ func (_m *FileInfoStore) Get(id string) (*model.FileInfo, error) {
return r0, r1 return r0, r1
} }
// GetByIds provides a mock function with given fields: ids
func (_m *FileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
ret := _m.Called(ids)
var r0 []*model.FileInfo
if rf, ok := ret.Get(0).(func([]string) []*model.FileInfo); ok {
r0 = rf(ids)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.FileInfo)
}
}
var r1 error
if rf, ok := ret.Get(1).(func([]string) error); ok {
r1 = rf(ids)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetByPath provides a mock function with given fields: path // GetByPath provides a mock function with given fields: path
func (_m *FileInfoStore) GetByPath(path string) (*model.FileInfo, error) { func (_m *FileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
ret := _m.Called(path) ret := _m.Called(path)
@@ -253,6 +276,29 @@ func (_m *FileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, error) {
return r0, r1 return r0, r1
} }
// Search provides a mock function with given fields: paramsList, userId, teamId, page, perPage
func (_m *FileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) {
ret := _m.Called(paramsList, userId, teamId, page, perPage)
var r0 *model.FileInfoList
if rf, ok := ret.Get(0).(func([]*model.SearchParams, string, string, int, int) *model.FileInfoList); ok {
r0 = rf(paramsList, userId, teamId, page, perPage)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.FileInfoList)
}
}
var r1 error
if rf, ok := ret.Get(1).(func([]*model.SearchParams, string, string, int, int) error); ok {
r1 = rf(paramsList, userId, teamId, page, perPage)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// SetContent provides a mock function with given fields: fileId, content // SetContent provides a mock function with given fields: fileId, content
func (_m *FileInfoStore) SetContent(fileId string, content string) error { func (_m *FileInfoStore) SetContent(fileId string, content string) error {
ret := _m.Called(fileId, content) ret := _m.Called(fileId, content)

Просмотреть файл

@@ -2734,6 +2734,22 @@ func (s *TimerLayerFileInfoStore) Get(id string) (*model.FileInfo, error) {
return result, err return result, err
} }
func (s *TimerLayerFileInfoStore) GetByIds(ids []string) ([]*model.FileInfo, error) {
start := timemodule.Now()
result, err := s.FileInfoStore.GetByIds(ids)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.GetByIds", success, elapsed)
}
return result, err
}
func (s *TimerLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) { func (s *TimerLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
start := timemodule.Now() start := timemodule.Now()
@@ -2877,6 +2893,22 @@ func (s *TimerLayerFileInfoStore) Save(info *model.FileInfo) (*model.FileInfo, e
return result, err return result, err
} }
func (s *TimerLayerFileInfoStore) Search(paramsList []*model.SearchParams, userId string, teamId string, page int, perPage int) (*model.FileInfoList, error) {
start := timemodule.Now()
result, err := s.FileInfoStore.Search(paramsList, userId, teamId, page, perPage)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
success := "false"
if err == nil {
success = "true"
}
s.Root.Metrics.ObserveStoreMethodDuration("FileInfoStore.Search", success, elapsed)
}
return result, err
}
func (s *TimerLayerFileInfoStore) SetContent(fileId string, content string) error { func (s *TimerLayerFileInfoStore) SetContent(fileId string, content string) error {
start := timemodule.Now() start := timemodule.Now()