mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

MM-41260: Revamp ES/Bleve batching logic (#19841)

Просмотр исходного кода 
The older method used to reply completely on timestamps
to take batches of items in a timestamp range and then
just incrementing the timestamp. This led to handling
edge-cases such as more items than the batch count, all
having the same timestamp.

Additionally, relying on timestamp as the page cursor
meant that indexing was not very efficient if you had
several items spread out across large spans of time.

To get away from all of that we use a proper cursor-based
approach consisting of createAt+Id. With this, we move
completely to a constant page size where we can fetch
a given number of objects irrespective of when they
were created. This makes indexing much more faster and
efficient.

https://mattermost.atlassian.net/browse/MM-41260

```release-note
Elasticsearch and Bleve indexing have been revamped to be much
more efficient and faster. The config parameter BulkIndexingTimeWindowSeconds
for both elasticsearch and bleve have been removed.
A new config parameter called BatchSize has been introduced instead.
This parameter controls the number of objects that
can be indexed in a single batch. This makes things
more efficient and maintains a constant workload.
```
Этот коммит содержится в:
Agniva De Sarker
2022-03-31 10:46:01 +05:30
коммит произвёл GitHub
родитель 9adf06e122
Коммит f8a3119426
27 изменённых файлов: 384 добавлений и 376 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

16
store/opentracinglayer/opentracinglayer.go
Просмотреть файл

@@ -1105,7 +1105,7 @@ func (s *OpenTracingLayerChannelStore) GetChannels(teamID string, userID string,
return result, err
}
func (s *OpenTracingLayerChannelStore) GetChannelsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.Channel, error) {
func (s *OpenTracingLayerChannelStore) GetChannelsBatchForIndexing(startTime int64, startChannelID string, limit int) ([]*model.Channel, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ChannelStore.GetChannelsBatchForIndexing")
s.Root.Store.SetContext(newCtx)
@@ -1114,7 +1114,7 @@ func (s *OpenTracingLayerChannelStore) GetChannelsBatchForIndexing(startTime int
}()
defer span.Finish()
result, err := s.ChannelStore.GetChannelsBatchForIndexing(startTime, endTime, limit)
result, err := s.ChannelStore.GetChannelsBatchForIndexing(startTime, startChannelID, limit)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
@@ -3318,7 +3318,7 @@ func (s *OpenTracingLayerFileInfoStore) GetByPath(path string) (*model.FileInfo,
return result, err
}
func (s *OpenTracingLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) {
func (s *OpenTracingLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "FileInfoStore.GetFilesBatchForIndexing")
s.Root.Store.SetContext(newCtx)
@@ -3327,7 +3327,7 @@ func (s *OpenTracingLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64
}()
defer span.Finish()
result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit)
result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, limit)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
@@ -5700,7 +5700,7 @@ func (s *OpenTracingLayerPostStore) GetPostsAfter(options model.GetPostsOptions)
return result, err
}
func (s *OpenTracingLayerPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, error) {
func (s *OpenTracingLayerPostStore) GetPostsBatchForIndexing(startTime int64, startPostID string, limit int) ([]*model.PostForIndexing, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "PostStore.GetPostsBatchForIndexing")
s.Root.Store.SetContext(newCtx)
@@ -5709,7 +5709,7 @@ func (s *OpenTracingLayerPostStore) GetPostsBatchForIndexing(startTime int64, en
}()
defer span.Finish()
result, err := s.PostStore.GetPostsBatchForIndexing(startTime, endTime, limit)
result, err := s.PostStore.GetPostsBatchForIndexing(startTime, startPostID, limit)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)
@@ -10610,7 +10610,7 @@ func (s *OpenTracingLayerUserStore) GetUnreadCountForChannel(userID string, chan
return result, err
}
func (s *OpenTracingLayerUserStore) GetUsersBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.UserForIndexing, error) {
func (s *OpenTracingLayerUserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "UserStore.GetUsersBatchForIndexing")
s.Root.Store.SetContext(newCtx)
@@ -10619,7 +10619,7 @@ func (s *OpenTracingLayerUserStore) GetUsersBatchForIndexing(startTime int64, en
}()
defer span.Finish()
result, err := s.UserStore.GetUsersBatchForIndexing(startTime, endTime, limit)
result, err := s.UserStore.GetUsersBatchForIndexing(startTime, startFileID, limit)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)

16
store/retrylayer/retrylayer.go
Просмотреть файл

@@ -1234,11 +1234,11 @@ func (s *RetryLayerChannelStore) GetChannels(teamID string, userID string, opts
}
func (s *RetryLayerChannelStore) GetChannelsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.Channel, error) {
func (s *RetryLayerChannelStore) GetChannelsBatchForIndexing(startTime int64, startChannelID string, limit int) ([]*model.Channel, error) {
tries := 0
for {
result, err := s.ChannelStore.GetChannelsBatchForIndexing(startTime, endTime, limit)
result, err := s.ChannelStore.GetChannelsBatchForIndexing(startTime, startChannelID, limit)
if err == nil {
return result, nil
}
@@ -3715,11 +3715,11 @@ func (s *RetryLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error
}
func (s *RetryLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) {
func (s *RetryLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
tries := 0
for {
result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit)
result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, limit)
if err == nil {
return result, nil
}
@@ -6448,11 +6448,11 @@ func (s *RetryLayerPostStore) GetPostsAfter(options model.GetPostsOptions) (*mod
}
func (s *RetryLayerPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, error) {
func (s *RetryLayerPostStore) GetPostsBatchForIndexing(startTime int64, startPostID string, limit int) ([]*model.PostForIndexing, error) {
tries := 0
for {
result, err := s.PostStore.GetPostsBatchForIndexing(startTime, endTime, limit)
result, err := s.PostStore.GetPostsBatchForIndexing(startTime, startPostID, limit)
if err == nil {
return result, nil
}
@@ -12103,11 +12103,11 @@ func (s *RetryLayerUserStore) GetUnreadCountForChannel(userID string, channelID
}
func (s *RetryLayerUserStore) GetUsersBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.UserForIndexing, error) {
func (s *RetryLayerUserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
tries := 0
for {
result, err := s.UserStore.GetUsersBatchForIndexing(startTime, endTime, limit)
result, err := s.UserStore.GetUsersBatchForIndexing(startTime, startFileID, limit)
if err == nil {
return result, nil
}

12
store/sqlstore/channel_store.go
Просмотреть файл

@@ -3949,23 +3949,23 @@ func (s SqlChannelStore) GetAllDirectChannelsForExportAfter(limit int, afterId s
return directChannelsForExport, nil
}
func (s SqlChannelStore) GetChannelsBatchForIndexing(startTime, endTime int64, limit int) ([]*model.Channel, error) {
func (s SqlChannelStore) GetChannelsBatchForIndexing(startTime int64, startChannelID string, limit int) ([]*model.Channel, error) {
query :=
`SELECT
*
FROM
Channels
WHERE
CreateAt >= ?
AND
CreateAt < ?
CreateAt > ?
OR
(CreateAt = ? AND Id > ?)
ORDER BY
CreateAt
CreateAt ASC, Id ASC
LIMIT
?`
channels := []*model.Channel{}
err := s.GetSearchReplicaX().Select(&channels, query, startTime, endTime, limit)
err := s.GetSearchReplicaX().Select(&channels, query, startTime, startTime, startChannelID, limit)
if err != nil {
return nil, errors.Wrap(err, "failed to find Channels")
}

14
store/sqlstore/file_info_store.go
Просмотреть файл

@@ -688,17 +688,23 @@ func (fs SqlFileInfoStore) CountAll() (int64, error) {
return count, nil
}
func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime, endTime int64, limit int) ([]*model.FileForIndexing, error) {
func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
files := []*model.FileForIndexing{}
sql, args, _ := fs.getQueryBuilder().
Select(append(fs.queryFields, "Coalesce(p.ChannelId, '') AS ChannelId")...).
From("FileInfo").
LeftJoin("Posts AS p ON FileInfo.PostId = p.Id").
Where(sq.GtOrEq{"FileInfo.CreateAt": startTime}).
Where(sq.Lt{"FileInfo.CreateAt": endTime}).
OrderBy("FileInfo.CreateAt").
Where(sq.Or{
sq.Gt{"FileInfo.CreateAt": startTime},
sq.And{
sq.Eq{"FileInfo.CreateAt": startTime},
sq.Gt{"FileInfo.Id": startFileID},
},
}).
OrderBy("FileInfo.CreateAt ASC, FileInfo.Id ASC").
Limit(uint64(limit)).
ToSql()
err := fs.GetSearchReplicaX().Select(&files, sql, args...)
if err != nil {
return nil, errors.Wrap(err, "failed to find Files")

29
store/sqlstore/post_store.go
Просмотреть файл

@@ -2198,22 +2198,26 @@ func (s *SqlPostStore) GetPostsByIds(postIds []string) ([]*model.Post, error) {
return posts, nil
}
func (s *SqlPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, error) {
func (s *SqlPostStore) GetPostsBatchForIndexing(startTime int64, startPostID string, limit int) ([]*model.PostForIndexing, error) {
posts := []*model.PostForIndexing{}
err := s.GetSearchReplicaX().Select(&posts,
`SELECT
PostsQuery.*, Channels.TeamId, ParentPosts.CreateAt ParentCreateAt
table := "Posts"
// We force this index to avoid any chances of index merge intersection.
if s.DriverName() == model.DatabaseDriverMysql {
table += " USE INDEX(idx_posts_create_at_id)"
}
query := `SELECT
PostsQuery.*, Channels.TeamId
FROM (
SELECT
*
FROM
Posts
` + table + `
WHERE
Posts.CreateAt >= ?
AND
Posts.CreateAt < ?
Posts.CreateAt > ?
OR
(Posts.CreateAt = ? AND Posts.Id > ?)
ORDER BY
CreateAt ASC
CreateAt ASC, Id ASC
LIMIT
?
)
@@ -2223,11 +2227,8 @@ func (s *SqlPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64,
Channels
ON
PostsQuery.ChannelId = Channels.Id
LEFT JOIN
Posts ParentPosts
ON
PostsQuery.RootId = ParentPosts.Id`,
startTime, endTime, limit)
ORDER BY CreateAt ASC, Id ASC`
err := s.GetSearchReplicaX().Select(&posts, query, startTime, startTime, startPostID, limit)
if err != nil {
return nil, errors.Wrap(err, "failed to find Posts")

13
store/sqlstore/user_store.go
Просмотреть файл

@@ -1671,12 +1671,17 @@ func (us SqlUserStore) InferSystemInstallDate() (int64, error) {
return createAt, nil
}
func (us SqlUserStore) GetUsersBatchForIndexing(startTime, endTime int64, limit int) ([]*model.UserForIndexing, error) {
func (us SqlUserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
users := []*model.User{}
usersQuery, args, _ := us.usersQuery.
Where(sq.GtOrEq{"u.CreateAt": startTime}).
Where(sq.Lt{"u.CreateAt": endTime}).
OrderBy("u.CreateAt").
Where(sq.Or{
sq.Gt{"u.CreateAt": startTime},
sq.And{
sq.Eq{"u.CreateAt": startTime},
sq.Gt{"u.Id": startFileID},
},
}).
OrderBy("u.CreateAt ASC, u.Id ASC").
Limit(uint64(limit)).
ToSql()
err := us.GetSearchReplicaX().Select(&users, usersQuery, args...)

8
store/store.go
Просмотреть файл

@@ -267,7 +267,7 @@ type ChannelStore interface {
GetAllDirectChannelsForExportAfter(limit int, afterID string) ([]*model.DirectChannelForExport, error)
GetChannelMembersForExport(userID string, teamID string) ([]*model.ChannelMemberForExport, error)
RemoveAllDeactivatedMembers(channelID string) error
GetChannelsBatchForIndexing(startTime, endTime int64, limit int) ([]*model.Channel, error)
GetChannelsBatchForIndexing(startTime int64, startChannelID string, limit int) ([]*model.Channel, error)
UserBelongsToChannels(userID string, channelIds []string) (bool, error)
// UpdateMembersRole sets all of the given team members to admins and all of the other members of the team to
@@ -351,7 +351,7 @@ type PostStore interface {
Overwrite(post *model.Post) (*model.Post, error)
OverwriteMultiple(posts []*model.Post) ([]*model.Post, int, error)
GetPostsByIds(postIds []string) ([]*model.Post, error)
GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, error)
GetPostsBatchForIndexing(startTime int64, startPostID string, limit int) ([]*model.PostForIndexing, error)
PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error)
DeleteOrphanedRows(limit int) (deleted int64, err error)
PermanentDeleteBatch(endTime int64, limit int64) (int64, error)
@@ -430,7 +430,7 @@ type UserStore interface {
ClearAllCustomRoleAssignments() error
InferSystemInstallDate() (int64, error)
GetAllAfter(limit int, afterID string) ([]*model.User, error)
GetUsersBatchForIndexing(startTime, endTime int64, limit int) ([]*model.UserForIndexing, error)
GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error)
Count(options model.UserCountOptions) (int64, error)
GetTeamGroupUsers(teamID string) ([]*model.User, error)
GetChannelGroupUsers(channelID string) ([]*model.User, error)
@@ -657,7 +657,7 @@ type FileInfoStore interface {
SetContent(fileID, content string) error
Search(paramsList []*model.SearchParams, userID, teamID string, page, perPage int) (*model.FileInfoList, error)
CountAll() (int64, error)
GetFilesBatchForIndexing(startTime, endTime int64, limit int) ([]*model.FileForIndexing, error)
GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error)
ClearCaches()
}

18
store/storetest/channel_store.go
Просмотреть файл

@@ -7601,7 +7601,6 @@ func testChannelStoreGetChannelsBatchForIndexing(t *testing.T, ss store.Store) {
require.NoError(t, nErr)
time.Sleep(10 * time.Millisecond)
startTime := c2.CreateAt
c3 := &model.Channel{}
c3.DisplayName = "Channel3"
@@ -7633,23 +7632,20 @@ func testChannelStoreGetChannelsBatchForIndexing(t *testing.T, ss store.Store) {
_, nErr = ss.Channel().Save(c6, -1)
require.NoError(t, nErr)
endTime := c6.CreateAt
// First and last channel should be outside the range
channels, err := ss.Channel().GetChannelsBatchForIndexing(startTime, endTime, 1000)
channels, err := ss.Channel().GetChannelsBatchForIndexing(c1.CreateAt, "", 4)
assert.NoError(t, err)
assert.ElementsMatch(t, []*model.Channel{c2, c3, c4, c5}, channels)
assert.Len(t, channels, 4)
// Update the endTime, last channel should be in
endTime = model.GetMillis()
channels, err = ss.Channel().GetChannelsBatchForIndexing(startTime, endTime, 1000)
// From 4th createat+id
channels, err = ss.Channel().GetChannelsBatchForIndexing(channels[3].CreateAt, channels[3].Id, 5)
assert.NoError(t, err)
assert.ElementsMatch(t, []*model.Channel{c2, c3, c4, c5, c6}, channels)
assert.Len(t, channels, 2)
// Testing the limit
channels, err = ss.Channel().GetChannelsBatchForIndexing(startTime, endTime, 2)
channels, err = ss.Channel().GetChannelsBatchForIndexing(channels[1].CreateAt, channels[1].Id, 1)
assert.NoError(t, err)
assert.ElementsMatch(t, []*model.Channel{c2, c3}, channels)
assert.Len(t, channels, 0)
}
func testGroupSyncedChannelCount(t *testing.T, ss store.Store) {

51
store/storetest/file_info_store.go
Просмотреть файл

@@ -673,42 +673,23 @@ func testFileInfoStoreGetFilesBatchForIndexing(t *testing.T, ss store.Store) {
ss.FileInfo().PermanentDelete(f3.Id)
}()
t.Run("get all files", func(t *testing.T) {
r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt, model.GetMillis()+100000, 100)
require.NoError(t, err)
require.Len(t, r, 3, "Expected 3 posts in results. Got %v", len(r))
for _, f := range r {
if f.Id == f1.Id {
require.Equal(t, f.ChannelId, o1.ChannelId, "Unexpected channel ID")
require.Equal(t, f.Path, "file1.txt", "Unexpected filename")
} else if f.Id == f2.Id {
require.Equal(t, f.ChannelId, o2.ChannelId, "Unexpected channel ID")
require.Equal(t, f.Path, "file2.txt", "Unexpected filename")
} else if f.Id == f3.Id {
require.Equal(t, f.ChannelId, o3.ChannelId, "Unexpected channel ID")
require.Equal(t, f.Path, "file3.txt", "Unexpected filename")
} else {
require.Fail(t, "unexpected file returned")
}
}
})
// Getting all
r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt-1, "", 100)
require.NoError(t, err)
require.Len(t, r, 3, "Expected 3 posts in results. Got %v", len(r))
t.Run("get files after certain date", func(t *testing.T) {
r, err := ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt+1, model.GetMillis()+100000, 100)
require.NoError(t, err)
require.Len(t, r, 2, "Expected 2 posts in results. Got %v", len(r))
for _, f := range r {
if f.Id == f2.Id {
require.Equal(t, f.ChannelId, o2.ChannelId, "Unexpected channel ID")
require.Equal(t, f.Path, "file2.txt", "Unexpected filename")
} else if f.Id == f3.Id {
require.Equal(t, f.ChannelId, o3.ChannelId, "Unexpected channel ID")
require.Equal(t, f.Path, "file3.txt", "Unexpected filename")
} else {
require.Fail(t, "unexpected file returned")
}
}
})
// Testing pagination
r, err = ss.FileInfo().GetFilesBatchForIndexing(f1.CreateAt-1, "", 2)
require.NoError(t, err)
require.Len(t, r, 2, "Expected 2 posts in results. Got %v", len(r))
r, err = ss.FileInfo().GetFilesBatchForIndexing(r[1].CreateAt, r[1].Id, 2)
require.NoError(t, err)
require.Len(t, r, 1, "Expected 1 post in results. Got %v", len(r))
r, err = ss.FileInfo().GetFilesBatchForIndexing(r[0].CreateAt, r[0].Id, 2)
require.NoError(t, err)
require.Len(t, r, 0, "Expected 0 posts in results. Got %v", len(r))
}
func testFileInfoStoreCountAll(t *testing.T, ss store.Store) {

14
store/storetest/mocks/ChannelStore.go
Просмотреть файл

@@ -697,13 +697,13 @@ func (_m *ChannelStore) GetChannels(teamID string, userID string, opts *model.Ch
return r0, r1
}
// GetChannelsBatchForIndexing provides a mock function with given fields: startTime, endTime, limit
func (_m *ChannelStore) GetChannelsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.Channel, error) {
ret := _m.Called(startTime, endTime, limit)
// GetChannelsBatchForIndexing provides a mock function with given fields: startTime, startChannelID, limit
func (_m *ChannelStore) GetChannelsBatchForIndexing(startTime int64, startChannelID string, limit int) ([]*model.Channel, error) {
ret := _m.Called(startTime, startChannelID, limit)
var r0 []*model.Channel
if rf, ok := ret.Get(0).(func(int64, int64, int) []*model.Channel); ok {
r0 = rf(startTime, endTime, limit)
if rf, ok := ret.Get(0).(func(int64, string, int) []*model.Channel); ok {
r0 = rf(startTime, startChannelID, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.Channel)
@@ -711,8 +711,8 @@ func (_m *ChannelStore) GetChannelsBatchForIndexing(startTime int64, endTime int
}
var r1 error
if rf, ok := ret.Get(1).(func(int64, int64, int) error); ok {
r1 = rf(startTime, endTime, limit)
if rf, ok := ret.Get(1).(func(int64, string, int) error); ok {
r1 = rf(startTime, startChannelID, limit)
} else {
r1 = ret.Error(1)
}

14
store/storetest/mocks/FileInfoStore.go
Просмотреть файл

@@ -144,13 +144,13 @@ func (_m *FileInfoStore) GetByPath(path string) (*model.FileInfo, error) {
return r0, r1
}
// GetFilesBatchForIndexing provides a mock function with given fields: startTime, endTime, limit
func (_m *FileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) {
ret := _m.Called(startTime, endTime, limit)
// GetFilesBatchForIndexing provides a mock function with given fields: startTime, startFileID, limit
func (_m *FileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
ret := _m.Called(startTime, startFileID, limit)
var r0 []*model.FileForIndexing
if rf, ok := ret.Get(0).(func(int64, int64, int) []*model.FileForIndexing); ok {
r0 = rf(startTime, endTime, limit)
if rf, ok := ret.Get(0).(func(int64, string, int) []*model.FileForIndexing); ok {
r0 = rf(startTime, startFileID, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.FileForIndexing)
@@ -158,8 +158,8 @@ func (_m *FileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64
}
var r1 error
if rf, ok := ret.Get(1).(func(int64, int64, int) error); ok {
r1 = rf(startTime, endTime, limit)
if rf, ok := ret.Get(1).(func(int64, string, int) error); ok {
r1 = rf(startTime, startFileID, limit)
} else {
r1 = ret.Error(1)
}

14
store/storetest/mocks/PostStore.go
Просмотреть файл

@@ -465,13 +465,13 @@ func (_m *PostStore) GetPostsAfter(options model.GetPostsOptions) (*model.PostLi
return r0, r1
}
// GetPostsBatchForIndexing provides a mock function with given fields: startTime, endTime, limit
func (_m *PostStore) GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, error) {
ret := _m.Called(startTime, endTime, limit)
// GetPostsBatchForIndexing provides a mock function with given fields: startTime, startPostID, limit
func (_m *PostStore) GetPostsBatchForIndexing(startTime int64, startPostID string, limit int) ([]*model.PostForIndexing, error) {
ret := _m.Called(startTime, startPostID, limit)
var r0 []*model.PostForIndexing
if rf, ok := ret.Get(0).(func(int64, int64, int) []*model.PostForIndexing); ok {
r0 = rf(startTime, endTime, limit)
if rf, ok := ret.Get(0).(func(int64, string, int) []*model.PostForIndexing); ok {
r0 = rf(startTime, startPostID, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.PostForIndexing)
@@ -479,8 +479,8 @@ func (_m *PostStore) GetPostsBatchForIndexing(startTime int64, endTime int64, li
}
var r1 error
if rf, ok := ret.Get(1).(func(int64, int64, int) error); ok {
r1 = rf(startTime, endTime, limit)
if rf, ok := ret.Get(1).(func(int64, string, int) error); ok {
r1 = rf(startTime, startPostID, limit)
} else {
r1 = ret.Error(1)
}

14
store/storetest/mocks/UserStore.go
Просмотреть файл

@@ -979,13 +979,13 @@ func (_m *UserStore) GetUnreadCountForChannel(userID string, channelID string) (
return r0, r1
}
// GetUsersBatchForIndexing provides a mock function with given fields: startTime, endTime, limit
func (_m *UserStore) GetUsersBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.UserForIndexing, error) {
ret := _m.Called(startTime, endTime, limit)
// GetUsersBatchForIndexing provides a mock function with given fields: startTime, startFileID, limit
func (_m *UserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
ret := _m.Called(startTime, startFileID, limit)
var r0 []*model.UserForIndexing
if rf, ok := ret.Get(0).(func(int64, int64, int) []*model.UserForIndexing); ok {
r0 = rf(startTime, endTime, limit)
if rf, ok := ret.Get(0).(func(int64, string, int) []*model.UserForIndexing); ok {
r0 = rf(startTime, startFileID, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.UserForIndexing)
@@ -993,8 +993,8 @@ func (_m *UserStore) GetUsersBatchForIndexing(startTime int64, endTime int64, li
}
var r1 error
if rf, ok := ret.Get(1).(func(int64, int64, int) error); ok {
r1 = rf(startTime, endTime, limit)
if rf, ok := ret.Get(1).(func(int64, string, int) error); ok {
r1 = rf(startTime, startFileID, limit)
} else {
r1 = ret.Error(1)
}

38
store/storetest/post_store.go
Просмотреть файл

@@ -2953,7 +2953,7 @@ func testPostStoreGetPostsBatchForIndexing(t *testing.T, ss store.Store) {
o2.ChannelId = c2.Id
o2.UserId = model.NewId()
o2.Message = NewTestId()
o2, err = ss.Post().Save(o2)
_, err = ss.Post().Save(o2)
require.NoError(t, err)
o3 := &model.Post{}
@@ -2961,26 +2961,30 @@ func testPostStoreGetPostsBatchForIndexing(t *testing.T, ss store.Store) {
o3.UserId = model.NewId()
o3.RootId = o1.Id
o3.Message = NewTestId()
o3, err = ss.Post().Save(o3)
_, err = ss.Post().Save(o3)
require.NoError(t, err)
r, err := ss.Post().GetPostsBatchForIndexing(o1.CreateAt, model.GetMillis()+100000, 100)
// Getting all
r, err := ss.Post().GetPostsBatchForIndexing(o1.CreateAt-1, "", 100)
require.NoError(t, err)
require.Len(t, r, 3, "Expected 3 posts in results. Got %v", len(r))
for _, p := range r {
if p.Id == o1.Id {
require.Equal(t, p.TeamId, c1.TeamId, "Unexpected team ID")
require.Nil(t, p.ParentCreateAt, "Unexpected parent create at")
} else if p.Id == o2.Id {
require.Equal(t, p.TeamId, c2.TeamId, "Unexpected team ID")
require.Nil(t, p.ParentCreateAt, "Unexpected parent create at")
} else if p.Id == o3.Id {
require.Equal(t, p.TeamId, c1.TeamId, "Unexpected team ID")
require.Equal(t, *p.ParentCreateAt, o1.CreateAt, "Unexpected parent create at")
} else {
require.Fail(t, "unexpected post returned")
}
}
// Testing pagination
r, err = ss.Post().GetPostsBatchForIndexing(o1.CreateAt-1, "", 1)
require.NoError(t, err)
require.Len(t, r, 1, "Expected 1 post in results. Got %v", len(r))
r, err = ss.Post().GetPostsBatchForIndexing(r[0].CreateAt, r[0].Id, 1)
require.NoError(t, err)
require.Len(t, r, 1, "Expected 1 post in results. Got %v", len(r))
r, err = ss.Post().GetPostsBatchForIndexing(r[0].CreateAt, r[0].Id, 1)
require.NoError(t, err)
require.Len(t, r, 1, "Expected 1 post in results. Got %v", len(r))
r, err = ss.Post().GetPostsBatchForIndexing(r[0].CreateAt, r[0].Id, 1)
require.NoError(t, err)
require.Len(t, r, 0, "Expected 0 post in results. Got %v", len(r))
}
func testPostStorePermanentDeleteBatch(t *testing.T, ss store.Store) {

45
store/storetest/user_store.go
Просмотреть файл

@@ -4716,7 +4716,6 @@ func testUserStoreGetUsersBatchForIndexing(t *testing.T, ss store.Store) {
})
require.NoError(t, err)
startTime := u2.CreateAt
time.Sleep(time.Millisecond)
u3, err := ss.User().Save(&model.User{
@@ -4744,47 +4743,23 @@ func testUserStoreGetUsersBatchForIndexing(t *testing.T, ss store.Store) {
})
require.NoError(t, err)
endTime := u3.CreateAt
// First and last user should be outside the range
res1List, err := ss.User().GetUsersBatchForIndexing(startTime, endTime, 100)
// Getting all users
res1List, err := ss.User().GetUsersBatchForIndexing(u1.CreateAt-1, "", 100)
require.NoError(t, err)
assert.Len(t, res1List, 3)
assert.Len(t, res1List, 1)
assert.Equal(t, res1List[0].Username, u2.Username)
assert.ElementsMatch(t, res1List[0].TeamsIds, []string{t1.Id})
assert.ElementsMatch(t, res1List[0].ChannelsIds, []string{cPub1.Id, cPub2.Id})
// Update startTime to include first user
startTime = u1.CreateAt
res2List, err := ss.User().GetUsersBatchForIndexing(startTime, endTime, 100)
// Testing pagination
res2List, err := ss.User().GetUsersBatchForIndexing(u1.CreateAt-1, "", 1)
require.NoError(t, err)
assert.Len(t, res2List, 1)
res2List, err = ss.User().GetUsersBatchForIndexing(res2List[0].CreateAt, res2List[0].Id, 2)
require.NoError(t, err)
assert.Len(t, res2List, 2)
assert.Equal(t, res2List[0].Username, u1.Username)
assert.Equal(t, res2List[0].ChannelsIds, []string{})
assert.Equal(t, res2List[0].TeamsIds, []string{})
assert.Equal(t, res2List[1].Username, u2.Username)
// Update endTime to include last user
endTime = model.GetMillis()
res3List, err := ss.User().GetUsersBatchForIndexing(startTime, endTime, 100)
res2List, err = ss.User().GetUsersBatchForIndexing(res2List[1].CreateAt, res2List[1].Id, 2)
require.NoError(t, err)
assert.Len(t, res3List, 3)
assert.Equal(t, res3List[0].Username, u1.Username)
assert.Equal(t, res3List[1].Username, u2.Username)
assert.Equal(t, res3List[2].Username, u3.Username)
assert.ElementsMatch(t, res3List[2].TeamsIds, []string{})
assert.ElementsMatch(t, res3List[2].ChannelsIds, []string{cPub2.Id})
// Testing the limit
res4List, err := ss.User().GetUsersBatchForIndexing(startTime, endTime, 2)
require.NoError(t, err)
assert.Len(t, res4List, 2)
assert.Equal(t, res4List[0].Username, u1.Username)
assert.Equal(t, res4List[1].Username, u2.Username)
assert.Len(t, res2List, 0)
}
func testUserStoreGetTeamGroupUsers(t *testing.T, ss store.Store) {

16
store/timerlayer/timerlayer.go
Просмотреть файл

@@ -1029,10 +1029,10 @@ func (s *TimerLayerChannelStore) GetChannels(teamID string, userID string, opts
return result, err
}
func (s *TimerLayerChannelStore) GetChannelsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.Channel, error) {
func (s *TimerLayerChannelStore) GetChannelsBatchForIndexing(startTime int64, startChannelID string, limit int) ([]*model.Channel, error) {
start := timemodule.Now()
result, err := s.ChannelStore.GetChannelsBatchForIndexing(startTime, endTime, limit)
result, err := s.ChannelStore.GetChannelsBatchForIndexing(startTime, startChannelID, limit)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
@@ -3036,10 +3036,10 @@ func (s *TimerLayerFileInfoStore) GetByPath(path string) (*model.FileInfo, error
return result, err
}
func (s *TimerLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.FileForIndexing, error) {
func (s *TimerLayerFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
start := timemodule.Now()
result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, endTime, limit)
result, err := s.FileInfoStore.GetFilesBatchForIndexing(startTime, startFileID, limit)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
@@ -5162,10 +5162,10 @@ func (s *TimerLayerPostStore) GetPostsAfter(options model.GetPostsOptions) (*mod
return result, err
}
func (s *TimerLayerPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, error) {
func (s *TimerLayerPostStore) GetPostsBatchForIndexing(startTime int64, startPostID string, limit int) ([]*model.PostForIndexing, error) {
start := timemodule.Now()
result, err := s.PostStore.GetPostsBatchForIndexing(startTime, endTime, limit)
result, err := s.PostStore.GetPostsBatchForIndexing(startTime, startPostID, limit)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {
@@ -9557,10 +9557,10 @@ func (s *TimerLayerUserStore) GetUnreadCountForChannel(userID string, channelID
return result, err
}
func (s *TimerLayerUserStore) GetUsersBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.UserForIndexing, error) {
func (s *TimerLayerUserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
start := timemodule.Now()
result, err := s.UserStore.GetUsersBatchForIndexing(startTime, endTime, limit)
result, err := s.UserStore.GetUsersBatchForIndexing(startTime, startFileID, limit)
elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
if s.Root.Metrics != nil {