MM-41260: Revamp ES/Bleve batching logic (#19841)

The older method used to reply completely on timestamps
to take batches of items in a timestamp range and then
just incrementing the timestamp. This led to handling
edge-cases such as more items than the batch count, all
having the same timestamp.

Additionally, relying on timestamp as the page cursor
meant that indexing was not very efficient if you had
several items spread out across large spans of time.

To get away from all of that we use a proper cursor-based
approach consisting of createAt+Id. With this, we move
completely to a constant page size where we can fetch
a given number of objects irrespective of when they
were created. This makes indexing much more faster and
efficient.

https://mattermost.atlassian.net/browse/MM-41260

```release-note
Elasticsearch and Bleve indexing have been revamped to be much
more efficient and faster. The config parameter BulkIndexingTimeWindowSeconds
for both elasticsearch and bleve have been removed.
A new config parameter called BatchSize has been introduced instead.
This parameter controls the number of objects that
can be indexed in a single batch. This makes things
more efficient and maintains a constant workload.
```

store/sqlstore/channel_store.go

@@ -3949,23 +3949,23 @@ func (s SqlChannelStore) GetAllDirectChannelsForExportAfter(limit int, afterId s
 	return directChannelsForExport, nil
 }
 
-func (s SqlChannelStore) GetChannelsBatchForIndexing(startTime, endTime int64, limit int) ([]*model.Channel, error) {
+func (s SqlChannelStore) GetChannelsBatchForIndexing(startTime int64, startChannelID string, limit int) ([]*model.Channel, error) {
 	query :=
 		`SELECT
 			 *
 		 FROM
 			 Channels
 		 WHERE
-			 CreateAt >= ?
-		 AND
-			 CreateAt < ?
+			 CreateAt > ?
+		OR
+			(CreateAt = ? AND Id > ?)
 		 ORDER BY
-			 CreateAt
+			 CreateAt ASC, Id ASC
 		 LIMIT
 			 ?`
 
 	channels := []*model.Channel{}
-	err := s.GetSearchReplicaX().Select(&channels, query, startTime, endTime, limit)
+	err := s.GetSearchReplicaX().Select(&channels, query, startTime, startTime, startChannelID, limit)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find Channels")
 	}

store/sqlstore/file_info_store.go

@@ -688,17 +688,23 @@ func (fs SqlFileInfoStore) CountAll() (int64, error) {
 	return count, nil
 }
 
-func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime, endTime int64, limit int) ([]*model.FileForIndexing, error) {
+func (fs SqlFileInfoStore) GetFilesBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.FileForIndexing, error) {
 	files := []*model.FileForIndexing{}
 	sql, args, _ := fs.getQueryBuilder().
 		Select(append(fs.queryFields, "Coalesce(p.ChannelId, '') AS ChannelId")...).
 		From("FileInfo").
 		LeftJoin("Posts AS p ON FileInfo.PostId = p.Id").
-		Where(sq.GtOrEq{"FileInfo.CreateAt": startTime}).
-		Where(sq.Lt{"FileInfo.CreateAt": endTime}).
-		OrderBy("FileInfo.CreateAt").
+		Where(sq.Or{
+			sq.Gt{"FileInfo.CreateAt": startTime},
+			sq.And{
+				sq.Eq{"FileInfo.CreateAt": startTime},
+				sq.Gt{"FileInfo.Id": startFileID},
+			},
+		}).
+		OrderBy("FileInfo.CreateAt ASC, FileInfo.Id ASC").
 		Limit(uint64(limit)).
 		ToSql()
+
 	err := fs.GetSearchReplicaX().Select(&files, sql, args...)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find Files")

store/sqlstore/post_store.go

@@ -2198,22 +2198,26 @@ func (s *SqlPostStore) GetPostsByIds(postIds []string) ([]*model.Post, error) {
 	return posts, nil
 }
 
-func (s *SqlPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64, limit int) ([]*model.PostForIndexing, error) {
+func (s *SqlPostStore) GetPostsBatchForIndexing(startTime int64, startPostID string, limit int) ([]*model.PostForIndexing, error) {
 	posts := []*model.PostForIndexing{}
-	err := s.GetSearchReplicaX().Select(&posts,
-		`SELECT
-			PostsQuery.*, Channels.TeamId, ParentPosts.CreateAt ParentCreateAt
+	table := "Posts"
+	// We force this index to avoid any chances of index merge intersection.
+	if s.DriverName() == model.DatabaseDriverMysql {
+		table += " USE INDEX(idx_posts_create_at_id)"
+	}
+	query := `SELECT
+			PostsQuery.*, Channels.TeamId
 		FROM (
 			SELECT
 				*
 			FROM
-				Posts
+				` + table + `
 			WHERE
-				Posts.CreateAt >= ?
-			AND
-				Posts.CreateAt < ?
+				Posts.CreateAt > ?
+			OR
+				(Posts.CreateAt = ? AND Posts.Id > ?)
 			ORDER BY
-				CreateAt ASC
+				CreateAt ASC, Id ASC
 			LIMIT
 				?
 			)
@@ -2223,11 +2227,8 @@ func (s *SqlPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64,
 			Channels
 		ON
 			PostsQuery.ChannelId = Channels.Id
-		LEFT JOIN
-			Posts ParentPosts
-		ON
-			PostsQuery.RootId = ParentPosts.Id`,
-		startTime, endTime, limit)
+		ORDER BY CreateAt ASC, Id ASC`
+	err := s.GetSearchReplicaX().Select(&posts, query, startTime, startTime, startPostID, limit)
 
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to find Posts")

store/sqlstore/user_store.go

@@ -1671,12 +1671,17 @@ func (us SqlUserStore) InferSystemInstallDate() (int64, error) {
 	return createAt, nil
 }
 
-func (us SqlUserStore) GetUsersBatchForIndexing(startTime, endTime int64, limit int) ([]*model.UserForIndexing, error) {
+func (us SqlUserStore) GetUsersBatchForIndexing(startTime int64, startFileID string, limit int) ([]*model.UserForIndexing, error) {
 	users := []*model.User{}
 	usersQuery, args, _ := us.usersQuery.
-		Where(sq.GtOrEq{"u.CreateAt": startTime}).
-		Where(sq.Lt{"u.CreateAt": endTime}).
-		OrderBy("u.CreateAt").
+		Where(sq.Or{
+			sq.Gt{"u.CreateAt": startTime},
+			sq.And{
+				sq.Eq{"u.CreateAt": startTime},
+				sq.Gt{"u.Id": startFileID},
+			},
+		}).
+		OrderBy("u.CreateAt ASC, u.Id ASC").
 		Limit(uint64(limit)).
 		ToSql()
 	err := us.GetSearchReplicaX().Select(&users, usersQuery, args...)