Optimize queries and indexes on posts table (#13217)

* Optimize queries and indexes on posts table

Added RootId column to two indexes. Query in getParentPosts split into
two queries - for big channels can be ~1000x times faster in MySQL, but
a bit slower in PostgreSQL.
Rewritten query in GetPostsSince for MySQL - around 20% faster.
Benchmark results (/1 - 140k posts in channel, /2 - 800 posts):

benchmark                                                         old ns/op      new ns/op     delta
BenchmarkPosts/postgres/GetFlaggedPostsForTeam/1-16               12068006       12130615      +0.52%
BenchmarkPosts/postgres/GetFlaggedPostsForChannel/1-16            7334992        7388359       +0.73%
BenchmarkPosts/postgres/GetPosts(skipThreads=true)/1-16           1845547        1979362       +7.25%
BenchmarkPosts/postgres/GetPosts(skipThreads=false)/1-16          2260061        2595112       +14.82%
BenchmarkPosts/postgres/GetPostsSince(skipThreads=true)/1-16      38510212       40625368      +5.49%
BenchmarkPosts/postgres/GetPostsSince(skipThreads=false)/1-16     32389821       32581044      +0.59%
BenchmarkPosts/postgres/GetFlaggedPostsForTeam/2-16               1604215        1584941       -1.20%
BenchmarkPosts/postgres/GetFlaggedPostsForChannel/2-16            1278623        1277473       -0.09%
BenchmarkPosts/postgres/GetPosts(skipThreads=true)/2-16           1921049        1984581       +3.31%
BenchmarkPosts/postgres/GetPosts(skipThreads=false)/2-16          3478147        3000086       -13.74%
BenchmarkPosts/postgres/GetPostsSince(skipThreads=true)/2-16      4813332        5198276       +8.00%
BenchmarkPosts/postgres/GetPostsSince(skipThreads=false)/2-16     3475847        3816201       +9.79%
BenchmarkPosts/mysql/GetFlaggedPostsForTeam/1-16                  9674132        9708361       +0.35%
BenchmarkPosts/mysql/GetFlaggedPostsForChannel/1-16               5780763        5818874       +0.66%
BenchmarkPosts/mysql/GetPosts(skipThreads=true)/1-16              2261194        2268826       +0.34%
BenchmarkPosts/mysql/GetPosts(skipThreads=false)/1-16             2371804023     3184120       -99.87%
BenchmarkPosts/mysql/GetPostsSince(skipThreads=true)/1-16         35552813       27709811      -22.06%
BenchmarkPosts/mysql/GetPostsSince(skipThreads=false)/1-16        28758400       22622865      -21.33%
BenchmarkPosts/mysql/GetFlaggedPostsForTeam/2-16                  1174064        1205933       +2.71%
BenchmarkPosts/mysql/GetFlaggedPostsForChannel/2-16               1007026        1091551       +8.39%
BenchmarkPosts/mysql/GetPosts(skipThreads=true)/2-16              2274397        2408730       +5.91%
BenchmarkPosts/mysql/GetPosts(skipThreads=false)/2-16             7454395        2542741       -65.89%
BenchmarkPosts/mysql/GetPostsSince(skipThreads=true)/2-16         8879200        6843435       -22.93%
BenchmarkPosts/mysql/GetPostsSince(skipThreads=false)/2-16        6293932        5373276       -14.63%

* old version of getParentsPosts for PostgreSQL

because new version using two queries is around 15% slower.

* review fixes

* revert index changes

store/sqlstore/post_store.go

@@ -455,37 +455,69 @@ func (s *SqlPostStore) GetPostsSince(options model.GetPostsSinceOptions, allowFr
 	replyCountQuery1 := ""
 	replyCountQuery2 := ""
 	if options.SkipFetchThreads {
-		replyCountQuery1 = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE p1.RootId = '' AND Posts.RootId = p1.Id AND Posts.DeleteAt = 0) as ReplyCount`
+		replyCountQuery1 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p1.RootId = '' AND Posts.RootId = p1.Id AND Posts.DeleteAt = 0) as ReplyCount`
-		replyCountQuery2 = ` ,(SELECT COUNT(Posts.Id) FROM Posts WHERE p2.RootId = '' AND Posts.RootId = p2.Id AND Posts.DeleteAt = 0) as ReplyCount`
+		replyCountQuery2 = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p2.RootId = '' AND Posts.RootId = p2.Id AND Posts.DeleteAt = 0) as ReplyCount`
 	}
+	var query string
 
-	_, err := s.GetReplica().Select(&posts,
+	// union of IDs and then join to get full posts is faster in mysql
-		`(SELECT
+	if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
-			*`+replyCountQuery1+`
+		query = `SELECT *` + replyCountQuery1 + ` FROM Posts p1 JOIN (
-		FROM
-			Posts p1
-		WHERE
-			(UpdateAt > :Time
-				AND ChannelId = :ChannelId)
-			LIMIT 1000)
-		UNION
 			(SELECT
-			    *`+replyCountQuery2+`
+              Id
-			FROM
+			  FROM
-			    Posts p2
+				  Posts p2
-			WHERE
+			  WHERE
-			    Id
+				  (UpdateAt > :Time
-			IN
+					  AND ChannelId = :ChannelId)
-			    (SELECT * FROM (SELECT
+				  LIMIT 1000)
-			        RootId
+			  UNION
-			    FROM
+				  (SELECT
-			        Posts
+					  Id
-			    WHERE
+				  FROM
-			        UpdateAt > :Time
+					  Posts p3
-						AND ChannelId = :ChannelId
+				  WHERE
-				LIMIT 1000) temp_tab))
+					  Id
-		ORDER BY CreateAt DESC`,
+				  IN
-		map[string]interface{}{"ChannelId": options.ChannelId, "Time": options.Time})
+					  (SELECT * FROM (SELECT
+						  RootId
+					  FROM
+						  Posts
+					  WHERE
+						  UpdateAt > :Time
+							  AND ChannelId = :ChannelId
+					  LIMIT 1000) temp_tab))
+			) j ON p1.Id = j.Id
+          ORDER BY CreateAt DESC`
+	} else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
+		query = `
+			(SELECT
+                       *` + replyCountQuery1 + `
+               FROM
+                       Posts p1
+               WHERE
+                       (UpdateAt > :Time
+                               AND ChannelId = :ChannelId)
+                       LIMIT 1000)
+               UNION
+                       (SELECT
+                           *` + replyCountQuery2 + `
+                       FROM
+                           Posts p2
+                       WHERE
+                           Id
+                       IN
+                           (SELECT * FROM (SELECT
+                               RootId
+                           FROM
+                               Posts
+                           WHERE
+                               UpdateAt > :Time
+                                               AND ChannelId = :ChannelId
+                               LIMIT 1000) temp_tab))
+               ORDER BY CreateAt DESC`
+	}
+	_, err := s.GetReplica().Select(&posts, query, map[string]interface{}{"ChannelId": options.ChannelId, "Time": options.Time})
 
 	if err != nil {
 		return nil, model.NewAppError("SqlPostStore.GetPostsSince", "store.sql_post.get_posts_since.app_error", nil, "channelId="+options.ChannelId+err.Error(), http.StatusInternalServerError)
@@ -697,6 +729,70 @@ func (s *SqlPostStore) getRootPosts(channelId string, offset int, limit int, ski
 }
 
 func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, *model.AppError) {
+	if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
+		return s.getParentsPostsPostgreSQL(channelId, offset, limit, skipFetchThreads)
+	}
+
+	// query parent Ids first
+	var roots []*struct {
+		RootId string
+	}
+	rootQuery := `
+		SELECT DISTINCT
+			q.RootId
+		FROM
+			(SELECT
+				RootId
+			FROM
+				Posts
+			WHERE
+				ChannelId = :ChannelId
+					AND DeleteAt = 0
+			ORDER BY CreateAt DESC
+			LIMIT :Limit OFFSET :Offset) q
+		WHERE q.RootId != ''`
+
+	_, err := s.GetReplica().Select(&roots, rootQuery, map[string]interface{}{"ChannelId": channelId, "Offset": offset, "Limit": limit})
+	if err != nil {
+		return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError)
+	}
+	if len(roots) == 0 {
+		return nil, nil
+	}
+	params := make(map[string]interface{})
+	placeholders := make([]string, len(roots))
+	for idx, r := range roots {
+		key := fmt.Sprintf(":Root%v", idx)
+		params[key[1:]] = r.RootId
+		placeholders[idx] = key
+	}
+	placeholderString := strings.Join(placeholders, ", ")
+	params["ChannelId"] = channelId
+	replyCountQuery := ""
+	whereStatement := "p.Id IN (" + placeholderString + ")"
+	if skipFetchThreads {
+		replyCountQuery = `, (SELECT COUNT(Posts.Id) FROM Posts WHERE p.RootId = '' AND Posts.RootId = p.Id AND Posts.DeleteAt = 0) as ReplyCount`
+	} else {
+		whereStatement += " OR p.RootId IN (" + placeholderString + ")"
+	}
+	var posts []*model.Post
+	_, err = s.GetReplica().Select(&posts, `
+		SELECT p.*`+replyCountQuery+`
+		FROM
+			Posts p
+		WHERE
+			(`+whereStatement+`)
+				AND ChannelId = :ChannelId
+				AND DeleteAt = 0
+		ORDER BY CreateAt`,
+		params)
+	if err != nil {
+		return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError)
+	}
+	return posts, nil
+}
+
+func (s *SqlPostStore) getParentsPostsPostgreSQL(channelId string, offset int, limit int, skipFetchThreads bool) ([]*model.Post, *model.AppError) {
 	var posts []*model.Post
 	replyCountQuery := ""
 	onStatement := "q1.RootId = q2.Id"
@@ -707,27 +803,27 @@ func (s *SqlPostStore) getParentsPosts(channelId string, offset int, limit int,
 	}
 	_, err := s.GetReplica().Select(&posts,
 		`SELECT q2.*`+replyCountQuery+`
-		FROM
+        FROM
-			Posts q2
+            Posts q2
-				INNER JOIN
+                INNER JOIN
-			(SELECT DISTINCT
+            (SELECT DISTINCT
-				q3.RootId
+                q3.RootId
-			FROM
+            FROM
-				(SELECT
+                (SELECT
-					RootId
+                    RootId
-				FROM
+                FROM
-					Posts
+                    Posts
-				WHERE
+                WHERE
-					ChannelId = :ChannelId1
+                    ChannelId = :ChannelId1
-						AND DeleteAt = 0
+                        AND DeleteAt = 0
-				ORDER BY CreateAt DESC
+                ORDER BY CreateAt DESC
-				LIMIT :Limit OFFSET :Offset) q3
+                LIMIT :Limit OFFSET :Offset) q3
-			WHERE q3.RootId != '') q1
+            WHERE q3.RootId != '') q1
-			ON `+onStatement+`
+            ON `+onStatement+`
-		WHERE
+        WHERE
-			ChannelId = :ChannelId2
+            ChannelId = :ChannelId2
-				AND DeleteAt = 0
+                AND DeleteAt = 0
-		ORDER BY CreateAt`,
+        ORDER BY CreateAt`,
 		map[string]interface{}{"ChannelId1": channelId, "Offset": offset, "Limit": limit, "ChannelId2": channelId})
 	if err != nil {
 		return nil, model.NewAppError("SqlPostStore.GetLinearPosts", "store.sql_post.get_parents_posts.app_error", nil, "channelId="+channelId+" err="+err.Error(), http.StatusInternalServerError)