[MM-30831] granular data retention wireup (#17417)

* pre-checkout commit * add API endpoints for retention policies * allow deleting multiple teams/channels from a policy in a single request * pre-checkout commit * add auditing in API functions * add permission checks * update the store layers * update storetest * add check constraint on PostDuration column * pre-checkout commit * add query to delete posts under the scope of a granular retention policy * add suggestions from sbishel * allow clients to specify channels/teams when creating a new policy * remove foreign keys referencing Channels and Teams tables * add checks for whether teams and channels exist * pre-checkout commit * remove data referencing the Posts table * pre-checkout commit * write data store tests * sort results of buildGetPoliciesQuery * add missing test cases for teams * pre-checkout commit * add Client4 methods for data retention policy endpoints * add uint and uint64 to app/layer_generators * make granular policies override global policies * fix lint errors * pre-checkout commit * add license to top of files * add tests for data store layer * add missing test cases for store layer * run make i18n-extract * add query to delete ChannelMemberHistory * work in progress * add test for old reply to old post * fix lint error * use COALESCE on each Posts column * begin implementing orphaned rows worker * split PR * pre-checkout commit * use RetentionPolicyWithTeamAndChannelCounts * update app and api layers * run make i18n-extract * add RetentionPolicy to retrylayer_test.go * Revert "split PR" This reverts commit b316f03dd307a30deae931944ca7e4a1cc904605. * fix errors caused by revert * add suggestions from sbishel * fix copy-paste error * fix lint errors * pre-checkout commit * add function to delete orphaned rows * use -1 for infinite retention * remove check constraint * copy i18n entries from master * re-run tests with newer enterprise branch * add team data to channel list * add search for channels and teams in a policy * add store tests for channel and team search * add suggestions from mkraft * run make einterfaces-mocks * fix lint errors * add suggestions from mkraft * move removeOrphanedRows method to wireup branch * Revert "move removeOrphanedRows method to wireup branch" This reverts commit 94605c9b4a5378ffa44a3dec4d3f8e3306b9d33e. * use DeleteOrphanedRows where possible * run make i18n-extract * use COMPLIANCE permissions * run make migrations-bindadta * clean up teams before test * fix tests for TestRetentionPolicyStore * add API endpoints for mobile * fix lint error * fix some of the lint errors * move user/data_retention endpoints to data_retention.go * Revert "fix some of the lint errors" This reverts commit b5b2dc27566c427187db942c5c0afe319e7679c4. * add exclude_policy_constrained parameter for /channels and /teams * fix lint errors * add policy_id field to GET endpoints for channels and teams * use PolicyWithTeamID in RetentionPolicy layer * fix lint errors * run make i18n-extract * update mock call in telemetry_test.go * return status:OK in JSON instead of 204 * pre-checkout commit * add policy_id field on channels/teams * fix lint errors * use sq.Eq instead of '?' * use new subsection permissions * update channels and teams endpoints to use new subsection permissions * add extra search opts for channels in a policy * fix lint errors * allow negative post duration in patch * remove DELETE FROM query in retention policy tests * use *int64 for PostDuration * re-run CI tests * use 3-step deletion strategy for each table * fix lint errors * run make store-layers * re-run CI tests * add test with channel, team and global policies * use common function for SQL queries * add pagination test * use struct for args to common SQL function * fix lint errors * run make i18n-extract * check if Channels.TeamId is "" or nil * use three OR clauses * write separate genericRetentionPoliciesDeletion function * add config setting for BatchSize * add telemetry for BatchSize * use feature flag * add old i18n messages back in * re-run CI tests * update call signature in storetest * MM-30831: Adds constant for retention default batch size. * MM-30831: Removes comment re: optimization. * MM-30831: Converts days to milliseconds. * MM-30831: Reverts change to test. * Revert "MM-30831: Reverts change to test." This reverts commit 6d14275a1ceae682bb9e17ec69b39252b44e0c0c. * Revert "MM-30831: Converts days to milliseconds." This reverts commit a0cb6ec09d854a05194c1daee1c5333f260231c3. * MM-30831: Fixes tests. * MM-30381: Fix for change to method sig. Co-authored-by: Max Erenberg <max.erenberg@mattermost.com> Co-authored-by: Mattermod <mattermod@users.noreply.github.com> Co-authored-by: Martin Kraft <martin@upspin.org>
2021-06-23 07:55:12 -04:00
--- a/store/sqlstore/channel_member_history_store.go
+++ b/store/sqlstore/channel_member_history_store.go
@@ -165,6 +165,46 @@ func (s SqlChannelMemberHistoryStore) getFromChannelMembersTable(startTime int64
 	return histories, nil
 }

+// PermanentDeleteBatchForRetentionPolicies deletes a batch of records which are affected by
+// the global or a granular retention policy.
+// See `genericPermanentDeleteBatchForRetentionPolicies` for details.
+func (s SqlChannelMemberHistoryStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	builder := s.getQueryBuilder().
+		Select("ChannelMemberHistory.ChannelId, ChannelMemberHistory.UserId, ChannelMemberHistory.JoinTime").
+		From("ChannelMemberHistory")
+	return genericPermanentDeleteBatchForRetentionPolicies(RetentionPolicyBatchDeletionInfo{
+		BaseBuilder:         builder,
+		Table:               "ChannelMemberHistory",
+		TimeColumn:          "LeaveTime",
+		PrimaryKeys:         []string{"ChannelId", "UserId", "JoinTime"},
+		ChannelIDTable:      "ChannelMemberHistory",
+		NowMillis:           now,
+		GlobalPolicyEndTime: globalPolicyEndTime,
+		Limit:               limit,
+	}, s.SqlStore, cursor)
+}
+
+// DeleteOrphanedRows removes entries from ChannelMemberHistory when a corresponding channel no longer exists.
+func (s SqlChannelMemberHistoryStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
+	// We need the extra level of nesting to deal with MySQL's locking
+	const query = `
+	DELETE FROM ChannelMemberHistory WHERE (ChannelId, UserId, JoinTime) IN (
+		SELECT * FROM (
+			SELECT ChannelId, UserId, JoinTime FROM ChannelMemberHistory
+			LEFT JOIN Channels ON ChannelMemberHistory.ChannelId = Channels.Id
+			WHERE Channels.Id IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	props := map[string]interface{}{"Limit": limit}
+	result, err := s.GetMaster().Exec(query, props)
+	if err != nil {
+		return
+	}
+	deleted, err = result.RowsAffected()
+	return
+}
+
 func (s SqlChannelMemberHistoryStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
 	var (
 		query string
--- a/store/sqlstore/post_store.go
+++ b/store/sqlstore/post_store.go
@@ -7,6 +7,7 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"reflect"
 	"regexp"
 	"strconv"
 	"strings"
@@ -42,8 +43,34 @@ type postWithExtra struct {
 func (s *SqlPostStore) ClearCaches() {
 }

-func postSliceColumns() []string {
-	return []string{"Id", "CreateAt", "UpdateAt", "EditAt", "DeleteAt", "IsPinned", "UserId", "ChannelId", "RootId", "ParentId", "OriginalId", "Message", "Type", "Props", "Hashtags", "Filenames", "FileIds", "HasReactions", "RemoteId"}
+func postSliceColumnsWithTypes() []struct {
+	Name string
+	Type reflect.Kind
+} {
+	return []struct {
+		Name string
+		Type reflect.Kind
+	}{
+		{"Id", reflect.String},
+		{"CreateAt", reflect.Int64},
+		{"UpdateAt", reflect.Int64},
+		{"EditAt", reflect.Int64},
+		{"DeleteAt", reflect.Int64},
+		{"IsPinned", reflect.Bool},
+		{"UserId", reflect.String},
+		{"ChannelId", reflect.String},
+		{"RootId", reflect.String},
+		{"ParentId", reflect.String},
+		{"OriginalId", reflect.String},
+		{"Message", reflect.String},
+		{"Type", reflect.String},
+		{"Props", reflect.Map},
+		{"Hashtags", reflect.String},
+		{"Filenames", reflect.Slice},
+		{"FileIds", reflect.Slice},
+		{"HasReactions", reflect.Bool},
+		{"RemoteId", reflect.String},
+	}
 }

 func postToSlice(post *model.Post) []interface{} {
@@ -70,6 +97,37 @@ func postToSlice(post *model.Post) []interface{} {
 	}
 }

+func postSliceColumns() []string {
+	colInfos := postSliceColumnsWithTypes()
+	cols := make([]string, len(colInfos))
+	for i, colInfo := range colInfos {
+		cols[i] = colInfo.Name
+	}
+	return cols
+}
+
+func postSliceCoalesceQuery() string {
+	colInfos := postSliceColumnsWithTypes()
+	cols := make([]string, len(colInfos))
+	for i, colInfo := range colInfos {
+		var defaultValue string
+		switch colInfo.Type {
+		case reflect.String:
+			defaultValue = "''"
+		case reflect.Int64:
+			defaultValue = "0"
+		case reflect.Bool:
+			defaultValue = "false"
+		case reflect.Map:
+			defaultValue = "'{}'"
+		case reflect.Slice:
+			defaultValue = "'[]'"
+		}
+		cols[i] = "COALESCE(Posts." + colInfo.Name + "," + defaultValue + ") AS " + colInfo.Name
+	}
+	return strings.Join(cols, ",")
+}
+
 func newSqlPostStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) store.PostStore {
 	s := &SqlPostStore{
 		SqlStore:          sqlStore,
@@ -1927,6 +1985,46 @@ func (s *SqlPostStore) GetPostsBatchForIndexing(startTime int64, endTime int64,
 	return posts, nil
 }

+// PermanentDeleteBatchForRetentionPolicies deletes a batch of records which are affected by
+// the global or a granular retention policy.
+// See `genericPermanentDeleteBatchForRetentionPolicies` for details.
+func (s *SqlPostStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	builder := s.getQueryBuilder().
+		Select("Posts.Id").
+		From("Posts")
+	return genericPermanentDeleteBatchForRetentionPolicies(RetentionPolicyBatchDeletionInfo{
+		BaseBuilder:         builder,
+		Table:               "Posts",
+		TimeColumn:          "CreateAt",
+		PrimaryKeys:         []string{"Id"},
+		ChannelIDTable:      "Posts",
+		NowMillis:           now,
+		GlobalPolicyEndTime: globalPolicyEndTime,
+		Limit:               limit,
+	}, s.SqlStore, cursor)
+}
+
+// DeleteOrphanedRows removes entries from Posts when a corresponding channel no longer exists.
+func (s *SqlPostStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
+	// We need the extra level of nesting to deal with MySQL's locking
+	const query = `
+	DELETE FROM Posts WHERE Id IN (
+		SELECT * FROM (
+			SELECT Posts.Id FROM Posts
+			LEFT JOIN Channels ON Posts.ChannelId = Channels.Id
+			WHERE Channels.Id IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	props := map[string]interface{}{"Limit": limit}
+	result, err := s.GetMaster().Exec(query, props)
+	if err != nil {
+		return
+	}
+	deleted, err = result.RowsAffected()
+	return
+}
+
 func (s *SqlPostStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
 	var query string
 	if s.DriverName() == "postgres" {
--- a/store/sqlstore/preference_store.go
+++ b/store/sqlstore/preference_store.go
@@ -265,6 +265,28 @@ func (s SqlPreferenceStore) DeleteCategoryAndName(category string, name string)
 	return nil
 }

+// DeleteOrphanedRows removes entries from Preferences (flagged post) when a
+// corresponding post no longer exists.
+func (s *SqlPreferenceStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
+	// We need the extra level of nesting to deal with MySQL's locking
+	const query = `
+	DELETE FROM Preferences WHERE Name IN (
+		SELECT * FROM (
+			SELECT Preferences.Name FROM Preferences
+			LEFT JOIN Posts ON Preferences.Name = Posts.Id
+			WHERE Posts.Id IS NULL AND Category = :Category
+			LIMIT :Limit
+		) AS A
+	)`
+	props := map[string]interface{}{"Limit": limit, "Category": model.PREFERENCE_CATEGORY_FLAGGED_POST}
+	result, err := s.GetMaster().Exec(query, props)
+	if err != nil {
+		return
+	}
+	deleted, err = result.RowsAffected()
+	return
+}
+
 func (s SqlPreferenceStore) CleanupFlagsBatch(limit int64) (int64, error) {
 	if limit < 0 {
 		// uint64 does not throw an error, it overflows if it is negative.
--- a/store/sqlstore/reaction_store.go
+++ b/store/sqlstore/reaction_store.go
@@ -209,6 +209,27 @@ func (s *SqlReactionStore) DeleteAllWithEmojiName(emojiName string) error {
 	return nil
 }

+// DeleteOrphanedRows removes entries from Reactions when a corresponding post no longer exists.
+func (s *SqlReactionStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
+	// We need the extra level of nesting to deal with MySQL's locking
+	const query = `
+	DELETE FROM Reactions WHERE PostId IN (
+		SELECT * FROM (
+			SELECT PostId FROM Reactions
+			LEFT JOIN Posts ON Reactions.PostId = Posts.Id
+			WHERE Posts.Id IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	props := map[string]interface{}{"Limit": limit}
+	result, err := s.GetMaster().Exec(query, props)
+	if err != nil {
+		return
+	}
+	deleted, err = result.RowsAffected()
+	return
+}
+
 func (s *SqlReactionStore) PermanentDeleteBatch(endTime int64, limit int64) (int64, error) {
 	var query string
 	if s.DriverName() == "postgres" {
--- a/store/sqlstore/retention_policy_store.go
+++ b/store/sqlstore/retention_policy_store.go
@@ -5,6 +5,8 @@ package sqlstore

 import (
 	"database/sql"
+	"strconv"
+	"strings"

 	sq "github.com/Masterminds/squirrel"
 	"github.com/go-sql-driver/mysql"
@@ -541,6 +543,49 @@ func (s *SqlRetentionPolicyStore) RemoveTeams(policyId string, teamIds []string)
 	return err
 }

+// DeleteOrphanedRows removes entries from RetentionPoliciesChannels and RetentionPoliciesTeams
+// where a channel or team no longer exists.
+func (s *SqlRetentionPolicyStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
+	// We need the extra level of nesting to deal with MySQL's locking
+	const rpcDeleteQuery = `
+	DELETE FROM RetentionPoliciesChannels WHERE ChannelId IN (
+		SELECT * FROM (
+			SELECT ChannelId FROM RetentionPoliciesChannels
+			LEFT JOIN Channels ON RetentionPoliciesChannels.ChannelId = Channels.Id
+			WHERE Channels.Id IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	const rptDeleteQuery = `
+	DELETE FROM RetentionPoliciesTeams WHERE TeamId IN (
+		SELECT * FROM (
+			SELECT TeamId FROM RetentionPoliciesTeams
+			LEFT JOIN Teams ON RetentionPoliciesTeams.TeamId = Teams.Id
+			WHERE Teams.Id IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	props := map[string]interface{}{"Limit": limit}
+	result, err := s.GetMaster().Exec(rpcDeleteQuery, props)
+	if err != nil {
+		return
+	}
+	rpcDeleted, err := result.RowsAffected()
+	if err != nil {
+		return
+	}
+	result, err = s.GetMaster().Exec(rptDeleteQuery, props)
+	if err != nil {
+		return
+	}
+	rptDeleted, err := result.RowsAffected()
+	if err != nil {
+		return
+	}
+	deleted = rpcDeleted + rptDeleted
+	return
+}
+
 func (s *SqlRetentionPolicyStore) GetTeamPoliciesForUser(userID string, offset, limit int) (policies []*model.RetentionPolicyForTeam, err error) {
 	const query = `
 	SELECT Teams.Id, RetentionPolicies.PostDuration
@@ -606,3 +651,173 @@ func (s *SqlRetentionPolicyStore) GetChannelPoliciesCountForUser(userID string)
 	props := map[string]interface{}{"UserId": userID}
 	return s.GetReplica().SelectInt(query, props)
 }
+
+// RetentionPolicyBatchDeletionInfo gives information on how to delete records
+// under a retention policy; see `genericPermanentDeleteBatchForRetentionPolicies`.
+//
+// `BaseBuilder` should already have selected the primary key(s) for the main table
+// and should be joined to a table with a ChannelId column, which will be used to join
+// on the Channels table.
+// `Table` is the name of the table from which records are being deleted.
+// `TimeColumn` is the name of the column which contains the timestamp of the record.
+// `PrimaryKeys` contains the primary keys of `table`. It should be the same as the
+// `From` clause in `baseBuilder`.
+// `ChannelIDTable` is the table which contains the ChannelId column, it may be the
+// same as `table`, or will be different if a join was used.
+// `NowMillis` must be a Unix timestamp in milliseconds and is used by the granular
+// policies; if `nowMillis - timestamp(record)` is greater than
+// the post duration of a granular policy, than the record will be deleted.
+// `GlobalPolicyEndTime` is used by the global policy; any record older than this time
+// will be deleted by the global policy if it does not fall under a granular policy.
+// To disable the granular policies, set `NowMillis` to 0.
+// To disable the global policy, set `GlobalPolicyEndTime` to 0.
+type RetentionPolicyBatchDeletionInfo struct {
+	BaseBuilder         sq.SelectBuilder
+	Table               string
+	TimeColumn          string
+	PrimaryKeys         []string
+	ChannelIDTable      string
+	NowMillis           int64
+	GlobalPolicyEndTime int64
+	Limit               int64
+}
+
+// genericPermanentDeleteBatchForRetentionPolicies is a helper function for tables
+// which need to delete records for granular and global policies.
+func genericPermanentDeleteBatchForRetentionPolicies(
+	r RetentionPolicyBatchDeletionInfo,
+	s *SqlStore,
+	cursor model.RetentionPolicyCursor,
+) (int64, model.RetentionPolicyCursor, error) {
+	baseBuilder := r.BaseBuilder.InnerJoin("Channels ON " + r.ChannelIDTable + ".ChannelId = Channels.Id")
+
+	scopedTimeColumn := r.Table + "." + r.TimeColumn
+	nowStr := strconv.FormatInt(r.NowMillis, 10)
+	// A record falls under the scope of a granular retention policy if:
+	// 1. The policy's post duration is >= 0
+	// 2. The record's lifespan has not exceeded the policy's post duration
+	const millisecondsInADay = 24 * 60 * 60 * 1000
+	fallsUnderGranularPolicy := sq.And{
+		sq.GtOrEq{"RetentionPolicies.PostDuration": 0},
+		sq.Expr(nowStr + " - " + scopedTimeColumn + " > RetentionPolicies.PostDuration * " + strconv.FormatInt(millisecondsInADay, 10)),
+	}
+
+	// If the caller wants to disable the global policy from running
+	if r.GlobalPolicyEndTime <= 0 {
+		cursor.GlobalPoliciesDone = true
+	}
+	// If the caller wants to disable the granular policies from running
+	if r.NowMillis <= 0 {
+		cursor.ChannelPoliciesDone = true
+		cursor.TeamPoliciesDone = true
+	}
+
+	var totalRowsAffected int64
+
+	// First, delete all of the records which fall under the scope of a channel-specific policy
+	if !cursor.ChannelPoliciesDone {
+		channelPoliciesBuilder := baseBuilder.
+			InnerJoin("RetentionPoliciesChannels ON " + r.ChannelIDTable + ".ChannelId = RetentionPoliciesChannels.ChannelId").
+			InnerJoin("RetentionPolicies ON RetentionPoliciesChannels.PolicyId = RetentionPolicies.Id").
+			Where(fallsUnderGranularPolicy).
+			Limit(uint64(r.Limit))
+		rowsAffected, err := genericRetentionPoliciesDeletion(channelPoliciesBuilder, r, s)
+		if err != nil {
+			return 0, cursor, err
+		}
+		if rowsAffected < r.Limit {
+			cursor.ChannelPoliciesDone = true
+		}
+		totalRowsAffected += rowsAffected
+		r.Limit -= rowsAffected
+	}
+
+	// Next, delete all of the records which fall under the scope of a team-specific policy
+	if cursor.ChannelPoliciesDone && !cursor.TeamPoliciesDone {
+		// Channel-specific policies override team-specific policies.
+		teamPoliciesBuilder := baseBuilder.
+			LeftJoin("RetentionPoliciesChannels ON " + r.ChannelIDTable + ".ChannelId = RetentionPoliciesChannels.ChannelId").
+			InnerJoin("RetentionPoliciesTeams ON Channels.TeamId = RetentionPoliciesTeams.TeamId").
+			InnerJoin("RetentionPolicies ON RetentionPoliciesTeams.PolicyId = RetentionPolicies.Id").
+			Where(sq.And{
+				sq.Eq{"RetentionPoliciesChannels.PolicyId": nil},
+				sq.Expr("RetentionPoliciesTeams.PolicyId = RetentionPolicies.Id"),
+			}).
+			Where(fallsUnderGranularPolicy).
+			Limit(uint64(r.Limit))
+		rowsAffected, err := genericRetentionPoliciesDeletion(teamPoliciesBuilder, r, s)
+		if err != nil {
+			return 0, cursor, err
+		}
+		if rowsAffected < r.Limit {
+			cursor.TeamPoliciesDone = true
+		}
+		totalRowsAffected += rowsAffected
+		r.Limit -= rowsAffected
+	}
+
+	// Finally, delete all of the records which fall under the scope of the global policy
+	if cursor.ChannelPoliciesDone && cursor.TeamPoliciesDone && !cursor.GlobalPoliciesDone {
+		// Granular policies override the global policy.
+		globalPolicyBuilder := baseBuilder.
+			LeftJoin("RetentionPoliciesChannels ON " + r.ChannelIDTable + ".ChannelId = RetentionPoliciesChannels.ChannelId").
+			LeftJoin("RetentionPoliciesTeams ON Channels.TeamId = RetentionPoliciesTeams.TeamId").
+			LeftJoin("RetentionPolicies ON RetentionPoliciesChannels.PolicyId = RetentionPolicies.Id").
+			Where(sq.And{
+				sq.Eq{"RetentionPoliciesChannels.PolicyId": nil},
+				sq.Eq{"RetentionPoliciesTeams.PolicyId": nil},
+			}).
+			Where(sq.Lt{scopedTimeColumn: r.GlobalPolicyEndTime}).
+			Limit(uint64(r.Limit))
+		rowsAffected, err := genericRetentionPoliciesDeletion(globalPolicyBuilder, r, s)
+		if err != nil {
+			return 0, cursor, err
+		}
+		if rowsAffected < r.Limit {
+			cursor.GlobalPoliciesDone = true
+		}
+		totalRowsAffected += rowsAffected
+	}
+
+	return totalRowsAffected, cursor, nil
+}
+
+// genericRetentionPoliciesDeletion actually executes the DELETE query using a sq.SelectBuilder
+// which selects the rows to delete.
+func genericRetentionPoliciesDeletion(
+	builder sq.SelectBuilder,
+	r RetentionPolicyBatchDeletionInfo,
+	s *SqlStore,
+) (rowsAffected int64, err error) {
+	query, args, err := builder.ToSql()
+	if err != nil {
+		return 0, errors.Wrap(err, r.Table+"_tosql")
+	}
+	if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
+		primaryKeysStr := "(" + strings.Join(r.PrimaryKeys, ",") + ")"
+		query = `
+		DELETE FROM ` + r.Table + ` WHERE ` + primaryKeysStr + ` IN (
+		` + query + `
+		)`
+	} else {
+		// MySQL does not support the LIMIT clause in a subquery with IN
+		clauses := make([]string, len(r.PrimaryKeys))
+		for i, key := range r.PrimaryKeys {
+			clauses[i] = r.Table + "." + key + " = A." + key
+		}
+		joinClause := strings.Join(clauses, " AND ")
+		query = `
+		DELETE ` + r.Table + ` FROM ` + r.Table + ` INNER JOIN (
+		` + query + `
+		) AS A ON ` + joinClause
+	}
+	result, err := s.GetMaster().Exec(query, args...)
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to delete "+r.Table)
+	}
+	rowsAffected, err = result.RowsAffected()
+	if err != nil {
+		return 0, errors.Wrap(err, "failed to get rows affected for "+r.Table)
+	}
+	return
+}
--- a/store/sqlstore/thread_store.go
+++ b/store/sqlstore/thread_store.go
@@ -132,12 +132,15 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get

 	unreadRepliesQuery := "SELECT COUNT(Posts.Id) From Posts Where Posts.RootId=ThreadMemberships.PostId AND Posts.CreateAt >= ThreadMemberships.LastViewed"
 	fetchConditions := sq.And{
-		sq.Or{sq.Eq{"Channels.TeamId": teamId}, sq.Eq{"Channels.TeamId": ""}},
+		sq.Or{sq.Eq{"Channels.TeamId": teamId}, sq.Eq{"Channels.TeamId": ""}, sq.Eq{"Channels.TeamId": nil}},
 		sq.Eq{"ThreadMemberships.UserId": userId},
 		sq.Eq{"ThreadMemberships.Following": true},
 	}
 	if !opts.Deleted {
-		fetchConditions = sq.And{fetchConditions, sq.Eq{"Posts.DeleteAt": 0}}
+		fetchConditions = sq.And{
+			fetchConditions,
+			sq.Eq{"COALESCE(Posts.DeleteAt, 0)": 0},
+		}
 	}

 	pageSize := uint64(30)
@@ -217,7 +220,10 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get
 		}
 		var threads []*JoinedThread
 		query, args, _ := s.getQueryBuilder().
-			Select("Threads.*, Posts.*, ThreadMemberships.LastViewed as LastViewedAt, ThreadMemberships.UnreadMentions as UnreadMentions").
+			Select(`Threads.*,
+				` + postSliceCoalesceQuery() + `,
+				ThreadMemberships.LastViewed as LastViewedAt,
+				ThreadMemberships.UnreadMentions as UnreadMentions`).
 			From("Threads").
 			Column(sq.Alias(sq.Expr(unreadRepliesQuery), "UnreadReplies")).
 			LeftJoin("Posts ON Posts.Id = Threads.PostId").
@@ -309,7 +315,7 @@ func (s *SqlThreadStore) GetThreadsForUser(userId, teamId string, opts model.Get
 			UnreadReplies:  thread.UnreadReplies,
 			UnreadMentions: thread.UnreadMentions,
 			Participants:   participants,
-			Post:           &thread.Post,
+			Post:           thread.Post.ToNilIfInvalid(),
 		})
 	}

@@ -401,7 +407,7 @@ func (s *SqlThreadStore) GetThreadForUser(teamId string, threadMembership *model
 		UnreadReplies:  thread.UnreadReplies,
 		UnreadMentions: thread.UnreadMentions,
 		Participants:   users,
-		Post:           &thread.Post,
+		Post:           thread.Post.ToNilIfInvalid(),
 	}

 	return result, nil
@@ -691,3 +697,86 @@ func (s *SqlThreadStore) GetPosts(threadId string, since int64) ([]*model.Post,
 	}
 	return result, nil
 }
+
+// PermanentDeleteBatchForRetentionPolicies deletes a batch of records which are affected by
+// the global or a granular retention policy.
+// See `genericPermanentDeleteBatchForRetentionPolicies` for details.
+func (s *SqlThreadStore) PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	builder := s.getQueryBuilder().
+		Select("Threads.PostId").
+		From("Threads")
+	return genericPermanentDeleteBatchForRetentionPolicies(RetentionPolicyBatchDeletionInfo{
+		BaseBuilder:         builder,
+		Table:               "Threads",
+		TimeColumn:          "LastReplyAt",
+		PrimaryKeys:         []string{"PostId"},
+		ChannelIDTable:      "Threads",
+		NowMillis:           now,
+		GlobalPolicyEndTime: globalPolicyEndTime,
+		Limit:               limit,
+	}, s.SqlStore, cursor)
+}
+
+// PermanentDeleteBatchThreadMembershipsForRetentionPolicies deletes a batch of records
+// which are affected by the global or a granular retention policy.
+// See `genericPermanentDeleteBatchForRetentionPolicies` for details.
+func (s *SqlThreadStore) PermanentDeleteBatchThreadMembershipsForRetentionPolicies(now, globalPolicyEndTime, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	builder := s.getQueryBuilder().
+		Select("ThreadMemberships.PostId").
+		From("ThreadMemberships").
+		InnerJoin("Threads ON ThreadMemberships.PostId = Threads.PostId")
+	return genericPermanentDeleteBatchForRetentionPolicies(RetentionPolicyBatchDeletionInfo{
+		BaseBuilder:         builder,
+		Table:               "ThreadMemberships",
+		TimeColumn:          "LastUpdated",
+		PrimaryKeys:         []string{"PostId"},
+		ChannelIDTable:      "Threads",
+		NowMillis:           now,
+		GlobalPolicyEndTime: globalPolicyEndTime,
+		Limit:               limit,
+	}, s.SqlStore, cursor)
+}
+
+// DeleteOrphanedRows removes orphaned rows from Threads and ThreadMemberships
+func (s *SqlThreadStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
+	// We need the extra level of nesting to deal with MySQL's locking
+	const threadsQuery = `
+	DELETE FROM Threads WHERE PostId IN (
+		SELECT * FROM (
+			SELECT Threads.PostId FROM Threads
+			LEFT JOIN Channels ON Threads.ChannelId = Channels.Id
+			WHERE Channels.Id IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	// We only delete a thread membership if the entire thread no longer exists,
+	// not if the root post has been deleted
+	const threadMembershipsQuery = `
+	DELETE FROM ThreadMemberships WHERE PostId IN (
+		SELECT * FROM (
+			SELECT ThreadMemberships.PostId FROM ThreadMemberships
+			LEFT JOIN Threads ON ThreadMemberships.PostId = Threads.PostId
+			WHERE Threads.PostId IS NULL
+			LIMIT :Limit
+		) AS A
+	)`
+	props := map[string]interface{}{"Limit": limit}
+	result, err := s.GetMaster().Exec(threadsQuery, props)
+	if err != nil {
+		return
+	}
+	rpcDeleted, err := result.RowsAffected()
+	if err != nil {
+		return
+	}
+	result, err = s.GetMaster().Exec(threadMembershipsQuery, props)
+	if err != nil {
+		return
+	}
+	rptDeleted, err := result.RowsAffected()
+	if err != nil {
+		return
+	}
+	deleted = rpcDeleted + rptDeleted
+	return
+}