[MM-30831] granular data retention wireup (#17417)
* pre-checkout commit * add API endpoints for retention policies * allow deleting multiple teams/channels from a policy in a single request * pre-checkout commit * add auditing in API functions * add permission checks * update the store layers * update storetest * add check constraint on PostDuration column * pre-checkout commit * add query to delete posts under the scope of a granular retention policy * add suggestions from sbishel * allow clients to specify channels/teams when creating a new policy * remove foreign keys referencing Channels and Teams tables * add checks for whether teams and channels exist * pre-checkout commit * remove data referencing the Posts table * pre-checkout commit * write data store tests * sort results of buildGetPoliciesQuery * add missing test cases for teams * pre-checkout commit * add Client4 methods for data retention policy endpoints * add uint and uint64 to app/layer_generators * make granular policies override global policies * fix lint errors * pre-checkout commit * add license to top of files * add tests for data store layer * add missing test cases for store layer * run make i18n-extract * add query to delete ChannelMemberHistory * work in progress * add test for old reply to old post * fix lint error * use COALESCE on each Posts column * begin implementing orphaned rows worker * split PR * pre-checkout commit * use RetentionPolicyWithTeamAndChannelCounts * update app and api layers * run make i18n-extract * add RetentionPolicy to retrylayer_test.go * Revert "split PR" This reverts commit b316f03dd307a30deae931944ca7e4a1cc904605. * fix errors caused by revert * add suggestions from sbishel * fix copy-paste error * fix lint errors * pre-checkout commit * add function to delete orphaned rows * use -1 for infinite retention * remove check constraint * copy i18n entries from master * re-run tests with newer enterprise branch * add team data to channel list * add search for channels and teams in a policy * add store tests for channel and team search * add suggestions from mkraft * run make einterfaces-mocks * fix lint errors * add suggestions from mkraft * move removeOrphanedRows method to wireup branch * Revert "move removeOrphanedRows method to wireup branch" This reverts commit 94605c9b4a5378ffa44a3dec4d3f8e3306b9d33e. * use DeleteOrphanedRows where possible * run make i18n-extract * use COMPLIANCE permissions * run make migrations-bindadta * clean up teams before test * fix tests for TestRetentionPolicyStore * add API endpoints for mobile * fix lint error * fix some of the lint errors * move user/data_retention endpoints to data_retention.go * Revert "fix some of the lint errors" This reverts commit b5b2dc27566c427187db942c5c0afe319e7679c4. * add exclude_policy_constrained parameter for /channels and /teams * fix lint errors * add policy_id field to GET endpoints for channels and teams * use PolicyWithTeamID in RetentionPolicy layer * fix lint errors * run make i18n-extract * update mock call in telemetry_test.go * return status:OK in JSON instead of 204 * pre-checkout commit * add policy_id field on channels/teams * fix lint errors * use sq.Eq instead of '?' * use new subsection permissions * update channels and teams endpoints to use new subsection permissions * add extra search opts for channels in a policy * fix lint errors * allow negative post duration in patch * remove DELETE FROM query in retention policy tests * use *int64 for PostDuration * re-run CI tests * use 3-step deletion strategy for each table * fix lint errors * run make store-layers * re-run CI tests * add test with channel, team and global policies * use common function for SQL queries * add pagination test * use struct for args to common SQL function * fix lint errors * run make i18n-extract * check if Channels.TeamId is "" or nil * use three OR clauses * write separate genericRetentionPoliciesDeletion function * add config setting for BatchSize * add telemetry for BatchSize * use feature flag * add old i18n messages back in * re-run CI tests * update call signature in storetest * MM-30831: Adds constant for retention default batch size. * MM-30831: Removes comment re: optimization. * MM-30831: Converts days to milliseconds. * MM-30831: Reverts change to test. * Revert "MM-30831: Reverts change to test." This reverts commit 6d14275a1ceae682bb9e17ec69b39252b44e0c0c. * Revert "MM-30831: Converts days to milliseconds." This reverts commit a0cb6ec09d854a05194c1daee1c5333f260231c3. * MM-30831: Fixes tests. * MM-30381: Fix for change to method sig. Co-authored-by: Max Erenberg <max.erenberg@mattermost.com> Co-authored-by: Mattermod <mattermod@users.noreply.github.com> Co-authored-by: Martin Kraft <martin@upspin.org>
Этот коммит содержится в:
Max Erenberg
коммит произвёл
GitHub
GitHub
родитель
71ef1c5386
Коммит
58d5d51f7a
@@ -17,6 +17,12 @@ import (
|
||||
|
||||
func TestThreadStore(t *testing.T, ss store.Store, s SqlStore) {
|
||||
t.Run("ThreadStorePopulation", func(t *testing.T) { testThreadStorePopulation(t, ss) })
|
||||
t.Run("ThreadStorePermanentDeleteBatchForRetentionPolicies", func(t *testing.T) {
|
||||
testThreadStorePermanentDeleteBatchForRetentionPolicies(t, ss)
|
||||
})
|
||||
t.Run("ThreadStorePermanentDeleteBatchThreadMembershipsForRetentionPolicies", func(t *testing.T) {
|
||||
testThreadStorePermanentDeleteBatchThreadMembershipsForRetentionPolicies(t, ss)
|
||||
})
|
||||
}
|
||||
|
||||
func testThreadStorePopulation(t *testing.T, ss store.Store) {
|
||||
@@ -411,3 +417,185 @@ func testThreadStorePopulation(t *testing.T, ss store.Store) {
|
||||
require.Equal(t, int64(0), th.UnreadReplies)
|
||||
})
|
||||
}
|
||||
|
||||
func threadStoreCreateReply(t *testing.T, ss store.Store, channelID, postID string, createAt int64) *model.Post {
|
||||
reply, err := ss.Post().Save(&model.Post{
|
||||
ChannelId: channelID,
|
||||
UserId: model.NewId(),
|
||||
CreateAt: createAt,
|
||||
RootId: postID,
|
||||
ParentId: postID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return reply
|
||||
}
|
||||
|
||||
func testThreadStorePermanentDeleteBatchForRetentionPolicies(t *testing.T, ss store.Store) {
|
||||
const limit = 1000
|
||||
team, err := ss.Team().Save(&model.Team{
|
||||
DisplayName: "DisplayName",
|
||||
Name: "team" + model.NewId(),
|
||||
Email: MakeEmail(),
|
||||
Type: model.TEAM_OPEN,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
channel, err := ss.Channel().Save(&model.Channel{
|
||||
TeamId: team.Id,
|
||||
DisplayName: "DisplayName",
|
||||
Name: "channel" + model.NewId(),
|
||||
Type: model.CHANNEL_OPEN,
|
||||
}, -1)
|
||||
require.NoError(t, err)
|
||||
|
||||
post, err := ss.Post().Save(&model.Post{
|
||||
ChannelId: channel.Id,
|
||||
UserId: model.NewId(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
threadStoreCreateReply(t, ss, channel.Id, post.Id, 2000)
|
||||
|
||||
thread, err := ss.Thread().Get(post.Id)
|
||||
require.NoError(t, err)
|
||||
|
||||
channelPolicy, err := ss.RetentionPolicy().Save(&model.RetentionPolicyWithTeamAndChannelIDs{
|
||||
RetentionPolicy: model.RetentionPolicy{
|
||||
DisplayName: "DisplayName",
|
||||
PostDuration: model.NewInt64(30),
|
||||
},
|
||||
ChannelIDs: []string{channel.Id},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
nowMillis := thread.LastReplyAt + *channelPolicy.PostDuration*24*60*60*1000 + 1
|
||||
_, _, err = ss.Thread().PermanentDeleteBatchForRetentionPolicies(nowMillis, 0, limit, model.RetentionPolicyCursor{})
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Thread().Get(post.Id)
|
||||
require.Error(t, err, "thread should have been deleted by channel policy")
|
||||
|
||||
// create a new thread
|
||||
threadStoreCreateReply(t, ss, channel.Id, post.Id, 2000)
|
||||
thread, err = ss.Thread().Get(post.Id)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Create a team policy which is stricter than the channel policy
|
||||
teamPolicy, err := ss.RetentionPolicy().Save(&model.RetentionPolicyWithTeamAndChannelIDs{
|
||||
RetentionPolicy: model.RetentionPolicy{
|
||||
DisplayName: "DisplayName",
|
||||
PostDuration: model.NewInt64(20),
|
||||
},
|
||||
TeamIDs: []string{team.Id},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
nowMillis = thread.LastReplyAt + *teamPolicy.PostDuration*24*60*60*1000 + 1
|
||||
_, _, err = ss.Thread().PermanentDeleteBatchForRetentionPolicies(nowMillis, 0, limit, model.RetentionPolicyCursor{})
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Thread().Get(post.Id)
|
||||
require.NoError(t, err, "channel policy should have overridden team policy")
|
||||
|
||||
// Delete channel policy and re-run team policy
|
||||
err = ss.RetentionPolicy().Delete(channelPolicy.ID)
|
||||
require.NoError(t, err)
|
||||
_, _, err = ss.Thread().PermanentDeleteBatchForRetentionPolicies(nowMillis, 0, limit, model.RetentionPolicyCursor{})
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Thread().Get(post.Id)
|
||||
require.Error(t, err, "thread should have been deleted by team policy")
|
||||
}
|
||||
|
||||
func testThreadStorePermanentDeleteBatchThreadMembershipsForRetentionPolicies(t *testing.T, ss store.Store) {
|
||||
const limit = 1000
|
||||
userID := model.NewId()
|
||||
createThreadMembership := func(userID, postID string) *model.ThreadMembership {
|
||||
opts := store.ThreadMembershipOpts{
|
||||
Following: true,
|
||||
IncrementMentions: false,
|
||||
UpdateFollowing: true,
|
||||
UpdateViewedTimestamp: false,
|
||||
UpdateParticipants: false,
|
||||
}
|
||||
_, err := ss.Thread().MaintainMembership(userID, postID, opts)
|
||||
require.NoError(t, err)
|
||||
threadMembership, err := ss.Thread().GetMembershipForUser(userID, postID)
|
||||
require.NoError(t, err)
|
||||
return threadMembership
|
||||
}
|
||||
team, err := ss.Team().Save(&model.Team{
|
||||
DisplayName: "DisplayName",
|
||||
Name: "team" + model.NewId(),
|
||||
Email: MakeEmail(),
|
||||
Type: model.TEAM_OPEN,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
channel, err := ss.Channel().Save(&model.Channel{
|
||||
TeamId: team.Id,
|
||||
DisplayName: "DisplayName",
|
||||
Name: "channel" + model.NewId(),
|
||||
Type: model.CHANNEL_OPEN,
|
||||
}, -1)
|
||||
require.NoError(t, err)
|
||||
post, err := ss.Post().Save(&model.Post{
|
||||
ChannelId: channel.Id,
|
||||
UserId: model.NewId(),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
threadStoreCreateReply(t, ss, channel.Id, post.Id, 2000)
|
||||
|
||||
threadMembership := createThreadMembership(userID, post.Id)
|
||||
|
||||
channelPolicy, err := ss.RetentionPolicy().Save(&model.RetentionPolicyWithTeamAndChannelIDs{
|
||||
RetentionPolicy: model.RetentionPolicy{
|
||||
DisplayName: "DisplayName",
|
||||
PostDuration: model.NewInt64(30),
|
||||
},
|
||||
ChannelIDs: []string{channel.Id},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
nowMillis := threadMembership.LastUpdated + *channelPolicy.PostDuration*24*60*60*1000 + 1
|
||||
_, _, err = ss.Thread().PermanentDeleteBatchThreadMembershipsForRetentionPolicies(nowMillis, 0, limit, model.RetentionPolicyCursor{})
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Thread().GetMembershipForUser(userID, post.Id)
|
||||
require.Error(t, err, "thread membership should have been deleted by channel policy")
|
||||
|
||||
// create a new thread membership
|
||||
threadMembership = createThreadMembership(userID, post.Id)
|
||||
|
||||
// Create a team policy which is stricter than the channel policy
|
||||
teamPolicy, err := ss.RetentionPolicy().Save(&model.RetentionPolicyWithTeamAndChannelIDs{
|
||||
RetentionPolicy: model.RetentionPolicy{
|
||||
DisplayName: "DisplayName",
|
||||
PostDuration: model.NewInt64(20),
|
||||
},
|
||||
TeamIDs: []string{team.Id},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
nowMillis = threadMembership.LastUpdated + *teamPolicy.PostDuration*24*60*60*1000 + 1
|
||||
_, _, err = ss.Thread().PermanentDeleteBatchThreadMembershipsForRetentionPolicies(nowMillis, 0, limit, model.RetentionPolicyCursor{})
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Thread().GetMembershipForUser(userID, post.Id)
|
||||
require.NoError(t, err, "channel policy should have overridden team policy")
|
||||
|
||||
// Delete channel policy and re-run team policy
|
||||
err = ss.RetentionPolicy().Delete(channelPolicy.ID)
|
||||
require.NoError(t, err)
|
||||
_, _, err = ss.Thread().PermanentDeleteBatchThreadMembershipsForRetentionPolicies(nowMillis, 0, limit, model.RetentionPolicyCursor{})
|
||||
require.NoError(t, err)
|
||||
_, err = ss.Thread().GetMembershipForUser(userID, post.Id)
|
||||
require.Error(t, err, "thread membership should have been deleted by team policy")
|
||||
|
||||
// create a new thread membership
|
||||
threadMembership = createThreadMembership(userID, post.Id)
|
||||
|
||||
// Delete team policy and thread
|
||||
err = ss.RetentionPolicy().Delete(teamPolicy.ID)
|
||||
require.NoError(t, err)
|
||||
err = ss.Thread().Delete(post.Id)
|
||||
require.NoError(t, err)
|
||||
|
||||
deleted, err := ss.Thread().DeleteOrphanedRows(1000)
|
||||
require.NoError(t, err)
|
||||
require.NotZero(t, deleted)
|
||||
_, err = ss.Thread().GetMembershipForUser(userID, post.Id)
|
||||
require.Error(t, err, "thread membership should have been deleted because thread no longer exists")
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user