mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[MM-53291] Data retention improvements (#24253)

Просмотр исходного кода 
* adding new migration for RetentionIdsForDeletion, changing logic for deleting orphaned reactions. Updating delete user and channel endpoints to remove respective reactions
Этот коммит содержится в:
Ben Cooke
2023-09-06 08:25:27 -04:00
коммит произвёл GitHub
родитель d13429aa92
Коммит 791ee40568
27 изменённых файлов: 820 добавлений и 275 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

3
server/channels/store/storetest/channel_member_history_store.go
Просмотреть файл

@@ -389,6 +389,9 @@ func testPermanentDeleteBatchForRetentionPolicies(t *testing.T, ss store.Store)
result, err := ss.ChannelMemberHistory().GetUsersInChannelDuring(joinTime, leaveTime, channel.Id)
require.NoError(t, err)
require.Empty(t, result, "history should have been deleted by channel policy")
rows, err := ss.RetentionPolicy().GetIdsForDeletionByTableName("ChannelMemberHistory", 1000)
require.NoError(t, err)
require.Equal(t, 0, len(rows))
}
func testGetChannelsLeftSince(t *testing.T, ss store.Store) {

24
server/channels/store/storetest/mocks/PostStore.go
Просмотреть файл

@@ -113,30 +113,6 @@ func (_m *PostStore) Delete(postID string, timestamp int64, deleteByID string) e
return r0
}
// DeleteOrphanedRows provides a mock function with given fields: limit
func (_m *PostStore) DeleteOrphanedRows(limit int) (int64, error) {
ret := _m.Called(limit)
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(int) (int64, error)); ok {
return rf(limit)
}
if rf, ok := ret.Get(0).(func(int) int64); ok {
r0 = rf(limit)
} else {
r0 = ret.Get(0).(int64)
}
if rf, ok := ret.Get(1).(func(int) error); ok {
r1 = rf(limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Get provides a mock function with given fields: ctx, id, opts, userID, sanitizeOptions
func (_m *PostStore) Get(ctx context.Context, id string, opts model.GetPostsOptions, userID string, sanitizeOptions map[string]bool) (*model.PostList, error) {
ret := _m.Called(ctx, id, opts, userID, sanitizeOptions)

40
server/channels/store/storetest/mocks/ReactionStore.go
Просмотреть файл

@@ -80,28 +80,18 @@ func (_m *ReactionStore) DeleteAllWithEmojiName(emojiName string) error {
return r0
}
// DeleteOrphanedRows provides a mock function with given fields: limit
func (_m *ReactionStore) DeleteOrphanedRows(limit int) (int64, error) {
ret := _m.Called(limit)
// DeleteOrphanedRowsByIds provides a mock function with given fields: r
func (_m *ReactionStore) DeleteOrphanedRowsByIds(r *model.RetentionIdsForDeletion) error {
ret := _m.Called(r)
var r0 int64
var r1 error
if rf, ok := ret.Get(0).(func(int) (int64, error)); ok {
return rf(limit)
}
if rf, ok := ret.Get(0).(func(int) int64); ok {
r0 = rf(limit)
var r0 error
if rf, ok := ret.Get(0).(func(*model.RetentionIdsForDeletion) error); ok {
r0 = rf(r)
} else {
r0 = ret.Get(0).(int64)
r0 = ret.Error(0)
}
if rf, ok := ret.Get(1).(func(int) error); ok {
r1 = rf(limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
return r0
}
// GetForPost provides a mock function with given fields: postID, allowFromCache
@@ -180,6 +170,20 @@ func (_m *ReactionStore) PermanentDeleteBatch(endTime int64, limit int64) (int64
return r0, r1
}
// PermanentDeleteByUser provides a mock function with given fields: userID
func (_m *ReactionStore) PermanentDeleteByUser(userID string) error {
ret := _m.Called(userID)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(userID)
} else {
r0 = ret.Error(0)
}
return r0
}
// Save provides a mock function with given fields: reaction
func (_m *ReactionStore) Save(reaction *model.Reaction) (*model.Reaction, error) {
ret := _m.Called(reaction)

26
server/channels/store/storetest/mocks/RetentionPolicyStore.go
Просмотреть файл

@@ -256,6 +256,32 @@ func (_m *RetentionPolicyStore) GetCount() (int64, error) {
return r0, r1
}
// GetIdsForDeletionByTableName provides a mock function with given fields: tableName, limit
func (_m *RetentionPolicyStore) GetIdsForDeletionByTableName(tableName string, limit int) ([]*model.RetentionIdsForDeletion, error) {
ret := _m.Called(tableName, limit)
var r0 []*model.RetentionIdsForDeletion
var r1 error
if rf, ok := ret.Get(0).(func(string, int) ([]*model.RetentionIdsForDeletion, error)); ok {
return rf(tableName, limit)
}
if rf, ok := ret.Get(0).(func(string, int) []*model.RetentionIdsForDeletion); ok {
r0 = rf(tableName, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.RetentionIdsForDeletion)
}
}
if rf, ok := ret.Get(1).(func(string, int) error); ok {
r1 = rf(tableName, limit)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// GetTeamPoliciesCountForUser provides a mock function with given fields: userID
func (_m *RetentionPolicyStore) GetTeamPoliciesCountForUser(userID string) (int64, error) {
ret := _m.Called(userID)

105
server/channels/store/storetest/post_store.go
Просмотреть файл

@@ -1411,6 +1411,30 @@ func testPostStorePermDelete1Level(t *testing.T, ss store.Store) {
o2, err = ss.Post().Save(o2)
require.NoError(t, err)
r1 := &model.Reaction{}
r1.ChannelId = o1.ChannelId
r1.UserId = o2.UserId
r1.PostId = o1.Id
r1.EmojiName = "smile"
r1, err = ss.Reaction().Save(r1)
require.NoError(t, err)
r2 := &model.Reaction{}
r2.ChannelId = o1.ChannelId
r2.UserId = o1.UserId
r2.PostId = o2.Id
r2.EmojiName = "wave"
_, err = ss.Reaction().Save(r2)
require.NoError(t, err)
r3 := &model.Reaction{}
r3.ChannelId = o1.ChannelId
r3.UserId = model.NewId()
r3.PostId = o1.Id
r3.EmojiName = "sad"
r3, err = ss.Reaction().Save(r3)
require.NoError(t, err)
channel2, err := ss.Channel().Save(&model.Channel{
TeamId: teamId,
DisplayName: "DisplayName2",
@@ -1425,6 +1449,14 @@ func testPostStorePermDelete1Level(t *testing.T, ss store.Store) {
o3, err = ss.Post().Save(o3)
require.NoError(t, err)
r4 := &model.Reaction{}
r4.ChannelId = channel2.Id
r4.UserId = model.NewId()
r4.PostId = o3.Id
r4.EmojiName = "angry"
_, err = ss.Reaction().Save(r4)
require.NoError(t, err)
channel3, err := ss.Channel().Save(&model.Channel{
TeamId: teamId,
DisplayName: "DisplayName3",
@@ -1475,9 +1507,21 @@ func testPostStorePermDelete1Level(t *testing.T, ss store.Store) {
_, err = ss.Post().Get(context.Background(), o1.Id, model.GetPostsOptions{}, "", map[string]bool{})
require.NoError(t, err, "Deleted id shouldn't have failed")
reactions, err := ss.Reaction().GetForPost(o1.Id, false)
require.NoError(t, err, "Reactions should exist")
require.Equal(t, 2, len(reactions))
emojis := []string{r1.EmojiName, r3.EmojiName}
for _, reaction := range reactions {
require.Contains(t, emojis, reaction.EmojiName)
}
_, err = ss.Post().Get(context.Background(), o2.Id, model.GetPostsOptions{}, "", map[string]bool{})
require.Error(t, err, "Deleted id should have failed")
reactions, err = ss.Reaction().GetForPost(o2.Id, false)
require.NoError(t, err, "No error for not found")
require.Equal(t, 0, len(reactions))
thread, err = ss.Thread().Get(o5.Id)
require.NoError(t, err)
require.NotEmpty(t, thread)
@@ -1489,6 +1533,17 @@ func testPostStorePermDelete1Level(t *testing.T, ss store.Store) {
require.NoError(t, err)
require.Nil(t, thread)
reactions, err = ss.Reaction().GetForPost(o3.Id, false)
require.NoError(t, err, "No error for not found")
require.Equal(t, 0, len(reactions))
reactions, err = ss.Reaction().GetForPost(o1.Id, false)
require.NoError(t, err, "Reactions should exist")
require.Equal(t, 2, len(reactions))
for _, reaction := range reactions {
require.Contains(t, emojis, reaction.EmojiName)
}
_, err = ss.Post().Get(context.Background(), o3.Id, model.GetPostsOptions{}, "", map[string]bool{})
require.Error(t, err, "Deleted id should have failed")
@@ -3926,8 +3981,9 @@ func testPostStorePermanentDeleteBatch(t *testing.T, ss store.Store) {
o3, err = ss.Post().Save(o3)
require.NoError(t, err)
_, _, err = ss.Post().PermanentDeleteBatchForRetentionPolicies(0, 2000, 1000, model.RetentionPolicyCursor{})
deleted, _, err := ss.Post().PermanentDeleteBatchForRetentionPolicies(0, 2000, 1000, model.RetentionPolicyCursor{})
require.NoError(t, err)
require.Equal(t, int64(2), deleted)
_, err = ss.Post().Get(context.Background(), o1.Id, model.GetPostsOptions{}, "", map[string]bool{})
require.Error(t, err, "Should have not found post 1 after purge")
@@ -3938,6 +3994,14 @@ func testPostStorePermanentDeleteBatch(t *testing.T, ss store.Store) {
_, err = ss.Post().Get(context.Background(), o3.Id, model.GetPostsOptions{}, "", map[string]bool{})
require.NoError(t, err, "Should have found post 3 after purge")
rows, err := ss.RetentionPolicy().GetIdsForDeletionByTableName("Posts", 1000)
require.NoError(t, err)
require.Equal(t, 1, len(rows))
require.Equal(t, 2, len(rows[0].Ids))
// Clean up retention ids table
err = ss.Reaction().DeleteOrphanedRowsByIds(rows[0])
require.NoError(t, err)
t.Run("with pagination", func(t *testing.T) {
for i := 0; i < 3; i++ {
_, err = ss.Post().Save(&model.Post{
@@ -3950,13 +4014,31 @@ func testPostStorePermanentDeleteBatch(t *testing.T, ss store.Store) {
}
cursor := model.RetentionPolicyCursor{}
deleted, cursor, err := ss.Post().PermanentDeleteBatchForRetentionPolicies(0, 2, 2, cursor)
deleted, cursor, err = ss.Post().PermanentDeleteBatchForRetentionPolicies(0, 2, 2, cursor)
require.NoError(t, err)
require.Equal(t, int64(2), deleted)
rows, err = ss.RetentionPolicy().GetIdsForDeletionByTableName("Posts", 1000)
require.NoError(t, err)
require.Equal(t, 1, len(rows))
require.Equal(t, 2, len(rows[0].Ids))
// Clean up retention ids table
err = ss.Reaction().DeleteOrphanedRowsByIds(rows[0])
require.NoError(t, err)
deleted, _, err = ss.Post().PermanentDeleteBatchForRetentionPolicies(0, 2, 2, cursor)
require.NoError(t, err)
require.Equal(t, int64(1), deleted)
rows, err = ss.RetentionPolicy().GetIdsForDeletionByTableName("Posts", 1000)
require.NoError(t, err)
require.Equal(t, 1, len(rows))
require.Equal(t, 1, len(rows[0].Ids))
// Clean up retention ids table
err = ss.Reaction().DeleteOrphanedRowsByIds(rows[0])
require.NoError(t, err)
})
t.Run("with data retention policies", func(t *testing.T) {
@@ -4024,6 +4106,14 @@ func testPostStorePermanentDeleteBatch(t *testing.T, ss store.Store) {
err2 = ss.RetentionPolicy().Delete(teamPolicy.ID)
require.NoError(t, err2)
// Clean up retention ids table
rows, err = ss.RetentionPolicy().GetIdsForDeletionByTableName("Posts", 1000)
require.NoError(t, err)
for _, row := range rows {
err = ss.Reaction().DeleteOrphanedRowsByIds(row)
require.NoError(t, err)
}
})
t.Run("with channel, team and global policies", func(t *testing.T) {
@@ -4089,6 +4179,17 @@ func testPostStorePermanentDeleteBatch(t *testing.T, ss store.Store) {
deleted, _, err2 := ss.Post().PermanentDeleteBatchForRetentionPolicies(nowMillis, 2, 1000, model.RetentionPolicyCursor{})
require.NoError(t, err2)
require.Equal(t, int64(3), deleted)
rows, err = ss.RetentionPolicy().GetIdsForDeletionByTableName("Posts", 1000)
require.NoError(t, err)
// Each policy would generate it's own row
require.Equal(t, 3, len(rows))
// Clean up retention ids table
for _, row := range rows {
err = ss.Reaction().DeleteOrphanedRowsByIds(row)
require.NoError(t, err)
}
})
}

8
server/channels/store/storetest/preference_store.go
Просмотреть файл

@@ -384,6 +384,14 @@ func testPreferenceDeleteOrphanedRows(t *testing.T, ss store.Store) {
_, _, nErr = ss.Post().PermanentDeleteBatchForRetentionPolicies(0, 2000, limit, model.RetentionPolicyCursor{})
assert.NoError(t, nErr)
rows, err := ss.RetentionPolicy().GetIdsForDeletionByTableName("Posts", 1000)
require.NoError(t, err)
require.Equal(t, 1, len(rows))
// Clean up retention ids table
err = ss.Reaction().DeleteOrphanedRowsByIds(rows[0])
require.NoError(t, err)
_, nErr = ss.Preference().DeleteOrphanedRows(limit)
assert.NoError(t, nErr)

100
server/channels/store/storetest/reaction_store.go
Просмотреть файл

@@ -24,6 +24,7 @@ func TestReactionStore(t *testing.T, ss store.Store, s SqlStore) {
t.Run("ReactionGetForPost", func(t *testing.T) { testReactionGetForPost(t, ss) })
t.Run("ReactionGetForPostSince", func(t *testing.T) { testReactionGetForPostSince(t, ss, s) })
t.Run("ReactionDeleteAllWithEmojiName", func(t *testing.T) { testReactionDeleteAllWithEmojiName(t, ss, s) })
t.Run("PermanentDeleteByUser", func(t *testing.T) { testPermanentDeleteByUser(t, ss) })
t.Run("PermanentDeleteBatch", func(t *testing.T) { testReactionStorePermanentDeleteBatch(t, ss) })
t.Run("ReactionBulkGetForPosts", func(t *testing.T) { testReactionBulkGetForPosts(t, ss) })
t.Run("ReactionDeadlock", func(t *testing.T) { testReactionDeadlock(t, ss) })
@@ -563,6 +564,91 @@ func testReactionDeleteAllWithEmojiName(t *testing.T, ss store.Store, s SqlStore
}
func testPermanentDeleteByUser(t *testing.T, ss store.Store) {
userId := model.NewId()
post, err1 := ss.Post().Save(&model.Post{
ChannelId: model.NewId(),
UserId: model.NewId(),
})
require.NoError(t, err1)
post2, err2 := ss.Post().Save(&model.Post{
ChannelId: model.NewId(),
UserId: model.NewId(),
})
require.NoError(t, err2)
post3, err3 := ss.Post().Save(&model.Post{
ChannelId: model.NewId(),
UserId: model.NewId(),
})
require.NoError(t, err3)
reactions := []*model.Reaction{
{
UserId: userId,
PostId: post.Id,
EmojiName: "happy",
},
{
UserId: model.NewId(),
PostId: post.Id,
EmojiName: "smile",
},
{
UserId: model.NewId(),
PostId: post.Id,
EmojiName: "sad",
},
{
UserId: userId,
PostId: post2.Id,
EmojiName: "angry",
},
{
UserId: userId,
PostId: post3.Id,
EmojiName: "joy",
},
}
for _, reaction := range reactions {
_, err := ss.Reaction().Save(reaction)
require.NoError(t, err)
}
err := ss.Reaction().PermanentDeleteByUser(userId)
require.NoError(t, err)
// check that the reactions were deleted
returned, err := ss.Reaction().GetForPost(post.Id, false)
require.NoError(t, err)
require.Len(t, returned, 2, "should only have removed reaction for user")
for _, reaction := range returned {
assert.NotEqual(t, reaction.EmojiName, "happy", "should've removed reaction with emoji name")
}
returned, err = ss.Reaction().GetForPost(post2.Id, false)
require.NoError(t, err)
require.Len(t, returned, 0, "should have removed reaction for user")
returned, err = ss.Reaction().GetForPost(post3.Id, false)
require.NoError(t, err)
require.Len(t, returned, 0, "should remove reaction for user")
// check that the posts are updated
postList, err := ss.Post().Get(context.Background(), post.Id, model.GetPostsOptions{}, "", map[string]bool{})
require.NoError(t, err)
assert.True(t, postList.Posts[post.Id].HasReactions, "post should still have reactions")
postList, err = ss.Post().Get(context.Background(), post2.Id, model.GetPostsOptions{}, "", map[string]bool{})
require.NoError(t, err)
assert.False(t, postList.Posts[post2.Id].HasReactions, "post shouldn't have reactions any more")
postList, err = ss.Post().Get(context.Background(), post3.Id, model.GetPostsOptions{}, "", map[string]bool{})
require.NoError(t, err)
assert.False(t, postList.Posts[post3.Id].HasReactions, "post shouldn't have reactions any more")
}
func testReactionStorePermanentDeleteBatch(t *testing.T, ss store.Store) {
const limit = 1000
team, err := ss.Team().Save(&model.Team{
@@ -620,8 +706,20 @@ func testReactionStorePermanentDeleteBatch(t *testing.T, ss store.Store) {
_, _, err = ss.Post().PermanentDeleteBatchForRetentionPolicies(0, 2000, limit, model.RetentionPolicyCursor{})
require.NoError(t, err)
_, err = ss.Reaction().DeleteOrphanedRows(limit)
rows, err := ss.RetentionPolicy().GetIdsForDeletionByTableName("Posts", 1000)
require.NoError(t, err)
require.Equal(t, 1, len(rows))
require.Equal(t, 1, len(rows[0].Ids))
require.Contains(t, rows[0].Ids, olderPost.Id)
for _, row := range rows {
err = ss.Reaction().DeleteOrphanedRowsByIds(row)
require.NoError(t, err)
}
rows, err = ss.RetentionPolicy().GetIdsForDeletionByTableName("Posts", 1000)
require.NoError(t, err)
require.Equal(t, 0, len(rows))
returned, err := ss.Reaction().GetForPost(olderPost.Id, false)
require.NoError(t, err)