[MM-53428] Delete empty drafts on upsert (#24046)
* [MM-53428] Delete empty drafts on upsert * Add migrations to fix existing drafts * Fix CI * Delete empty drafts entirely from the DB * Fix lint * Implement batch migration for deleting drafts * Missing store layers * Add updated mock * Remove unnecessary test * PR feedback * Add check for cluster migration * Fix MySQL * Don't check for len<2 * Bit of PR feedback * Use query builder for parameters * PR feedback * More PR feedback * Merge'd * unit test GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration * simplified builder interface * fix DeleteEmptyDraftsByCreateAtAndUserId for MySQL * rework as batch migration worker * fix typo * log ip address on version mismatches too * simplify reset semantics * remove trace log in favour of low spam * document parameters for clarity --------- Co-authored-by: Mattermost Build <build@mattermost.com> Co-authored-by: Jesse Hallam <jesse.hallam@gmail.com>
Этот коммит содержится в:
Devin Binnie
коммит произвёл
GitHub
GitHub
родитель
760dfe41f9
Коммит
89492a6a46
@@ -0,0 +1,85 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package delete_empty_drafts_migration
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/jobs"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/store"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
timeBetweenBatches = 1 * time.Second
|
||||
)
|
||||
|
||||
// MakeWorker creates a batch migration worker to delete empty drafts.
|
||||
func MakeWorker(jobServer *jobs.JobServer, store store.Store, app jobs.BatchMigrationWorkerAppIFace) model.Worker {
|
||||
return jobs.MakeBatchMigrationWorker(
|
||||
jobServer,
|
||||
store,
|
||||
app,
|
||||
model.MigrationKeyDeleteEmptyDrafts,
|
||||
timeBetweenBatches,
|
||||
doDeleteEmptyDraftsMigrationBatch,
|
||||
)
|
||||
}
|
||||
|
||||
// parseJobMetadata parses the opaque job metadata to return the information needed to decide which
|
||||
// batch to process next.
|
||||
func parseJobMetadata(data model.StringMap) (int64, string, error) {
|
||||
createAt := int64(0)
|
||||
if data["create_at"] != "" {
|
||||
parsedCreateAt, parseErr := strconv.ParseInt(data["create_at"], 10, 64)
|
||||
if parseErr != nil {
|
||||
return 0, "", errors.Wrap(parseErr, "failed to parse create_at")
|
||||
}
|
||||
createAt = parsedCreateAt
|
||||
}
|
||||
|
||||
userID := data["user_id"]
|
||||
|
||||
return createAt, userID, nil
|
||||
}
|
||||
|
||||
// makeJobMetadata encodes the information needed to decide which batch to process next back into
|
||||
// the opaque job metadata.
|
||||
func makeJobMetadata(createAt int64, userID string) model.StringMap {
|
||||
data := make(model.StringMap)
|
||||
data["create_at"] = strconv.FormatInt(createAt, 10)
|
||||
data["user_id"] = userID
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
// doDeleteEmptyDraftsMigrationBatch iterates through all drafts, deleting empty drafts within each
|
||||
// batch keyed by the compound primary key (createAt, userID)
|
||||
func doDeleteEmptyDraftsMigrationBatch(data model.StringMap, store store.Store) (model.StringMap, bool, error) {
|
||||
createAt, userID, err := parseJobMetadata(data)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrap(err, "failed to parse job metadata")
|
||||
}
|
||||
|
||||
// Determine the /next/ (createAt, userId) by finding the last record in the batch we're
|
||||
// about to delete.
|
||||
nextCreateAt, nextUserID, err := store.Draft().GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration(createAt, userID)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrapf(err, "failed to get the next batch (create_at=%v, user_id=%v)", createAt, userID)
|
||||
}
|
||||
|
||||
// If we get the nil values, it means the batch was empty and we're done.
|
||||
if nextCreateAt == 0 && nextUserID == "" {
|
||||
return nil, true, nil
|
||||
}
|
||||
|
||||
err = store.Draft().DeleteEmptyDraftsByCreateAtAndUserId(createAt, userID)
|
||||
if err != nil {
|
||||
return nil, false, errors.Wrapf(err, "failed to delete empty drafts (create_at=%v, user_id=%v)", createAt, userID)
|
||||
}
|
||||
|
||||
return makeJobMetadata(nextCreateAt, nextUserID), false, nil
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See LICENSE.txt for license information.
|
||||
|
||||
package delete_empty_drafts_migration
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/mattermost/mattermost/server/public/model"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/store/storetest"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestJobMetadata(t *testing.T) {
|
||||
t.Run("parse nil data", func(t *testing.T) {
|
||||
var data model.StringMap
|
||||
createAt, userID, err := parseJobMetadata(data)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, createAt)
|
||||
assert.Empty(t, userID)
|
||||
})
|
||||
|
||||
t.Run("parse invalid create_at", func(t *testing.T) {
|
||||
data := make(model.StringMap)
|
||||
data["user_id"] = "user_id"
|
||||
data["create_at"] = "invalid"
|
||||
_, _, err := parseJobMetadata(data)
|
||||
require.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("parse valid", func(t *testing.T) {
|
||||
data := make(model.StringMap)
|
||||
data["user_id"] = "user_id"
|
||||
data["create_at"] = "1695918431"
|
||||
|
||||
createAt, userID, err := parseJobMetadata(data)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 1695918431, createAt)
|
||||
assert.Equal(t, "user_id", userID)
|
||||
})
|
||||
|
||||
t.Run("parse/make", func(t *testing.T) {
|
||||
data := makeJobMetadata(1695918431, "user_id")
|
||||
assert.Equal(t, "1695918431", data["create_at"])
|
||||
assert.Equal(t, "user_id", data["user_id"])
|
||||
|
||||
createAt, userID, err := parseJobMetadata(data)
|
||||
require.NoError(t, err)
|
||||
assert.EqualValues(t, 1695918431, createAt)
|
||||
assert.Equal(t, "user_id", userID)
|
||||
})
|
||||
}
|
||||
|
||||
func TestDoDeleteEmptyDraftsMigrationBatch(t *testing.T) {
|
||||
t.Run("invalid job metadata", func(t *testing.T) {
|
||||
mockStore := &storetest.Store{}
|
||||
t.Cleanup(func() {
|
||||
mockStore.AssertExpectations(t)
|
||||
})
|
||||
|
||||
data := make(model.StringMap)
|
||||
data["user_id"] = "user_id"
|
||||
data["create_at"] = "invalid"
|
||||
data, done, err := doDeleteEmptyDraftsMigrationBatch(data, mockStore)
|
||||
require.Error(t, err)
|
||||
assert.False(t, done)
|
||||
assert.Nil(t, data)
|
||||
})
|
||||
|
||||
t.Run("failure getting next offset", func(t *testing.T) {
|
||||
mockStore := &storetest.Store{}
|
||||
t.Cleanup(func() {
|
||||
mockStore.AssertExpectations(t)
|
||||
})
|
||||
|
||||
createAt, userID := int64(1695920000), "user_id_1"
|
||||
nextCreateAt, nextUserID := int64(0), ""
|
||||
|
||||
mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, errors.New("failure"))
|
||||
|
||||
data, done, err := doDeleteEmptyDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
|
||||
require.EqualError(t, err, "failed to get the next batch (create_at=1695920000, user_id=user_id_1): failure")
|
||||
assert.False(t, done)
|
||||
assert.Nil(t, data)
|
||||
})
|
||||
|
||||
t.Run("failure deleting batch", func(t *testing.T) {
|
||||
mockStore := &storetest.Store{}
|
||||
t.Cleanup(func() {
|
||||
mockStore.AssertExpectations(t)
|
||||
})
|
||||
|
||||
createAt, userID := int64(1695920000), "user_id_1"
|
||||
nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
|
||||
|
||||
mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
|
||||
mockStore.DraftStore.On("DeleteEmptyDraftsByCreateAtAndUserId", createAt, userID).Return(errors.New("failure"))
|
||||
|
||||
data, done, err := doDeleteEmptyDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
|
||||
require.EqualError(t, err, "failed to delete empty drafts (create_at=1695920000, user_id=user_id_1): failure")
|
||||
assert.False(t, done)
|
||||
assert.Nil(t, data)
|
||||
})
|
||||
|
||||
t.Run("do first batch (nil job metadata)", func(t *testing.T) {
|
||||
mockStore := &storetest.Store{}
|
||||
t.Cleanup(func() {
|
||||
mockStore.AssertExpectations(t)
|
||||
})
|
||||
|
||||
createAt, userID := int64(0), ""
|
||||
nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
|
||||
|
||||
mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
|
||||
mockStore.DraftStore.On("DeleteEmptyDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
|
||||
|
||||
data, done, err := doDeleteEmptyDraftsMigrationBatch(nil, mockStore)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, done)
|
||||
assert.Equal(t, model.StringMap{
|
||||
"create_at": "1695922034",
|
||||
"user_id": "user_id_2",
|
||||
}, data)
|
||||
})
|
||||
|
||||
t.Run("do first batch (empty job metadata)", func(t *testing.T) {
|
||||
mockStore := &storetest.Store{}
|
||||
t.Cleanup(func() {
|
||||
mockStore.AssertExpectations(t)
|
||||
})
|
||||
|
||||
createAt, userID := int64(0), ""
|
||||
nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
|
||||
|
||||
mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
|
||||
mockStore.DraftStore.On("DeleteEmptyDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
|
||||
|
||||
data, done, err := doDeleteEmptyDraftsMigrationBatch(model.StringMap{}, mockStore)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, done)
|
||||
assert.Equal(t, makeJobMetadata(nextCreateAt, nextUserID), data)
|
||||
})
|
||||
|
||||
t.Run("do batch", func(t *testing.T) {
|
||||
mockStore := &storetest.Store{}
|
||||
t.Cleanup(func() {
|
||||
mockStore.AssertExpectations(t)
|
||||
})
|
||||
|
||||
createAt, userID := int64(1695922000), "user_id_1"
|
||||
nextCreateAt, nextUserID := int64(1695922034), "user_id_2"
|
||||
|
||||
mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
|
||||
mockStore.DraftStore.On("DeleteEmptyDraftsByCreateAtAndUserId", createAt, userID).Return(nil)
|
||||
|
||||
data, done, err := doDeleteEmptyDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, done)
|
||||
assert.Equal(t, makeJobMetadata(nextCreateAt, nextUserID), data)
|
||||
})
|
||||
|
||||
t.Run("done batches", func(t *testing.T) {
|
||||
mockStore := &storetest.Store{}
|
||||
t.Cleanup(func() {
|
||||
mockStore.AssertExpectations(t)
|
||||
})
|
||||
|
||||
createAt, userID := int64(1695922000), "user_id_1"
|
||||
nextCreateAt, nextUserID := int64(0), ""
|
||||
|
||||
mockStore.DraftStore.On("GetLastCreateAtAndUserIdValuesForEmptyDraftsMigration", createAt, userID).Return(nextCreateAt, nextUserID, nil)
|
||||
|
||||
data, done, err := doDeleteEmptyDraftsMigrationBatch(makeJobMetadata(createAt, userID), mockStore)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, done)
|
||||
assert.Nil(t, data)
|
||||
})
|
||||
}
|
||||
Ссылка в новой задаче
Block a user