* pre-checkout commit * add API endpoints for retention policies * allow deleting multiple teams/channels from a policy in a single request * pre-checkout commit * add auditing in API functions * add permission checks * update the store layers * update storetest * add check constraint on PostDuration column * pre-checkout commit * add query to delete posts under the scope of a granular retention policy * add suggestions from sbishel * allow clients to specify channels/teams when creating a new policy * remove foreign keys referencing Channels and Teams tables * add checks for whether teams and channels exist * pre-checkout commit * remove data referencing the Posts table * pre-checkout commit * write data store tests * sort results of buildGetPoliciesQuery * add missing test cases for teams * pre-checkout commit * add Client4 methods for data retention policy endpoints * add uint and uint64 to app/layer_generators * make granular policies override global policies * fix lint errors * pre-checkout commit * add license to top of files * add tests for data store layer * add missing test cases for store layer * run make i18n-extract * add query to delete ChannelMemberHistory * work in progress * add test for old reply to old post * fix lint error * use COALESCE on each Posts column * begin implementing orphaned rows worker * split PR * pre-checkout commit * use RetentionPolicyWithTeamAndChannelCounts * update app and api layers * run make i18n-extract * add RetentionPolicy to retrylayer_test.go * Revert "split PR" This reverts commit b316f03dd307a30deae931944ca7e4a1cc904605. * fix errors caused by revert * add suggestions from sbishel * fix copy-paste error * fix lint errors * pre-checkout commit * add function to delete orphaned rows * use -1 for infinite retention * remove check constraint * copy i18n entries from master * re-run tests with newer enterprise branch * add team data to channel list * add search for channels and teams in a policy * add store tests for channel and team search * add suggestions from mkraft * run make einterfaces-mocks * fix lint errors * add suggestions from mkraft * move removeOrphanedRows method to wireup branch * Revert "move removeOrphanedRows method to wireup branch" This reverts commit 94605c9b4a5378ffa44a3dec4d3f8e3306b9d33e. * use DeleteOrphanedRows where possible * run make i18n-extract * use COMPLIANCE permissions * run make migrations-bindadta * clean up teams before test * fix tests for TestRetentionPolicyStore * add API endpoints for mobile * fix lint error * fix some of the lint errors * move user/data_retention endpoints to data_retention.go * Revert "fix some of the lint errors" This reverts commit b5b2dc27566c427187db942c5c0afe319e7679c4. * add exclude_policy_constrained parameter for /channels and /teams * fix lint errors * add policy_id field to GET endpoints for channels and teams * use PolicyWithTeamID in RetentionPolicy layer * fix lint errors * run make i18n-extract * update mock call in telemetry_test.go * return status:OK in JSON instead of 204 * pre-checkout commit * add policy_id field on channels/teams * fix lint errors * use sq.Eq instead of '?' * use new subsection permissions * update channels and teams endpoints to use new subsection permissions * add extra search opts for channels in a policy * fix lint errors * allow negative post duration in patch * remove DELETE FROM query in retention policy tests * use *int64 for PostDuration * re-run CI tests * use 3-step deletion strategy for each table * fix lint errors * run make store-layers * re-run CI tests * add test with channel, team and global policies * use common function for SQL queries * add pagination test * use struct for args to common SQL function * fix lint errors * run make i18n-extract * check if Channels.TeamId is "" or nil * use three OR clauses * write separate genericRetentionPoliciesDeletion function * add config setting for BatchSize * add telemetry for BatchSize * use feature flag * add old i18n messages back in * re-run CI tests * update call signature in storetest * MM-30831: Adds constant for retention default batch size. * MM-30831: Removes comment re: optimization. * MM-30831: Converts days to milliseconds. * MM-30831: Reverts change to test. * Revert "MM-30831: Reverts change to test." This reverts commit 6d14275a1ceae682bb9e17ec69b39252b44e0c0c. * Revert "MM-30831: Converts days to milliseconds." This reverts commit a0cb6ec09d854a05194c1daee1c5333f260231c3. * MM-30831: Fixes tests. * MM-30381: Fix for change to method sig. Co-authored-by: Max Erenberg <max.erenberg@mattermost.com> Co-authored-by: Mattermod <mattermod@users.noreply.github.com> Co-authored-by: Martin Kraft <martin@upspin.org>
329 строки
11 KiB
Go
329 строки
11 KiB
Go
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
|
// See LICENSE.txt for license information.
|
|
|
|
package sqlstore
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
sq "github.com/Masterminds/squirrel"
|
|
"github.com/mattermost/gorp"
|
|
"github.com/mattermost/mattermost-server/v5/model"
|
|
"github.com/mattermost/mattermost-server/v5/shared/mlog"
|
|
"github.com/mattermost/mattermost-server/v5/store"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
type SqlPreferenceStore struct {
|
|
*SqlStore
|
|
}
|
|
|
|
func newSqlPreferenceStore(sqlStore *SqlStore) store.PreferenceStore {
|
|
s := &SqlPreferenceStore{sqlStore}
|
|
|
|
for _, db := range sqlStore.GetAllConns() {
|
|
table := db.AddTableWithName(model.Preference{}, "Preferences").SetKeys(false, "UserId", "Category", "Name")
|
|
table.ColMap("UserId").SetMaxSize(26)
|
|
table.ColMap("Category").SetMaxSize(32)
|
|
table.ColMap("Name").SetMaxSize(32)
|
|
table.ColMap("Value").SetMaxSize(2000)
|
|
}
|
|
|
|
return s
|
|
}
|
|
|
|
func (s SqlPreferenceStore) createIndexesIfNotExists() {
|
|
s.CreateIndexIfNotExists("idx_preferences_category", "Preferences", "Category")
|
|
s.CreateIndexIfNotExists("idx_preferences_name", "Preferences", "Name")
|
|
}
|
|
|
|
func (s SqlPreferenceStore) deleteUnusedFeatures() {
|
|
mlog.Debug("Deleting any unused pre-release features")
|
|
sql, args, err := s.getQueryBuilder().
|
|
Delete("Preferences").
|
|
Where(sq.Eq{"Category": model.PREFERENCE_CATEGORY_ADVANCED_SETTINGS}).
|
|
Where(sq.Eq{"Value": "false"}).
|
|
Where(sq.Like{"Name": store.FeatureTogglePrefix + "%"}).ToSql()
|
|
if err != nil {
|
|
mlog.Warn(errors.Wrap(err, "could not build sql query to delete unused features!").Error())
|
|
}
|
|
if _, err = s.GetMaster().Exec(sql, args...); err != nil {
|
|
mlog.Warn("Failed to delete unused features", mlog.Err(err))
|
|
}
|
|
}
|
|
|
|
func (s SqlPreferenceStore) Save(preferences *model.Preferences) error {
|
|
// wrap in a transaction so that if one fails, everything fails
|
|
transaction, err := s.GetMaster().Begin()
|
|
if err != nil {
|
|
return errors.Wrap(err, "begin_transaction")
|
|
}
|
|
|
|
defer finalizeTransaction(transaction)
|
|
for _, preference := range *preferences {
|
|
preference := preference
|
|
if upsertErr := s.save(transaction, &preference); upsertErr != nil {
|
|
return upsertErr
|
|
}
|
|
}
|
|
|
|
if err := transaction.Commit(); err != nil {
|
|
// don't need to rollback here since the transaction is already closed
|
|
return errors.Wrap(err, "commit_transaction")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s SqlPreferenceStore) save(transaction *gorp.Transaction, preference *model.Preference) error {
|
|
preference.PreUpdate()
|
|
|
|
if err := preference.IsValid(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
|
|
queryString, args, err := s.getQueryBuilder().
|
|
Insert("Preferences").
|
|
Columns("UserId", "Category", "Name", "Value").
|
|
Values(preference.UserId, preference.Category, preference.Name, preference.Value).
|
|
SuffixExpr(sq.Expr("ON DUPLICATE KEY UPDATE Value = ?", preference.Value)).
|
|
ToSql()
|
|
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to generate sqlquery")
|
|
}
|
|
|
|
if _, err = transaction.Exec(queryString, args...); err != nil {
|
|
return errors.Wrap(err, "failed to save Preference")
|
|
}
|
|
return nil
|
|
} else if s.DriverName() == model.DATABASE_DRIVER_POSTGRES {
|
|
|
|
// postgres has no way to upsert values until version 9.5 and trying inserting and then updating causes transactions to abort
|
|
queryString, args, err := s.getQueryBuilder().
|
|
Select("count(0)").
|
|
From("Preferences").
|
|
Where(sq.Eq{"UserId": preference.UserId}).
|
|
Where(sq.Eq{"Category": preference.Category}).
|
|
Where(sq.Eq{"Name": preference.Name}).
|
|
ToSql()
|
|
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to generate sqlquery")
|
|
}
|
|
|
|
count, err := transaction.SelectInt(queryString, args...)
|
|
if err != nil {
|
|
return errors.Wrap(err, "failed to count Preferences")
|
|
}
|
|
|
|
if count == 1 {
|
|
return s.update(transaction, preference)
|
|
}
|
|
return s.insert(transaction, preference)
|
|
}
|
|
return store.NewErrNotImplemented("failed to update preference because of missing driver")
|
|
}
|
|
|
|
func (s SqlPreferenceStore) insert(transaction *gorp.Transaction, preference *model.Preference) error {
|
|
if err := transaction.Insert(preference); err != nil {
|
|
if IsUniqueConstraintError(err, []string{"UserId", "preferences_pkey"}) {
|
|
return store.NewErrInvalidInput("Preference", "<userId, category, name>", fmt.Sprintf("<%s, %s, %s>", preference.UserId, preference.Category, preference.Name))
|
|
}
|
|
return errors.Wrapf(err, "failed to save Preference with userId=%s, category=%s, name=%s", preference.UserId, preference.Category, preference.Name)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s SqlPreferenceStore) update(transaction *gorp.Transaction, preference *model.Preference) error {
|
|
if _, err := transaction.Update(preference); err != nil {
|
|
return errors.Wrapf(err, "failed to update Preference with userId=%s, category=%s, name=%s", preference.UserId, preference.Category, preference.Name)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s SqlPreferenceStore) Get(userId string, category string, name string) (*model.Preference, error) {
|
|
var preference *model.Preference
|
|
query, args, err := s.getQueryBuilder().
|
|
Select("*").
|
|
From("Preferences").
|
|
Where(sq.Eq{"UserId": userId}).
|
|
Where(sq.Eq{"Category": category}).
|
|
Where(sq.Eq{"Name": name}).
|
|
ToSql()
|
|
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "could not build sql query to get preference")
|
|
}
|
|
if err = s.GetReplica().SelectOne(&preference, query, args...); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to find Preference with userId=%s, category=%s, name=%s", userId, category, name)
|
|
}
|
|
|
|
return preference, nil
|
|
}
|
|
|
|
func (s SqlPreferenceStore) GetCategory(userId string, category string) (model.Preferences, error) {
|
|
var preferences model.Preferences
|
|
query, args, err := s.getQueryBuilder().
|
|
Select("*").
|
|
From("Preferences").
|
|
Where(sq.Eq{"UserId": userId}).
|
|
Where(sq.Eq{"Category": category}).
|
|
ToSql()
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "could not build sql query to get preference")
|
|
}
|
|
if _, err = s.GetReplica().Select(&preferences, query, args...); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to find Preference with userId=%s, category=%s", userId, category)
|
|
}
|
|
return preferences, nil
|
|
|
|
}
|
|
|
|
func (s SqlPreferenceStore) GetAll(userId string) (model.Preferences, error) {
|
|
var preferences model.Preferences
|
|
query, args, err := s.getQueryBuilder().
|
|
Select("*").
|
|
From("Preferences").
|
|
Where(sq.Eq{"UserId": userId}).
|
|
ToSql()
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "could not build sql query to get preference")
|
|
}
|
|
if _, err = s.GetReplica().Select(&preferences, query, args...); err != nil {
|
|
return nil, errors.Wrapf(err, "failed to find Preference with userId=%s", userId)
|
|
}
|
|
return preferences, nil
|
|
}
|
|
|
|
func (s SqlPreferenceStore) PermanentDeleteByUser(userId string) error {
|
|
sql, args, err := s.getQueryBuilder().
|
|
Delete("Preferences").
|
|
Where(sq.Eq{"UserId": userId}).ToSql()
|
|
if err != nil {
|
|
return errors.Wrap(err, "could not build sql query to get delete preference by user")
|
|
}
|
|
if _, err := s.GetMaster().Exec(sql, args...); err != nil {
|
|
return errors.Wrapf(err, "failed to delete Preference with userId=%s", userId)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s SqlPreferenceStore) Delete(userId, category, name string) error {
|
|
|
|
sql, args, err := s.getQueryBuilder().
|
|
Delete("Preferences").
|
|
Where(sq.Eq{"UserId": userId}).
|
|
Where(sq.Eq{"Category": category}).
|
|
Where(sq.Eq{"Name": name}).ToSql()
|
|
|
|
if err != nil {
|
|
return errors.Wrap(err, "could not build sql query to get delete preference")
|
|
}
|
|
|
|
if _, err = s.GetMaster().Exec(sql, args...); err != nil {
|
|
return errors.Wrapf(err, "failed to delete Preference with userId=%s, category=%s and name=%s", userId, category, name)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s SqlPreferenceStore) DeleteCategory(userId string, category string) error {
|
|
|
|
sql, args, err := s.getQueryBuilder().
|
|
Delete("Preferences").
|
|
Where(sq.Eq{"UserId": userId}).
|
|
Where(sq.Eq{"Category": category}).ToSql()
|
|
|
|
if err != nil {
|
|
return errors.Wrap(err, "could not build sql query to get delete preference by category")
|
|
}
|
|
|
|
if _, err = s.GetMaster().Exec(sql, args...); err != nil {
|
|
return errors.Wrapf(err, "failed to delete Preference with userId=%s and category=%s", userId, category)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s SqlPreferenceStore) DeleteCategoryAndName(category string, name string) error {
|
|
sql, args, err := s.getQueryBuilder().
|
|
Delete("Preferences").
|
|
Where(sq.Eq{"Name": name}).
|
|
Where(sq.Eq{"Category": category}).ToSql()
|
|
|
|
if err != nil {
|
|
return errors.Wrap(err, "could not build sql query to get delete preference by category and name")
|
|
}
|
|
|
|
if _, err = s.GetMaster().Exec(sql, args...); err != nil {
|
|
return errors.Wrapf(err, "failed to delete Preference with category=%s and name=%s", category, name)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// DeleteOrphanedRows removes entries from Preferences (flagged post) when a
|
|
// corresponding post no longer exists.
|
|
func (s *SqlPreferenceStore) DeleteOrphanedRows(limit int) (deleted int64, err error) {
|
|
// We need the extra level of nesting to deal with MySQL's locking
|
|
const query = `
|
|
DELETE FROM Preferences WHERE Name IN (
|
|
SELECT * FROM (
|
|
SELECT Preferences.Name FROM Preferences
|
|
LEFT JOIN Posts ON Preferences.Name = Posts.Id
|
|
WHERE Posts.Id IS NULL AND Category = :Category
|
|
LIMIT :Limit
|
|
) AS A
|
|
)`
|
|
props := map[string]interface{}{"Limit": limit, "Category": model.PREFERENCE_CATEGORY_FLAGGED_POST}
|
|
result, err := s.GetMaster().Exec(query, props)
|
|
if err != nil {
|
|
return
|
|
}
|
|
deleted, err = result.RowsAffected()
|
|
return
|
|
}
|
|
|
|
func (s SqlPreferenceStore) CleanupFlagsBatch(limit int64) (int64, error) {
|
|
if limit < 0 {
|
|
// uint64 does not throw an error, it overflows if it is negative.
|
|
// it is better to manually check here, or change the function type to uint64
|
|
return int64(0), errors.Errorf("Received a negative limit")
|
|
}
|
|
nameInQ, nameInArgs, err := sq.Select("*").
|
|
FromSelect(
|
|
sq.Select("Preferences.Name").
|
|
From("Preferences").
|
|
LeftJoin("Posts ON Preferences.Name = Posts.Id").
|
|
Where(sq.Eq{"Preferences.Category": model.PREFERENCE_CATEGORY_FLAGGED_POST}).
|
|
Where(sq.Eq{"Posts.Id": nil}).
|
|
Limit(uint64(limit)),
|
|
"t").
|
|
ToSql()
|
|
if err != nil {
|
|
return int64(0), errors.Wrap(err, "could not build nested sql query to delete preference")
|
|
}
|
|
query, args, err := s.getQueryBuilder().Delete("Preferences").
|
|
Where(sq.Eq{"Category": model.PREFERENCE_CATEGORY_FLAGGED_POST}).
|
|
Where(sq.Expr("name IN ("+nameInQ+")", nameInArgs...)).
|
|
ToSql()
|
|
|
|
if err != nil {
|
|
return int64(0), errors.Wrap(err, "could not build sql query to delete preference")
|
|
}
|
|
|
|
sqlResult, err := s.GetMaster().Exec(query, args...)
|
|
if err != nil {
|
|
return int64(0), errors.Wrap(err, "failed to delete Preference")
|
|
}
|
|
rowsAffected, err := sqlResult.RowsAffected()
|
|
if err != nil {
|
|
return int64(0), errors.Wrap(err, "unable to get rows affected")
|
|
}
|
|
|
|
return rowsAffected, nil
|
|
}
|