Files
mostlymatter/store/sqlstore/compliance_store.go
Jonathan 375c0632fa PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)
* Created message export scheduled task

* Added CLI command to immediately kick off an export job

* Added email addresses for users joining and leaving the channel to the export

* Added support for both MySQL and PostgreSQL

* Fixing gofmt error

* Added a new ChannelMemberHistory store and associated tests

* Updating the ChannelMemberHistory channel as users create/join/leave channels

* Added user email to the message export object so it can be included in the actiance export xml

* Don't fail to log a leave event if a corresponding join event wasn't logged

* Adding copyright notices

* Adding message export settings to daily diagnostics report

* Added System Console integration for message export

* Cleaned up TODOs

* Made batch size configurable

* Added export from timestamp to CLI command

* Made ChannelMemberHistory table updates best effort

* Added a context-based timeout option to the message export CLI

* Minor PR updates/improvements

* Removed unnecessary fields from MessageExport object to reduce query overhead

* Removed JSON functions from the message export query in an effort to optimize performance

* Changed the way that channel member history queries and purges work to better account for edge cases

* Fixing a test I missed with the last refactor

* Added file copy functionality to file backend, improved config validation, added default config values

* Fixed file copy tests

* More concise use of the testing libraries

* Fixed context leak error

* Changed default export path to correctly place an 'export' directory under the 'data' directory

* Can't delete records from a read replica

* Fixed copy file tests

* Start job workers when license is applied, if configured to do so

* Suggestions from the PR

* Moar unit tests

* Fixed test imports
2017-11-30 09:07:04 -05:00

247 строки
7.7 KiB
Go

// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package sqlstore
import (
"net/http"
"strconv"
"strings"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store"
)
type SqlComplianceStore struct {
SqlStore
}
func NewSqlComplianceStore(sqlStore SqlStore) store.ComplianceStore {
s := &SqlComplianceStore{sqlStore}
for _, db := range sqlStore.GetAllConns() {
table := db.AddTableWithName(model.Compliance{}, "Compliances").SetKeys(false, "Id")
table.ColMap("Id").SetMaxSize(26)
table.ColMap("UserId").SetMaxSize(26)
table.ColMap("Status").SetMaxSize(64)
table.ColMap("Desc").SetMaxSize(512)
table.ColMap("Type").SetMaxSize(64)
table.ColMap("Keywords").SetMaxSize(512)
table.ColMap("Emails").SetMaxSize(1024)
}
return s
}
func (s SqlComplianceStore) CreateIndexesIfNotExists() {
}
func (s SqlComplianceStore) Save(compliance *model.Compliance) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
compliance.PreSave()
if result.Err = compliance.IsValid(); result.Err != nil {
return
}
if err := s.GetMaster().Insert(compliance); err != nil {
result.Err = model.NewAppError("SqlComplianceStore.Save", "store.sql_compliance.save.saving.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
result.Data = compliance
}
})
}
func (us SqlComplianceStore) Update(compliance *model.Compliance) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
if result.Err = compliance.IsValid(); result.Err != nil {
return
}
if _, err := us.GetMaster().Update(compliance); err != nil {
result.Err = model.NewAppError("SqlComplianceStore.Update", "store.sql_compliance.save.saving.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
result.Data = compliance
}
})
}
func (s SqlComplianceStore) GetAll(offset, limit int) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
query := "SELECT * FROM Compliances ORDER BY CreateAt DESC LIMIT :Limit OFFSET :Offset"
var compliances model.Compliances
if _, err := s.GetReplica().Select(&compliances, query, map[string]interface{}{"Offset": offset, "Limit": limit}); err != nil {
result.Err = model.NewAppError("SqlComplianceStore.Get", "store.sql_compliance.get.finding.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
result.Data = compliances
}
})
}
func (us SqlComplianceStore) Get(id string) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
if obj, err := us.GetReplica().Get(model.Compliance{}, id); err != nil {
result.Err = model.NewAppError("SqlComplianceStore.Get", "store.sql_compliance.get.finding.app_error", nil, err.Error(), http.StatusInternalServerError)
} else if obj == nil {
result.Err = model.NewAppError("SqlComplianceStore.Get", "store.sql_compliance.get.finding.app_error", nil, err.Error(), http.StatusNotFound)
} else {
result.Data = obj.(*model.Compliance)
}
})
}
func (s SqlComplianceStore) ComplianceExport(job *model.Compliance) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
props := map[string]interface{}{"StartTime": job.StartAt, "EndTime": job.EndAt}
keywordQuery := ""
keywords := strings.Fields(strings.TrimSpace(strings.ToLower(strings.Replace(job.Keywords, ",", " ", -1))))
if len(keywords) > 0 {
keywordQuery = "AND ("
for index, keyword := range keywords {
if index >= 1 {
keywordQuery += " OR LOWER(Posts.Message) LIKE :Keyword" + strconv.Itoa(index)
} else {
keywordQuery += "LOWER(Posts.Message) LIKE :Keyword" + strconv.Itoa(index)
}
props["Keyword"+strconv.Itoa(index)] = "%" + keyword + "%"
}
keywordQuery += ")"
}
emailQuery := ""
emails := strings.Fields(strings.TrimSpace(strings.ToLower(strings.Replace(job.Emails, ",", " ", -1))))
if len(emails) > 0 {
emailQuery = "AND ("
for index, email := range emails {
if index >= 1 {
emailQuery += " OR Users.Email = :Email" + strconv.Itoa(index)
} else {
emailQuery += "Users.Email = :Email" + strconv.Itoa(index)
}
props["Email"+strconv.Itoa(index)] = email
}
emailQuery += ")"
}
query :=
`(SELECT
Teams.Name AS TeamName,
Teams.DisplayName AS TeamDisplayName,
Channels.Name AS ChannelName,
Channels.DisplayName AS ChannelDisplayName,
Users.Username AS UserUsername,
Users.Email AS UserEmail,
Users.Nickname AS UserNickname,
Posts.Id AS PostId,
Posts.CreateAt AS PostCreateAt,
Posts.UpdateAt AS PostUpdateAt,
Posts.DeleteAt AS PostDeleteAt,
Posts.RootId AS PostRootId,
Posts.ParentId AS PostParentId,
Posts.OriginalId AS PostOriginalId,
Posts.Message AS PostMessage,
Posts.Type AS PostType,
Posts.Props AS PostProps,
Posts.Hashtags AS PostHashtags,
Posts.FileIds AS PostFileIds
FROM
Teams,
Channels,
Users,
Posts
WHERE
Teams.Id = Channels.TeamId
AND Posts.ChannelId = Channels.Id
AND Posts.UserId = Users.Id
AND Posts.CreateAt > :StartTime
AND Posts.CreateAt <= :EndTime
` + emailQuery + `
` + keywordQuery + `)
UNION ALL
(SELECT
'direct-messages' AS TeamName,
'Direct Messages' AS TeamDisplayName,
Channels.Name AS ChannelName,
Channels.DisplayName AS ChannelDisplayName,
Users.Username AS UserUsername,
Users.Email AS UserEmail,
Users.Nickname AS UserNickname,
Posts.Id AS PostId,
Posts.CreateAt AS PostCreateAt,
Posts.UpdateAt AS PostUpdateAt,
Posts.DeleteAt AS PostDeleteAt,
Posts.RootId AS PostRootId,
Posts.ParentId AS PostParentId,
Posts.OriginalId AS PostOriginalId,
Posts.Message AS PostMessage,
Posts.Type AS PostType,
Posts.Props AS PostProps,
Posts.Hashtags AS PostHashtags,
Posts.FileIds AS PostFileIds
FROM
Channels,
Users,
Posts
WHERE
Channels.TeamId = ''
AND Posts.ChannelId = Channels.Id
AND Posts.UserId = Users.Id
AND Posts.CreateAt > :StartTime
AND Posts.CreateAt <= :EndTime
` + emailQuery + `
` + keywordQuery + `)
ORDER BY PostCreateAt
LIMIT 30000`
var cposts []*model.CompliancePost
if _, err := s.GetReplica().Select(&cposts, query, props); err != nil {
result.Err = model.NewAppError("SqlPostStore.ComplianceExport", "store.sql_post.compliance_export.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
result.Data = cposts
}
})
}
func (s SqlComplianceStore) MessageExport(after int64, limit int) store.StoreChannel {
return store.Do(func(result *store.StoreResult) {
props := map[string]interface{}{"StartTime": after, "Limit": limit}
query :=
`SELECT
Posts.Id AS PostId,
Posts.CreateAt AS PostCreateAt,
Posts.Message AS PostMessage,
Posts.Type AS PostType,
Posts.FileIds AS PostFileIds,
Channels.Id AS ChannelId,
Channels.DisplayName AS ChannelDisplayName,
Users.Id AS UserId,
Users.Email AS UserEmail
FROM
Posts
LEFT OUTER JOIN Channels ON Posts.ChannelId = Channels.Id
LEFT OUTER JOIN Users ON Posts.UserId = Users.Id
WHERE
Posts.CreateAt > :StartTime AND
Posts.Type = ''
ORDER BY PostCreateAt
LIMIT :Limit`
var cposts []*model.MessageExport
if _, err := s.GetReplica().Select(&cposts, query, props); err != nil {
result.Err = model.NewAppError("SqlComplianceStore.MessageExport", "store.sql_compliance.message_export.app_error", nil, err.Error(), http.StatusInternalServerError)
} else {
result.Data = cposts
}
})
}