Merge branch 'master' into MM-47853-true-up-review-telemetry-off-non-air-gapped

Этот коммит содержится в:
Mattermod
2022-12-23 23:12:20 +02:00
коммит произвёл GitHub
родитель bd2724e07d 7a9354cd09
Коммит b4d9d12856
22 изменённых файлов: 167 добавлений и 80 удалений

Просмотреть файл

@@ -3207,7 +3207,7 @@ func (s *OpenTracingLayerComplianceStore) GetAll(offset int, limit int) (model.C
return result, err
}
func (s *OpenTracingLayerComplianceStore) MessageExport(cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
func (s *OpenTracingLayerComplianceStore) MessageExport(ctx context.Context, cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
origCtx := s.Root.Store.Context()
span, newCtx := tracing.StartSpanWithParentByContext(s.Root.Store.Context(), "ComplianceStore.MessageExport")
s.Root.Store.SetContext(newCtx)
@@ -3216,7 +3216,7 @@ func (s *OpenTracingLayerComplianceStore) MessageExport(cursor model.MessageExpo
}()
defer span.Finish()
result, resultVar1, err := s.ComplianceStore.MessageExport(cursor, limit)
result, resultVar1, err := s.ComplianceStore.MessageExport(ctx, cursor, limit)
if err != nil {
span.LogFields(spanlog.Error(err))
ext.Error.Set(span, true)

Просмотреть файл

@@ -3577,11 +3577,11 @@ func (s *RetryLayerComplianceStore) GetAll(offset int, limit int) (model.Complia
}
func (s *RetryLayerComplianceStore) MessageExport(cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
func (s *RetryLayerComplianceStore) MessageExport(ctx context.Context, cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
tries := 0
for {
result, resultVar1, err := s.ComplianceStore.MessageExport(cursor, limit)
result, resultVar1, err := s.ComplianceStore.MessageExport(ctx, cursor, limit)
if err == nil {
return result, resultVar1, nil
}

Просмотреть файл

@@ -3850,7 +3850,31 @@ func (s SqlChannelStore) MigrateChannelMembers(fromChannelId string, fromUserId
defer finalizeTransactionX(transaction, &err)
channelMembers := []channelMember{}
if err := transaction.Select(&channelMembers, "SELECT * from ChannelMembers WHERE (ChannelId, UserId) > (?, ?) ORDER BY ChannelId, UserId LIMIT 100", fromChannelId, fromUserId); err != nil {
query := `
SELECT
ChannelId,
UserId,
Roles,
LastViewedAt,
MsgCount,
MentionCount,
MentionCountRoot,
COALESCE(UrgentMentionCount, 0) AS UrgentMentionCount,
MsgCountRoot,
NotifyProps,
LastUpdateAt,
SchemeUser,
SchemeAdmin,
SchemeGuest
FROM
ChannelMembers
WHERE
(ChannelId, UserId) > (?, ?)
ORDER BY ChannelId, UserId
LIMIT 100
`
if err := transaction.Select(&channelMembers, query, fromChannelId, fromUserId); err != nil {
return nil, errors.Wrap(err, "failed to find ChannelMembers")
}
@@ -3954,7 +3978,31 @@ func (s SqlChannelStore) ClearAllCustomRoleAssignments() (err error) {
}
channelMembers := []*channelMember{}
if err = transaction.Select(&channelMembers, "SELECT * from ChannelMembers WHERE (ChannelId, UserId) > (?, ?) ORDER BY ChannelId, UserId LIMIT 1000", lastChannelId, lastUserId); err != nil {
query := `
SELECT
ChannelId,
UserId,
Roles,
LastViewedAt,
MsgCount,
MentionCount,
MentionCountRoot,
COALESCE(UrgentMentionCount, 0) AS UrgentMentionCount,
MsgCountRoot,
NotifyProps,
LastUpdateAt,
SchemeUser,
SchemeAdmin,
SchemeGuest
FROM
ChannelMembers
WHERE
(ChannelId, UserId) > (?, ?)
ORDER BY ChannelId, UserId
LIMIT 1000
`
if err = transaction.Select(&channelMembers, query, lastChannelId, lastUserId); err != nil {
finalizeTransactionX(transaction, &err)
return errors.Wrap(err, "failed to find ChannelMembers")
}

Просмотреть файл

@@ -4,6 +4,7 @@
package sqlstore
import (
"context"
"database/sql"
"fmt"
"strings"
@@ -270,7 +271,7 @@ func (s SqlComplianceStore) ComplianceExport(job *model.Compliance, cursor model
return append(channelPosts, directMessagePosts...), cursor, nil
}
func (s SqlComplianceStore) MessageExport(cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
func (s SqlComplianceStore) MessageExport(ctx context.Context, cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
var args []any
args = append(args, model.ChannelTypeDirect, model.ChannelTypeGroup, cursor.LastPostUpdateAt, cursor.LastPostUpdateAt, cursor.LastPostId, limit)
query :=
@@ -317,7 +318,7 @@ func (s SqlComplianceStore) MessageExport(cursor model.MessageExportCursor, limi
LIMIT ?`
cposts := []*model.MessageExport{}
if err := s.GetReplicaX().Select(&cposts, query, args...); err != nil {
if err := s.GetReplicaX().SelectCtx(ctx, &cposts, query, args...); err != nil {
return nil, cursor, errors.Wrap(err, "unable to export messages")
}
if len(cposts) > 0 {

Просмотреть файл

@@ -224,8 +224,12 @@ func (w *sqlxDBWrapper) QueryX(query string, args ...any) (*sqlx.Rows, error) {
}
func (w *sqlxDBWrapper) Select(dest any, query string, args ...any) error {
return w.SelectCtx(context.Background(), dest, query, args...)
}
func (w *sqlxDBWrapper) SelectCtx(ctx context.Context, dest any, query string, args ...any) error {
query = w.DB.Rebind(query)
ctx, cancel := context.WithTimeout(context.Background(), w.queryTimeout)
ctx, cancel := context.WithTimeout(ctx, w.queryTimeout)
defer cancel()
if w.trace {

Просмотреть файл

@@ -9,7 +9,7 @@ import (
dbsql "database/sql"
"fmt"
"log"
"path/filepath"
"path"
"strconv"
"strings"
"sync"
@@ -228,7 +228,9 @@ func New(settings model.SqlSettings, metrics einterfaces.MetricsInterface) *SqlS
return store
}
func setupConnection(connType string, dataSource string, settings *model.SqlSettings) *dbsql.DB {
// SetupConnection sets up the connection to the database and pings it to make sure it's alive.
// It also applies any database configuration settings that are required.
func SetupConnection(connType string, dataSource string, settings *model.SqlSettings) *dbsql.DB {
db, err := dbsql.Open(*settings.DriverName, dataSource)
if err != nil {
mlog.Fatal("Failed to open SQL connection to err.", mlog.Err(err))
@@ -294,7 +296,7 @@ func (ss *SqlStore) initConnection() {
}
}
handle := setupConnection("master", dataSource, ss.settings)
handle := SetupConnection("master", dataSource, ss.settings)
ss.masterX = newSqlxDBWrapper(sqlx.NewDb(handle, ss.DriverName()),
time.Duration(*ss.settings.QueryTimeout)*time.Second,
*ss.settings.Trace)
@@ -305,7 +307,7 @@ func (ss *SqlStore) initConnection() {
if len(ss.settings.DataSourceReplicas) > 0 {
ss.ReplicaXs = make([]*sqlxDBWrapper, len(ss.settings.DataSourceReplicas))
for i, replica := range ss.settings.DataSourceReplicas {
handle := setupConnection(fmt.Sprintf("replica-%v", i), replica, ss.settings)
handle := SetupConnection(fmt.Sprintf("replica-%v", i), replica, ss.settings)
ss.ReplicaXs[i] = newSqlxDBWrapper(sqlx.NewDb(handle, ss.DriverName()),
time.Duration(*ss.settings.QueryTimeout)*time.Second,
*ss.settings.Trace)
@@ -318,7 +320,7 @@ func (ss *SqlStore) initConnection() {
if len(ss.settings.DataSourceSearchReplicas) > 0 {
ss.searchReplicaXs = make([]*sqlxDBWrapper, len(ss.settings.DataSourceSearchReplicas))
for i, replica := range ss.settings.DataSourceSearchReplicas {
handle := setupConnection(fmt.Sprintf("search-replica-%v", i), replica, ss.settings)
handle := SetupConnection(fmt.Sprintf("search-replica-%v", i), replica, ss.settings)
ss.searchReplicaXs[i] = newSqlxDBWrapper(sqlx.NewDb(handle, ss.DriverName()),
time.Duration(*ss.settings.QueryTimeout)*time.Second,
*ss.settings.Trace)
@@ -334,7 +336,7 @@ func (ss *SqlStore) initConnection() {
if src.DataSource == nil {
continue
}
ss.replicaLagHandles[i] = setupConnection(fmt.Sprintf(replicaLagPrefix+"-%d", i), *src.DataSource, ss.settings)
ss.replicaLagHandles[i] = SetupConnection(fmt.Sprintf(replicaLagPrefix+"-%d", i), *src.DataSource, ss.settings)
}
}
}
@@ -1047,7 +1049,7 @@ func (ss *SqlStore) hasLicense() bool {
func (ss *SqlStore) migrate(direction migrationDirection) error {
assets := db.Assets()
assetsList, err := assets.ReadDir(filepath.Join("migrations", ss.DriverName()))
assetsList, err := assets.ReadDir(path.Join("migrations", ss.DriverName()))
if err != nil {
return err
}
@@ -1060,7 +1062,7 @@ func (ss *SqlStore) migrate(direction migrationDirection) error {
src, err := mbindata.WithInstance(&mbindata.AssetSource{
Names: assetNamesForDriver,
AssetFunc: func(name string) ([]byte, error) {
return assets.ReadFile(filepath.Join("migrations", ss.DriverName(), name))
return assets.ReadFile(path.Join("migrations", ss.DriverName(), name))
},
})
if err != nil {
@@ -1079,7 +1081,7 @@ func (ss *SqlStore) migrate(direction migrationDirection) error {
if err != nil {
return err
}
db := setupConnection("master", dataSource, ss.settings)
db := SetupConnection("master", dataSource, ss.settings)
driver, err = ms.WithInstance(db)
defer db.Close()
case model.DatabaseDriverPostgres:

Просмотреть файл

@@ -550,7 +550,7 @@ type ComplianceStore interface {
Get(id string) (*model.Compliance, error)
GetAll(offset, limit int) (model.Compliances, error)
ComplianceExport(compliance *model.Compliance, cursor model.ComplianceExportCursor, limit int) ([]*model.CompliancePost, model.ComplianceExportCursor, error)
MessageExport(cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error)
MessageExport(ctx context.Context, cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error)
}
type OAuthStore interface {

Просмотреть файл

@@ -4,6 +4,7 @@
package storetest
import (
"context"
"encoding/json"
"testing"
"time"
@@ -399,7 +400,7 @@ func testMessageExportPublicChannel(t *testing.T, ss store.Store) {
// get the starting number of message export entries
startTime := model.GetMillis()
messages, _, err := ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
messages, _, err := ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
require.NoError(t, err)
assert.Equal(t, 0, len(messages))
@@ -469,7 +470,7 @@ func testMessageExportPublicChannel(t *testing.T, ss store.Store) {
// fetch the message exports for both posts that user1 sent
messageExportMap := map[string]model.MessageExport{}
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
require.NoError(t, err)
assert.Equal(t, 2, len(messages))
@@ -503,7 +504,7 @@ func testMessageExportPrivateChannel(t *testing.T, ss store.Store) {
// get the starting number of message export entries
startTime := model.GetMillis()
messages, _, err := ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
messages, _, err := ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
require.NoError(t, err)
assert.Equal(t, 0, len(messages))
@@ -573,7 +574,7 @@ func testMessageExportPrivateChannel(t *testing.T, ss store.Store) {
// fetch the message exports for both posts that user1 sent
messageExportMap := map[string]model.MessageExport{}
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
require.NoError(t, err)
assert.Equal(t, 2, len(messages))
@@ -609,7 +610,7 @@ func testMessageExportDirectMessageChannel(t *testing.T, ss store.Store) {
// get the starting number of message export entries
startTime := model.GetMillis()
messages, _, err := ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
messages, _, err := ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
require.NoError(t, err)
assert.Equal(t, 0, len(messages))
@@ -664,7 +665,7 @@ func testMessageExportDirectMessageChannel(t *testing.T, ss store.Store) {
// fetch the message export for the post that user1 sent
messageExportMap := map[string]model.MessageExport{}
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
require.NoError(t, err)
assert.Equal(t, 1, len(messages))
@@ -690,7 +691,7 @@ func testMessageExportGroupMessageChannel(t *testing.T, ss store.Store) {
// get the starting number of message export entries
startTime := model.GetMillis()
messages, _, err := ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
messages, _, err := ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
require.NoError(t, err)
assert.Equal(t, 0, len(messages))
@@ -762,7 +763,7 @@ func testMessageExportGroupMessageChannel(t *testing.T, ss store.Store) {
// fetch the message export for the post that user1 sent
messageExportMap := map[string]model.MessageExport{}
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 10}, 10)
require.NoError(t, err)
assert.Equal(t, 1, len(messages))
@@ -787,7 +788,7 @@ func testEditExportMessage(t *testing.T, ss store.Store) {
defer cleanupStoreState(t, ss)
// get the starting number of message export entries
startTime := model.GetMillis()
messages, _, err := ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
messages, _, err := ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 0, len(messages))
@@ -842,7 +843,7 @@ func testEditExportMessage(t *testing.T, ss store.Store) {
require.NoError(t, err)
// fetch the message exports from the start
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 2, len(messages))
@@ -879,7 +880,7 @@ func testEditAfterExportMessage(t *testing.T, ss store.Store) {
defer cleanupStoreState(t, ss)
// get the starting number of message export entries
startTime := model.GetMillis()
messages, _, err := ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
messages, _, err := ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 0, len(messages))
@@ -927,7 +928,7 @@ func testEditAfterExportMessage(t *testing.T, ss store.Store) {
require.NoError(t, err)
// fetch the message exports from the start
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 1, len(messages))
@@ -953,7 +954,7 @@ func testEditAfterExportMessage(t *testing.T, ss store.Store) {
require.NoError(t, err)
// fetch the message exports after edit
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: postEditTime - 1}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: postEditTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 2, len(messages))
@@ -990,7 +991,7 @@ func testDeleteExportMessage(t *testing.T, ss store.Store) {
defer cleanupStoreState(t, ss)
// get the starting number of message export entries
startTime := model.GetMillis()
messages, _, err := ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
messages, _, err := ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 0, len(messages))
@@ -1043,7 +1044,7 @@ func testDeleteExportMessage(t *testing.T, ss store.Store) {
require.NoError(t, err)
// fetch the message exports from the start
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 1, len(messages))
@@ -1075,7 +1076,7 @@ func testDeleteAfterExportMessage(t *testing.T, ss store.Store) {
defer cleanupStoreState(t, ss)
// get the starting number of message export entries
startTime := model.GetMillis()
messages, _, err := ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
messages, _, err := ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 0, len(messages))
@@ -1123,7 +1124,7 @@ func testDeleteAfterExportMessage(t *testing.T, ss store.Store) {
require.NoError(t, err)
// fetch the message exports from the start
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: startTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 1, len(messages))
@@ -1146,7 +1147,7 @@ func testDeleteAfterExportMessage(t *testing.T, ss store.Store) {
require.NoError(t, err)
// fetch the message exports after delete
messages, _, err = ss.Compliance().MessageExport(model.MessageExportCursor{LastPostUpdateAt: postDeleteTime - 1}, 10)
messages, _, err = ss.Compliance().MessageExport(context.Background(), model.MessageExportCursor{LastPostUpdateAt: postDeleteTime - 1}, 10)
require.NoError(t, err)
assert.Equal(t, 1, len(messages))

Просмотреть файл

@@ -5,6 +5,8 @@
package mocks
import (
context "context"
model "github.com/mattermost/mattermost-server/v6/model"
mock "github.com/stretchr/testify/mock"
)
@@ -90,13 +92,13 @@ func (_m *ComplianceStore) GetAll(offset int, limit int) (model.Compliances, err
return r0, r1
}
// MessageExport provides a mock function with given fields: cursor, limit
func (_m *ComplianceStore) MessageExport(cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
ret := _m.Called(cursor, limit)
// MessageExport provides a mock function with given fields: ctx, cursor, limit
func (_m *ComplianceStore) MessageExport(ctx context.Context, cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
ret := _m.Called(ctx, cursor, limit)
var r0 []*model.MessageExport
if rf, ok := ret.Get(0).(func(model.MessageExportCursor, int) []*model.MessageExport); ok {
r0 = rf(cursor, limit)
if rf, ok := ret.Get(0).(func(context.Context, model.MessageExportCursor, int) []*model.MessageExport); ok {
r0 = rf(ctx, cursor, limit)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*model.MessageExport)
@@ -104,15 +106,15 @@ func (_m *ComplianceStore) MessageExport(cursor model.MessageExportCursor, limit
}
var r1 model.MessageExportCursor
if rf, ok := ret.Get(1).(func(model.MessageExportCursor, int) model.MessageExportCursor); ok {
r1 = rf(cursor, limit)
if rf, ok := ret.Get(1).(func(context.Context, model.MessageExportCursor, int) model.MessageExportCursor); ok {
r1 = rf(ctx, cursor, limit)
} else {
r1 = ret.Get(1).(model.MessageExportCursor)
}
var r2 error
if rf, ok := ret.Get(2).(func(model.MessageExportCursor, int) error); ok {
r2 = rf(cursor, limit)
if rf, ok := ret.Get(2).(func(context.Context, model.MessageExportCursor, int) error); ok {
r2 = rf(ctx, cursor, limit)
} else {
r2 = ret.Error(2)
}

Просмотреть файл

@@ -2942,10 +2942,10 @@ func (s *TimerLayerComplianceStore) GetAll(offset int, limit int) (model.Complia
return result, err
}
func (s *TimerLayerComplianceStore) MessageExport(cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
func (s *TimerLayerComplianceStore) MessageExport(ctx context.Context, cursor model.MessageExportCursor, limit int) ([]*model.MessageExport, model.MessageExportCursor, error) {
start := time.Now()
result, resultVar1, err := s.ComplianceStore.MessageExport(cursor, limit)
result, resultVar1, err := s.ComplianceStore.MessageExport(ctx, cursor, limit)
elapsed := float64(time.Since(start)) / float64(time.Second)
if s.Root.Metrics != nil {