[MM-57693] Add schema dump to Support Packet (#31162)

Co-authored-by: Claude <noreply@anthropic.com>
Этот коммит содержится в:
Ben Schumacher
2025-06-19 11:33:55 +02:00
коммит произвёл GitHub
родитель 824d3b8259
Коммит 04a60b6609
9 изменённых файлов: 632 добавлений и 4 удалений

Просмотреть файл

@@ -26,6 +26,7 @@ func (a *App) GenerateSupportPacket(rctx request.CTX, options *model.SupportPack
"jobs": a.getSupportPacketJobList,
"permissions": a.getSupportPacketPermissionsInfo,
"plugins": a.getPluginsFile,
"schema": a.getSupportPacketDatabaseSchema,
}
var (
@@ -375,3 +376,24 @@ func (a *App) getSupportPacketMetadata(_ request.CTX) (*model.FileData, error) {
}
return fileData, nil
}
func (a *App) getSupportPacketDatabaseSchema(rctx request.CTX) (*model.FileData, error) {
if *a.Config().SqlSettings.DriverName != model.DatabaseDriverPostgres {
return nil, nil
}
schemaInfo, err := a.Srv().Store().GetSchemaDefinition()
if err != nil {
return nil, errors.Wrap(err, "failed to get schema definition")
}
schemaDump, err := yaml.Marshal(schemaInfo)
if err != nil {
return nil, errors.Wrap(err, "failed to marshal schema into YAML")
}
return &model.FileData{
Filename: "database_schema.yaml",
Body: schemaDump,
}, nil
}

Просмотреть файл

@@ -77,6 +77,11 @@ func TestGenerateSupportPacket(t *testing.T) {
"goroutines",
}
// database_schema.yaml is only generated for Postgres
if *th.App.Config().SqlSettings.DriverName == model.DatabaseDriverPostgres {
expectedFileNames = append(expectedFileNames, "database_schema.yaml")
}
expectedFileNamesWithLogs := append(expectedFileNames, []string{
"mattermost.log",
"notifications.log",
@@ -143,6 +148,9 @@ func TestGenerateSupportPacket(t *testing.T) {
mockStore.On("TotalMasterDbConnections").Return(30)
mockStore.On("TotalReadDbConnections").Return(20)
mockStore.On("TotalSearchDbConnections").Return(10)
mockStore.On("GetSchemaDefinition").Return(&model.SupportPacketDatabaseSchema{
Tables: []model.DatabaseTable{},
}, nil)
oldStore := th.App.Srv().Store()
t.Cleanup(func() {
@@ -687,3 +695,98 @@ func TestGetSupportPacketMetadata(t *testing.T) {
assert.NotEmpty(t, metadate.GeneratedAt)
})
}
func TestGetSupportPacketDatabaseSchema(t *testing.T) {
th := Setup(t)
defer th.TearDown()
// Mock store for testing
mockStore := &smocks.Store{}
originalStore := th.App.Srv().Store()
th.App.Srv().SetStore(mockStore)
defer th.App.Srv().SetStore(originalStore)
// Set up mock return for schema definition
mockStore.On("GetSchemaDefinition").Return(&model.SupportPacketDatabaseSchema{
Tables: []model.DatabaseTable{
{
Name: "users",
Columns: []model.DatabaseColumn{
{Name: "id", DataType: "varchar", IsNullable: false},
{Name: "username", DataType: "varchar", IsNullable: false},
{Name: "email", DataType: "varchar", IsNullable: false},
},
},
{
Name: "channels",
Columns: []model.DatabaseColumn{
{Name: "id", DataType: "varchar", IsNullable: false},
{Name: "name", DataType: "varchar", IsNullable: false},
},
},
{
Name: "teams",
Columns: []model.DatabaseColumn{
{Name: "id", DataType: "varchar", IsNullable: false},
{Name: "name", DataType: "varchar", IsNullable: false},
},
},
{
Name: "posts",
Columns: []model.DatabaseColumn{
{Name: "id", DataType: "varchar", IsNullable: false},
{Name: "message", DataType: "text", IsNullable: false},
},
},
},
}, nil)
// Test with Postgres
oldDriverName := *th.App.Config().SqlSettings.DriverName
*th.App.Config().SqlSettings.DriverName = model.DatabaseDriverPostgres
defer func() {
*th.App.Config().SqlSettings.DriverName = oldDriverName
}()
fileData, err := th.App.getSupportPacketDatabaseSchema(th.Context)
require.NoError(t, err)
require.NotNil(t, fileData)
assert.Equal(t, "database_schema.yaml", fileData.Filename)
assert.Positive(t, len(fileData.Body))
var schema model.SupportPacketDatabaseSchema
err = yaml.Unmarshal(fileData.Body, &schema)
require.NoError(t, err)
// Verify schema structure
assert.NotEmpty(t, schema.Tables)
// Verify that common tables are present
tableNames := make([]string, 0, len(schema.Tables))
for _, table := range schema.Tables {
tableNames = append(tableNames, table.Name)
}
// Verify some core tables
expectedTables := []string{"users", "channels", "teams", "posts"}
for _, expected := range expectedTables {
assert.Contains(t, tableNames, expected)
}
// Verify table structure
for _, table := range schema.Tables {
if table.Name == "users" {
// Check user table has key columns
columnNames := make([]string, 0, len(table.Columns))
for _, column := range table.Columns {
columnNames = append(columnNames, column.Name)
}
expectedColumns := []string{"id", "username", "email"}
for _, expected := range expectedColumns {
assert.Contains(t, columnNames, expected)
}
break
}
}
}

308
server/channels/store/sqlstore/schema_dump.go Обычный файл
Просмотреть файл

@@ -0,0 +1,308 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"database/sql"
"strings"
"github.com/hashicorp/go-multierror"
sq "github.com/mattermost/squirrel"
"github.com/pkg/errors"
"github.com/mattermost/mattermost/server/public/model"
)
// GetSchemaDefinition dumps the database schema.
// Only Postgres is supported.
func (ss *SqlStore) GetSchemaDefinition() (*model.SupportPacketDatabaseSchema, error) {
if ss.DriverName() != model.DatabaseDriverPostgres {
return nil, errors.New("schema dump is only supported for Postgres")
}
var schemaInfo model.SupportPacketDatabaseSchema
var rErr *multierror.Error
// Get the database collation
dbCollation, err := ss.getDatabaseCollation()
if err != nil {
rErr = multierror.Append(rErr, err)
} else {
schemaInfo.DatabaseCollation = dbCollation
}
// Get the database encoding
dbEncoding, err := ss.getDatabaseEncoding()
if err != nil {
rErr = multierror.Append(rErr, err)
} else {
schemaInfo.DatabaseEncoding = dbEncoding
}
// Get table options
tableOptions, err := ss.getTableOptions()
if err != nil {
rErr = multierror.Append(rErr, err)
}
// Get table schema information
tablesMap, tableCollations, err := ss.getTableSchemaInformation()
if err != nil {
rErr = multierror.Append(rErr, err)
}
// Get table indexes
tableIndexes, err := ss.getTableIndexes()
if err != nil {
rErr = multierror.Append(rErr, err)
}
// Process and combine table metadata
for _, table := range tablesMap {
// Add table collation if it exists
if collation, ok := tableCollations[table.Name]; ok {
table.Collation = collation
}
// Add table options if they exist
if options, ok := tableOptions[table.Name]; ok && len(options) > 0 {
table.Options = options
}
// Add table indexes if they exist
if indexes, ok := tableIndexes[table.Name]; ok {
table.Indexes = indexes
}
schemaInfo.Tables = append(schemaInfo.Tables, *table)
}
return &schemaInfo, rErr.ErrorOrNil()
}
// getDatabaseCollation retrieves the database collation for PostgreSQL
func (ss *SqlStore) getDatabaseCollation() (string, error) {
var dbCollation sql.NullString
collationQuery := sq.Select("datcollate").
From("pg_database").
Where(sq.Expr("datname = current_database()"))
sqlString, args, err := collationQuery.PlaceholderFormat(sq.Dollar).ToSql()
if err != nil {
return "", errors.Wrap(err, "failed to build database collation query")
}
err = ss.GetMaster().DB.QueryRow(sqlString, args...).Scan(&dbCollation)
if err != nil {
return "", errors.Wrap(err, "failed to get database collation")
}
if !dbCollation.Valid {
return "", nil
}
return dbCollation.String, nil
}
// getDatabaseEncoding retrieves the database encoding for PostgreSQL
func (ss *SqlStore) getDatabaseEncoding() (string, error) {
var dbEncoding sql.NullString
encodingQuery := sq.Select("pg_encoding_to_char(encoding)").
From("pg_database").
Where(sq.Expr("datname = current_database()"))
sqlString, args, err := encodingQuery.PlaceholderFormat(sq.Dollar).ToSql()
if err != nil {
return "", errors.Wrap(err, "failed to build database encoding query")
}
err = ss.GetMaster().DB.QueryRow(sqlString, args...).Scan(&dbEncoding)
if err != nil {
return "", errors.Wrap(err, "failed to get database encoding")
}
if !dbEncoding.Valid {
return "", nil
}
return dbEncoding.String, nil
}
// getTableOptions retrieves table-specific options from PostgreSQL system catalogs
func (ss *SqlStore) getTableOptions() (map[string]map[string]string, error) {
tableOptions := make(map[string]map[string]string)
optionsQuery := sq.Select("c.relname as table_name", "unnest(c.reloptions) as option_value").
From("pg_class c").
Join("pg_namespace n ON n.oid = c.relnamespace").
Where(sq.And{
sq.Expr("n.nspname = current_schema()"),
sq.Eq{"c.relkind": "r"},
sq.NotEq{"c.reloptions": nil},
})
optionsSql, optionsArgs, err := optionsQuery.PlaceholderFormat(sq.Dollar).ToSql()
if err != nil {
return nil, errors.Wrap(err, "failed to build table options query")
}
optionsRows, err := ss.GetMaster().DB.Query(optionsSql, optionsArgs...)
if err != nil {
return nil, errors.Wrap(err, "failed to query table options")
}
defer optionsRows.Close()
// Process table options
var rErr *multierror.Error
for optionsRows.Next() {
var tableName string
var optionValue string
err = optionsRows.Scan(&tableName, &optionValue)
if err != nil {
rErr = multierror.Append(rErr, errors.Wrap(err, "failed to scan database schema row"))
continue
}
// Parse option in format key=value
parts := strings.SplitN(optionValue, "=", 2)
if len(parts) != 2 {
continue
}
key := parts[0]
value := parts[1]
// Initialize the options map for this table if needed
if _, ok := tableOptions[tableName]; !ok {
tableOptions[tableName] = make(map[string]string)
}
// Add option to the table
tableOptions[tableName][key] = value
}
return tableOptions, rErr.ErrorOrNil()
}
// getTableSchemaInformation retrieves table and column information from information_schema
func (ss *SqlStore) getTableSchemaInformation() (map[string]*model.DatabaseTable, map[string]string, error) {
tablesMap := make(map[string]*model.DatabaseTable)
tableCollations := make(map[string]string)
schemaQuery := sq.Select(
"t.table_name",
"c.column_name",
"c.data_type",
"c.character_maximum_length",
"c.is_nullable",
"c.collation_name",
).
From("information_schema.tables t").
LeftJoin("information_schema.columns c ON t.table_name = c.table_name AND t.table_schema = c.table_schema").
Where(sq.Expr("t.table_schema = current_schema()")).
OrderBy("t.table_name", "c.ordinal_position")
schemaSql, schemaArgs, err := schemaQuery.PlaceholderFormat(sq.Dollar).ToSql()
if err != nil {
return nil, nil, errors.Wrap(err, "failed to build schema information query")
}
rows, err := ss.GetMaster().DB.Query(schemaSql, schemaArgs...)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to query schema information")
}
defer rows.Close()
var rErr *multierror.Error
for rows.Next() {
var tableName, columnName, dataType, isNullable string
var characterMaxLength sql.NullInt64
var collationName sql.NullString
err = rows.Scan(&tableName, &columnName, &dataType, &characterMaxLength, &isNullable, &collationName)
if err != nil {
rErr = multierror.Append(rErr, errors.Wrap(err, "failed to scan database schema row"))
continue
}
// Track collation names for tables.
// Only the first non-null collation encountered per table is stored in tableCollations.
if collationName.Valid && collationName.String != "" {
if _, ok := tableCollations[tableName]; !ok {
tableCollations[tableName] = collationName.String
}
}
// Initialize table in map if it doesn't exist
if _, ok := tablesMap[tableName]; !ok {
tablesMap[tableName] = &model.DatabaseTable{
Name: tableName,
Columns: []model.DatabaseColumn{},
}
}
// Add column to table
if columnName != "" {
maxLength := int64(0)
if characterMaxLength.Valid {
maxLength = characterMaxLength.Int64
}
tablesMap[tableName].Columns = append(tablesMap[tableName].Columns, model.DatabaseColumn{
Name: columnName,
DataType: dataType,
MaxLength: maxLength,
IsNullable: isNullable == "YES",
})
}
}
return tablesMap, tableCollations, rErr.ErrorOrNil()
}
// getTableIndexes retrieves index information for all tables
func (ss *SqlStore) getTableIndexes() (map[string][]model.DatabaseIndex, error) {
tableIndexes := make(map[string][]model.DatabaseIndex)
// Query pg_indexes for index information
indexQuery := sq.Select(
"tablename",
"indexname",
"indexdef",
).
From("pg_indexes").
Where(sq.Expr("schemaname = current_schema()"))
indexSql, indexArgs, err := indexQuery.PlaceholderFormat(sq.Dollar).ToSql()
if err != nil {
return nil, errors.Wrap(err, "failed to build index query")
}
rows, err := ss.GetMaster().DB.Query(indexSql, indexArgs...)
if err != nil {
return nil, errors.Wrap(err, "failed to query index information")
}
defer rows.Close()
var rErr *multierror.Error
for rows.Next() {
var tableName, indexName, indexDef string
err = rows.Scan(&tableName, &indexName, &indexDef)
if err != nil {
rErr = multierror.Append(rErr, errors.Wrap(err, "failed to scan index row"))
continue
}
index := model.DatabaseIndex{
Name: indexName,
Definition: indexDef,
}
tableIndexes[tableName] = append(tableIndexes[tableName], index)
}
return tableIndexes, rErr.ErrorOrNil()
}

Просмотреть файл

@@ -0,0 +1,128 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package sqlstore
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/store"
)
func TestGetSchemaDefinition(t *testing.T) {
StoreTest(t, func(t *testing.T, rctx request.CTX, ss store.Store) {
t.Run("MySQL", func(t *testing.T) {
if ss.(*SqlStore).DriverName() != model.DatabaseDriverMysql {
t.Skip("Skipping test as database is not MySQL")
}
// Schema dump is only supported for Postgres
schemaInfo, err := ss.GetSchemaDefinition()
require.Error(t, err)
require.Nil(t, schemaInfo)
assert.Contains(t, err.Error(), "only supported for Postgres")
})
t.Run("PostgreSQL", func(t *testing.T) {
if ss.(*SqlStore).DriverName() != model.DatabaseDriverPostgres {
t.Skip("Skipping test as database is not PostgreSQL")
}
schemaInfo, err := ss.GetSchemaDefinition()
require.NoError(t, err)
require.NotNil(t, schemaInfo)
// Verify database encoding is captured
assert.Equal(t, "UTF8", schemaInfo.DatabaseEncoding, "Database encoding should be captured")
// Verify schema structure
assert.NotEmpty(t, schemaInfo.Tables)
// Verify that columns are not duplicated
for _, table := range schemaInfo.Tables {
columnNames := make(map[string]bool)
for _, column := range table.Columns {
// Assert that this column name hasn't been seen before
assert.False(t, columnNames[column.Name], "Column %s in table %s is duplicated", column.Name, table.Name)
columnNames[column.Name] = true
}
}
// Verify that common tables are present
tableNames := make([]string, 0, len(schemaInfo.Tables))
for _, table := range schemaInfo.Tables {
tableNames = append(tableNames, table.Name)
}
// Verify some core tables
expectedTables := []string{"users", "channels", "teams", "posts"}
assert.Subset(t, tableNames, expectedTables)
// Verify table structure
for _, table := range schemaInfo.Tables {
if table.Name == "users" {
// Check user table has key columns
columnNames := make([]string, 0, len(table.Columns))
for _, column := range table.Columns {
columnNames = append(columnNames, column.Name)
}
expectedColumns := []string{"id", "username", "email"}
assert.Subset(t, columnNames, expectedColumns)
break
}
}
// Verify indexes are present for tables
for _, table := range schemaInfo.Tables {
if table.Name == "channels" {
// Check that indexes are present
assert.NotEmpty(t, table.Indexes, "channels table should have indexes")
assert.Equal(t, 11, len(table.Indexes), "channels table should have 11 indexes")
// Expected index definitions
expectedIndexDefs := map[string]string{
"idx_channels_delete_at": "CREATE INDEX idx_channels_delete_at ON public.channels USING btree (deleteat)",
"idx_channels_create_at": "CREATE INDEX idx_channels_create_at ON public.channels USING btree (createat)",
"channels_pkey": "CREATE UNIQUE INDEX channels_pkey ON public.channels USING btree (id)",
"channels_name_teamid_key": "CREATE UNIQUE INDEX channels_name_teamid_key ON public.channels USING btree (name, teamid)",
"idx_channels_displayname_lower": "CREATE INDEX idx_channels_displayname_lower ON public.channels USING btree (lower((displayname)::text))",
"idx_channels_name_lower": "CREATE INDEX idx_channels_name_lower ON public.channels USING btree (lower((name)::text))",
"idx_channels_update_at": "CREATE INDEX idx_channels_update_at ON public.channels USING btree (updateat)",
"idx_channel_search_txt": "CREATE INDEX idx_channel_search_txt ON public.channels USING gin (to_tsvector('english'::regconfig, (((((name)::text || ' '::text) || (displayname)::text) || ' '::text) || (purpose)::text)))",
"idx_channels_scheme_id": "CREATE INDEX idx_channels_scheme_id ON public.channels USING btree (schemeid)",
"idx_channels_team_id_display_name": "CREATE INDEX idx_channels_team_id_display_name ON public.channels USING btree (teamid, displayname)",
"idx_channels_team_id_type": "CREATE INDEX idx_channels_team_id_type ON public.channels USING btree (teamid, type)",
}
// Verify all expected indexes are present with correct definitions
foundIndexes := make(map[string]bool)
for _, index := range table.Indexes {
foundIndexes[index.Name] = true
// Verify definition is not empty
assert.NotEmpty(t, index.Definition, "Index %s should have a definition", index.Name)
// Check if this is an expected index and verify definition
expectedDef, ok := expectedIndexDefs[index.Name]
require.Truef(t, ok, "Unexpected definition found: %s", index.Name)
assert.Equal(t, expectedDef, index.Definition, "Index %s has incorrect definition", index.Name)
}
// Verify all expected indexes were found
for expectedName := range expectedIndexDefs {
assert.True(t, foundIndexes[expectedName], "Expected index %s not found", expectedName)
}
break
}
}
})
})
}

Просмотреть файл

@@ -6,7 +6,6 @@ package sqlstore
import (
"context"
"database/sql"
dbsql "database/sql"
"fmt"
"path"
"strconv"
@@ -134,7 +133,7 @@ type SqlStore struct {
searchReplicaXs []*atomic.Pointer[sqlxDBWrapper]
replicaLagHandles []*dbsql.DB
replicaLagHandles []*sql.DB
stores SqlStoreStores
settings *model.SqlSettings
lockedToMaster bool
@@ -353,7 +352,7 @@ func (ss *SqlStore) initConnection() error {
}
if len(ss.settings.ReplicaLagSettings) > 0 {
ss.replicaLagHandles = make([]*dbsql.DB, 0, len(ss.settings.ReplicaLagSettings))
ss.replicaLagHandles = make([]*sql.DB, 0, len(ss.settings.ReplicaLagSettings))
for i, src := range ss.settings.ReplicaLagSettings {
if src.DataSource == nil {
continue
@@ -535,7 +534,7 @@ func (ss *SqlStore) monitorReplicas() {
}
}
func (ss *SqlStore) setDB(replica *atomic.Pointer[sqlxDBWrapper], handle *dbsql.DB, name string) {
func (ss *SqlStore) setDB(replica *atomic.Pointer[sqlxDBWrapper], handle *sql.DB, name string) {
replica.Store(newSqlxDBWrapper(sqlx.NewDb(handle, ss.DriverName()),
time.Duration(*ss.settings.QueryTimeout)*time.Second,
*ss.settings.Trace))

Просмотреть файл

@@ -97,6 +97,7 @@ type Store interface {
PropertyValue() PropertyValueStore
AccessControlPolicy() AccessControlPolicyStore
Attributes() AttributesStore
GetSchemaDefinition() (*model.SupportPacketDatabaseSchema, error)
}
type RetentionPolicyStore interface {

Просмотреть файл

@@ -528,6 +528,36 @@ func (_m *Store) GetLocalSchemaVersion() (int, error) {
return r0, r1
}
// GetSchemaDefinition provides a mock function with no fields
func (_m *Store) GetSchemaDefinition() (*model.SupportPacketDatabaseSchema, error) {
ret := _m.Called()
if len(ret) == 0 {
panic("no return value specified for GetSchemaDefinition")
}
var r0 *model.SupportPacketDatabaseSchema
var r1 error
if rf, ok := ret.Get(0).(func() (*model.SupportPacketDatabaseSchema, error)); ok {
return rf()
}
if rf, ok := ret.Get(0).(func() *model.SupportPacketDatabaseSchema); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*model.SupportPacketDatabaseSchema)
}
}
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Group provides a mock function with no fields
func (_m *Store) Group() store.GroupStore {
ret := _m.Called()

Просмотреть файл

@@ -163,6 +163,12 @@ func (s *Store) Attributes() store.AttributesStore {
return &s.AttributesStore
}
func (s *Store) GetSchemaDefinition() (*model.SupportPacketDatabaseSchema, error) {
return &model.SupportPacketDatabaseSchema{
Tables: []model.DatabaseTable{},
}, nil
}
func (s *Store) AssertExpectations(t mock.TestingT) bool {
return mock.AssertExpectationsForObjects(t,
&s.TeamStore,

Просмотреть файл

@@ -123,6 +123,37 @@ type SupportPacketPluginList struct {
Disabled []Manifest `json:"disabled"`
}
// SupportPacketDatabaseSchema contains the database schema information.
// It is included in the Support Packet.
type SupportPacketDatabaseSchema struct {
DatabaseCollation string `yaml:"database_collation,omitempty"`
DatabaseEncoding string `yaml:"database_encoding,omitempty"`
Tables []DatabaseTable `yaml:"tables"`
}
// DatabaseTable represents a table in the database schema.
type DatabaseTable struct {
Name string `yaml:"name"`
Collation string `yaml:"collation,omitempty"`
Options map[string]string `yaml:"options,omitempty"`
Columns []DatabaseColumn `yaml:"columns"`
Indexes []DatabaseIndex `yaml:"indexes,omitempty"`
}
// DatabaseColumn represents a column in a database table.
type DatabaseColumn struct {
Name string `yaml:"name"`
DataType string `yaml:"data_type"`
MaxLength int64 `yaml:"max_length,omitempty"`
IsNullable bool `yaml:"is_nullable"`
}
// DatabaseIndex represents an index in a database table.
type DatabaseIndex struct {
Name string `yaml:"name"`
Definition string `yaml:"definition"`
}
type FileData struct {
Filename string
Body []byte