diff --git a/go.mod b/go.mod index 16ed62d11c..2d85fb41a8 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( github.com/lib/pq v1.10.2 github.com/mailru/easyjson v0.7.7 // indirect github.com/mattermost/go-i18n v1.11.0 - github.com/mattermost/gorp v1.6.2-0.20210419141818-0904a6a388d3 + github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f github.com/mattermost/gosaml2 v0.3.3 github.com/mattermost/gziphandler v0.0.1 github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d diff --git a/go.sum b/go.sum index a3d88b27b2..5f44b0ffc3 100644 --- a/go.sum +++ b/go.sum @@ -643,8 +643,8 @@ github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQ github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattermost/go-i18n v1.11.0 h1:1hLKqn/ZvhZ80OekjVPGYcCrBfMz+YxNNgqS+beL7zE= github.com/mattermost/go-i18n v1.11.0/go.mod h1:RyS7FDNQlzF1PsjbJWHRI35exqaKGSO9qD4iv8QjE34= -github.com/mattermost/gorp v1.6.2-0.20210419141818-0904a6a388d3 h1:VW5cRIZheMbcC8w7RtJyt0hFckdcJ1cNpSUOELsQGE8= -github.com/mattermost/gorp v1.6.2-0.20210419141818-0904a6a388d3/go.mod h1:QCQ3U0M9T/BlAdjKFJo0I1oe/YAgbyjNdhU8bpOLafk= +github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f h1:J47te5ubXIEbEaXi/nwStKQ/sLniXACgZ7LhK365YIY= +github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f/go.mod h1:QCQ3U0M9T/BlAdjKFJo0I1oe/YAgbyjNdhU8bpOLafk= github.com/mattermost/gosaml2 v0.3.3 h1:ysWrjp08tpWmo6rV2MQ88Eag/Ttjuj91fZsvibF4/Tg= github.com/mattermost/gosaml2 v0.3.3/go.mod h1:Z429EIOiEi9kbq6yHoApfzlcXpa6dzRDc6pO+Vy2Ksk= github.com/mattermost/gziphandler v0.0.1 h1:uXHcXF5agnQ6bXabvpiwwwZOlCYoa7mKHH0lxns/o8w= diff --git a/store/sqlstore/channel_store.go b/store/sqlstore/channel_store.go index c3f799d7a8..23c58e8fab 100644 --- a/store/sqlstore/channel_store.go +++ b/store/sqlstore/channel_store.go @@ -391,7 +391,7 @@ func newSqlChannelStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface tablem.ColMap("ChannelId").SetMaxSize(26) tablem.ColMap("UserId").SetMaxSize(26) tablem.ColMap("Roles").SetMaxSize(64) - tablem.ColMap("NotifyProps").SetMaxSize(2000) + tablem.ColMap("NotifyProps").SetDataType(sqlStore.jsonDataType()) tablePublicChannels := db.AddTableWithName(publicChannel{}, "PublicChannels").SetKeys(false, "Id") tablePublicChannels.ColMap("Id").SetMaxSize(26) @@ -1898,45 +1898,27 @@ func (s SqlChannelStore) GetMemberCountsByGroup(ctx context.Context, channelID s selectStr := "GroupMembers.GroupId, COUNT(ChannelMembers.UserId) AS ChannelMemberCount" if includeTimezones { - // Length of default timezone (len {"automaticTimezone":"","manualTimezone":"","useAutomaticTimezone":"true"}) - defaultTimezoneLength := `74` - - // Beginning and end of the value for the automatic and manual timezones respectively - autoTimezone := `LOCATE(':', Users.Timezone) + 2` - autoTimezoneEnd := `LOCATE(',', Users.Timezone) - LOCATE(':', Users.Timezone) - 3` - manualTimezone := `LOCATE(',', Users.Timezone) + 19` - manualTimezoneEnd := `LOCATE('useAutomaticTimezone', Users.Timezone) - 22 - LOCATE(',', Users.Timezone)` - - if s.DriverName() == model.DatabaseDriverPostgres { - autoTimezone = `POSITION(':' IN Users.Timezone) + 2` - autoTimezoneEnd = `POSITION(',' IN Users.Timezone) - POSITION(':' IN Users.Timezone) - 3` - manualTimezone = `POSITION(',' IN Users.Timezone) + 19` - manualTimezoneEnd = `POSITION('useAutomaticTimezone' IN Users.Timezone) - 22 - POSITION(',' IN Users.Timezone)` - } - - selectStr = ` - GroupMembers.GroupId, - COUNT(ChannelMembers.UserId) AS ChannelMemberCount, - COUNT(DISTINCT + if s.DriverName() == model.DatabaseDriverMysql { + selectStr += `, + COUNT(DISTINCT ( - CASE WHEN Timezone like '%"useAutomaticTimezone":"true"}' AND LENGTH(Timezone) > ` + defaultTimezoneLength + ` - THEN - SUBSTRING( - Timezone - FROM ` + autoTimezone + ` - FOR ` + autoTimezoneEnd + ` - ) - WHEN Timezone like '%"useAutomaticTimezone":"false"}' AND LENGTH(Timezone) > ` + defaultTimezoneLength + ` - THEN - SUBSTRING( - Timezone - FROM ` + manualTimezone + ` - FOR ` + manualTimezoneEnd + ` - ) + CASE WHEN Timezone->"$.useAutomaticTimezone" = 'true' AND LENGTH(JSON_UNQUOTE(Timezone->"$.automaticTimezone")) > 0 + THEN Timezone->"$.automaticTimezone" + WHEN Timezone->"$.useAutomaticTimezone" = 'false' AND LENGTH(JSON_UNQUOTE(Timezone->"$.manualTimezone")) > 0 + THEN Timezone->"$.manualTimezone" END - ) - ) AS ChannelMemberTimezonesCount - ` + )) AS ChannelMemberTimezonesCount` + } else if s.DriverName() == model.DatabaseDriverPostgres { + selectStr += `, + COUNT(DISTINCT + ( + CASE WHEN Timezone->>'useAutomaticTimezone' = 'true' AND length(Timezone->>'automaticTimezone') > 0 + THEN Timezone->>'automaticTimezone' + WHEN Timezone->>'useAutomaticTimezone' = 'false' AND length(Timezone->>'manualTimezone') > 0 + THEN Timezone->>'manualTimezone' + END + )) AS ChannelMemberTimezonesCount` + } } query := s.getQueryBuilder(). @@ -1954,6 +1936,7 @@ func (s SqlChannelStore) GetMemberCountsByGroup(ctx context.Context, channelID s if err != nil { return nil, errors.Wrap(err, "channel_tosql") } + var data []*model.ChannelMemberCountByGroup if _, err = s.DBFromContext(ctx).Select(&data, queryString, args...); err != nil { return nil, errors.Wrapf(err, "failed to count ChannelMembers with channelId=%s", channelID) diff --git a/store/sqlstore/job_store.go b/store/sqlstore/job_store.go index b55e9caf05..bc26560b37 100644 --- a/store/sqlstore/job_store.go +++ b/store/sqlstore/job_store.go @@ -28,7 +28,7 @@ func newSqlJobStore(sqlStore *SqlStore) store.JobStore { table.ColMap("Id").SetMaxSize(26) table.ColMap("Type").SetMaxSize(32) table.ColMap("Status").SetMaxSize(32) - table.ColMap("Data").SetMaxSize(1024) + table.ColMap("Data").SetDataType(sqlStore.jsonDataType()) } return s diff --git a/store/sqlstore/link_metadata_store.go b/store/sqlstore/link_metadata_store.go index 46e57090e2..792eb04cdb 100644 --- a/store/sqlstore/link_metadata_store.go +++ b/store/sqlstore/link_metadata_store.go @@ -24,7 +24,7 @@ func newSqlLinkMetadataStore(sqlStore *SqlStore) store.LinkMetadataStore { table := db.AddTableWithName(model.LinkMetadata{}, "LinkMetadata").SetKeys(false, "Hash") table.ColMap("URL").SetMaxSize(2048) table.ColMap("Type").SetMaxSize(16) - table.ColMap("Data").SetMaxSize(4096) + table.ColMap("Data").SetDataType(sqlStore.jsonDataType()) } return s diff --git a/store/sqlstore/post_store.go b/store/sqlstore/post_store.go index 8be7397c58..9b76f6164c 100644 --- a/store/sqlstore/post_store.go +++ b/store/sqlstore/post_store.go @@ -146,7 +146,7 @@ func newSqlPostStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) s table.ColMap("Message").SetMaxSize(model.PostMessageMaxBytesV2) table.ColMap("Type").SetMaxSize(26) table.ColMap("Hashtags").SetMaxSize(1000) - table.ColMap("Props").SetMaxSize(8000) + table.ColMap("Props").SetDataType(sqlStore.jsonDataType()) table.ColMap("Filenames").SetMaxSize(model.PostFilenamesMaxRunes) table.ColMap("FileIds").SetMaxSize(300) table.ColMap("RemoteId").SetMaxSize(26) diff --git a/store/sqlstore/session_store.go b/store/sqlstore/session_store.go index dacec0c83e..702130217b 100644 --- a/store/sqlstore/session_store.go +++ b/store/sqlstore/session_store.go @@ -34,7 +34,7 @@ func newSqlSessionStore(sqlStore *SqlStore) store.SessionStore { table.ColMap("UserId").SetMaxSize(26) table.ColMap("DeviceId").SetMaxSize(512) table.ColMap("Roles").SetMaxSize(64) - table.ColMap("Props").SetMaxSize(1000) + table.ColMap("Props").SetDataType(sqlStore.jsonDataType()) } return us diff --git a/store/sqlstore/store.go b/store/sqlstore/store.go index 9516058055..aa334dc5aa 100644 --- a/store/sqlstore/store.go +++ b/store/sqlstore/store.go @@ -911,7 +911,12 @@ func (ss *SqlStore) AlterColumnTypeIfExists(tableName string, columnName string, if ss.DriverName() == model.DatabaseDriverMysql { _, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " MODIFY " + columnName + " " + mySqlColType) } else if ss.DriverName() == model.DatabaseDriverPostgres { - _, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + strings.ToLower(tableName) + " ALTER COLUMN " + strings.ToLower(columnName) + " TYPE " + postgresColType) + query := "ALTER TABLE " + strings.ToLower(tableName) + " ALTER COLUMN " + strings.ToLower(columnName) + " TYPE " + postgresColType + // We need to explicitly cast when moving from text based to jsonb datatypes. + if postgresColType == "jsonb" { + query += " USING " + strings.ToLower(columnName) + "::jsonb" + } + _, err = ss.GetMaster().ExecNoTimeout(query) } if err != nil { @@ -1742,3 +1747,10 @@ func versionString(v int, driver string) string { } return "" } + +func (ss *SqlStore) jsonDataType() string { + if ss.DriverName() == model.DatabaseDriverPostgres { + return "jsonb" + } + return "json" +} diff --git a/store/sqlstore/thread_store.go b/store/sqlstore/thread_store.go index d27c4e0750..e562276068 100644 --- a/store/sqlstore/thread_store.go +++ b/store/sqlstore/thread_store.go @@ -33,7 +33,7 @@ func newSqlThreadStore(sqlStore *SqlStore) store.ThreadStore { tableThreads := db.AddTableWithName(model.Thread{}, "Threads").SetKeys(false, "PostId") tableThreads.ColMap("PostId").SetMaxSize(26) tableThreads.ColMap("ChannelId").SetMaxSize(26) - tableThreads.ColMap("Participants").SetMaxSize(0) + tableThreads.ColMap("Participants").SetDataType(sqlStore.jsonDataType()) tableThreadMemberships := db.AddTableWithName(model.ThreadMembership{}, "ThreadMemberships").SetKeys(false, "PostId", "UserId") tableThreadMemberships.ColMap("PostId").SetMaxSize(26) tableThreadMemberships.ColMap("UserId").SetMaxSize(26) diff --git a/store/sqlstore/upgrade.go b/store/sqlstore/upgrade.go index f40f8110f4..dfabf01f90 100644 --- a/store/sqlstore/upgrade.go +++ b/store/sqlstore/upgrade.go @@ -212,6 +212,7 @@ func upgradeDatabase(sqlStore *SqlStore, currentModelVersionString string) error upgradeDatabaseToVersion536(sqlStore) upgradeDatabaseToVersion537(sqlStore) upgradeDatabaseToVersion538(sqlStore) + upgradeDatabaseToVersion600(sqlStore) return nil } @@ -1297,3 +1298,20 @@ func fixCRTChannelMembershipCounts(sqlStore *SqlStore) { mlog.Error("Error marking migration as done", mlog.Err(err)) } } + +func upgradeDatabaseToVersion600(sqlStore *SqlStore) { + // if shouldPerformUpgrade(sqlStore, Version5380, Version600) { + + sqlStore.AlterColumnTypeIfExists("ChannelMembers", "NotifyProps", "JSON", "jsonb") + sqlStore.AlterColumnTypeIfExists("Jobs", "Data", "JSON", "jsonb") + sqlStore.AlterColumnTypeIfExists("LinkMetadata", "Data", "JSON", "jsonb") + sqlStore.AlterColumnTypeIfExists("Posts", "Props", "JSON", "jsonb") + sqlStore.AlterColumnTypeIfExists("Sessions", "Props", "JSON", "jsonb") + sqlStore.AlterColumnTypeIfExists("Threads", "Participants", "JSON", "jsonb") + sqlStore.AlterColumnTypeIfExists("Users", "Props", "JSON", "jsonb") + sqlStore.AlterColumnTypeIfExists("Users", "NotifyProps", "JSON", "jsonb") + sqlStore.AlterColumnTypeIfExists("Users", "Timezone", "JSON", "jsonb") + + // saveSchemaVersion(sqlStore, Version600) + // } +} diff --git a/store/sqlstore/user_store.go b/store/sqlstore/user_store.go index 61ca9c3218..943c2c5d07 100644 --- a/store/sqlstore/user_store.go +++ b/store/sqlstore/user_store.go @@ -69,13 +69,13 @@ func newSqlUserStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) s table.ColMap("FirstName").SetMaxSize(64) table.ColMap("LastName").SetMaxSize(64) table.ColMap("Roles").SetMaxSize(256) - table.ColMap("Props").SetMaxSize(4000) - table.ColMap("NotifyProps").SetMaxSize(2000) + table.ColMap("Props").SetDataType(sqlStore.jsonDataType()) + table.ColMap("NotifyProps").SetDataType(sqlStore.jsonDataType()) table.ColMap("Locale").SetMaxSize(5) table.ColMap("MfaSecret").SetMaxSize(128) table.ColMap("RemoteId").SetMaxSize(26) table.ColMap("Position").SetMaxSize(128) - table.ColMap("Timezone").SetMaxSize(256) + table.ColMap("Timezone").SetDataType(sqlStore.jsonDataType()) } return us diff --git a/store/storetest/channel_store.go b/store/storetest/channel_store.go index 097c864c2d..ffadaf3051 100644 --- a/store/storetest/channel_store.go +++ b/store/storetest/channel_store.go @@ -4610,8 +4610,11 @@ func testGetMemberCountsByGroup(t *testing.T, ss store.Store) { } else if i == 3 || i == 4 { timeZone["manualTimezone"] = "PST" timeZone["useAutomaticTimezone"] = "false" - } else if i == 5 || i == 6 { - timeZone["autoTimezone"] = "PST" + } else if i == 5 { + timeZone["autoTimezone"] = "CET" + timeZone["useAutomaticTimezone"] = "true" + } else if i == 6 { + timeZone["automaticTimezone"] = "CET" timeZone["useAutomaticTimezone"] = "true" } else { // Give every user with auto timezone set to true a random manual timezone to ensure that manual timezone is not looked at if auto is set diff --git a/vendor/github.com/mattermost/gorp/column.go b/vendor/github.com/mattermost/gorp/column.go index abb1ebecc8..995324e72d 100644 --- a/vendor/github.com/mattermost/gorp/column.go +++ b/vendor/github.com/mattermost/gorp/column.go @@ -41,6 +41,9 @@ type ColumnMap struct { DefaultValue string + // The column data type. If set it overrides the one converted from the Go type. + DataType string + fieldName string gotype reflect.Type isPK bool @@ -86,6 +89,12 @@ func (c *ColumnMap) SetMaxSize(size int) *ColumnMap { return c } +// SetDataType allows to specify a custom data type for the column. +func (c *ColumnMap) SetDataType(dataType string) *ColumnMap { + c.DataType = dataType + return c +} + // SetDefaultConstraint adds " default 'value'" to the create table statements for this // column, if value is not nil. Not used elsewhere func (c *ColumnMap) SetDefaultConstraint(value *string) *ColumnMap { diff --git a/vendor/github.com/mattermost/gorp/table.go b/vendor/github.com/mattermost/gorp/table.go index f6b74fdd4e..0d1d0e4c89 100644 --- a/vendor/github.com/mattermost/gorp/table.go +++ b/vendor/github.com/mattermost/gorp/table.go @@ -215,7 +215,12 @@ func (t *TableMap) SqlForCreate(ifNotExists bool) string { if x > 0 { s.WriteString(", ") } - stype := dialect.ToSqlType(col.gotype, col.MaxSize, col.isAutoIncr) + + stype := col.DataType + if stype == "" { + stype = dialect.ToSqlType(col.gotype, col.MaxSize, col.isAutoIncr) + } + s.WriteString(fmt.Sprintf("%s %s", dialect.QuoteField(col.ColumnName), stype)) if col.isPK || col.isNotNull { diff --git a/vendor/modules.txt b/vendor/modules.txt index d737da9db8..cafd9fb3be 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -382,7 +382,7 @@ github.com/mattermost/go-i18n/i18n github.com/mattermost/go-i18n/i18n/bundle github.com/mattermost/go-i18n/i18n/language github.com/mattermost/go-i18n/i18n/translation -# github.com/mattermost/gorp v1.6.2-0.20210419141818-0904a6a388d3 +# github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f ## explicit github.com/mattermost/gorp # github.com/mattermost/gosaml2 v0.3.3