[MM-36995] Start using db native JSON datatypes (#17930)

* Start using db native JSON datatypes * Fix syntax to work on MySQL 5.7.12
2021-07-28 08:50:36 +02:00
--- a/go.mod
+++ b/go.mod
@@ -68,7 +68,7 @@ require (
 	github.com/lib/pq v1.10.2
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattermost/go-i18n v1.11.0
-	github.com/mattermost/gorp v1.6.2-0.20210419141818-0904a6a388d3
+	github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f
 	github.com/mattermost/gosaml2 v0.3.3
 	github.com/mattermost/gziphandler v0.0.1
 	github.com/mattermost/ldap v0.0.0-20201202150706-ee0e6284187d
--- a/go.sum
+++ b/go.sum
@@ -643,8 +643,8 @@ github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQ
 github.com/marstr/guid v0.0.0-20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattermost/go-i18n v1.11.0 h1:1hLKqn/ZvhZ80OekjVPGYcCrBfMz+YxNNgqS+beL7zE=
 github.com/mattermost/go-i18n v1.11.0/go.mod h1:RyS7FDNQlzF1PsjbJWHRI35exqaKGSO9qD4iv8QjE34=
-github.com/mattermost/gorp v1.6.2-0.20210419141818-0904a6a388d3 h1:VW5cRIZheMbcC8w7RtJyt0hFckdcJ1cNpSUOELsQGE8=
-github.com/mattermost/gorp v1.6.2-0.20210419141818-0904a6a388d3/go.mod h1:QCQ3U0M9T/BlAdjKFJo0I1oe/YAgbyjNdhU8bpOLafk=
+github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f h1:J47te5ubXIEbEaXi/nwStKQ/sLniXACgZ7LhK365YIY=
+github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f/go.mod h1:QCQ3U0M9T/BlAdjKFJo0I1oe/YAgbyjNdhU8bpOLafk=
 github.com/mattermost/gosaml2 v0.3.3 h1:ysWrjp08tpWmo6rV2MQ88Eag/Ttjuj91fZsvibF4/Tg=
 github.com/mattermost/gosaml2 v0.3.3/go.mod h1:Z429EIOiEi9kbq6yHoApfzlcXpa6dzRDc6pO+Vy2Ksk=
 github.com/mattermost/gziphandler v0.0.1 h1:uXHcXF5agnQ6bXabvpiwwwZOlCYoa7mKHH0lxns/o8w=
--- a/store/sqlstore/channel_store.go
+++ b/store/sqlstore/channel_store.go
@@ -391,7 +391,7 @@ func newSqlChannelStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface
 		tablem.ColMap("ChannelId").SetMaxSize(26)
 		tablem.ColMap("UserId").SetMaxSize(26)
 		tablem.ColMap("Roles").SetMaxSize(64)
-		tablem.ColMap("NotifyProps").SetMaxSize(2000)
+		tablem.ColMap("NotifyProps").SetDataType(sqlStore.jsonDataType())

 		tablePublicChannels := db.AddTableWithName(publicChannel{}, "PublicChannels").SetKeys(false, "Id")
 		tablePublicChannels.ColMap("Id").SetMaxSize(26)
@@ -1898,45 +1898,27 @@ func (s SqlChannelStore) GetMemberCountsByGroup(ctx context.Context, channelID s
 	selectStr := "GroupMembers.GroupId, COUNT(ChannelMembers.UserId) AS ChannelMemberCount"

 	if includeTimezones {
-		// Length of default timezone (len {"automaticTimezone":"","manualTimezone":"","useAutomaticTimezone":"true"})
-		defaultTimezoneLength := `74`
-
-		// Beginning and end of the value for the automatic and manual timezones respectively
-		autoTimezone := `LOCATE(':', Users.Timezone) + 2`
-		autoTimezoneEnd := `LOCATE(',', Users.Timezone) - LOCATE(':', Users.Timezone) - 3`
-		manualTimezone := `LOCATE(',', Users.Timezone) + 19`
-		manualTimezoneEnd := `LOCATE('useAutomaticTimezone', Users.Timezone) - 22 - LOCATE(',', Users.Timezone)`
-
-		if s.DriverName() == model.DatabaseDriverPostgres {
-			autoTimezone = `POSITION(':' IN Users.Timezone) + 2`
-			autoTimezoneEnd = `POSITION(',' IN Users.Timezone) - POSITION(':' IN Users.Timezone) - 3`
-			manualTimezone = `POSITION(',' IN Users.Timezone) + 19`
-			manualTimezoneEnd = `POSITION('useAutomaticTimezone' IN Users.Timezone) - 22 - POSITION(',' IN Users.Timezone)`
-		}
-
-		selectStr = `
-			GroupMembers.GroupId,
-			COUNT(ChannelMembers.UserId) AS ChannelMemberCount,
-			COUNT(DISTINCT
+		if s.DriverName() == model.DatabaseDriverMysql {
+			selectStr += `, 
+				COUNT(DISTINCT
 				(
-					CASE WHEN Timezone like '%"useAutomaticTimezone":"true"}' AND LENGTH(Timezone) > ` + defaultTimezoneLength + `
-					THEN
-					SUBSTRING(
-						Timezone
-						FROM ` + autoTimezone + `
-						FOR ` + autoTimezoneEnd + `
-					)
-					WHEN Timezone like '%"useAutomaticTimezone":"false"}' AND LENGTH(Timezone) > ` + defaultTimezoneLength + `
-					THEN
-						SUBSTRING(
-						Timezone
-						FROM ` + manualTimezone + `
-						FOR ` + manualTimezoneEnd + `
-					)
+					CASE WHEN Timezone->"$.useAutomaticTimezone" = 'true' AND LENGTH(JSON_UNQUOTE(Timezone->"$.automaticTimezone")) > 0
+					THEN Timezone->"$.automaticTimezone"
+					WHEN Timezone->"$.useAutomaticTimezone" = 'false' AND LENGTH(JSON_UNQUOTE(Timezone->"$.manualTimezone")) > 0
+					THEN Timezone->"$.manualTimezone"
 					END
-				)
-			) AS ChannelMemberTimezonesCount
-		`
+				)) AS ChannelMemberTimezonesCount`
+		} else if s.DriverName() == model.DatabaseDriverPostgres {
+			selectStr += `, 
+				COUNT(DISTINCT
+				(
+					CASE WHEN Timezone->>'useAutomaticTimezone' = 'true' AND length(Timezone->>'automaticTimezone') > 0
+					THEN Timezone->>'automaticTimezone'
+					WHEN Timezone->>'useAutomaticTimezone' = 'false' AND length(Timezone->>'manualTimezone') > 0
+					THEN Timezone->>'manualTimezone'
+					END
+				)) AS ChannelMemberTimezonesCount`
+		}
 	}

 	query := s.getQueryBuilder().
@@ -1954,6 +1936,7 @@ func (s SqlChannelStore) GetMemberCountsByGroup(ctx context.Context, channelID s
 	if err != nil {
 		return nil, errors.Wrap(err, "channel_tosql")
 	}
+
 	var data []*model.ChannelMemberCountByGroup
 	if _, err = s.DBFromContext(ctx).Select(&data, queryString, args...); err != nil {
 		return nil, errors.Wrapf(err, "failed to count ChannelMembers with channelId=%s", channelID)
--- a/store/sqlstore/job_store.go
+++ b/store/sqlstore/job_store.go
@@ -28,7 +28,7 @@ func newSqlJobStore(sqlStore *SqlStore) store.JobStore {
 		table.ColMap("Id").SetMaxSize(26)
 		table.ColMap("Type").SetMaxSize(32)
 		table.ColMap("Status").SetMaxSize(32)
-		table.ColMap("Data").SetMaxSize(1024)
+		table.ColMap("Data").SetDataType(sqlStore.jsonDataType())
 	}

 	return s
--- a/store/sqlstore/link_metadata_store.go
+++ b/store/sqlstore/link_metadata_store.go
@@ -24,7 +24,7 @@ func newSqlLinkMetadataStore(sqlStore *SqlStore) store.LinkMetadataStore {
 		table := db.AddTableWithName(model.LinkMetadata{}, "LinkMetadata").SetKeys(false, "Hash")
 		table.ColMap("URL").SetMaxSize(2048)
 		table.ColMap("Type").SetMaxSize(16)
-		table.ColMap("Data").SetMaxSize(4096)
+		table.ColMap("Data").SetDataType(sqlStore.jsonDataType())
 	}

 	return s
--- a/store/sqlstore/post_store.go
+++ b/store/sqlstore/post_store.go
@@ -146,7 +146,7 @@ func newSqlPostStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) s
 		table.ColMap("Message").SetMaxSize(model.PostMessageMaxBytesV2)
 		table.ColMap("Type").SetMaxSize(26)
 		table.ColMap("Hashtags").SetMaxSize(1000)
-		table.ColMap("Props").SetMaxSize(8000)
+		table.ColMap("Props").SetDataType(sqlStore.jsonDataType())
 		table.ColMap("Filenames").SetMaxSize(model.PostFilenamesMaxRunes)
 		table.ColMap("FileIds").SetMaxSize(300)
 		table.ColMap("RemoteId").SetMaxSize(26)
--- a/store/sqlstore/session_store.go
+++ b/store/sqlstore/session_store.go
@@ -34,7 +34,7 @@ func newSqlSessionStore(sqlStore *SqlStore) store.SessionStore {
 		table.ColMap("UserId").SetMaxSize(26)
 		table.ColMap("DeviceId").SetMaxSize(512)
 		table.ColMap("Roles").SetMaxSize(64)
-		table.ColMap("Props").SetMaxSize(1000)
+		table.ColMap("Props").SetDataType(sqlStore.jsonDataType())
 	}

 	return us
--- a/store/sqlstore/store.go
+++ b/store/sqlstore/store.go
@@ -911,7 +911,12 @@ func (ss *SqlStore) AlterColumnTypeIfExists(tableName string, columnName string,
 	if ss.DriverName() == model.DatabaseDriverMysql {
 		_, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + tableName + " MODIFY " + columnName + " " + mySqlColType)
 	} else if ss.DriverName() == model.DatabaseDriverPostgres {
-		_, err = ss.GetMaster().ExecNoTimeout("ALTER TABLE " + strings.ToLower(tableName) + " ALTER COLUMN " + strings.ToLower(columnName) + " TYPE " + postgresColType)
+		query := "ALTER TABLE " + strings.ToLower(tableName) + " ALTER COLUMN " + strings.ToLower(columnName) + " TYPE " + postgresColType
+		// We need to explicitly cast when moving from text based to jsonb datatypes.
+		if postgresColType == "jsonb" {
+			query += " USING " + strings.ToLower(columnName) + "::jsonb"
+		}
+		_, err = ss.GetMaster().ExecNoTimeout(query)
 	}

 	if err != nil {
@@ -1742,3 +1747,10 @@ func versionString(v int, driver string) string {
 	}
 	return ""
 }
+
+func (ss *SqlStore) jsonDataType() string {
+	if ss.DriverName() == model.DatabaseDriverPostgres {
+		return "jsonb"
+	}
+	return "json"
+}
--- a/store/sqlstore/thread_store.go
+++ b/store/sqlstore/thread_store.go
@@ -33,7 +33,7 @@ func newSqlThreadStore(sqlStore *SqlStore) store.ThreadStore {
 		tableThreads := db.AddTableWithName(model.Thread{}, "Threads").SetKeys(false, "PostId")
 		tableThreads.ColMap("PostId").SetMaxSize(26)
 		tableThreads.ColMap("ChannelId").SetMaxSize(26)
-		tableThreads.ColMap("Participants").SetMaxSize(0)
+		tableThreads.ColMap("Participants").SetDataType(sqlStore.jsonDataType())
 		tableThreadMemberships := db.AddTableWithName(model.ThreadMembership{}, "ThreadMemberships").SetKeys(false, "PostId", "UserId")
 		tableThreadMemberships.ColMap("PostId").SetMaxSize(26)
 		tableThreadMemberships.ColMap("UserId").SetMaxSize(26)
--- a/store/sqlstore/upgrade.go
+++ b/store/sqlstore/upgrade.go
@@ -212,6 +212,7 @@ func upgradeDatabase(sqlStore *SqlStore, currentModelVersionString string) error
 	upgradeDatabaseToVersion536(sqlStore)
 	upgradeDatabaseToVersion537(sqlStore)
 	upgradeDatabaseToVersion538(sqlStore)
+	upgradeDatabaseToVersion600(sqlStore)

 	return nil
 }
@@ -1297,3 +1298,20 @@ func fixCRTChannelMembershipCounts(sqlStore *SqlStore) {
 		mlog.Error("Error marking migration as done", mlog.Err(err))
 	}
 }
+
+func upgradeDatabaseToVersion600(sqlStore *SqlStore) {
+	// if shouldPerformUpgrade(sqlStore, Version5380, Version600) {
+
+	sqlStore.AlterColumnTypeIfExists("ChannelMembers", "NotifyProps", "JSON", "jsonb")
+	sqlStore.AlterColumnTypeIfExists("Jobs", "Data", "JSON", "jsonb")
+	sqlStore.AlterColumnTypeIfExists("LinkMetadata", "Data", "JSON", "jsonb")
+	sqlStore.AlterColumnTypeIfExists("Posts", "Props", "JSON", "jsonb")
+	sqlStore.AlterColumnTypeIfExists("Sessions", "Props", "JSON", "jsonb")
+	sqlStore.AlterColumnTypeIfExists("Threads", "Participants", "JSON", "jsonb")
+	sqlStore.AlterColumnTypeIfExists("Users", "Props", "JSON", "jsonb")
+	sqlStore.AlterColumnTypeIfExists("Users", "NotifyProps", "JSON", "jsonb")
+	sqlStore.AlterColumnTypeIfExists("Users", "Timezone", "JSON", "jsonb")
+
+	// saveSchemaVersion(sqlStore, Version600)
+	// }
+}
--- a/store/sqlstore/user_store.go
+++ b/store/sqlstore/user_store.go
@@ -69,13 +69,13 @@ func newSqlUserStore(sqlStore *SqlStore, metrics einterfaces.MetricsInterface) s
 		table.ColMap("FirstName").SetMaxSize(64)
 		table.ColMap("LastName").SetMaxSize(64)
 		table.ColMap("Roles").SetMaxSize(256)
-		table.ColMap("Props").SetMaxSize(4000)
-		table.ColMap("NotifyProps").SetMaxSize(2000)
+		table.ColMap("Props").SetDataType(sqlStore.jsonDataType())
+		table.ColMap("NotifyProps").SetDataType(sqlStore.jsonDataType())
 		table.ColMap("Locale").SetMaxSize(5)
 		table.ColMap("MfaSecret").SetMaxSize(128)
 		table.ColMap("RemoteId").SetMaxSize(26)
 		table.ColMap("Position").SetMaxSize(128)
-		table.ColMap("Timezone").SetMaxSize(256)
+		table.ColMap("Timezone").SetDataType(sqlStore.jsonDataType())
 	}

 	return us
--- a/store/storetest/channel_store.go
+++ b/store/storetest/channel_store.go
@@ -4610,8 +4610,11 @@ func testGetMemberCountsByGroup(t *testing.T, ss store.Store) {
 		} else if i == 3 || i == 4 {
 			timeZone["manualTimezone"] = "PST"
 			timeZone["useAutomaticTimezone"] = "false"
-		} else if i == 5 || i == 6 {
-			timeZone["autoTimezone"] = "PST"
+		} else if i == 5 {
+			timeZone["autoTimezone"] = "CET"
+			timeZone["useAutomaticTimezone"] = "true"
+		} else if i == 6 {
+			timeZone["automaticTimezone"] = "CET"
 			timeZone["useAutomaticTimezone"] = "true"
 		} else {
 			// Give every user with auto timezone set to true a random manual timezone to ensure that manual timezone is not looked at if auto is set
--- a/vendor/github.com/mattermost/gorp/column.go
+++ b/vendor/github.com/mattermost/gorp/column.go
@@ -41,6 +41,9 @@ type ColumnMap struct {

 	DefaultValue string

+	// The column data type. If set it overrides the one converted from the Go type.
+	DataType string
+
 	fieldName  string
 	gotype     reflect.Type
 	isPK       bool
@@ -86,6 +89,12 @@ func (c *ColumnMap) SetMaxSize(size int) *ColumnMap {
 	return c
 }

+// SetDataType allows to specify a custom data type for the column.
+func (c *ColumnMap) SetDataType(dataType string) *ColumnMap {
+	c.DataType = dataType
+	return c
+}
+
 // SetDefaultConstraint adds " default 'value'" to the create table statements for this
 // column, if value is not nil. Not used elsewhere
 func (c *ColumnMap) SetDefaultConstraint(value *string) *ColumnMap {
--- a/vendor/github.com/mattermost/gorp/table.go
+++ b/vendor/github.com/mattermost/gorp/table.go
@@ -215,7 +215,12 @@ func (t *TableMap) SqlForCreate(ifNotExists bool) string {
 			if x > 0 {
 				s.WriteString(", ")
 			}
-			stype := dialect.ToSqlType(col.gotype, col.MaxSize, col.isAutoIncr)
+
+			stype := col.DataType
+			if stype == "" {
+				stype = dialect.ToSqlType(col.gotype, col.MaxSize, col.isAutoIncr)
+			}
+
 			s.WriteString(fmt.Sprintf("%s %s", dialect.QuoteField(col.ColumnName), stype))

 			if col.isPK || col.isNotNull {
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -382,7 +382,7 @@ github.com/mattermost/go-i18n/i18n
 github.com/mattermost/go-i18n/i18n/bundle
 github.com/mattermost/go-i18n/i18n/language
 github.com/mattermost/go-i18n/i18n/translation
-# github.com/mattermost/gorp v1.6.2-0.20210419141818-0904a6a388d3
+# github.com/mattermost/gorp v1.6.2-0.20210714143452-8b50f5209a7f
 ## explicit
 github.com/mattermost/gorp
 # github.com/mattermost/gosaml2 v0.3.3