MM-13664 Cache external link metadata when populating post metadata (#10128)

* MM-13664 Added LinkMetadata types

* MM-13664 Use LinkMetadata when populating post metadata

* Fix unused import

* Fix index name on SQLite

* Finish adding unit tests

* Address feedback

* Increase max length of URL column to 2048 characters

store/sqlstore/link_metadata_store.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package sqlstore
+
+import (
+	"database/sql"
+	"net/http"
+
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/store"
+)
+
+type SqlLinkMetadataStore struct {
+	SqlStore
+}
+
+func NewSqlLinkMetadataStore(sqlStore SqlStore) store.LinkMetadataStore {
+	s := &SqlLinkMetadataStore{sqlStore}
+
+	for _, db := range sqlStore.GetAllConns() {
+		table := db.AddTableWithName(model.LinkMetadata{}, "LinkMetadata").SetKeys(false, "Hash")
+		table.ColMap("URL").SetMaxSize(2048)
+		table.ColMap("Type").SetMaxSize(16)
+		table.ColMap("Data").SetMaxSize(4096)
+	}
+
+	return s
+}
+
+func (s SqlLinkMetadataStore) CreateIndexesIfNotExists() {
+	if s.DriverName() == model.DATABASE_DRIVER_MYSQL {
+		s.CreateCompositeIndexIfNotExists("idx_link_metadata_url_timestamp", "LinkMetadata", []string{"URL(512)", "Timestamp"})
+	} else {
+		s.CreateCompositeIndexIfNotExists("idx_link_metadata_url_timestamp", "LinkMetadata", []string{"URL", "Timestamp"})
+	}
+}
+
+func (s SqlLinkMetadataStore) Save(metadata *model.LinkMetadata) store.StoreChannel {
+	return store.Do(func(result *store.StoreResult) {
+		if result.Err = metadata.IsValid(); result.Err != nil {
+			return
+		}
+
+		metadata.PreSave()
+
+		err := s.GetMaster().Insert(metadata)
+		if err != nil {
+			result.Err = model.NewAppError("SqlLinkMetadataStore.Save", "store.sql_link_metadata.save.app_error", nil, "url="+metadata.URL+", "+err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		result.Data = metadata
+	})
+}
+
+func (s SqlLinkMetadataStore) Get(url string, timestamp int64) store.StoreChannel {
+	return store.Do(func(result *store.StoreResult) {
+		var metadata *model.LinkMetadata
+
+		err := s.GetReplica().SelectOne(&metadata,
+			`SELECT
+				*
+			FROM
+				LinkMetadata
+			WHERE
+				URL = :URL
+				AND Timestamp = :Timestamp`, map[string]interface{}{"URL": url, "Timestamp": timestamp})
+		if err != nil {
+			result.Err = model.NewAppError("SqlLinkMetadataStore.Get", "store.sql_link_metadata.get.app_error", nil, "url="+url+", "+err.Error(), http.StatusInternalServerError)
+
+			if err == sql.ErrNoRows {
+				result.Err.StatusCode = http.StatusNotFound
+			}
+
+			return
+		}
+
+		err = metadata.DeserializeDataToConcreteType()
+		if err != nil {
+			result.Err = model.NewAppError("SqlLinkMetadataStore.Get", "store.sql_link_metadata.get.app_error", nil, "url="+url+", "+err.Error(), http.StatusInternalServerError)
+
+			return
+		}
+
+		result.Data = metadata
+	})
+}

store/sqlstore/link_metadata_store_test.go

@@ -0,0 +1,14 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package sqlstore
+
+import (
+	"testing"
+
+	"github.com/mattermost/mattermost-server/store/storetest"
+)
+
+func TestLinkMetadataStore(t *testing.T) {
+	StoreTest(t, storetest.TestLinkMetadataStore)
+}

store/sqlstore/store.go

@@ -96,4 +96,5 @@ type SqlStore interface {
 	Scheme() store.SchemeStore
 	TermsOfService() store.TermsOfServiceStore
 	UserTermsOfService() store.UserTermsOfServiceStore
+	LinkMetadata() store.LinkMetadataStore
 }

store/sqlstore/supplier.go

@@ -15,6 +15,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/dyatlov/go-opengraph/opengraph"
 	"github.com/go-sql-driver/mysql"
 	"github.com/lib/pq"
 	"github.com/mattermost/gorp"
@@ -95,6 +96,7 @@ type SqlSupplierOldStores struct {
 	TermsOfService       store.TermsOfServiceStore
 	group                store.GroupStore
 	UserTermsOfService   store.UserTermsOfServiceStore
+	linkMetadata         store.LinkMetadataStore
 }
 
 type SqlSupplier struct {
@@ -145,6 +147,7 @@ func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInter
 	supplier.oldStores.plugin = NewSqlPluginStore(supplier)
 	supplier.oldStores.TermsOfService = NewSqlTermsOfServiceStore(supplier, metrics)
 	supplier.oldStores.UserTermsOfService = NewSqlUserTermsOfServiceStore(supplier)
+	supplier.oldStores.linkMetadata = NewSqlLinkMetadataStore(supplier)
 
 	initSqlSupplierReactions(supplier)
 	initSqlSupplierRoles(supplier)
@@ -183,6 +186,7 @@ func NewSqlSupplier(settings model.SqlSettings, metrics einterfaces.MetricsInter
 	supplier.oldStores.plugin.(*SqlPluginStore).CreateIndexesIfNotExists()
 	supplier.oldStores.TermsOfService.(SqlTermsOfServiceStore).CreateIndexesIfNotExists()
 	supplier.oldStores.UserTermsOfService.(SqlUserTermsOfServiceStore).CreateIndexesIfNotExists()
+	supplier.oldStores.linkMetadata.(*SqlLinkMetadataStore).CreateIndexesIfNotExists()
 
 	supplier.CreateIndexesIfNotExistsGroups()
 
@@ -1032,6 +1036,10 @@ func (ss *SqlSupplier) Group() store.GroupStore {
 	return ss.oldStores.group
 }
 
+func (ss *SqlSupplier) LinkMetadata() store.LinkMetadataStore {
+	return ss.oldStores.linkMetadata
+}
+
 func (ss *SqlSupplier) DropAllTables() {
 	ss.master.TruncateTables()
 }
@@ -1051,6 +1059,10 @@ func (me mattermConverter) ToDb(val interface{}) (interface{}, error) {
 		return model.StringInterfaceToJson(t), nil
 	case map[string]interface{}:
 		return model.StringInterfaceToJson(model.StringInterface(t)), nil
+	case JSONSerializable:
+		return t.ToJson(), nil
+	case *opengraph.OpenGraph:
+		return json.Marshal(t)
 	}
 
 	return val, nil
@@ -1113,6 +1125,10 @@ func (me mattermConverter) FromDb(target interface{}) (gorp.CustomScanner, bool)
 	return gorp.CustomScanner{}, false
 }
 
+type JSONSerializable interface {
+	ToJson() string
+}
+
 func convertMySQLFullTextColumnsToPostgres(columnNames string) string {
 	columns := strings.Split(columnNames, ", ")
 	concatenatedColumnNames := ""