MM-13664 Cache external link metadata when populating post metadata (#10128)
* MM-13664 Added LinkMetadata types * MM-13664 Use LinkMetadata when populating post metadata * Fix unused import * Fix index name on SQLite * Finish adding unit tests * Address feedback * Increase max length of URL column to 2048 characters
Этот коммит содержится в:
Harrison Healey
коммит произвёл
Carlos Tadeu Panato Junior
Carlos Tadeu Panato Junior
родитель
5c76e90a83
Коммит
26684716aa
148
model/link_metadata.go
Обычный файл
148
model/link_metadata.go
Обычный файл
@@ -0,0 +1,148 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"hash/fnv"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/dyatlov/go-opengraph/opengraph"
|
||||
)
|
||||
|
||||
const (
|
||||
LINK_METADATA_TYPE_IMAGE LinkMetadataType = "image"
|
||||
LINK_METADATA_TYPE_NONE LinkMetadataType = "none"
|
||||
LINK_METADATA_TYPE_OPENGRAPH LinkMetadataType = "opengraph"
|
||||
)
|
||||
|
||||
type LinkMetadataType string
|
||||
|
||||
// LinkMetadata stores arbitrary data about a link posted in a message. This includes dimensions of linked images
|
||||
// and OpenGraph metadata.
|
||||
type LinkMetadata struct {
|
||||
// Hash is a value computed from the URL and Timestamp for use as a primary key in the database.
|
||||
Hash int64
|
||||
|
||||
URL string
|
||||
Timestamp int64
|
||||
Type LinkMetadataType
|
||||
|
||||
// Data is the actual metadata for the link. It should contain data of one of the following types:
|
||||
// - *model.PostImage if the linked content is an image
|
||||
// - *opengraph.OpenGraph if the linked content is an HTML document
|
||||
// - nil if the linked content has no metadata
|
||||
Data interface{}
|
||||
}
|
||||
|
||||
func (o *LinkMetadata) PreSave() {
|
||||
o.Hash = GenerateLinkMetadataHash(o.URL, o.Timestamp)
|
||||
}
|
||||
|
||||
func (o *LinkMetadata) IsValid() *AppError {
|
||||
if o.URL == "" {
|
||||
return NewAppError("LinkMetadata.IsValid", "model.link_metadata.is_valid.url.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
if o.Timestamp == 0 || !isRoundedToNearestHour(o.Timestamp) {
|
||||
return NewAppError("LinkMetadata.IsValid", "model.link_metadata.is_valid.timestamp.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
switch o.Type {
|
||||
case LINK_METADATA_TYPE_IMAGE:
|
||||
if o.Data == nil {
|
||||
return NewAppError("LinkMetadata.IsValid", "model.link_metadata.is_valid.data.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
if _, ok := o.Data.(*PostImage); !ok {
|
||||
return NewAppError("LinkMetadata.IsValid", "model.link_metadata.is_valid.data_type.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
case LINK_METADATA_TYPE_NONE:
|
||||
if o.Data != nil {
|
||||
return NewAppError("LinkMetadata.IsValid", "model.link_metadata.is_valid.data_type.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
case LINK_METADATA_TYPE_OPENGRAPH:
|
||||
if o.Data == nil {
|
||||
return NewAppError("LinkMetadata.IsValid", "model.link_metadata.is_valid.data.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
if _, ok := o.Data.(*opengraph.OpenGraph); !ok {
|
||||
return NewAppError("LinkMetadata.IsValid", "model.link_metadata.is_valid.data_type.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
default:
|
||||
return NewAppError("LinkMetadata.IsValid", "model.link_metadata.is_valid.type.app_error", nil, "", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeserializeDataToConcreteType converts o.Data from JSON into properly structured data. This is intended to be used
|
||||
// after getting a LinkMetadata object that has been stored in the database.
|
||||
func (o *LinkMetadata) DeserializeDataToConcreteType() error {
|
||||
var b []byte
|
||||
switch t := o.Data.(type) {
|
||||
case []byte:
|
||||
// MySQL uses a byte slice for JSON
|
||||
b = t
|
||||
case string:
|
||||
// Postgres uses a string for JSON
|
||||
b = []byte(t)
|
||||
}
|
||||
|
||||
if b == nil {
|
||||
// Data doesn't need to be fixed
|
||||
return nil
|
||||
}
|
||||
|
||||
var data interface{}
|
||||
var err error
|
||||
|
||||
switch o.Type {
|
||||
case LINK_METADATA_TYPE_IMAGE:
|
||||
image := &PostImage{}
|
||||
|
||||
err = json.Unmarshal(b, &image)
|
||||
|
||||
data = image
|
||||
case LINK_METADATA_TYPE_OPENGRAPH:
|
||||
og := &opengraph.OpenGraph{}
|
||||
|
||||
json.Unmarshal(b, &og)
|
||||
|
||||
data = og
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
o.Data = data
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FloorToNearestHour takes a timestamp (in milliseconds) and returns it rounded to the previous hour in UTC.
|
||||
func FloorToNearestHour(ms int64) int64 {
|
||||
t := time.Unix(0, ms*int64(1000*1000))
|
||||
|
||||
return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, t.Location()).UnixNano() / int64(time.Millisecond)
|
||||
}
|
||||
|
||||
// isRoundedToNearestHour returns true if the given timestamp (in milliseconds) has been rounded to the nearest hour in UTC.
|
||||
func isRoundedToNearestHour(ms int64) bool {
|
||||
return FloorToNearestHour(ms) == ms
|
||||
}
|
||||
|
||||
// GenerateLinkMetadataHash generates a unique hash for a given URL and timestamp for use as a database key.
|
||||
func GenerateLinkMetadataHash(url string, timestamp int64) int64 {
|
||||
hash := fnv.New32()
|
||||
|
||||
// Note that we ignore write errors here because the Hash interface says that its Write will never return an error
|
||||
binary.Write(hash, binary.LittleEndian, timestamp)
|
||||
hash.Write([]byte(url))
|
||||
|
||||
return int64(hash.Sum32())
|
||||
}
|
||||
227
model/link_metadata_test.go
Обычный файл
227
model/link_metadata_test.go
Обычный файл
@@ -0,0 +1,227 @@
|
||||
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
|
||||
// See License.txt for license information.
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"github.com/dyatlov/go-opengraph/opengraph"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestLinkMetadataIsValid(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
Name string
|
||||
Metadata *LinkMetadata
|
||||
Expected bool
|
||||
}{
|
||||
{
|
||||
Name: "should be valid image metadata",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800000,
|
||||
Type: LINK_METADATA_TYPE_IMAGE,
|
||||
Data: &PostImage{},
|
||||
},
|
||||
Expected: true,
|
||||
},
|
||||
{
|
||||
Name: "should be valid opengraph metadata",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800000,
|
||||
Type: LINK_METADATA_TYPE_OPENGRAPH,
|
||||
Data: &opengraph.OpenGraph{},
|
||||
},
|
||||
Expected: true,
|
||||
},
|
||||
{
|
||||
Name: "should be valid with no metadata",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800000,
|
||||
Type: LINK_METADATA_TYPE_NONE,
|
||||
Data: nil,
|
||||
},
|
||||
Expected: true,
|
||||
},
|
||||
{
|
||||
Name: "should be invalid because of empty URL",
|
||||
Metadata: &LinkMetadata{
|
||||
Timestamp: 1546300800000,
|
||||
Type: LINK_METADATA_TYPE_IMAGE,
|
||||
Data: &PostImage{},
|
||||
},
|
||||
Expected: false,
|
||||
},
|
||||
{
|
||||
Name: "should be invalid because of empty timestamp",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Type: LINK_METADATA_TYPE_IMAGE,
|
||||
Data: &PostImage{},
|
||||
},
|
||||
Expected: false,
|
||||
},
|
||||
{
|
||||
Name: "should be invalid because of unrounded timestamp",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800001,
|
||||
Type: LINK_METADATA_TYPE_IMAGE,
|
||||
Data: &PostImage{},
|
||||
},
|
||||
Expected: false,
|
||||
},
|
||||
{
|
||||
Name: "should be invalid because of invalid type",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800000,
|
||||
Type: "garbage",
|
||||
Data: &PostImage{},
|
||||
},
|
||||
Expected: false,
|
||||
},
|
||||
{
|
||||
Name: "should be invalid because of empty data",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800000,
|
||||
Type: LINK_METADATA_TYPE_IMAGE,
|
||||
},
|
||||
Expected: false,
|
||||
},
|
||||
{
|
||||
Name: "should be invalid because of mismatched data and type, image type and opengraph data",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800000,
|
||||
Type: LINK_METADATA_TYPE_IMAGE,
|
||||
Data: &opengraph.OpenGraph{},
|
||||
},
|
||||
Expected: false,
|
||||
},
|
||||
{
|
||||
Name: "should be invalid because of mismatched data and type, opengraph type and image data",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800000,
|
||||
Type: LINK_METADATA_TYPE_OPENGRAPH,
|
||||
Data: &PostImage{},
|
||||
},
|
||||
Expected: false,
|
||||
},
|
||||
{
|
||||
Name: "should be invalid because of mismatched data and type, image type and random data",
|
||||
Metadata: &LinkMetadata{
|
||||
URL: "http://example.com",
|
||||
Timestamp: 1546300800000,
|
||||
Type: LINK_METADATA_TYPE_OPENGRAPH,
|
||||
Data: &Channel{},
|
||||
},
|
||||
Expected: false,
|
||||
},
|
||||
} {
|
||||
t.Run(test.Name, func(t *testing.T) {
|
||||
err := test.Metadata.IsValid()
|
||||
|
||||
if test.Expected {
|
||||
assert.Nil(t, err)
|
||||
} else {
|
||||
assert.NotNil(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLinkMetadataDeserializeDataToConcreteType(t *testing.T) {
|
||||
t.Run("should convert []byte to PostImage", func(t *testing.T) {
|
||||
image := &PostImage{
|
||||
Height: 400,
|
||||
Width: 500,
|
||||
}
|
||||
|
||||
metadata := &LinkMetadata{
|
||||
Type: LINK_METADATA_TYPE_IMAGE,
|
||||
Data: []byte(image.ToJson()),
|
||||
}
|
||||
|
||||
require.IsType(t, []byte{}, metadata.Data)
|
||||
|
||||
err := metadata.DeserializeDataToConcreteType()
|
||||
|
||||
assert.Nil(t, err)
|
||||
assert.IsType(t, &PostImage{}, metadata.Data)
|
||||
assert.Equal(t, *image, *metadata.Data.(*PostImage))
|
||||
})
|
||||
|
||||
t.Run("should convert string to OpenGraph", func(t *testing.T) {
|
||||
og := &opengraph.OpenGraph{
|
||||
URL: "http://example.com",
|
||||
Description: "Hello, world!",
|
||||
Images: []*opengraph.Image{
|
||||
{
|
||||
URL: "http://example.com/image.png",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
b, err := json.Marshal(og)
|
||||
|
||||
require.Nil(t, err)
|
||||
|
||||
metadata := &LinkMetadata{
|
||||
Type: LINK_METADATA_TYPE_OPENGRAPH,
|
||||
Data: b,
|
||||
}
|
||||
|
||||
require.IsType(t, []byte{}, metadata.Data)
|
||||
|
||||
err = metadata.DeserializeDataToConcreteType()
|
||||
|
||||
assert.Nil(t, err)
|
||||
assert.IsType(t, &opengraph.OpenGraph{}, metadata.Data)
|
||||
assert.Equal(t, *og, *metadata.Data.(*opengraph.OpenGraph))
|
||||
})
|
||||
|
||||
t.Run("should ignore data of the correct type", func(t *testing.T) {
|
||||
metadata := &LinkMetadata{
|
||||
Type: LINK_METADATA_TYPE_OPENGRAPH,
|
||||
Data: 1234,
|
||||
}
|
||||
|
||||
err := metadata.DeserializeDataToConcreteType()
|
||||
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("should ignore an invalid type", func(t *testing.T) {
|
||||
metadata := &LinkMetadata{
|
||||
Type: "garbage",
|
||||
Data: "garbage",
|
||||
}
|
||||
|
||||
err := metadata.DeserializeDataToConcreteType()
|
||||
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("should return error for invalid data", func(t *testing.T) {
|
||||
metadata := &LinkMetadata{
|
||||
Type: LINK_METADATA_TYPE_IMAGE,
|
||||
Data: "garbage",
|
||||
}
|
||||
|
||||
err := metadata.DeserializeDataToConcreteType()
|
||||
|
||||
assert.NotNil(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFloorToNearestHour(t *testing.T) {
|
||||
assert.True(t, isRoundedToNearestHour(FloorToNearestHour(1546346096000)))
|
||||
}
|
||||
@@ -3,6 +3,10 @@
|
||||
|
||||
package model
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
)
|
||||
|
||||
type PostMetadata struct {
|
||||
// Embeds holds information required to render content embedded in the post. This includes the OpenGraph metadata
|
||||
// for links in the post.
|
||||
@@ -28,3 +32,8 @@ type PostImage struct {
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
func (o *PostImage) ToJson() string {
|
||||
b, _ := json.Marshal(o)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user