MM-11272 Add OpenGraph and image dimension metadata to posts (#9313)

* Move OpenGraph code into its own file

* Move OpenGraph image proxying to app layer

* Move test file code out of api4 package

* MM-11272 Add OpenGraph and image dimension metadata to posts

app/opengraph.go

@@ -0,0 +1,116 @@
+// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package app
+
+import (
+	"io"
+	"net/url"
+
+	"github.com/dyatlov/go-opengraph/opengraph"
+	"github.com/mattermost/mattermost-server/mlog"
+	"golang.org/x/net/html/charset"
+)
+
+func (a *App) GetOpenGraphMetadata(requestURL string) *opengraph.OpenGraph {
+	res, err := a.HTTPClient(false).Get(requestURL)
+	if err != nil {
+		mlog.Error("GetOpenGraphMetadata request failed", mlog.String("requestURL", requestURL), mlog.Any("err", err))
+		return nil
+	}
+	defer consumeAndClose(res)
+
+	return a.ParseOpenGraphMetadata(requestURL, res.Body, res.Header.Get("Content-Type"))
+}
+
+func (a *App) ParseOpenGraphMetadata(requestURL string, body io.Reader, contentType string) *opengraph.OpenGraph {
+	og := opengraph.NewOpenGraph()
+
+	body = forceHTMLEncodingToUTF8(body, contentType)
+
+	if err := og.ProcessHTML(body); err != nil {
+		mlog.Error("ParseOpenGraphMetadata processing failed", mlog.String("requestURL", requestURL), mlog.Any("err", err))
+	}
+
+	makeOpenGraphURLsAbsolute(og, requestURL)
+
+	// If image proxy enabled modify open graph data to feed though proxy
+	if toProxyURL := a.ImageProxyAdder(); toProxyURL != nil {
+		og = OpenGraphDataWithProxyAddedToImageURLs(og, toProxyURL)
+	}
+
+	// The URL should be the link the user provided in their message, not a redirected one.
+	if og.URL != "" {
+		og.URL = requestURL
+	}
+
+	return og
+}
+
+func forceHTMLEncodingToUTF8(body io.Reader, contentType string) io.Reader {
+	r, err := charset.NewReader(body, contentType)
+	if err != nil {
+		mlog.Error("forceHTMLEncodingToUTF8 failed to convert", mlog.String("contentType", contentType), mlog.Any("err", err))
+		return body
+	}
+	return r
+}
+
+func makeOpenGraphURLsAbsolute(og *opengraph.OpenGraph, requestURL string) {
+	parsedRequestURL, err := url.Parse(requestURL)
+	if err != nil {
+		mlog.Warn("makeOpenGraphURLsAbsolute failed to parse url", mlog.String("requestURL", requestURL), mlog.Any("err", err))
+		return
+	}
+
+	makeURLAbsolute := func(resultURL string) string {
+		if resultURL == "" {
+			return resultURL
+		}
+
+		parsedResultURL, err := url.Parse(resultURL)
+		if err != nil {
+			mlog.Warn("makeOpenGraphURLsAbsolute failed to parse result", mlog.String("requestURL", requestURL), mlog.Any("err", err))
+			return resultURL
+		}
+
+		if parsedResultURL.IsAbs() {
+			return resultURL
+		}
+
+		return parsedRequestURL.ResolveReference(parsedResultURL).String()
+	}
+
+	og.URL = makeURLAbsolute(og.URL)
+
+	for _, image := range og.Images {
+		image.URL = makeURLAbsolute(image.URL)
+		image.SecureURL = makeURLAbsolute(image.SecureURL)
+	}
+
+	for _, audio := range og.Audios {
+		audio.URL = makeURLAbsolute(audio.URL)
+		audio.SecureURL = makeURLAbsolute(audio.SecureURL)
+	}
+
+	for _, video := range og.Videos {
+		video.URL = makeURLAbsolute(video.URL)
+		video.SecureURL = makeURLAbsolute(video.SecureURL)
+	}
+}
+
+func OpenGraphDataWithProxyAddedToImageURLs(ogdata *opengraph.OpenGraph, toProxyURL func(string) string) *opengraph.OpenGraph {
+	for _, image := range ogdata.Images {
+		var url string
+		if image.SecureURL != "" {
+			url = image.SecureURL
+		} else {
+			url = image.URL
+		}
+
+		image.URL = ""
+		image.SecureURL = toProxyURL(url)
+	}
+
+	return ogdata
+}