MM-60351 Use oEmbed for YouTube links (#28312)

* Split up handling of permalinks and other links in getLinkMetadata

* MM-60351 Use oEmbed for YouTube links

* Explicitly request json from the oEmbed provider

* Fix linter

* Fix type of CacheAge field

* Address feedback

server/channels/app/oembed/endpoint.go

@@ -0,0 +1,42 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package oembed
+
+import (
+	"net/url"
+	"regexp"
+)
+
+//go:generate go run ./generator/providers_generator.go
+
+type ProviderEndpoint struct {
+	URL      string
+	Patterns []*regexp.Regexp
+}
+
+func (e *ProviderEndpoint) GetProviderURL(requestURL string) string {
+	// This error is checked when generating the list of providers
+	url, _ := url.Parse(e.URL)
+
+	query := url.Query()
+	query.Add("format", "json")
+	query.Add("url", requestURL)
+	url.RawQuery = query.Encode()
+
+	return url.String()
+}
+
+// FindEndpointForURL returns a ProviderEndpoint for a given URL if it matches one that's supported by us. Returns nil
+// if none of the supported providers match the given URL.
+func FindEndpointForURL(requestURL string) *ProviderEndpoint {
+	for _, provider := range providers {
+		for _, pattern := range provider.Patterns {
+			if pattern.MatchString(requestURL) {
+				return provider
+			}
+		}
+	}
+
+	return nil
+}

server/channels/app/oembed/endpoint_test.go

@@ -0,0 +1,63 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package oembed
+
+import (
+	"slices"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFindEndpointForURL(t *testing.T) {
+	youtubeProvider := providers[slices.IndexFunc(providers, func(provider *ProviderEndpoint) bool {
+		return provider.URL == "https://www.youtube.com/oembed"
+	})]
+
+	for _, testCase := range []struct {
+		Name     string
+		Input    string
+		Expected *ProviderEndpoint
+	}{
+		{
+			Name:     "random URL",
+			Input:    "https://example.com/some/random.url",
+			Expected: nil,
+		},
+		{
+			Name:     "YouTube home page",
+			Input:    "https://www.youtube.com",
+			Expected: nil,
+		},
+		{
+			Name:     "YouTube video",
+			Input:    "https://www.youtube.com/watch?v=szfZfQFUSnU",
+			Expected: youtubeProvider,
+		},
+		{
+			Name:     "YouTube video with short link and tracking information",
+			Input:    "https://youtu.be/Qq3zukqBFqQ?si=iK_TPT20H30mH90G",
+			Expected: youtubeProvider,
+		},
+		{
+			Name:     "YouTube video with playlist",
+			Input:    "https://www.youtube.com/watch?v=Qq3zukqBFqQ&list=PL-jqvaPsjQpMqnRgFEw_3fuGQbcVDTpaM",
+			Expected: youtubeProvider,
+		},
+		{
+			Name:     "YouTube playlist",
+			Input:    "https://www.youtube.com/playlist?list=PL-jqvaPsjQpMqnRgFEw_3fuGQbcVDTpaM",
+			Expected: youtubeProvider,
+		},
+		{
+			Name:     "YouTube channel",
+			Input:    "https://www.youtube.com/@MattermostHQ",
+			Expected: nil,
+		},
+	} {
+		t.Run(testCase.Name, func(t *testing.T) {
+			assert.Equal(t, testCase.Expected, FindEndpointForURL(testCase.Input))
+		})
+	}
+}

server/channels/app/oembed/generator/.gitignore

@@ -0,0 +1 @@
+providers.json

server/channels/app/oembed/generator/providers.go.tmpl

@@ -0,0 +1,30 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// Code generated by "go generate ./channels/app/oembed"
+// DO NOT EDIT
+
+//go:generate go run ./generator/providers_generator.go
+
+package oembed
+
+import (
+	"regexp"
+)
+
+var providers []*ProviderEndpoint
+
+func init() {
+	providers = []*ProviderEndpoint{
+		{{- range .Endpoints }}
+		{
+			URL: "{{ .URL }}",
+			Patterns: []*regexp.Regexp{
+				{{- range .Patterns }}
+				regexp.MustCompile(`{{ . }}`),
+				{{- end }}
+			},
+		},
+		{{- end }}
+	}
+}

server/channels/app/oembed/generator/providers_generator.go

@@ -0,0 +1,166 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"os"
+	"regexp"
+	"slices"
+	"strings"
+	"text/template"
+
+	"github.com/mattermost/mattermost/server/v8/channels/app/oembed"
+	"github.com/pkg/errors"
+)
+
+// To update the list of oEmbed providers that we support:
+// 1. Download the latest providers.json file from https://oembed.com/providers.json and place it in this folder
+// 2. If desired, update supportedProviders below to add the names of additional oEmbed providers that we want to use
+// 3. Run `go generate ./channels/app/oembed` from the server folder
+
+var (
+	// supportedProviders contains the names of all of the oEmbed providers that we currently support.
+	//
+	// As of writing, we're only going to support YouTube because they've stopped giving us the required OpenGraph
+	// metadata. When we want to support oEmbed embeds for other providers, this will need to be updated.
+	supportedProviders = []string{
+		"YouTube",
+	}
+
+	outputTemplate = template.Must(template.New("providers.go.tmpl").ParseFiles("./generator/providers.go.tmpl"))
+)
+
+type oEmbedProvider struct {
+	ProviderName string            `json:"provider_name"`
+	ProviderURL  string            `json:"provider_url"`
+	Endpoints    []*oEmbedEndpoint `json:"endpoints"`
+}
+
+type oEmbedEndpoint struct {
+	Schemes   []string `json:"schemes,omitempty"`
+	URL       string   `json:"url"`
+	Discovery bool     `json:"discovery,omitempty"`
+	Formats   []string `json:"formats,omitempty"`
+}
+
+func main() {
+	inputJson, err := os.ReadFile("./generator/providers.json")
+	if err != nil {
+		panic(errors.Wrap(err, "Unable to read providers.json. Did you forget to put it next to providers_generator.go?"))
+	}
+
+	outputFile, err := os.Create("./providers_gen.go")
+	if err != nil {
+		panic(errors.Wrap(err, "Unable to open output file"))
+	}
+	defer outputFile.Close()
+
+	var input []*oEmbedProvider
+	err = json.Unmarshal(inputJson, &input)
+	if err != nil {
+		panic(errors.Wrap(err, "Unable to read providers.json"))
+	}
+
+	var endpoints []*oembed.ProviderEndpoint
+	for _, inputProvider := range input {
+		if !slices.Contains(supportedProviders, inputProvider.ProviderName) {
+			continue
+		}
+
+		providerEndpoints, extractErr := extractEndpointsFromProvider(inputProvider)
+		if extractErr != nil {
+			panic(errors.Wrap(extractErr, "Unable to convert oEmbedProvider from providers.json to a ProviderEndpoint"))
+		}
+		endpoints = append(endpoints, providerEndpoints...)
+	}
+
+	err = outputTemplate.Execute(outputFile, map[string]any{
+		"Endpoints": endpoints,
+	})
+	if err != nil {
+		panic(errors.Wrap(err, "Unable to write file using template"))
+	}
+}
+
+// extractEndpointsFromProvider turns the data for one provider into providers.json into multiple, more compact ProviderEndpoints
+func extractEndpointsFromProvider(in *oEmbedProvider) ([]*oembed.ProviderEndpoint, error) {
+	var out []*oembed.ProviderEndpoint
+
+	for _, endpoint := range in.Endpoints {
+		// Ensure that the endpoint URL is valid so that we don't need to error check it at runtime
+		_, err := url.Parse(endpoint.URL)
+		if err != nil {
+			return nil, err
+		}
+
+		var patterns []*regexp.Regexp
+		for _, scheme := range endpoint.Schemes {
+			pattern, err := schemeToPattern(scheme)
+			if err != nil {
+				return nil, err
+			}
+
+			patterns = append(patterns, pattern)
+		}
+
+		if len(patterns) > 0 {
+			out = append(out, &oembed.ProviderEndpoint{
+				URL:      endpoint.URL,
+				Patterns: patterns,
+			})
+		}
+	}
+
+	return out, nil
+}
+
+func schemeToPattern(scheme string) (*regexp.Regexp, error) {
+	partsPattern := regexp.MustCompile(`^(\w+:(?://)?)([^/]*)(/[^?]*)?(\?[^?]*)?$`)
+	parts := partsPattern.FindStringSubmatch(scheme)
+	if parts == nil {
+		return nil, fmt.Errorf("unable to split scheme %s into parts", scheme)
+	} else if len(parts) != 5 {
+		return nil, fmt.Errorf("wrong number of parts for scheme %s", scheme)
+	}
+
+	protocol := parts[1]
+	if protocol != "http://" && protocol != "https://" && protocol != "spotify:" {
+		return nil, fmt.Errorf("unrecognized protocol %s for scheme %s", protocol, scheme)
+	}
+	domain := parts[2]
+	if domain == "" {
+		return nil, fmt.Errorf("no domain found for scheme %s", scheme)
+	}
+	path := parts[3]
+	if path == "" && protocol != "spotify:" {
+		return nil, fmt.Errorf("no path found for scheme %s", scheme)
+	}
+	query := parts[4]
+
+	// Replace any valid wildcards with a temporary character so that we can escape any regexp special characters
+	domain = strings.Replace(domain, "*", "%", -1)
+	path = strings.Replace(path, "*", "%", -1)
+	query = strings.Replace(query, "*", "%", -1)
+
+	// Escape any other special characters
+	protocol = regexp.QuoteMeta(protocol)
+	domain = regexp.QuoteMeta(domain)
+	path = regexp.QuoteMeta(path)
+	query = regexp.QuoteMeta(query)
+
+	// Replace the temporary character with the proper regexp to match a wildcard in that part of the URL
+	domain = strings.Replace(domain, "%", "[^/]*?", -1)
+	path = strings.Replace(path, "%", ".*?", -1)
+	query = strings.Replace(query, "%", ".*?", -1)
+
+	// Allow http schemes to match https URLs as well
+	if protocol == "http://" {
+		protocol = "https?://"
+	}
+
+	return regexp.Compile("^" + protocol + domain + path + query + "$")
+}

server/channels/app/oembed/oembed.go

@@ -0,0 +1,56 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+package oembed
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+)
+
+type OEmbedResponse struct {
+	// Type can be one of "photo", "video", "link", or "rich"
+	Type string `json:"type"`
+
+	// Fields that may be defined for any response type
+	Version         string `json:"version"`
+	Title           string `json:"title,omitempty"`
+	AuthorName      string `json:"author_name,omitempty"`
+	AuthorURL       string `json:"author_url,omitempty"`
+	ProviderName    string `json:"provider_name,omitempty"`
+	ProviderURL     string `json:"provider_url,omitempty"`
+	CacheAge        string `json:"cache_age,omitempty"`
+	ThumbnailURL    string `json:"thumbnail_url,omitempty"`
+	ThumbnailWidth  int    `json:"thumbnail_width,omitempty"`
+	ThumbnailHeight int    `json:"thumbnail_height,omitempty"`
+
+	// Fields that are required for responses with the type "photo"
+	URL string `json:"url"`
+
+	// Fields that are required for responses of the type "video" or "rich"
+	HTML string `json:"html"`
+
+	// Fields that are required for responses with the type "photo", "video", or "rich"
+	Width  int `json:"width"`
+	Height int `json:"height"`
+}
+
+func ResponseFromJSON(r io.Reader) (*OEmbedResponse, error) {
+	var response OEmbedResponse
+
+	err := json.NewDecoder(r).Decode(&response)
+	if err != nil {
+		return nil, err
+	}
+
+	// Do a quick smoke test to confirm that this is hopefully a valid oEmbed response
+	if response.Version != "1.0" {
+		return nil, fmt.Errorf("ResponseFromJson: Received unsupported response version %s", response.Version)
+	}
+	if response.Type != "photo" && response.Type != "video" && response.Type != "link" && response.Type != "rich" {
+		return nil, fmt.Errorf("ResponseFromJson: Received unsupported response type %s", response.Type)
+	}
+
+	return &response, nil
+}

server/channels/app/oembed/providers_gen.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See LICENSE.txt for license information.
+
+// Code generated by "go generate ./channels/app/oembed"
+// DO NOT EDIT
+
+//go:generate go run ./generator/providers_generator.go
+
+package oembed
+
+import (
+	"regexp"
+)
+
+var providers []*ProviderEndpoint
+
+func init() {
+	providers = []*ProviderEndpoint{
+		{
+			URL: "https://www.youtube.com/oembed",
+			Patterns: []*regexp.Regexp{
+				regexp.MustCompile(`^https://[^/]*?\.youtube\.com/watch.*?$`),
+				regexp.MustCompile(`^https://[^/]*?\.youtube\.com/v/.*?$`),
+				regexp.MustCompile(`^https://youtu\.be/.*?$`),
+				regexp.MustCompile(`^https://[^/]*?\.youtube\.com/playlist\?list=.*?$`),
+				regexp.MustCompile(`^https://youtube\.com/playlist\?list=.*?$`),
+				regexp.MustCompile(`^https://[^/]*?\.youtube\.com/shorts.*?$`),
+				regexp.MustCompile(`^https://youtube\.com/shorts.*?$`),
+				regexp.MustCompile(`^https://[^/]*?\.youtube\.com/embed/.*?$`),
+				regexp.MustCompile(`^https://[^/]*?\.youtube\.com/live.*?$`),
+				regexp.MustCompile(`^https://youtube\.com/live.*?$`),
+			},
+		},
+	}
+}