[MM-31899] Use a custom user-agent when previewing links (#17186)

* Use a custom user-agent when previewing links

Many websites block requests made with Go's default user-agent. We had
previous special-cased Twitter links to use a nonstandard user-agent.
This makes that behavior apply everywhere and also customizes the
user-agent to belong specifically to Mattermost.

* Correctly use custom transport for link previews

This allows us to use the custom user-agent defined in
services/httpservice/client.go.

* Stop leaking server version in custom user-agent

Since the custom user-agent is now used when previewing links, exposing
the server version could provide a vector for a malicious actor to
gather information about private deployments. To avoid this, we switch
to a generic string.

* Remove extraneous Transport creation

MakeClient already creates a transport for us, so this is unnecessary.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

app/post_metadata.go

@@ -422,12 +422,6 @@ func (a *App) getLinkMetadata(requestURL string, timestamp int64, isNewPost bool
 
 		client := a.HTTPService().MakeClient(false)
 		client.Timeout = time.Duration(*a.Config().ExperimentalSettings.LinkMetadataTimeoutMilliseconds) * time.Millisecond
-		mmTransport := a.HTTPService().MakeTransport(false)
-		client.Transport = mmTransport.Transport
-
-		if strings.HasPrefix(requestURL, "https://twitter.com/") || strings.HasPrefix(requestURL, "https://mobile.twitter.com/") {
-			request.Header.Add("User-Agent", "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)")
-		}
 
 		var res *http.Response
 		res, err = client.Do(request)

services/httpservice/client.go

@@ -10,8 +10,6 @@ import (
 	"net"
 	"net/http"
 	"time"
-
-	"github.com/mattermost/mattermost-server/v5/model"
 )
 
 const (
@@ -86,7 +84,7 @@ func init() {
 		}
 		reservedIPRanges = append(reservedIPRanges, parsed)
 	}
-	defaultUserAgent = "mattermost-" + model.CurrentVersion
+	defaultUserAgent = "Mattermost-Bot/1.1"
 }
 
 type DialContextFunction func(ctx context.Context, network, addr string) (net.Conn, error)