Move HTTPService and ConfigService into services package (#9422)

* Move HTTPService and ConfigService into utils package

* Re-add StaticConfigService

* Move config and http services into their own packages

services/httpservice/client.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package httpservice
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"net"
+	"net/http"
+	"time"
+)
+
+const (
+	connectTimeout = 3 * time.Second
+	requestTimeout = 30 * time.Second
+)
+
+var reservedIPRanges []*net.IPNet
+
+func IsReservedIP(ip net.IP) bool {
+	for _, ipRange := range reservedIPRanges {
+		if ipRange.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+func init() {
+	for _, cidr := range []string{
+		// See https://tools.ietf.org/html/rfc6890
+		"0.0.0.0/8",      // This host on this network
+		"10.0.0.0/8",     // Private-Use
+		"127.0.0.0/8",    // Loopback
+		"169.254.0.0/16", // Link Local
+		"172.16.0.0/12",  // Private-Use Networks
+		"192.168.0.0/16", // Private-Use Networks
+		"::/128",         // Unspecified Address
+		"::1/128",        // Loopback Address
+		"fc00::/7",       // Unique-Local
+		"fe80::/10",      // Linked-Scoped Unicast
+	} {
+		_, parsed, err := net.ParseCIDR(cidr)
+		if err != nil {
+			panic(err)
+		}
+		reservedIPRanges = append(reservedIPRanges, parsed)
+	}
+}
+
+type DialContextFunction func(ctx context.Context, network, addr string) (net.Conn, error)
+
+var AddressForbidden error = errors.New("address forbidden, you may need to set AllowedUntrustedInternalConnections to allow an integration access to your internal network")
+
+func dialContextFilter(dial DialContextFunction, allowHost func(host string) bool, allowIP func(ip net.IP) bool) DialContextFunction {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		host, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+
+		if allowHost != nil && allowHost(host) {
+			return dial(ctx, network, addr)
+		}
+
+		ips, err := net.LookupIP(host)
+		if err != nil {
+			return nil, err
+		}
+
+		var firstErr error
+		for _, ip := range ips {
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			default:
+			}
+
+			if allowIP == nil || !allowIP(ip) {
+				continue
+			}
+
+			conn, err := dial(ctx, network, net.JoinHostPort(ip.String(), port))
+			if err == nil {
+				return conn, nil
+			}
+			if firstErr == nil {
+				firstErr = err
+			}
+		}
+		if firstErr == nil {
+			return nil, AddressForbidden
+		}
+		return nil, firstErr
+	}
+}
+
+// NewHTTPClient returns a variation the default implementation of Client.
+// It uses a Transport with the same settings as the default Transport
+// but with the following modifications:
+// - shorter timeout for dial and TLS handshake (defined as constant
+//   "connectTimeout")
+// - timeout for the end-to-end request (defined as constant
+//   "requestTimeout")
+func NewHTTPClient(enableInsecureConnections bool, allowHost func(host string) bool, allowIP func(ip net.IP) bool) *http.Client {
+	dialContext := (&net.Dialer{
+		Timeout:   connectTimeout,
+		KeepAlive: 30 * time.Second,
+	}).DialContext
+
+	if allowHost != nil || allowIP != nil {
+		dialContext = dialContextFilter(dialContext, allowHost, allowIP)
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy:                 http.ProxyFromEnvironment,
+			DialContext:           dialContext,
+			MaxIdleConns:          100,
+			IdleConnTimeout:       90 * time.Second,
+			TLSHandshakeTimeout:   connectTimeout,
+			ExpectContinueTimeout: 1 * time.Second,
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: enableInsecureConnections,
+			},
+		},
+		Timeout: requestTimeout,
+	}
+
+	return client
+}

services/httpservice/client_test.go

@@ -0,0 +1,120 @@
+// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package httpservice
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+)
+
+func TestHTTPClient(t *testing.T) {
+	for _, allowInternal := range []bool{true, false} {
+		c := NewHTTPClient(false, func(_ string) bool { return false }, func(ip net.IP) bool { return allowInternal || !IsReservedIP(ip) })
+		for _, tc := range []struct {
+			URL        string
+			IsInternal bool
+		}{
+			{
+				URL:        "https://google.com",
+				IsInternal: false,
+			},
+			{
+				URL:        "https://127.0.0.1",
+				IsInternal: true,
+			},
+		} {
+			_, err := c.Get(tc.URL)
+			if !tc.IsInternal {
+				if err != nil {
+					t.Fatal("google is down?")
+				}
+			} else {
+				allowed := !tc.IsInternal || allowInternal
+				success := err == nil
+				switch e := err.(type) {
+				case *net.OpError:
+					success = e.Err != AddressForbidden
+				case *url.Error:
+					success = e.Err != AddressForbidden
+				}
+				if success != allowed {
+					t.Fatalf("failed for %v. allowed: %v, success %v", tc.URL, allowed, success)
+				}
+			}
+		}
+	}
+}
+
+func TestHTTPClientWithProxy(t *testing.T) {
+	proxy := createProxyServer()
+	defer proxy.Close()
+
+	c := NewHTTPClient(true, nil, nil)
+	purl, _ := url.Parse(proxy.URL)
+	c.Transport.(*http.Transport).Proxy = http.ProxyURL(purl)
+
+	resp, err := c.Get("http://acme.com")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(body) != "proxy" {
+		t.FailNow()
+	}
+}
+
+func createProxyServer() *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		w.Header().Set("Content-Type", "text/plain; charset=us-ascii")
+		fmt.Fprint(w, "proxy")
+	}))
+}
+
+func TestDialContextFilter(t *testing.T) {
+	for _, tc := range []struct {
+		Addr    string
+		IsValid bool
+	}{
+		{
+			Addr:    "google.com:80",
+			IsValid: true,
+		},
+		{
+			Addr:    "8.8.8.8:53",
+			IsValid: true,
+		},
+		{
+			Addr: "127.0.0.1:80",
+		},
+		{
+			Addr:    "10.0.0.1:80",
+			IsValid: true,
+		},
+	} {
+		didDial := false
+		filter := dialContextFilter(func(ctx context.Context, network, addr string) (net.Conn, error) {
+			didDial = true
+			return nil, nil
+		}, func(host string) bool { return host == "10.0.0.1" }, func(ip net.IP) bool { return !IsReservedIP(ip) })
+		_, err := filter(context.Background(), "", tc.Addr)
+		switch {
+		case tc.IsValid == (err == AddressForbidden) || (err != nil && err != AddressForbidden):
+			t.Errorf("unexpected err for %v (%v)", tc.Addr, err)
+		case tc.IsValid != didDial:
+			t.Errorf("unexpected didDial for %v", tc.Addr)
+		}
+	}
+}

services/httpservice/httpservice.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package httpservice
+
+import (
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/mattermost/mattermost-server/services/configservice"
+)
+
+// Wraps the functionality for creating a new http.Client to encapsulate that and allow it to be mocked when testing
+type HTTPService interface {
+	MakeClient(trustURLs bool) *http.Client
+	Close()
+}
+
+type HTTPServiceImpl struct {
+	configService configservice.ConfigService
+}
+
+func MakeHTTPService(configService configservice.ConfigService) HTTPService {
+	return &HTTPServiceImpl{configService}
+}
+
+func (h *HTTPServiceImpl) MakeClient(trustURLs bool) *http.Client {
+	insecure := h.configService.Config().ServiceSettings.EnableInsecureOutgoingConnections != nil && *h.configService.Config().ServiceSettings.EnableInsecureOutgoingConnections
+
+	if trustURLs {
+		return NewHTTPClient(insecure, nil, nil)
+	}
+
+	allowHost := func(host string) bool {
+		if h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections == nil {
+			return false
+		}
+		for _, allowed := range strings.Fields(*h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections) {
+			if host == allowed {
+				return true
+			}
+		}
+		return false
+	}
+
+	allowIP := func(ip net.IP) bool {
+		if !IsReservedIP(ip) {
+			return true
+		}
+		if h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections == nil {
+			return false
+		}
+		for _, allowed := range strings.Fields(*h.configService.Config().ServiceSettings.AllowedUntrustedInternalConnections) {
+			if _, ipRange, err := net.ParseCIDR(allowed); err == nil && ipRange.Contains(ip) {
+				return true
+			}
+		}
+		return false
+	}
+
+	return NewHTTPClient(insecure, allowHost, allowIP)
+}
+
+func (h *HTTPServiceImpl) Close() {
+	// Does nothing, but allows this to be overridden when mocking the service
+}