MM-28859 Add feature flag managment system using split.io and remove viper. (#15954)

* Add feature flag managment system using split.io and remove viper.

* Fixing tests.

* Attempt to fix postgres tests.

* Fix watch filepath for advanced logging.

* Review fixes.

* Some error wrapping.

* Remove unessisary store interface.

* Desanitize SplitKey

* Simplify.

* Review feedback.

* Rename split mlog adatper to split logger.

* fsInner

* Style.

* Restore oldcfg test.

* Downgrading non-actionable feature flag errors to warnings.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

vendor/github.com/splitio/go-split-commons/v2/service/api/auth.go

@@ -0,0 +1,39 @@
+package api
+
+import (
+	"encoding/json"
+
+	"github.com/splitio/go-split-commons/v2/conf"
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// AuthAPIClient struct is responsible for authenticating client for push services
+type AuthAPIClient struct {
+	client Client
+	logger logging.LoggerInterface
+}
+
+// NewAuthAPIClient instantiates and return an AuthAPIClient
+func NewAuthAPIClient(apikey string, cfg conf.AdvancedConfig, logger logging.LoggerInterface, metadata dtos.Metadata) *AuthAPIClient {
+	return &AuthAPIClient{
+		client: NewHTTPClient(apikey, cfg, cfg.AuthServiceURL, logger, metadata),
+		logger: logger,
+	}
+}
+
+// Authenticate performs authentication for push services
+func (a *AuthAPIClient) Authenticate() (*dtos.Token, error) {
+	raw, err := a.client.Get("/api/auth")
+	if err != nil {
+		a.logger.Error("Error while authenticating for streaming", err)
+		return nil, err
+	}
+
+	token := dtos.Token{}
+	err = json.Unmarshal(raw, &token)
+	if err != nil {
+		return nil, err
+	}
+	return &token, nil
+}

vendor/github.com/splitio/go-split-commons/v2/service/api/client.go

@@ -0,0 +1,157 @@
+package api
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"time"
+
+	"github.com/splitio/go-split-commons/v2/conf"
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// Client interface for HTTPClient
+type Client interface {
+	Get(service string) ([]byte, error)
+	Post(service string, body []byte, headers map[string]string) error
+}
+
+// HTTPClient structure to wrap up the net/http.Client
+type HTTPClient struct {
+	url        string
+	httpClient *http.Client
+	headers    map[string]string
+	logger     logging.LoggerInterface
+	apikey     string
+	metadata   dtos.Metadata
+}
+
+// NewHTTPClient instance of HttpClient
+func NewHTTPClient(
+	apikey string,
+	cfg conf.AdvancedConfig,
+	endpoint string,
+	logger logging.LoggerInterface,
+	metadata dtos.Metadata,
+) Client {
+	var timeout int
+	timeout = cfg.HTTPTimeout
+	client := &http.Client{Timeout: time.Duration(timeout) * time.Second}
+	return &HTTPClient{
+		url:        endpoint,
+		httpClient: client,
+		logger:     logger,
+		apikey:     apikey,
+		metadata:   metadata,
+	}
+}
+
+// Get method is a get call to an url
+func (c *HTTPClient) Get(service string) ([]byte, error) {
+	serviceURL := c.url + service
+	c.logger.Debug("[GET] ", serviceURL)
+	req, _ := http.NewRequest("GET", serviceURL, nil)
+
+	authorization := c.apikey
+	c.logger.Debug("Authorization [ApiKey]: ", logging.ObfuscateAPIKey(authorization))
+	req.Header.Add("Accept-Encoding", "gzip")
+	req.Header.Add("Content-Type", "application/json")
+
+	c.logger.Debug(fmt.Sprintf("Headers: %v", req.Header))
+
+	req.Header.Add("Authorization", "Bearer "+authorization)
+
+	req.Header.Add("SplitSDKVersion", c.metadata.SDKVersion)
+	req.Header.Add("SplitSDKMachineName", c.metadata.MachineName)
+	req.Header.Add("SplitSDKMachineIP", c.metadata.MachineIP)
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		c.logger.Error("Error requesting data to API: ", req.URL.String(), err.Error())
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	// Check that the server actually sent compressed data
+	var reader io.ReadCloser
+	switch resp.Header.Get("Content-Encoding") {
+	case "gzip":
+		reader, _ = gzip.NewReader(resp.Body)
+		defer reader.Close()
+	default:
+		reader = resp.Body
+	}
+
+	body, err := ioutil.ReadAll(reader)
+	if err != nil {
+		c.logger.Error(err.Error())
+		return nil, err
+	}
+
+	c.logger.Verbose("[RESPONSE_BODY]", string(body), "[END_RESPONSE_BODY]")
+
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+		return body, nil
+	}
+
+	c.logger.Error(fmt.Sprintf("GET method: Status Code: %d - %s", resp.StatusCode, resp.Status))
+	return nil, &dtos.HTTPError{
+		Code:    resp.StatusCode,
+		Message: resp.Status,
+	}
+}
+
+// Post performs a HTTP POST request
+func (c *HTTPClient) Post(service string, body []byte, headers map[string]string) error {
+
+	serviceURL := c.url + service
+	c.logger.Debug("[POST] ", serviceURL)
+	req, _ := http.NewRequest("POST", serviceURL, bytes.NewBuffer(body))
+	//****************
+	req.Close = true // To prevent EOF error when connection is closed
+	//****************
+	authorization := c.apikey
+	c.logger.Debug("Authorization [ApiKey]: ", logging.ObfuscateAPIKey(authorization))
+
+	req.Header.Add("Accept-Encoding", "gzip")
+	req.Header.Add("Content-Type", "application/json")
+
+	for headerName, headerValue := range headers {
+		req.Header.Add(headerName, headerValue)
+	}
+
+	c.logger.Debug(fmt.Sprintf("Headers: %v", req.Header))
+
+	req.Header.Add("Authorization", "Bearer "+authorization)
+
+	c.logger.Verbose("[REQUEST_BODY]", string(body), "[END_REQUEST_BODY]")
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		c.logger.Error("Error posting data to API: ", req.URL.String(), err.Error())
+		return err
+	}
+	defer resp.Body.Close()
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		c.logger.Error(err.Error())
+		return err
+	}
+
+	c.logger.Verbose("[RESPONSE_BODY]", string(respBody), "[END_RESPONSE_BODY]")
+
+	if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+		return nil
+	}
+
+	c.logger.Error(fmt.Sprintf("POST method: Status Code: %d - %s", resp.StatusCode, resp.Status))
+	return &dtos.HTTPError{
+		Code:    resp.StatusCode,
+		Message: resp.Status,
+	}
+}

vendor/github.com/splitio/go-split-commons/v2/service/api/http_fetchers.go

@@ -0,0 +1,110 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"strconv"
+
+	"github.com/splitio/go-split-commons/v2/conf"
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+type httpFetcherBase struct {
+	client Client
+	logger logging.LoggerInterface
+}
+
+func (h *httpFetcherBase) fetchRaw(url string, since int64) ([]byte, error) {
+	var bufferQuery bytes.Buffer
+	bufferQuery.WriteString(url)
+
+	if since >= -1 {
+		bufferQuery.WriteString("?since=")
+		bufferQuery.WriteString(strconv.FormatInt(since, 10))
+	}
+	data, err := h.client.Get(bufferQuery.String())
+	if err != nil {
+		return nil, err
+	}
+	return data, nil
+}
+
+// HTTPSplitFetcher struct is responsible for fetching splits from the backend via HTTP protocol
+type HTTPSplitFetcher struct {
+	httpFetcherBase
+}
+
+// NewHTTPSplitFetcher instantiates and return an HTTPSplitFetcher
+func NewHTTPSplitFetcher(
+	apikey string,
+	cfg conf.AdvancedConfig,
+	logger logging.LoggerInterface,
+	metadata dtos.Metadata,
+) *HTTPSplitFetcher {
+	return &HTTPSplitFetcher{
+		httpFetcherBase: httpFetcherBase{
+			client: NewHTTPClient(apikey, cfg, cfg.SdkURL, logger, metadata),
+			logger: logger,
+		},
+	}
+}
+
+// Fetch makes an http call to the split backend and returns the list of updated splits
+func (f *HTTPSplitFetcher) Fetch(since int64) (*dtos.SplitChangesDTO, error) {
+	data, err := f.fetchRaw("/splitChanges", since)
+	if err != nil {
+		f.logger.Error("Error fetching split changes ", err)
+		return nil, err
+	}
+
+	var splitChangesDto dtos.SplitChangesDTO
+	err = json.Unmarshal(data, &splitChangesDto)
+	if err != nil {
+		f.logger.Error("Error parsing split changes JSON ", err)
+		return nil, err
+	}
+
+	return &splitChangesDto, nil
+}
+
+// HTTPSegmentFetcher struct is responsible for fetching segment by name from the API via HTTP method
+type HTTPSegmentFetcher struct {
+	httpFetcherBase
+}
+
+// NewHTTPSegmentFetcher instantiates and returns a new HTTPSegmentFetcher.
+func NewHTTPSegmentFetcher(
+	apikey string,
+	cfg conf.AdvancedConfig,
+	logger logging.LoggerInterface,
+	metadata dtos.Metadata,
+) *HTTPSegmentFetcher {
+	return &HTTPSegmentFetcher{
+		httpFetcherBase: httpFetcherBase{
+			client: NewHTTPClient(apikey, cfg, cfg.SdkURL, logger, metadata),
+			logger: logger,
+		},
+	}
+}
+
+// Fetch issues a GET request to the split backend and returns the contents of a particular segment
+func (f *HTTPSegmentFetcher) Fetch(segmentName string, since int64) (*dtos.SegmentChangesDTO, error) {
+	var bufferQuery bytes.Buffer
+	bufferQuery.WriteString("/segmentChanges/")
+	bufferQuery.WriteString(segmentName)
+
+	data, err := f.fetchRaw(bufferQuery.String(), since)
+	if err != nil {
+		f.logger.Error(err.Error())
+		return nil, err
+	}
+	var segmentChangesDto dtos.SegmentChangesDTO
+	err = json.Unmarshal(data, &segmentChangesDto)
+	if err != nil {
+		f.logger.Error("Error parsing segment changes JSON for segment ", segmentName, err)
+		return nil, err
+	}
+
+	return &segmentChangesDto, nil
+}

vendor/github.com/splitio/go-split-commons/v2/service/api/http_recorders.go

@@ -0,0 +1,197 @@
+package api
+
+import (
+	"encoding/json"
+
+	"github.com/splitio/go-split-commons/v2/conf"
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+type httpRecorderBase struct {
+	client Client
+	logger logging.LoggerInterface
+}
+
+// RecordRaw records raw data
+func (h *httpRecorderBase) RecordRaw(url string, data []byte, metadata dtos.Metadata, extraHeaders map[string]string) error {
+	headers := make(map[string]string)
+	headers["SplitSDKVersion"] = metadata.SDKVersion
+	if metadata.MachineName != "NA" && metadata.MachineName != "unknown" {
+		headers["SplitSDKMachineName"] = metadata.MachineName
+	}
+	if metadata.MachineIP != "NA" && metadata.MachineIP != "unknown" {
+		headers["SplitSDKMachineIP"] = metadata.MachineIP
+	}
+	if extraHeaders != nil {
+		for header, value := range extraHeaders {
+			headers[header] = value
+		}
+	}
+	return h.client.Post(url, data, headers)
+}
+
+// HTTPImpressionRecorder is a struct responsible for submitting impression bulks to the backend
+type HTTPImpressionRecorder struct {
+	httpRecorderBase
+}
+
+// Record sends an array (or slice) of impressionsRecord to the backend
+func (i *HTTPImpressionRecorder) Record(impressions []dtos.ImpressionsDTO, metadata dtos.Metadata, extraHeaders map[string]string) error {
+	data, err := json.Marshal(impressions)
+	if err != nil {
+		i.logger.Error("Error marshaling JSON", err.Error())
+		return err
+	}
+
+	err = i.RecordRaw("/testImpressions/bulk", data, metadata, extraHeaders)
+	if err != nil {
+		i.logger.Error("Error posting impressions", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+// RecordImpressionsCount sens impressionsCount
+func (i *HTTPImpressionRecorder) RecordImpressionsCount(pf dtos.ImpressionsCountDTO, metadata dtos.Metadata) error {
+	if len(pf.PerFeature) == 0 {
+		return nil
+	}
+	data, err := json.Marshal(pf)
+	if err != nil {
+		i.logger.Error("Error marshaling JSON", err.Error())
+		return err
+	}
+
+	err = i.RecordRaw("/testImpressions/count", data, metadata, nil)
+	if err != nil {
+		i.logger.Error("Error posting impressionsCount", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+// NewHTTPImpressionRecorder instantiates an HTTPImpressionRecorder
+func NewHTTPImpressionRecorder(
+	apikey string,
+	cfg conf.AdvancedConfig,
+	logger logging.LoggerInterface,
+) *HTTPImpressionRecorder {
+	client := NewHTTPClient(apikey, cfg, cfg.EventsURL, logger, dtos.Metadata{})
+	return &HTTPImpressionRecorder{
+		httpRecorderBase: httpRecorderBase{
+			client: client,
+			logger: logger,
+		},
+	}
+}
+
+// HTTPMetricsRecorder is a struct responsible for submitting metrics (latency, gauge, counters) to the backend
+type HTTPMetricsRecorder struct {
+	httpRecorderBase
+}
+
+// RecordCounters method submits counter metrics to the backend
+func (m *HTTPMetricsRecorder) RecordCounters(counters []dtos.CounterDTO, metadata dtos.Metadata) error {
+	data, err := json.Marshal(counters)
+	if err != nil {
+		m.logger.Error("Error marshaling JSON", err.Error())
+		return err
+	}
+
+	err = m.RecordRaw("/metrics/counters", data, metadata, nil)
+	if err != nil {
+		m.logger.Error("Error posting counters", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+// RecordLatencies method submits latency metrics to the backend
+func (m *HTTPMetricsRecorder) RecordLatencies(latencies []dtos.LatenciesDTO, metadata dtos.Metadata) error {
+	data, err := json.Marshal(latencies)
+	if err != nil {
+		m.logger.Error("Error marshaling JSON", err.Error())
+		return err
+	}
+
+	err = m.RecordRaw("/metrics/times", data, metadata, nil)
+	if err != nil {
+		m.logger.Error("Error posting latencies", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+// RecordGauge method submits gauge metrics to the backend
+func (m *HTTPMetricsRecorder) RecordGauge(gauge dtos.GaugeDTO, metadata dtos.Metadata) error {
+	data, err := json.Marshal(gauge)
+	if err != nil {
+		m.logger.Error("Error marshaling JSON", err.Error())
+		return err
+	}
+
+	err = m.RecordRaw("/metrics/gauge", data, metadata, nil)
+	if err != nil {
+		m.logger.Error("Error posting gauges", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+// NewHTTPMetricsRecorder instantiates an HTTPMetricsRecorder
+func NewHTTPMetricsRecorder(
+	apikey string,
+	cfg conf.AdvancedConfig,
+	logger logging.LoggerInterface,
+) *HTTPMetricsRecorder {
+	client := NewHTTPClient(apikey, cfg, cfg.EventsURL, logger, dtos.Metadata{})
+	return &HTTPMetricsRecorder{
+		httpRecorderBase: httpRecorderBase{
+			client: client,
+			logger: logger,
+		},
+	}
+}
+
+// HTTPEventsRecorder is a struct responsible for submitting events bulks to the backend
+type HTTPEventsRecorder struct {
+	httpRecorderBase
+}
+
+// Record sends an array (or slice) of dtos.EventDTO to the backend
+func (i *HTTPEventsRecorder) Record(events []dtos.EventDTO, metadata dtos.Metadata) error {
+	data, err := json.Marshal(events)
+	if err != nil {
+		i.logger.Error("Error marshaling JSON", err.Error())
+		return err
+	}
+
+	err = i.RecordRaw("/events/bulk", data, metadata, nil)
+	if err != nil {
+		i.logger.Error("Error posting events", err.Error())
+		return err
+	}
+
+	return nil
+}
+
+// NewHTTPEventsRecorder instantiates an HTTPEventsRecorder
+func NewHTTPEventsRecorder(
+	apikey string,
+	cfg conf.AdvancedConfig,
+	logger logging.LoggerInterface,
+) *HTTPEventsRecorder {
+	client := NewHTTPClient(apikey, cfg, cfg.EventsURL, logger, dtos.Metadata{})
+	return &HTTPEventsRecorder{
+		httpRecorderBase: httpRecorderBase{
+			client: client,
+			logger: logger,
+		},
+	}
+}

vendor/github.com/splitio/go-split-commons/v2/service/api/sse/client.go

@@ -0,0 +1,127 @@
+package sse
+
+import (
+	"strings"
+	"sync"
+	"sync/atomic"
+
+	"github.com/splitio/go-split-commons/v2/conf"
+	"github.com/splitio/go-toolkit/v3/logging"
+	"github.com/splitio/go-toolkit/v3/sse"
+)
+
+const (
+	version   = "1.1"
+	keepAlive = 120
+)
+
+// StreamingClient struct
+type StreamingClient struct {
+	mutex           *sync.RWMutex
+	sseClient       *sse.SSEClient
+	sseStatus       chan int
+	streamingStatus chan<- int
+	running         atomic.Value
+	logger          logging.LoggerInterface
+	stopped         chan struct{}
+}
+
+// NewStreamingClient creates new SSE Client
+func NewStreamingClient(cfg *conf.AdvancedConfig, streamingStatus chan int, logger logging.LoggerInterface) *StreamingClient {
+	sseStatus := make(chan int, 1)
+	sseClient, _ := sse.NewSSEClient(cfg.StreamingServiceURL, sseStatus, keepAlive, logger)
+	running := atomic.Value{}
+	running.Store(false)
+
+	return &StreamingClient{
+		mutex:           &sync.RWMutex{},
+		sseClient:       sseClient,
+		sseStatus:       sseStatus,
+		streamingStatus: streamingStatus,
+		logger:          logger,
+		running:         running,
+		stopped:         make(chan struct{}, 1),
+	}
+}
+
+// ConnectStreaming connects to streaming
+func (s *StreamingClient) ConnectStreaming(token string, channelList []string, handleIncomingMessage func(e map[string]interface{})) {
+	params := make(map[string]string)
+	params["channels"] = strings.Join(append(channelList), ",")
+	params["accessToken"] = token
+	params["v"] = version
+
+	httpHandlerExited := make(chan struct{}, 1)
+	go func() {
+		s.sseClient.Do(params, handleIncomingMessage)
+		httpHandlerExited <- struct{}{}
+	}()
+
+	// Consume remaining message in completion signaling channel if any:
+	select {
+	case <-s.stopped:
+	default:
+	}
+	select {
+	case <-s.sseStatus:
+	default:
+	}
+
+	go func() {
+		defer func() { // When this goroutine exits, StopStreaming is freed
+			select {
+			case s.stopped <- struct{}{}:
+			default:
+			}
+		}()
+		for {
+			select {
+			case <-httpHandlerExited:
+				return
+			case status := <-s.sseStatus:
+				switch status {
+				case sse.OK:
+					s.logger.Info("SSE OK")
+					s.running.Store(true)
+					s.streamingStatus <- sse.OK
+				case sse.ErrorConnectToStreaming:
+					s.logger.Error("Error connecting to streaming")
+					s.streamingStatus <- sse.ErrorConnectToStreaming
+				case sse.ErrorKeepAlive:
+					s.logger.Error("Connection timed out")
+					s.streamingStatus <- sse.ErrorKeepAlive
+				case sse.ErrorOnClientCreation:
+					s.logger.Error("Could not create client for streaming")
+					s.streamingStatus <- sse.ErrorOnClientCreation
+				case sse.ErrorReadingStream:
+					s.logger.Error("Error reading streaming buffer")
+					s.streamingStatus <- sse.ErrorReadingStream
+				case sse.ErrorRequestPerformed:
+					s.logger.Error("Error performing request when connect to stream service")
+					s.streamingStatus <- sse.ErrorRequestPerformed
+				case sse.ErrorInternal:
+					s.logger.Error("Internal Error when connect to stream service")
+					s.streamingStatus <- sse.ErrorInternal
+				default:
+					s.logger.Error("Unexpected error occured with streaming")
+					s.streamingStatus <- sse.ErrorUnexpected
+				}
+			}
+		}
+	}()
+}
+
+// StopStreaming stops streaming
+func (s *StreamingClient) StopStreaming(blocking bool) {
+	s.sseClient.Shutdown()
+	s.logger.Info("Stopped streaming")
+	s.running.Store(false)
+	if blocking {
+		<-s.stopped
+	}
+}
+
+// IsRunning returns true if it's running
+func (s *StreamingClient) IsRunning() bool {
+	return s.running.Load().(bool)
+}