MM-28859 Add feature flag managment system using split.io and remove viper. (#15954)

* Add feature flag managment system using split.io and remove viper.

* Fixing tests.

* Attempt to fix postgres tests.

* Fix watch filepath for advanced logging.

* Review fixes.

* Some error wrapping.

* Remove unessisary store interface.

* Desanitize SplitKey

* Simplify.

* Review feedback.

* Rename split mlog adatper to split logger.

* fsInner

* Style.

* Restore oldcfg test.

* Downgrading non-actionable feature flag errors to warnings.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

vendor/github.com/splitio/go-split-commons/v2/synchronizer/interface.go

@@ -0,0 +1,12 @@
+package synchronizer
+
+// Synchronizer interface for syncing data to and from splits servers
+type Synchronizer interface {
+	SyncAll() error
+	SynchronizeSplits(till *int64) error
+	SynchronizeSegment(segmentName string, till *int64) error
+	StartPeriodicFetching()
+	StopPeriodicFetching()
+	StartPeriodicDataRecording()
+	StopPeriodicDataRecording()
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/local.go

@@ -0,0 +1,80 @@
+package synchronizer
+
+import (
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-split-commons/v2/service"
+	"github.com/splitio/go-split-commons/v2/storage"
+	storageMock "github.com/splitio/go-split-commons/v2/storage/mocks"
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/split"
+	"github.com/splitio/go-split-commons/v2/tasks"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// Local implements Local Synchronizer
+type Local struct {
+	splitTasks          SplitTasks
+	workers             Workers
+	logger              logging.LoggerInterface
+	inMememoryFullQueue chan string
+}
+
+// NewLocal creates new Local
+func NewLocal(
+	period int,
+	splitAPI *service.SplitAPI,
+	splitStorage storage.SplitStorage,
+	logger logging.LoggerInterface,
+) Synchronizer {
+	metricStorageMock := storageMock.MockMetricStorage{
+		IncCounterCall:   func(key string) {},
+		IncLatencyCall:   func(metricName string, index int) {},
+		PopCountersCall:  func() []dtos.CounterDTO { return make([]dtos.CounterDTO, 0, 0) },
+		PopGaugesCall:    func() []dtos.GaugeDTO { return make([]dtos.GaugeDTO, 0, 0) },
+		PopLatenciesCall: func() []dtos.LatenciesDTO { return make([]dtos.LatenciesDTO, 0, 0) },
+		PutGaugeCall:     func(key string, gauge float64) {},
+	}
+	metricsWrapper := storage.NewMetricWrapper(metricStorageMock, nil, logger)
+	workers := Workers{
+		SplitFetcher: split.NewSplitFetcher(splitStorage, splitAPI.SplitFetcher, metricsWrapper, logger),
+	}
+	return &Local{
+		splitTasks: SplitTasks{
+			SplitSyncTask: tasks.NewFetchSplitsTask(workers.SplitFetcher, period, logger),
+		},
+		workers: workers,
+		logger:  logger,
+	}
+}
+
+// SyncAll syncs splits and segments
+func (s *Local) SyncAll() error {
+	return s.workers.SplitFetcher.SynchronizeSplits(nil)
+}
+
+// StartPeriodicFetching starts periodic fetchers tasks
+func (s *Local) StartPeriodicFetching() {
+	s.splitTasks.SplitSyncTask.Start()
+}
+
+// StopPeriodicFetching stops periodic fetchers tasks
+func (s *Local) StopPeriodicFetching() {
+	s.splitTasks.SplitSyncTask.Stop(false)
+}
+
+// StartPeriodicDataRecording starts periodic recorders tasks
+func (s *Local) StartPeriodicDataRecording() {
+}
+
+// StopPeriodicDataRecording stops periodic recorders tasks
+func (s *Local) StopPeriodicDataRecording() {
+}
+
+// SynchronizeSplits syncs splits
+func (s *Local) SynchronizeSplits(till *int64) error {
+	return s.workers.SplitFetcher.SynchronizeSplits(till)
+}
+
+// SynchronizeSegment syncs segment
+func (s *Local) SynchronizeSegment(name string, till *int64) error {
+	return nil
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/manager.go

@@ -0,0 +1,184 @@
+package synchronizer
+
+import (
+	"errors"
+	"sync/atomic"
+
+	"github.com/splitio/go-split-commons/v2/conf"
+	"github.com/splitio/go-split-commons/v2/push"
+	"github.com/splitio/go-split-commons/v2/service"
+	"github.com/splitio/go-split-commons/v2/storage"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+const (
+	// Ready represents ready
+	Ready = iota
+	// StreamingReady ready
+	StreamingReady
+	// Error represents some error in SSE streaming
+	Error
+)
+
+const (
+	// Idle flags
+	Idle = iota
+	// Streaming flags
+	Streaming
+	// Polling flags
+	Polling
+)
+
+// Manager struct
+type Manager struct {
+	synchronizer    Synchronizer
+	logger          logging.LoggerInterface
+	config          conf.AdvancedConfig
+	pushManager     push.Manager
+	managerStatus   chan int
+	streamingStatus chan int
+	status          atomic.Value
+}
+
+// NewSynchronizerManager creates new sync manager
+func NewSynchronizerManager(
+	synchronizer Synchronizer,
+	logger logging.LoggerInterface,
+	config conf.AdvancedConfig,
+	authClient service.AuthClient,
+	splitStorage storage.SplitStorage,
+	managerStatus chan int,
+) (*Manager, error) {
+	if managerStatus == nil || cap(managerStatus) < 1 {
+		return nil, errors.New("Status channel cannot be nil nor having capacity")
+	}
+
+	status := atomic.Value{}
+	status.Store(Idle)
+	manager := &Manager{
+		synchronizer:  synchronizer,
+		logger:        logger,
+		config:        config,
+		managerStatus: managerStatus,
+		status:        status,
+	}
+	if config.StreamingEnabled {
+		streamingStatus := make(chan int, 1000)
+		pushManager, err := push.NewPushManager(logger, synchronizer.SynchronizeSegment, synchronizer.SynchronizeSplits, splitStorage, &config, streamingStatus, authClient)
+		if err != nil {
+			return nil, err
+		}
+		manager.pushManager = pushManager
+		manager.streamingStatus = streamingStatus
+	}
+
+	return manager, nil
+}
+
+func (s *Manager) startPolling() {
+	s.status.Store(Polling)
+	s.pushManager.StopWorkers()
+	s.synchronizer.StartPeriodicFetching()
+}
+
+// IsRunning returns true if is in Streaming or Polling
+func (s *Manager) IsRunning() bool {
+	return s.status.Load().(int) != Idle
+}
+
+// Start starts synchronization through Split
+func (s *Manager) Start() {
+	if s.IsRunning() {
+		s.logger.Info("Manager is already running, skipping start")
+		return
+	}
+	select {
+	case <-s.managerStatus:
+		// Discarding previous status before starting
+	default:
+	}
+	err := s.synchronizer.SyncAll()
+	if err != nil {
+		s.managerStatus <- Error
+		return
+	}
+	s.logger.Debug("SyncAll Ready")
+	s.managerStatus <- Ready
+	s.synchronizer.StartPeriodicDataRecording()
+	if s.config.StreamingEnabled {
+		s.logger.Info("Start Streaming")
+		go s.pushManager.Start()
+		// Listens Streaming Status
+		for {
+			status := <-s.streamingStatus
+			switch status {
+			// Backoff is running -> start polling until auth is ok
+			case push.BackoffAuth:
+				fallthrough
+			// Backoff is running -> start polling until sse is connected
+			case push.BackoffSSE:
+				if s.status.Load().(int) != Polling {
+					s.logger.Info("Start periodic polling due backoff")
+					s.startPolling()
+				}
+			// SSE Streaming and workers are ready
+			case push.Ready:
+				// If Ready comes eventually when Backoff is done and polling is running
+				if s.status.Load().(int) == Polling {
+					s.synchronizer.StopPeriodicFetching()
+				}
+				s.logger.Info("SSE Streaming is ready")
+				s.status.Store(Streaming)
+				go s.synchronizer.SyncAll()
+			case push.StreamingDisabled:
+				fallthrough
+			// NonRetriableError occurs and it will switch to polling
+			case push.NonRetriableError:
+				s.pushManager.Stop()
+				s.logger.Info("Start periodic polling in Streaming")
+				s.startPolling()
+				return
+			// Publisher sends that there is no Notification Managers available
+			case push.PushIsDown:
+				// If streaming is already running, proceeding to stop workers
+				// and keeping SSE running
+				if s.status.Load().(int) == Streaming {
+					s.logger.Info("Start periodic polling in Streaming")
+					s.startPolling()
+				}
+			// Publisher sends that there are at least one Notification Manager available
+			case push.PushIsUp:
+				// If streaming is not already running, proceeding to start workers
+				if s.status.Load().(int) != Streaming {
+					s.logger.Info("Stop periodic polling")
+					s.pushManager.StartWorkers()
+					s.synchronizer.StopPeriodicFetching()
+					s.status.Store(Streaming)
+					go s.synchronizer.SyncAll()
+				}
+			// Reconnect received due error in streaming -> reconnecting
+			case push.Reconnect:
+				fallthrough
+			// Token expired -> reconnecting
+			case push.TokenExpiration:
+				s.pushManager.Stop()
+				go s.pushManager.Start()
+			}
+		}
+	} else {
+		s.logger.Info("Start periodic polling")
+		s.synchronizer.StartPeriodicFetching()
+		s.status.Store(Polling)
+	}
+}
+
+// Stop stop synchronizaation through Split
+func (s *Manager) Stop() {
+	s.logger.Info("STOPPING MANAGER TASKS")
+	if s.pushManager != nil && s.pushManager.IsRunning() {
+		s.pushManager.Stop()
+	}
+	s.synchronizer.StopPeriodicFetching()
+	s.synchronizer.StopPeriodicDataRecording()
+	s.status.Store(Idle)
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/synchronizer.go

@@ -0,0 +1,158 @@
+package synchronizer
+
+import (
+	"github.com/splitio/go-split-commons/v2/conf"
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/event"
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/impression"
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/impressionscount"
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/metric"
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/segment"
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/split"
+	"github.com/splitio/go-split-commons/v2/tasks"
+	"github.com/splitio/go-toolkit/v3/asynctask"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// SplitTasks struct for tasks
+type SplitTasks struct {
+	SplitSyncTask            *asynctask.AsyncTask
+	SegmentSyncTask          *asynctask.AsyncTask
+	TelemetrySyncTask        *asynctask.AsyncTask
+	ImpressionSyncTask       tasks.Task
+	EventSyncTask            tasks.Task
+	ImpressionsCountSyncTask *asynctask.AsyncTask
+}
+
+// Workers struct for workers
+type Workers struct {
+	SplitFetcher             split.SplitFetcher
+	SegmentFetcher           segment.SegmentFetcher
+	TelemetryRecorder        metric.MetricRecorder
+	ImpressionRecorder       impression.ImpressionRecorder
+	EventRecorder            event.EventRecorder
+	ImpressionsCountRecorder impressionscount.ImpressionsCountRecorder
+}
+
+// SynchronizerImpl implements Synchronizer
+type SynchronizerImpl struct {
+	splitTasks          SplitTasks
+	workers             Workers
+	logger              logging.LoggerInterface
+	inMememoryFullQueue chan string
+	impressionBulkSize  int64
+	eventBulkSize       int64
+}
+
+// NewSynchronizer creates new SynchronizerImpl
+func NewSynchronizer(
+	confAdvanced conf.AdvancedConfig,
+	splitTasks SplitTasks,
+	workers Workers,
+	logger logging.LoggerInterface,
+	inMememoryFullQueue chan string,
+) Synchronizer {
+	return &SynchronizerImpl{
+		impressionBulkSize:  confAdvanced.ImpressionsBulkSize,
+		eventBulkSize:       confAdvanced.EventsBulkSize,
+		splitTasks:          splitTasks,
+		workers:             workers,
+		logger:              logger,
+		inMememoryFullQueue: inMememoryFullQueue,
+	}
+}
+
+func (s *SynchronizerImpl) dataFlusher() {
+	for true {
+		msg := <-s.inMememoryFullQueue
+		switch msg {
+		case "EVENTS_FULL":
+			s.logger.Debug("FLUSHING storage queue")
+			err := s.workers.EventRecorder.SynchronizeEvents(s.eventBulkSize)
+			if err != nil {
+				s.logger.Error("Error flushing storage queue", err)
+			}
+			break
+		case "IMPRESSIONS_FULL":
+			s.logger.Debug("FLUSHING storage queue")
+			err := s.workers.ImpressionRecorder.SynchronizeImpressions(s.impressionBulkSize)
+			if err != nil {
+				s.logger.Error("Error flushing storage queue", err)
+			}
+		}
+	}
+}
+
+// SyncAll syncs splits and segments
+func (s *SynchronizerImpl) SyncAll() error {
+	err := s.workers.SplitFetcher.SynchronizeSplits(nil)
+	if err != nil {
+		return err
+	}
+	return s.workers.SegmentFetcher.SynchronizeSegments()
+}
+
+// StartPeriodicFetching starts periodic fetchers tasks
+func (s *SynchronizerImpl) StartPeriodicFetching() {
+	if s.splitTasks.SplitSyncTask != nil {
+		s.splitTasks.SplitSyncTask.Start()
+	}
+	if s.splitTasks.SegmentSyncTask != nil {
+		s.splitTasks.SegmentSyncTask.Start()
+	}
+}
+
+// StopPeriodicFetching stops periodic fetchers tasks
+func (s *SynchronizerImpl) StopPeriodicFetching() {
+	if s.splitTasks.SplitSyncTask != nil {
+		s.splitTasks.SplitSyncTask.Stop(false)
+	}
+	if s.splitTasks.SegmentSyncTask != nil {
+		s.splitTasks.SegmentSyncTask.Stop(true)
+	}
+}
+
+// StartPeriodicDataRecording starts periodic recorders tasks
+func (s *SynchronizerImpl) StartPeriodicDataRecording() {
+	if s.inMememoryFullQueue != nil {
+		go s.dataFlusher()
+	}
+
+	if s.splitTasks.ImpressionSyncTask != nil {
+		s.splitTasks.ImpressionSyncTask.Start()
+	}
+	if s.splitTasks.TelemetrySyncTask != nil {
+		s.splitTasks.TelemetrySyncTask.Start()
+	}
+	if s.splitTasks.EventSyncTask != nil {
+		s.splitTasks.EventSyncTask.Start()
+	}
+	if s.splitTasks.ImpressionsCountSyncTask != nil {
+		s.splitTasks.ImpressionsCountSyncTask.Start()
+	}
+}
+
+// StopPeriodicDataRecording stops periodic recorders tasks
+func (s *SynchronizerImpl) StopPeriodicDataRecording() {
+	if s.splitTasks.ImpressionSyncTask != nil {
+		s.splitTasks.ImpressionSyncTask.Stop(true)
+	}
+	if s.splitTasks.TelemetrySyncTask != nil {
+		s.splitTasks.TelemetrySyncTask.Stop(false)
+	}
+	if s.splitTasks.EventSyncTask != nil {
+		s.splitTasks.EventSyncTask.Stop(true)
+	}
+	if s.splitTasks.ImpressionsCountSyncTask != nil {
+		s.splitTasks.ImpressionsCountSyncTask.Stop(true)
+	}
+}
+
+// SynchronizeSplits syncs splits
+func (s *SynchronizerImpl) SynchronizeSplits(till *int64) error {
+	return s.workers.SplitFetcher.SynchronizeSplits(till)
+}
+
+// SynchronizeSegment syncs segment
+func (s *SynchronizerImpl) SynchronizeSegment(name string, till *int64) error {
+	return s.workers.SegmentFetcher.SynchronizeSegment(name, till)
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/event/interface.go

@@ -0,0 +1,7 @@
+package event
+
+// EventRecorder interface
+type EventRecorder interface {
+	SynchronizeEvents(bulkSize int64) error
+	FlushEvents(bulkSize int64) error
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/event/single.go

@@ -0,0 +1,76 @@
+package event
+
+import (
+	"errors"
+	"time"
+
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-split-commons/v2/service"
+	"github.com/splitio/go-split-commons/v2/storage"
+	"github.com/splitio/go-split-commons/v2/util"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// RecorderSingle struct for event sync
+type RecorderSingle struct {
+	eventStorage   storage.EventStorageConsumer
+	eventRecorder  service.EventsRecorder
+	metricsWrapper *storage.MetricWrapper
+	logger         logging.LoggerInterface
+	metadata       dtos.Metadata
+}
+
+// NewEventRecorderSingle creates new event synchronizer for posting events
+func NewEventRecorderSingle(
+	eventStorage storage.EventStorageConsumer,
+	eventRecorder service.EventsRecorder,
+	metricsWrapper *storage.MetricWrapper,
+	logger logging.LoggerInterface,
+	metadata dtos.Metadata,
+) EventRecorder {
+	return &RecorderSingle{
+		eventStorage:   eventStorage,
+		eventRecorder:  eventRecorder,
+		metricsWrapper: metricsWrapper,
+		logger:         logger,
+		metadata:       metadata,
+	}
+}
+
+// SynchronizeEvents syncs events
+func (e *RecorderSingle) SynchronizeEvents(bulkSize int64) error {
+	queuedEvents, err := e.eventStorage.PopN(bulkSize)
+	if err != nil {
+		e.logger.Error("Error reading events queue", err)
+		return errors.New("Error reading events queue")
+	}
+
+	if len(queuedEvents) == 0 {
+		e.logger.Debug("No events fetched from queue. Nothing to send")
+		return nil
+	}
+
+	before := time.Now()
+	err = e.eventRecorder.Record(queuedEvents, e.metadata)
+	if err != nil {
+		if httpError, ok := err.(*dtos.HTTPError); ok {
+			e.metricsWrapper.StoreCounters(storage.PostEventsCounter, string(httpError.Code))
+		}
+		return err
+	}
+	bucket := util.Bucket(time.Now().Sub(before).Nanoseconds())
+	e.metricsWrapper.StoreLatencies(storage.PostEventsLatency, bucket)
+	e.metricsWrapper.StoreCounters(storage.PostEventsCounter, "ok")
+	return nil
+}
+
+// FlushEvents flushes events
+func (e *RecorderSingle) FlushEvents(bulkSize int64) error {
+	for !e.eventStorage.Empty() {
+		err := e.SynchronizeEvents(bulkSize)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/impression/interface.go

@@ -0,0 +1,7 @@
+package impression
+
+// ImpressionRecorder interface
+type ImpressionRecorder interface {
+	SynchronizeImpressions(bulkSize int64) error
+	FlushImpressions(bulkSize int64) error
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/impression/single.go

@@ -0,0 +1,117 @@
+package impression
+
+import (
+	"errors"
+	"time"
+
+	"github.com/splitio/go-split-commons/v2/conf"
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-split-commons/v2/service"
+	"github.com/splitio/go-split-commons/v2/storage"
+	"github.com/splitio/go-split-commons/v2/util"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+const (
+	maxImpressionCacheSize  = 500000
+	splitSDKImpressionsMode = "SplitSDKImpressionsMode"
+)
+
+// RecorderSingle struct for impression sync
+type RecorderSingle struct {
+	impressionStorage  storage.ImpressionStorageConsumer
+	impressionRecorder service.ImpressionsRecorder
+	metricsWrapper     *storage.MetricWrapper
+	logger             logging.LoggerInterface
+	metadata           dtos.Metadata
+	mode               string
+}
+
+// NewRecorderSingle creates new impression synchronizer for posting impressions
+func NewRecorderSingle(
+	impressionStorage storage.ImpressionStorageConsumer,
+	impressionRecorder service.ImpressionsRecorder,
+	metricsWrapper *storage.MetricWrapper,
+	logger logging.LoggerInterface,
+	metadata dtos.Metadata,
+	managerConfig conf.ManagerConfig,
+) ImpressionRecorder {
+	mode := conf.ImpressionsModeOptimized
+	if !util.ShouldBeOptimized(managerConfig) {
+		mode = conf.ImpressionsModeDebug
+	}
+	return &RecorderSingle{
+		impressionStorage:  impressionStorage,
+		impressionRecorder: impressionRecorder,
+		metricsWrapper:     metricsWrapper,
+		logger:             logger,
+		metadata:           metadata,
+		mode:               mode,
+	}
+}
+
+// SynchronizeImpressions syncs impressions
+func (i *RecorderSingle) SynchronizeImpressions(bulkSize int64) error {
+	queuedImpressions, err := i.impressionStorage.PopN(bulkSize)
+	if err != nil {
+		i.logger.Error("Error reading impressions queue", err)
+		return errors.New("Error reading impressions queue")
+	}
+
+	if len(queuedImpressions) == 0 {
+		i.logger.Debug("No impressions fetched from queue. Nothing to send")
+		return nil
+	}
+
+	impressionsToPost := make(map[string][]dtos.ImpressionDTO)
+	for _, impression := range queuedImpressions {
+		keyImpression := dtos.ImpressionDTO{
+			KeyName:      impression.KeyName,
+			Treatment:    impression.Treatment,
+			Time:         impression.Time,
+			ChangeNumber: impression.ChangeNumber,
+			Label:        impression.Label,
+			BucketingKey: impression.BucketingKey,
+			Pt:           impression.Pt,
+		}
+		v, ok := impressionsToPost[impression.FeatureName]
+		if ok {
+			v = append(v, keyImpression)
+		} else {
+			v = []dtos.ImpressionDTO{keyImpression}
+		}
+		impressionsToPost[impression.FeatureName] = v
+	}
+
+	bulkImpressions := make([]dtos.ImpressionsDTO, 0)
+	for testName, testImpressions := range impressionsToPost {
+		bulkImpressions = append(bulkImpressions, dtos.ImpressionsDTO{
+			TestName:       testName,
+			KeyImpressions: testImpressions,
+		})
+	}
+
+	before := time.Now()
+	err = i.impressionRecorder.Record(bulkImpressions, i.metadata, map[string]string{splitSDKImpressionsMode: i.mode})
+	if err != nil {
+		if httpError, ok := err.(*dtos.HTTPError); ok {
+			i.metricsWrapper.StoreCounters(storage.TestImpressionsCounter, string(httpError.Code))
+		}
+		return err
+	}
+	bucket := util.Bucket(time.Now().Sub(before).Nanoseconds())
+	i.metricsWrapper.StoreLatencies(storage.TestImpressionsLatency, bucket)
+	i.metricsWrapper.StoreCounters(storage.TestImpressionsCounter, "ok")
+	return nil
+}
+
+// FlushImpressions flushes impressions
+func (i *RecorderSingle) FlushImpressions(bulkSize int64) error {
+	for !i.impressionStorage.Empty() {
+		err := i.SynchronizeImpressions(bulkSize)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/impressionscount/interface.go

@@ -0,0 +1,6 @@
+package impressionscount
+
+// ImpressionsCountRecorder interface
+type ImpressionsCountRecorder interface {
+	SynchronizeImpressionsCount() error
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/impressionscount/single.go

@@ -0,0 +1,51 @@
+package impressionscount
+
+import (
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-split-commons/v2/provisional"
+	"github.com/splitio/go-split-commons/v2/service"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// RecorderSingle struct for impressionsCount sync
+type RecorderSingle struct {
+	impressionsCounter *provisional.ImpressionsCounter
+	impressionRecorder service.ImpressionsRecorder
+	metadata           dtos.Metadata
+	logger             logging.LoggerInterface
+}
+
+// NewRecorderSingle creates new impressionsCount synchronizer for posting impressionsCount
+func NewRecorderSingle(
+	impressionsCounter *provisional.ImpressionsCounter,
+	impressionRecorder service.ImpressionsRecorder,
+	metadata dtos.Metadata,
+	logger logging.LoggerInterface,
+) ImpressionsCountRecorder {
+	return &RecorderSingle{
+		impressionsCounter: impressionsCounter,
+		impressionRecorder: impressionRecorder,
+		metadata:           metadata,
+		logger:             logger,
+	}
+}
+
+// SynchronizeImpressionsCount syncs imp counts
+func (m *RecorderSingle) SynchronizeImpressionsCount() error {
+	impressionsCount := m.impressionsCounter.PopAll()
+
+	impressionsInTimeFrame := make([]dtos.ImpressionsInTimeFrameDTO, 0)
+	for key, count := range impressionsCount {
+		impressionInTimeFrame := dtos.ImpressionsInTimeFrameDTO{
+			FeatureName: key.FeatureName,
+			RawCount:    count,
+			TimeFrame:   key.TimeFrame,
+		}
+		impressionsInTimeFrame = append(impressionsInTimeFrame, impressionInTimeFrame)
+	}
+
+	pf := dtos.ImpressionsCountDTO{
+		PerFeature: impressionsInTimeFrame,
+	}
+	return m.impressionRecorder.RecordImpressionsCount(pf, m.metadata)
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/metric/interface.go

@@ -0,0 +1,6 @@
+package metric
+
+// MetricRecorder interface
+type MetricRecorder interface {
+	SynchronizeTelemetry() error
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/metric/single.go

@@ -0,0 +1,74 @@
+package metric
+
+import (
+	"errors"
+
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-split-commons/v2/service"
+	"github.com/splitio/go-split-commons/v2/storage"
+)
+
+// RecorderSingle struct for metric sync
+type RecorderSingle struct {
+	metricStorage  storage.MetricsStorageConsumer
+	metricRecorder service.MetricsRecorder
+	metadata       dtos.Metadata
+}
+
+// NewRecorderSingle creates new metric synchronizer for posting metrics
+func NewRecorderSingle(
+	metricStorage storage.MetricsStorageConsumer,
+	metricRecorder service.MetricsRecorder,
+	metadata dtos.Metadata,
+) MetricRecorder {
+	return &RecorderSingle{
+		metricStorage:  metricStorage,
+		metricRecorder: metricRecorder,
+		metadata:       metadata,
+	}
+}
+
+func (m *RecorderSingle) synchronizeLatencies() error {
+	latencies := m.metricStorage.PopLatencies()
+	if len(latencies) > 0 {
+		err := m.metricRecorder.RecordLatencies(latencies, m.metadata)
+		return err
+	}
+	return nil
+}
+
+func (m *RecorderSingle) synchronizeGauges() error {
+	var errs []error
+	for _, gauge := range m.metricStorage.PopGauges() {
+		err := m.metricRecorder.RecordGauge(gauge, m.metadata)
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if len(errs) > 0 {
+		return errors.New("Some gauges could not be posted")
+	}
+	return nil
+}
+
+func (m *RecorderSingle) synchronizeCounters() error {
+	counters := m.metricStorage.PopCounters()
+	if len(counters) > 0 {
+		err := m.metricRecorder.RecordCounters(counters, m.metadata)
+		return err
+	}
+	return nil
+}
+
+// SynchronizeTelemetry syncs telemetry
+func (m *RecorderSingle) SynchronizeTelemetry() error {
+	err := m.synchronizeGauges()
+	if err != nil {
+		return err
+	}
+	err = m.synchronizeLatencies()
+	if err != nil {
+		return err
+	}
+	return m.synchronizeCounters()
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/segment/interface.go

@@ -0,0 +1,8 @@
+package segment
+
+// SegmentFetcher interface
+type SegmentFetcher interface {
+	SynchronizeSegment(name string, till *int64) error
+	SynchronizeSegments() error
+	SegmentNames() []interface{}
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/segment/segment.go

@@ -0,0 +1,139 @@
+package segment
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-split-commons/v2/service"
+	"github.com/splitio/go-split-commons/v2/storage"
+	"github.com/splitio/go-split-commons/v2/util"
+	"github.com/splitio/go-toolkit/v3/datastructures/set"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// SegmentFetcherSimple struct for segment sync
+type SegmentFetcherSimple struct {
+	splitStorage   storage.SplitStorageConsumer
+	segmentStorage storage.SegmentStorage
+	segmentFetcher service.SegmentFetcher
+	metricsWrapper *storage.MetricWrapper
+	logger         logging.LoggerInterface
+}
+
+// NewSegmentFetcher creates new segment synchronizer for processing segment updates
+func NewSegmentFetcher(
+	splitStorage storage.SplitStorage,
+	segmentStorage storage.SegmentStorage,
+	segmentFetcher service.SegmentFetcher,
+	metricsWrapper *storage.MetricWrapper,
+	logger logging.LoggerInterface,
+) SegmentFetcher {
+	return &SegmentFetcherSimple{
+		splitStorage:   splitStorage,
+		segmentStorage: segmentStorage,
+		segmentFetcher: segmentFetcher,
+		metricsWrapper: metricsWrapper,
+		logger:         logger,
+	}
+}
+
+func (s *SegmentFetcherSimple) processUpdate(segmentChanges *dtos.SegmentChangesDTO) {
+	name := segmentChanges.Name
+	oldSegment := s.segmentStorage.Keys(name)
+	if oldSegment == nil {
+		keys := set.NewSet()
+		for _, key := range segmentChanges.Added {
+			keys.Add(key)
+		}
+		s.logger.Debug(fmt.Sprintf("Segment [%s] doesn't exist now, it will add (%d) keys", name, keys.Size()))
+		s.segmentStorage.Update(name, keys, set.NewSet(), segmentChanges.Till)
+	} else {
+		toAdd := set.NewSet()
+		toRemove := set.NewSet()
+		// Segment exists, must add new members and remove old ones
+		for _, key := range segmentChanges.Added {
+			toAdd.Add(key)
+		}
+		for _, key := range segmentChanges.Removed {
+			toRemove.Add(key)
+		}
+		if toAdd.Size() > 0 || toRemove.Size() > 0 {
+			s.logger.Debug(fmt.Sprintf("Segment [%s] exists, it will be updated. %d keys added, %d keys removed", name, toAdd.Size(), toRemove.Size()))
+			s.segmentStorage.Update(name, toAdd, toRemove, segmentChanges.Till)
+		}
+	}
+}
+
+// SynchronizeSegment syncs segment
+func (s *SegmentFetcherSimple) SynchronizeSegment(name string, till *int64) error {
+	for {
+		s.logger.Debug(fmt.Sprintf("Synchronizing segment %s", name))
+		changeNumber, _ := s.segmentStorage.ChangeNumber(name)
+		if changeNumber == 0 {
+			changeNumber = -1
+		}
+		if till != nil && *till < changeNumber {
+			return nil
+		}
+
+		before := time.Now()
+		segmentChanges, err := s.segmentFetcher.Fetch(name, changeNumber)
+		if err != nil {
+			if httpError, ok := err.(*dtos.HTTPError); ok {
+				s.metricsWrapper.StoreCounters(storage.SegmentChangesCounter, string(httpError.Code))
+			}
+			return err
+		}
+
+		s.processUpdate(segmentChanges)
+		bucket := util.Bucket(time.Now().Sub(before).Nanoseconds())
+		s.metricsWrapper.StoreLatencies(storage.SegmentChangesLatency, bucket)
+		s.metricsWrapper.StoreCounters(storage.SegmentChangesCounter, "ok")
+		if segmentChanges.Till == segmentChanges.Since || (till != nil && segmentChanges.Till >= *till) {
+			return nil
+		}
+	}
+}
+
+// SynchronizeSegments syncs segments at once
+func (s *SegmentFetcherSimple) SynchronizeSegments() error {
+	// @TODO: add delays
+	segmentNames := s.splitStorage.SegmentNames().List()
+	s.logger.Debug("Segment Sync", segmentNames)
+	wg := sync.WaitGroup{}
+	wg.Add(len(segmentNames))
+	failedSegments := set.NewThreadSafeSet()
+	for _, name := range segmentNames {
+		conv, ok := name.(string)
+		if !ok {
+			s.logger.Warning("Skipping non-string segment present in storage at initialization-time!")
+			continue
+		}
+		go func(segmentName string) {
+			defer wg.Done() // Make sure the "finished" signal is always sent
+			ready := false
+			var err error
+			for !ready {
+				err = s.SynchronizeSegment(segmentName, nil)
+				if err != nil {
+					failedSegments.Add(segmentName)
+				}
+				return
+			}
+		}(conv)
+	}
+	wg.Wait()
+
+	if failedSegments.Size() > 0 {
+		return fmt.Errorf("The following segments failed to be fetched %v", failedSegments.List())
+	}
+
+	return nil
+}
+
+// SegmentNames returns all segments
+func (s *SegmentFetcherSimple) SegmentNames() []interface{} {
+	return s.splitStorage.SegmentNames().List()
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/split/interface.go

@@ -0,0 +1,6 @@
+package split
+
+// SplitFetcher interface
+type SplitFetcher interface {
+	SynchronizeSplits(till *int64) error
+}

vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/split/split.go

@@ -0,0 +1,84 @@
+package split
+
+import (
+	"time"
+
+	"github.com/splitio/go-split-commons/v2/dtos"
+	"github.com/splitio/go-split-commons/v2/service"
+	"github.com/splitio/go-split-commons/v2/storage"
+	"github.com/splitio/go-split-commons/v2/util"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// SplitFetcherSimple struct for split sync
+type SplitFetcherSimple struct {
+	splitStorage   storage.SplitStorage
+	splitFetcher   service.SplitFetcher
+	metricsWrapper *storage.MetricWrapper
+	logger         logging.LoggerInterface
+}
+
+// NewSplitFetcher creates new split synchronizer for processing split updates
+func NewSplitFetcher(
+	splitStorage storage.SplitStorage,
+	splitFetcher service.SplitFetcher,
+	metricsWrapper *storage.MetricWrapper,
+	logger logging.LoggerInterface,
+) SplitFetcher {
+	return &SplitFetcherSimple{
+		splitStorage:   splitStorage,
+		splitFetcher:   splitFetcher,
+		metricsWrapper: metricsWrapper,
+		logger:         logger,
+	}
+}
+
+func (s *SplitFetcherSimple) processUpdate(splits *dtos.SplitChangesDTO) {
+	inactiveSplits := make([]dtos.SplitDTO, 0)
+	activeSplits := make([]dtos.SplitDTO, 0)
+	for _, split := range splits.Splits {
+		if split.Status == "ACTIVE" {
+			activeSplits = append(activeSplits, split)
+		} else {
+			inactiveSplits = append(inactiveSplits, split)
+		}
+	}
+
+	// Add/Update active splits
+	s.splitStorage.PutMany(activeSplits, splits.Till)
+
+	// Remove inactive splits
+	for _, split := range inactiveSplits {
+		s.splitStorage.Remove(split.Name)
+	}
+}
+
+// SynchronizeSplits syncs splits
+func (s *SplitFetcherSimple) SynchronizeSplits(till *int64) error {
+	// @TODO: add delays
+	for {
+		changeNumber, _ := s.splitStorage.ChangeNumber()
+		if changeNumber == 0 {
+			changeNumber = -1
+		}
+		if till != nil && *till < changeNumber {
+			return nil
+		}
+
+		before := time.Now()
+		splits, err := s.splitFetcher.Fetch(changeNumber)
+		if err != nil {
+			if httpError, ok := err.(*dtos.HTTPError); ok {
+				s.metricsWrapper.StoreCounters(storage.SplitChangesCounter, string(httpError.Code))
+			}
+			return err
+		}
+		s.processUpdate(splits)
+		bucket := util.Bucket(time.Now().Sub(before).Nanoseconds())
+		s.metricsWrapper.StoreCounters(storage.SplitChangesCounter, "ok")
+		s.metricsWrapper.StoreLatencies(storage.SplitChangesLatency, bucket)
+		if splits.Till == splits.Since || (till != nil && splits.Till >= *till) {
+			return nil
+		}
+	}
+}