MM-28859 Add feature flag managment system using split.io and remove viper. (#15954)

* Add feature flag managment system using split.io and remove viper.

* Fixing tests.

* Attempt to fix postgres tests.

* Fix watch filepath for advanced logging.

* Review fixes.

* Some error wrapping.

* Remove unessisary store interface.

* Desanitize SplitKey

* Simplify.

* Review feedback.

* Rename split mlog adatper to split logger.

* fsInner

* Style.

* Restore oldcfg test.

* Downgrading non-actionable feature flag errors to warnings.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

vendor/github.com/splitio/go-split-commons/v2/tasks/eventsync.go

@@ -0,0 +1,52 @@
+package tasks
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/event"
+	"github.com/splitio/go-toolkit/v3/asynctask"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// NewRecordEventsTask creates a new events recording task
+func NewRecordEventsTask(
+	synchronizer event.EventRecorder,
+	bulkSize int64,
+	period int,
+	logger logging.LoggerInterface,
+) Task {
+	record := func(logger logging.LoggerInterface) error {
+		return synchronizer.SynchronizeEvents(bulkSize)
+	}
+
+	onStop := func(logger logging.LoggerInterface) {
+		// All this function does is flush events which will clear the storage
+		synchronizer.FlushEvents(bulkSize)
+	}
+
+	return asynctask.NewAsyncTask("SubmitEvents", record, period, nil, onStop, logger)
+}
+
+// NewRecordEventsTasks creates a new splits fetching and storing task
+func NewRecordEventsTasks(
+	recorder event.EventRecorder,
+	bulkSize int64,
+	period int,
+	logger logging.LoggerInterface,
+	totalTasks int) Task {
+	record := func(logger logging.LoggerInterface) error {
+		return recorder.SynchronizeEvents(bulkSize)
+	}
+
+	tasks := make([]Task, 0, totalTasks)
+	for i := 0; i < totalTasks; i++ {
+		logger.Info(fmt.Sprintf("Creating SubmitEvents_%d", i))
+		tasks = append(tasks, asynctask.NewAsyncTask(fmt.Sprintf("SubmitEvents_%d", i), record, period, nil, nil, logger))
+	}
+	return MultipleTask{
+		logger: logger,
+		tasks:  tasks,
+		wg:     &sync.WaitGroup{},
+	}
+}

vendor/github.com/splitio/go-split-commons/v2/tasks/impressionscountsync.go

@@ -0,0 +1,27 @@
+package tasks
+
+import (
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/impressionscount"
+	"github.com/splitio/go-toolkit/v3/asynctask"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+const (
+	period = 1800 // 30 min
+)
+
+// NewRecordImpressionsCountTask creates a new impressionsCount recording task
+func NewRecordImpressionsCountTask(
+	recorder impressionscount.ImpressionsCountRecorder,
+	logger logging.LoggerInterface,
+) *asynctask.AsyncTask {
+	record := func(logger logging.LoggerInterface) error {
+		return recorder.SynchronizeImpressionsCount()
+	}
+
+	onStop := func(logger logging.LoggerInterface) {
+		recorder.SynchronizeImpressionsCount()
+	}
+
+	return asynctask.NewAsyncTask("SubmitImpressionsCount", record, period, nil, onStop, logger)
+}

vendor/github.com/splitio/go-split-commons/v2/tasks/impressionsync.go

@@ -0,0 +1,52 @@
+package tasks
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/impression"
+	"github.com/splitio/go-toolkit/v3/asynctask"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// NewRecordImpressionsTask creates a new splits fetching and storing task
+func NewRecordImpressionsTask(
+	recorder impression.ImpressionRecorder,
+	period int,
+	logger logging.LoggerInterface,
+	bulkSize int64,
+) Task {
+	record := func(logger logging.LoggerInterface) error {
+		return recorder.SynchronizeImpressions(bulkSize)
+	}
+
+	onStop := func(logger logging.LoggerInterface) {
+		// All this function does is flush impressions which will clear the storage
+		recorder.FlushImpressions(bulkSize)
+	}
+
+	return asynctask.NewAsyncTask("SubmitImpressions", record, period, nil, onStop, logger)
+}
+
+// NewRecordImpressionsTasks creates a new splits fetching and storing task
+func NewRecordImpressionsTasks(
+	recorder impression.ImpressionRecorder,
+	period int,
+	logger logging.LoggerInterface,
+	bulkSize int64,
+	totalTasks int) Task {
+	record := func(logger logging.LoggerInterface) error {
+		return recorder.SynchronizeImpressions(bulkSize)
+	}
+
+	tasks := make([]Task, 0, totalTasks)
+	for i := 0; i < totalTasks; i++ {
+		logger.Info(fmt.Sprintf("Creating SubmitImpressions_%d", i))
+		tasks = append(tasks, asynctask.NewAsyncTask(fmt.Sprintf("SubmitImpressions_%d", i), record, period, nil, nil, logger))
+	}
+	return MultipleTask{
+		logger: logger,
+		tasks:  tasks,
+		wg:     &sync.WaitGroup{},
+	}
+}

vendor/github.com/splitio/go-split-commons/v2/tasks/interface.go

@@ -0,0 +1,53 @@
+package tasks
+
+import (
+	"sync"
+
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// Task interface
+type Task interface {
+	Start()
+	Stop(blocking bool) error
+	IsRunning() bool
+}
+
+// MultipleTask struct
+type MultipleTask struct {
+	tasks  []Task
+	logger logging.LoggerInterface
+	wg     *sync.WaitGroup
+}
+
+// IsRunning method
+func (m MultipleTask) IsRunning() bool {
+	for _, t := range m.tasks {
+		if t.IsRunning() {
+			return true
+		}
+	}
+	return false
+}
+
+// Start method
+func (m MultipleTask) Start() {
+	for _, t := range m.tasks {
+		m.wg.Add(1)
+		t.Start()
+	}
+}
+
+// Stop method
+func (m MultipleTask) Stop(blocking bool) error {
+	for _, t := range m.tasks {
+		go func(t Task) {
+			t.Stop(blocking)
+			m.wg.Done()
+		}(t)
+	}
+	if blocking {
+		m.wg.Wait()
+	}
+	return nil
+}

vendor/github.com/splitio/go-split-commons/v2/tasks/metricsync.go

@@ -0,0 +1,23 @@
+package tasks
+
+import (
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/metric"
+	"github.com/splitio/go-toolkit/v3/asynctask"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// NewRecordTelemetryTask creates a new telemtry recording task
+func NewRecordTelemetryTask(
+	recorder metric.MetricRecorder,
+	period int,
+	logger logging.LoggerInterface,
+) *asynctask.AsyncTask {
+	record := func(logger logging.LoggerInterface) error {
+		return recorder.SynchronizeTelemetry()
+	}
+
+	onStop := func(l logging.LoggerInterface) {
+		record(logger)
+	}
+	return asynctask.NewAsyncTask("SubmitTelemetry", record, period, nil, onStop, logger)
+}

vendor/github.com/splitio/go-split-commons/v2/tasks/segmentsync.go

@@ -0,0 +1,81 @@
+package tasks
+
+import (
+	"errors"
+	"fmt"
+	"sync/atomic"
+
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/segment"
+	"github.com/splitio/go-toolkit/v3/asynctask"
+	"github.com/splitio/go-toolkit/v3/logging"
+	"github.com/splitio/go-toolkit/v3/workerpool"
+)
+
+func updateSegments(
+	fetcher segment.SegmentFetcher,
+	admin *workerpool.WorkerAdmin,
+	logger logging.LoggerInterface,
+) error {
+	segmentList := fetcher.SegmentNames()
+	for _, name := range segmentList {
+		ok := admin.QueueMessage(name)
+		if !ok {
+			logger.Error(
+				fmt.Sprintf("Segment %s could not be added because the job queue is full.\n", name),
+				fmt.Sprintf(
+					"You currently have %d segments and the queue size is %d.\n",
+					len(segmentList),
+					admin.QueueSize(),
+				),
+				"Please consider updating the segment queue size accordingly in the configuration options",
+			)
+		}
+	}
+	return nil
+}
+
+// NewFetchSegmentsTask creates a new segment fetching and storing task
+func NewFetchSegmentsTask(
+	fetcher segment.SegmentFetcher,
+	period int,
+	workerCount int,
+	queueSize int,
+	logger logging.LoggerInterface,
+) *asynctask.AsyncTask {
+
+	admin := atomic.Value{}
+
+	// After all segments are in sync, add workers to the pool that will keep them up to date
+	// periodically
+	onInit := func(logger logging.LoggerInterface) error {
+		admin.Store(workerpool.NewWorkerAdmin(queueSize, logger))
+		for i := 0; i < workerCount; i++ {
+			worker := NewSegmentWorker(
+				fmt.Sprintf("SegmentWorker_%d", i),
+				0,
+				fetcher.SynchronizeSegment,
+			)
+			admin.Load().(*workerpool.WorkerAdmin).AddWorker(worker)
+		}
+		return nil
+	}
+
+	update := func(logger logging.LoggerInterface) error {
+		wa, ok := admin.Load().(*workerpool.WorkerAdmin)
+		if !ok || wa == nil {
+			return errors.New("unable to type-assert worker manager")
+		}
+		return updateSegments(fetcher, wa, logger)
+	}
+
+	cleanup := func(logger logging.LoggerInterface) {
+		wa, ok := admin.Load().(*workerpool.WorkerAdmin)
+		if !ok || wa == nil {
+			logger.Error("unable to type-assert worker manager")
+			return
+		}
+		wa.StopAll(true)
+	}
+
+	return asynctask.NewAsyncTask("UpdateSegments", update, period, onInit, cleanup, logger)
+}

vendor/github.com/splitio/go-split-commons/v2/tasks/splitsync.go

@@ -0,0 +1,20 @@
+package tasks
+
+import (
+	"github.com/splitio/go-split-commons/v2/synchronizer/worker/split"
+	"github.com/splitio/go-toolkit/v3/asynctask"
+	"github.com/splitio/go-toolkit/v3/logging"
+)
+
+// NewFetchSplitsTask creates a new splits fetching and storing task
+func NewFetchSplitsTask(
+	fetcher split.SplitFetcher,
+	period int,
+	logger logging.LoggerInterface,
+) *asynctask.AsyncTask {
+	update := func(logger logging.LoggerInterface) error {
+		return fetcher.SynchronizeSplits(nil)
+	}
+
+	return asynctask.NewAsyncTask("UpdateSplits", update, period, nil, nil, logger)
+}

vendor/github.com/splitio/go-split-commons/v2/tasks/worker.go

@@ -0,0 +1,47 @@
+package tasks
+
+import (
+	"errors"
+)
+
+// SegmentWorker struct contains resources and functions for fetching segments and storing them
+type SegmentWorker struct {
+	name        string
+	failureTime int64
+	toExecute   func(name string, till *int64) error
+}
+
+// NewSegmentWorker some
+func NewSegmentWorker(name string, failureTime int64, toExecute func(name string, till *int64) error) *SegmentWorker {
+	return &SegmentWorker{
+		name:        name,
+		failureTime: failureTime,
+		toExecute:   toExecute,
+	}
+}
+
+// Name Returns the name of the worker
+func (w *SegmentWorker) Name() string {
+	return w.name
+}
+
+// FailureTime Returns how much time should be waited after an error, before the worker resumes execution
+func (w *SegmentWorker) FailureTime() int64 {
+	return w.failureTime
+}
+
+// DoWork performs the actual work and returns an error if something goes wrong
+func (w *SegmentWorker) DoWork(msg interface{}) error {
+	segmentName, ok := msg.(string)
+	if !ok {
+		return errors.New("segment name popped from queue is not a string")
+	}
+
+	return w.toExecute(segmentName, nil)
+}
+
+// OnError callback does nothing
+func (w *SegmentWorker) OnError(e error) {}
+
+// Cleanup callback does nothing
+func (w *SegmentWorker) Cleanup() error { return nil }