https://mattermost.atlassian.net/browse/MM-34932

```release-note
NONE
```
Этот коммит содержится в:
Agniva De Sarker
2021-06-03 12:59:05 +05:30
коммит произвёл GitHub
родитель fff09bf210
Коммит 3299a72adb
776 изменённых файлов: 40081 добавлений и 44702 удалений

44
vendor/github.com/splitio/go-split-commons/v3/storage/inmemory/helpers.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,44 @@
package inmemory
import (
"errors"
"fmt"
"sync/atomic"
)
// ErrorOutOfBounds err
var ErrorOutOfBounds error = errors.New("out of bounds")
// AtomicInt64Slice var
type AtomicInt64Slice []int64
// NewAtomicInt64Slice create slice
func NewAtomicInt64Slice(size int64) (AtomicInt64Slice, error) {
if size <= 0 {
return nil, fmt.Errorf("invalid array size: %d", size)
}
return make([]int64, size), nil
}
// Incr increments inx count
func (a AtomicInt64Slice) Incr(index int) {
atomic.AddInt64(&a[index], 1)
}
// FetchAndClearOne returns previous and reset
func (a AtomicInt64Slice) FetchAndClearOne(index int) (int64, error) {
if index >= len(a) || index < 0 {
return 0, ErrorOutOfBounds
}
return atomic.SwapInt64(&a[index], 0), nil
}
// FetchAndClearAll returns all and reset
func (a AtomicInt64Slice) FetchAndClearAll() []int64 {
toRet := make([]int64, len(a))
for index := 0; index < len(a); index++ {
toRet[index] = atomic.SwapInt64(&a[index], 0)
}
return toRet
}

Просмотреть файл

@@ -86,3 +86,14 @@ func (m *MMSegmentStorage) Update(name string, toAdd *set.ThreadUnsafeSet, toRem
// CountRemovedKeys method
func (m *MMSegmentStorage) CountRemovedKeys(segmentName string) int64 { return 0 }
// SegmentKeysCount
func (m *MMSegmentStorage) SegmentKeysCount() int64 {
m.mutex.RLock()
defer m.mutex.RUnlock()
var toReturn int64 = 0
for _, keys := range m.data {
toReturn = toReturn + int64(keys.Size())
}
return toReturn
}

Просмотреть файл

Просмотреть файл

@@ -6,6 +6,8 @@ import (
"sync"
"github.com/splitio/go-split-commons/v3/dtos"
"github.com/splitio/go-split-commons/v3/storage"
"github.com/splitio/go-split-commons/v3/telemetry"
"github.com/splitio/go-toolkit/v4/logging"
)
@@ -13,13 +15,14 @@ import (
const MaxAccumulatedBytes = 5 * 1024 * 1024
// NewMQEventsStorage returns an instance of MQEventsStorage
func NewMQEventsStorage(queueSize int, isFull chan string, logger logging.LoggerInterface) *MQEventsStorage {
func NewMQEventsStorage(queueSize int, isFull chan string, logger logging.LoggerInterface, runtimeTelemetry storage.TelemetryRuntimeProducer) *MQEventsStorage {
return &MQEventsStorage{
queue: list.New(),
size: queueSize,
mutexQueue: &sync.Mutex{},
fullChan: isFull,
logger: logger,
queue: list.New(),
size: queueSize,
mutexQueue: &sync.Mutex{},
fullChan: isFull,
logger: logger,
runtimeTelemetry: runtimeTelemetry,
}
}
@@ -36,6 +39,7 @@ type MQEventsStorage struct {
mutexQueue *sync.Mutex
fullChan chan string //only write channel
logger logging.LoggerInterface
runtimeTelemetry storage.TelemetryRuntimeProducer
}
func (s *MQEventsStorage) sendSignalIsFull() {
@@ -55,12 +59,14 @@ func (s *MQEventsStorage) Push(event dtos.EventDTO, size int) error {
defer s.mutexQueue.Unlock()
if s.queue.Len()+1 > s.size {
s.runtimeTelemetry.RecordEventsStats(telemetry.EventsDropped, 1)
s.sendSignalIsFull()
return ErrorMaxSizeReached
}
// Add element
s.queue.PushBack(eventWrapper{event: event, size: size})
s.runtimeTelemetry.RecordEventsStats(telemetry.EventsQueued, 1)
s.accumulatedBytes += size
if s.queue.Len() == s.size || s.accumulatedBytes >= MaxAccumulatedBytes {
s.sendSignalIsFull()

Просмотреть файл

@@ -5,27 +5,31 @@ import (
"sync"
"github.com/splitio/go-split-commons/v3/dtos"
"github.com/splitio/go-split-commons/v3/storage"
"github.com/splitio/go-split-commons/v3/telemetry"
"github.com/splitio/go-toolkit/v4/logging"
)
// NewMQImpressionsStorage returns an instance of MQEventsStorage
func NewMQImpressionsStorage(queueSize int, isFull chan<- string, logger logging.LoggerInterface) *MQImpressionsStorage {
func NewMQImpressionsStorage(queueSize int, isFull chan<- string, logger logging.LoggerInterface, runtimeTelemetry storage.TelemetryRuntimeProducer) *MQImpressionsStorage {
return &MQImpressionsStorage{
queue: list.New(),
size: queueSize,
mutexQueue: &sync.Mutex{},
fullChan: isFull,
logger: logger,
queue: list.New(),
size: queueSize,
mutexQueue: &sync.Mutex{},
fullChan: isFull,
logger: logger,
runtimeTelemetry: runtimeTelemetry,
}
}
// MQImpressionsStorage in memory events storage
type MQImpressionsStorage struct {
queue *list.List
size int
mutexQueue *sync.Mutex
fullChan chan<- string //only write channel
logger logging.LoggerInterface
queue *list.List
size int
mutexQueue *sync.Mutex
fullChan chan<- string //only write channel
logger logging.LoggerInterface
runtimeTelemetry storage.TelemetryRuntimeProducer
}
func (s *MQImpressionsStorage) sendSignalIsFull() {
@@ -59,13 +63,18 @@ func (s *MQImpressionsStorage) LogImpressions(impressions []dtos.Impression) err
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
impressionsToAdd := len(impressions)
for _, impression := range impressions {
if s.queue.Len()+1 > s.size {
s.sendSignalIsFull()
s.runtimeTelemetry.RecordImpressionsStats(telemetry.ImpressionsDropped, int64(impressionsToAdd))
return ErrorMaxSizeReached
}
// Add element
s.queue.PushBack(impression)
s.runtimeTelemetry.RecordImpressionsStats(telemetry.ImpressionsQueued, 1)
impressionsToAdd--
if s.queue.Len() == s.size {
s.sendSignalIsFull()

480
vendor/github.com/splitio/go-split-commons/v3/storage/inmemory/telemetry.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,480 @@
package inmemory
import (
"fmt"
"sync"
"sync/atomic"
"github.com/splitio/go-split-commons/v3/dtos"
"github.com/splitio/go-split-commons/v3/storage"
constants "github.com/splitio/go-split-commons/v3/telemetry"
)
type latencies struct {
// MethodLatencies
treatment AtomicInt64Slice
treatments AtomicInt64Slice
treatmentWithConfig AtomicInt64Slice
treatmentsWithConfig AtomicInt64Slice
track AtomicInt64Slice
// HTTPLatencies
splits AtomicInt64Slice
segments AtomicInt64Slice
impressions AtomicInt64Slice
impressionsCount AtomicInt64Slice
events AtomicInt64Slice
telemetry AtomicInt64Slice
token AtomicInt64Slice
}
type counters struct {
// Evaluation Counters
treatment int64
treatments int64
treatmentWithConfig int64
treatmentsWithConfig int64
track int64
// Push Counters
authRejections int64
tokenRefreshes int64
// Factory Counters
burTimeouts int64
nonReadyUsages int64
}
type records struct {
// Impressions Data
impressionsQueued int64
impressionsDropped int64
impressionsDeduped int64
// Events Data
eventsQueued int64
eventsDropped int64
// LastSynchronization
splits int64
segments int64
impressions int64
impressionsCount int64
events int64
token int64
telemetry int64
// SDK
session int64
}
// TelemetryStorage In Memory Telemetry Storage struct
type TelemetryStorage struct {
counters counters
httpErrors dtos.HTTPErrors
mutexHTTPErrors sync.RWMutex
latencies latencies
records records
streamingEvents []dtos.StreamingEvent // Max Length 20
mutexStreamingEvents sync.RWMutex
tags []string
mutexTags sync.RWMutex
}
// NewTelemetryStorage builds in memory telemetry storage
func NewTelemetryStorage() (storage.TelemetryStorage, error) {
treatmentLatencies, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
treatmentWithConfigLatencies, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
treatmentsLatencies, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
treatmentsWithConfigLatencies, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
track, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
splits, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
segments, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
impressions, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
impressionsCount, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
events, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
telemetry, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
token, err := NewAtomicInt64Slice(constants.LatencyBucketCount)
if err != nil {
return nil, fmt.Errorf("could not create InMemory Storage, %w", err)
}
return &TelemetryStorage{
counters: counters{},
httpErrors: dtos.HTTPErrors{
Splits: make(map[int]int64),
Segments: make(map[int]int64),
Impressions: make(map[int]int64),
ImpressionsCount: make(map[int]int64),
Events: make(map[int]int64),
Token: make(map[int]int64),
Telemetry: make(map[int]int64),
},
mutexHTTPErrors: sync.RWMutex{},
latencies: latencies{
treatment: treatmentLatencies,
treatmentWithConfig: treatmentWithConfigLatencies,
treatments: treatmentsLatencies,
treatmentsWithConfig: treatmentsWithConfigLatencies,
track: track,
splits: splits,
segments: segments,
impressions: impressions,
impressionsCount: impressionsCount,
events: events,
token: token,
telemetry: telemetry,
},
records: records{},
streamingEvents: make([]dtos.StreamingEvent, 0, constants.MaxStreamingEvents),
mutexStreamingEvents: sync.RWMutex{},
tags: make([]string, 0, constants.MaxTags),
mutexTags: sync.RWMutex{},
}, nil
}
// TELEMETRY STORAGE PRODUCER
func (i *TelemetryStorage) RecordConfigData(configData dtos.Config) error {
// No-Op. Config Data will be sent directly to Split Servers. No need to store.
return nil
}
// RecordLatency stores latency for method
func (i *TelemetryStorage) RecordLatency(method string, latency int64) {
bucket := constants.Bucket(latency)
switch method {
case constants.Treatment:
i.latencies.treatment.Incr(bucket)
case constants.Treatments:
i.latencies.treatments.Incr(bucket)
case constants.TreatmentWithConfig:
i.latencies.treatmentWithConfig.Incr(bucket)
case constants.TreatmentsWithConfig:
i.latencies.treatmentsWithConfig.Incr(bucket)
case constants.Track:
i.latencies.track.Incr(bucket)
}
}
// RecordException stores exceptions for method
func (i *TelemetryStorage) RecordException(method string) {
switch method {
case constants.Treatment:
atomic.AddInt64(&i.counters.treatment, 1)
case constants.Treatments:
atomic.AddInt64(&i.counters.treatments, 1)
case constants.TreatmentWithConfig:
atomic.AddInt64(&i.counters.treatmentWithConfig, 1)
case constants.TreatmentsWithConfig:
atomic.AddInt64(&i.counters.treatmentsWithConfig, 1)
case constants.Track:
atomic.AddInt64(&i.counters.track, 1)
}
}
// RecordImpressionsStats records impressions by type
func (i *TelemetryStorage) RecordImpressionsStats(dataType int, count int64) {
switch dataType {
case constants.ImpressionsDropped:
atomic.AddInt64(&i.records.impressionsDropped, count)
case constants.ImpressionsDeduped:
atomic.AddInt64(&i.records.impressionsDeduped, count)
case constants.ImpressionsQueued:
atomic.AddInt64(&i.records.impressionsQueued, count)
}
}
// RecordEventsStats recirds events by type
func (i *TelemetryStorage) RecordEventsStats(dataType int, count int64) {
switch dataType {
case constants.EventsDropped:
atomic.AddInt64(&i.records.eventsDropped, count)
case constants.EventsQueued:
atomic.AddInt64(&i.records.eventsQueued, count)
}
}
// RecordSuccessfulSync records sync for resource
func (i *TelemetryStorage) RecordSuccessfulSync(resource int, timestamp int64) {
switch resource {
case constants.SplitSync:
atomic.StoreInt64(&i.records.splits, timestamp)
case constants.SegmentSync:
atomic.StoreInt64(&i.records.segments, timestamp)
case constants.ImpressionSync:
atomic.StoreInt64(&i.records.impressions, timestamp)
case constants.ImpressionCountSync:
atomic.StoreInt64(&i.records.impressionsCount, timestamp)
case constants.EventSync:
atomic.StoreInt64(&i.records.events, timestamp)
case constants.TelemetrySync:
atomic.StoreInt64(&i.records.telemetry, timestamp)
case constants.TokenSync:
atomic.StoreInt64(&i.records.token, timestamp)
}
}
func (i *TelemetryStorage) createOrUpdate(status int, item map[int]int64) {
if item == nil {
item[status] = 1
return
}
item[status]++
}
// RecordSyncError records http error
func (i *TelemetryStorage) RecordSyncError(resource int, status int) {
i.mutexHTTPErrors.Lock()
defer i.mutexHTTPErrors.Unlock()
switch resource {
case constants.SplitSync:
i.createOrUpdate(status, i.httpErrors.Splits)
case constants.SegmentSync:
i.createOrUpdate(status, i.httpErrors.Segments)
case constants.ImpressionSync:
i.createOrUpdate(status, i.httpErrors.Impressions)
case constants.ImpressionCountSync:
i.createOrUpdate(status, i.httpErrors.ImpressionsCount)
case constants.EventSync:
i.createOrUpdate(status, i.httpErrors.Events)
case constants.TelemetrySync:
i.createOrUpdate(status, i.httpErrors.Telemetry)
case constants.TokenSync:
i.createOrUpdate(status, i.httpErrors.Token)
}
}
// RecordSyncLatency records http error
func (i *TelemetryStorage) RecordSyncLatency(resource int, latency int64) {
bucket := constants.Bucket(latency)
switch resource {
case constants.SplitSync:
i.latencies.splits.Incr(bucket)
case constants.SegmentSync:
i.latencies.segments.Incr(bucket)
case constants.ImpressionSync:
i.latencies.impressions.Incr(bucket)
case constants.ImpressionCountSync:
i.latencies.impressionsCount.Incr(bucket)
case constants.EventSync:
i.latencies.events.Incr(bucket)
case constants.TelemetrySync:
i.latencies.telemetry.Incr(bucket)
case constants.TokenSync:
i.latencies.token.Incr(bucket)
}
}
// RecordAuthRejections records auth rejections
func (i *TelemetryStorage) RecordAuthRejections() {
atomic.AddInt64(&i.counters.authRejections, 1)
}
// RecordTokenRefreshes records token
func (i *TelemetryStorage) RecordTokenRefreshes() {
atomic.AddInt64(&i.counters.tokenRefreshes, 1)
}
// RecordStreamingEvent appends new streaming event
func (i *TelemetryStorage) RecordStreamingEvent(event *dtos.StreamingEvent) {
if event == nil {
return
}
i.mutexStreamingEvents.Lock()
defer i.mutexStreamingEvents.Unlock()
if len(i.streamingEvents) < constants.MaxStreamingEvents {
i.streamingEvents = append(i.streamingEvents, *event)
}
}
// AddTag adds particular tag
func (i *TelemetryStorage) AddTag(tag string) {
i.mutexTags.Lock()
defer i.mutexTags.Unlock()
if len(i.tags) < constants.MaxTags {
i.tags = append(i.tags, tag)
}
}
// RecordSessionLength records session length
func (i *TelemetryStorage) RecordSessionLength(session int64) {
atomic.StoreInt64(&i.records.session, session)
}
// RecordNonReadyUsage records non ready usage
func (i *TelemetryStorage) RecordNonReadyUsage() {
atomic.AddInt64(&i.counters.nonReadyUsages, 1)
}
// RecordBURTimeout records bur timeodout
func (i *TelemetryStorage) RecordBURTimeout() {
atomic.AddInt64(&i.counters.burTimeouts, 1)
}
// TELEMETRY STORAGE CONSUMER
// PopLatencies gets and clears method latencies
func (i *TelemetryStorage) PopLatencies() dtos.MethodLatencies {
return dtos.MethodLatencies{
Treatment: i.latencies.treatment.FetchAndClearAll(),
Treatments: i.latencies.treatments.FetchAndClearAll(),
TreatmentWithConfig: i.latencies.treatmentWithConfig.FetchAndClearAll(),
TreatmentsWithConfig: i.latencies.treatmentsWithConfig.FetchAndClearAll(),
Track: i.latencies.track.FetchAndClearAll(),
}
}
// PopExceptions gets and clears method exceptions
func (i *TelemetryStorage) PopExceptions() dtos.MethodExceptions {
return dtos.MethodExceptions{
Treatment: atomic.SwapInt64(&i.counters.treatment, 0),
Treatments: atomic.SwapInt64(&i.counters.treatments, 0),
TreatmentWithConfig: atomic.SwapInt64(&i.counters.treatmentWithConfig, 0),
TreatmentsWithConfig: atomic.SwapInt64(&i.counters.treatmentsWithConfig, 0),
Track: atomic.SwapInt64(&i.counters.track, 0),
}
}
// GetImpressionsStats gets impressions by type
func (i *TelemetryStorage) GetImpressionsStats(dataType int) int64 {
switch dataType {
case constants.ImpressionsDropped:
return atomic.LoadInt64(&i.records.impressionsDropped)
case constants.ImpressionsDeduped:
return atomic.LoadInt64(&i.records.impressionsDeduped)
case constants.ImpressionsQueued:
return atomic.LoadInt64(&i.records.impressionsQueued)
}
return 0
}
// GetEventsStats gets events by type
func (i *TelemetryStorage) GetEventsStats(dataType int) int64 {
switch dataType {
case constants.EventsDropped:
return atomic.LoadInt64(&i.records.eventsDropped)
case constants.EventsQueued:
return atomic.LoadInt64(&i.records.eventsQueued)
}
return 0
}
// GetLastSynchronization gets last synchronization stats for fetchers and recorders
func (i *TelemetryStorage) GetLastSynchronization() dtos.LastSynchronization {
return dtos.LastSynchronization{
Splits: atomic.LoadInt64(&i.records.splits),
Segments: atomic.LoadInt64(&i.records.segments),
Impressions: atomic.LoadInt64(&i.records.impressions),
ImpressionsCount: atomic.LoadInt64(&i.records.impressionsCount),
Events: atomic.LoadInt64(&i.records.events),
Telemetry: atomic.LoadInt64(&i.records.telemetry),
Token: atomic.LoadInt64(&i.records.token),
}
}
// PopHTTPErrors gets http errors
func (i *TelemetryStorage) PopHTTPErrors() dtos.HTTPErrors {
i.mutexHTTPErrors.Lock()
defer i.mutexHTTPErrors.Unlock()
toReturn := i.httpErrors
i.httpErrors.Splits = make(map[int]int64)
i.httpErrors.Segments = make(map[int]int64)
i.httpErrors.Impressions = make(map[int]int64)
i.httpErrors.ImpressionsCount = make(map[int]int64)
i.httpErrors.Events = make(map[int]int64)
i.httpErrors.Telemetry = make(map[int]int64)
i.httpErrors.Token = make(map[int]int64)
return toReturn
}
// PopHTTPLatencies gets http latencies
func (i *TelemetryStorage) PopHTTPLatencies() dtos.HTTPLatencies {
return dtos.HTTPLatencies{
Splits: i.latencies.splits.FetchAndClearAll(),
Segments: i.latencies.segments.FetchAndClearAll(),
Impressions: i.latencies.impressions.FetchAndClearAll(),
ImpressionsCount: i.latencies.impressionsCount.FetchAndClearAll(),
Events: i.latencies.events.FetchAndClearAll(),
Telemetry: i.latencies.telemetry.FetchAndClearAll(),
Token: i.latencies.token.FetchAndClearAll(),
}
}
// PopAuthRejections gets total amount of auth rejections
func (i *TelemetryStorage) PopAuthRejections() int64 {
return atomic.SwapInt64(&i.counters.authRejections, 0)
}
// PopTokenRefreshes gets total amount of token refreshes
func (i *TelemetryStorage) PopTokenRefreshes() int64 {
return atomic.SwapInt64(&i.counters.tokenRefreshes, 0)
}
// PopStreamingEvents gets streamingEvents data
func (i *TelemetryStorage) PopStreamingEvents() []dtos.StreamingEvent {
i.mutexStreamingEvents.Lock()
defer i.mutexStreamingEvents.Unlock()
toReturn := i.streamingEvents
i.streamingEvents = make([]dtos.StreamingEvent, 0, constants.MaxStreamingEvents)
return toReturn
}
// PopTags gets total amount of tags
func (i *TelemetryStorage) PopTags() []string {
i.mutexTags.Lock()
defer i.mutexTags.Unlock()
toReturn := i.tags
i.tags = make([]string, 0, constants.MaxTags)
return toReturn
}
// GetSessionLength gets session duration
func (i *TelemetryStorage) GetSessionLength() int64 {
return atomic.LoadInt64(&i.records.session)
}
// GetNonReadyUsages gets non usages on ready
func (i *TelemetryStorage) GetNonReadyUsages() int64 {
return atomic.LoadInt64(&i.counters.nonReadyUsages)
}
// GetBURTimeouts gets timedouts data
func (i *TelemetryStorage) GetBURTimeouts() int64 {
return atomic.LoadInt64(&i.counters.burTimeouts)
}

105
vendor/github.com/splitio/go-split-commons/v3/storage/interfaces.go сгенерированный поставляемый
Просмотреть файл

@@ -36,6 +36,7 @@ type SegmentStorageConsumer interface {
CountRemovedKeys(segmentName string) int64
Keys(segmentName string) *set.ThreadUnsafeSet
SegmentContainsKey(segmentName string, key string) (bool, error)
SegmentKeysCount() int64
}
// ImpressionStorageProducer interface should be impemented by structs that accept incoming impressions
@@ -52,25 +53,6 @@ type ImpressionStorageConsumer interface {
PopNWithMetadata(n int64) ([]dtos.ImpressionQueueObject, error)
}
// MetricsStorageProducer interface should be impemented by structs that accept incoming metrics
type MetricsStorageProducer interface {
PutGauge(key string, gauge float64)
IncLatency(metricName string, index int)
IncCounter(key string)
}
// MetricsStorageConsumer interface should be implemented by structs that offer popping metrics
type MetricsStorageConsumer interface {
PeekCounters() map[string]int64
PeekLatencies() map[string][]int64
PopGauges() []dtos.GaugeDTO
PopLatencies() []dtos.LatenciesDTO
PopCounters() []dtos.CounterDTO
PopGaugesWithMetadata() (*dtos.GaugeDataBulk, error)
PopLatenciesWithMetadata() (*dtos.LatencyDataBulk, error)
PopCountersWithMetadata() (*dtos.CounterDataBulk, error)
}
// EventStorageProducer interface should be implemented by structs that accept incoming events
type EventStorageProducer interface {
Push(event dtos.EventDTO, size int) error
@@ -85,6 +67,79 @@ type EventStorageConsumer interface {
PopNWithMetadata(n int64) ([]dtos.QueueStoredEventDTO, error)
}
// TelemetryStorageProducer interface should be implemented by struct that accepts incoming telemetry
type TelemetryStorageProducer interface {
TelemetryConfigProducer
TelemetryEvaluationProducer
TelemetryRuntimeProducer
}
// TelemetryRedisProducer interface redis
type TelemetryRedisProducer interface {
TelemetryConfigProducer
TelemetryEvaluationProducer
}
// TelemetryConfigProducer interface for config data
type TelemetryConfigProducer interface {
RecordConfigData(configData dtos.Config) error
RecordNonReadyUsage()
RecordBURTimeout()
}
// TelemetryEvaluationProducer for evaluation
type TelemetryEvaluationProducer interface {
RecordLatency(method string, latency int64)
RecordException(method string)
}
// TelemetryRuntimeProducer for runtime stats
type TelemetryRuntimeProducer interface {
AddTag(tag string)
RecordImpressionsStats(dataType int, count int64)
RecordEventsStats(dataType int, count int64)
RecordSuccessfulSync(resource int, time int64)
RecordSyncError(resource int, status int)
RecordSyncLatency(resource int, latency int64)
RecordAuthRejections()
RecordTokenRefreshes()
RecordStreamingEvent(streamingEvent *dtos.StreamingEvent)
RecordSessionLength(session int64)
}
// TelemetryStorageConsumer interface should be implemented by structs that offer popping telemetry
type TelemetryStorageConsumer interface {
TelemetryConfigConsumer
TelemetryEvaluationConsumer
TelemetryRuntimeConsumer
}
// TelemetryConfigConsumer interface for config data
type TelemetryConfigConsumer interface {
GetNonReadyUsages() int64
GetBURTimeouts() int64
}
// TelemetryEvaluationConsumer for evaluation
type TelemetryEvaluationConsumer interface {
PopLatencies() dtos.MethodLatencies
PopExceptions() dtos.MethodExceptions
}
// TelemetryRuntimeConsumer for runtime stats
type TelemetryRuntimeConsumer interface {
GetImpressionsStats(dataType int) int64
GetEventsStats(dataType int) int64
GetLastSynchronization() dtos.LastSynchronization
PopHTTPErrors() dtos.HTTPErrors
PopHTTPLatencies() dtos.HTTPLatencies
PopAuthRejections() int64
PopTokenRefreshes() int64
PopStreamingEvents() []dtos.StreamingEvent
PopTags() []string
GetSessionLength() int64
}
// --- Wide Interfaces
// SplitStorage wraps consumer & producer interfaces
@@ -105,14 +160,14 @@ type ImpressionStorage interface {
ImpressionStorageProducer
}
// MetricsStorage wraps consumer and producer interfaces
type MetricsStorage interface {
MetricsStorageConsumer
MetricsStorageProducer
}
// EventsStorage wraps consumer and producer interfaces
type EventsStorage interface {
EventStorageConsumer
EventStorageProducer
}
// TelemetryStorage wraps consumer and producer interfaces
type TelemetryStorage interface {
TelemetryStorageConsumer
TelemetryStorageProducer
}

125
vendor/github.com/splitio/go-split-commons/v3/storage/metricWrapper.go сгенерированный поставляемый
Просмотреть файл

@@ -1,125 +0,0 @@
package storage
import (
"errors"
"strings"
"github.com/splitio/go-toolkit/v4/logging"
)
// MetricWrapper struct
type MetricWrapper struct {
Telemetry MetricsStorage
LocalTelemetry MetricsStorage
logger logging.LoggerInterface
}
const (
// SplitChangesCounter counters
SplitChangesCounter = iota
// SplitChangesLatency latencies
SplitChangesLatency
// SegmentChangesCounter counters
SegmentChangesCounter
// SegmentChangesLatency latencies
SegmentChangesLatency
// TestImpressionsCounter counter
TestImpressionsCounter
// TestImpressionsLatency latencies
TestImpressionsLatency
// PostEventsCounter counter
PostEventsCounter
//PostEventsLatency latencies
PostEventsLatency
// MySegmentsCounter counters
MySegmentsCounter
// MySegmentsLatency latencies
MySegmentsLatency
)
const (
counter = "backend::request.{status}"
splitChangesCounter = "splitChangeFetcher.status.{status}"
splitChangesLatency = "splitChangeFetcher.time"
localSplitChangesLatency = "backend::/api/splitChanges"
segmentChangesCounter = "segmentChangeFetcher.status.{status}"
segmentChangesLatency = "segmentChangeFetcher.time"
localSegmentChangesLatency = "backend::/api/segmentChanges"
testImpressionsCounter = "testImpressions.status.{status}"
testImpressionsLatency = "testImpressions.time"
localTestImpressionsLatency = "backend::/api/testImpressions/bulk"
postEventsCounter = "events.status.{status}"
postEventsLatency = "events.time"
localPostEventsLatency = "backend::/api/events/bulk"
mySegmentsCounter = "mySegments.status.{status}"
mySegmentsLatency = "mySegments.time"
localMySegmentsLatency = "backend::/api/mySegments"
)
// NewMetricWrapper builds new wrapper
func NewMetricWrapper(telemetry MetricsStorage, localTelemetry MetricsStorage, logger logging.LoggerInterface) *MetricWrapper {
return &MetricWrapper{
LocalTelemetry: localTelemetry,
logger: logger,
Telemetry: telemetry,
}
}
func (m *MetricWrapper) getKey(key int) (string, string, error) {
switch key {
case SplitChangesCounter:
return splitChangesCounter, counter, nil
case SplitChangesLatency:
return splitChangesLatency, localSplitChangesLatency, nil
case SegmentChangesCounter:
return segmentChangesCounter, counter, nil
case SegmentChangesLatency:
return segmentChangesLatency, localSegmentChangesLatency, nil
case TestImpressionsCounter:
return testImpressionsCounter, counter, nil
case TestImpressionsLatency:
return testImpressionsLatency, localTestImpressionsLatency, nil
case PostEventsCounter:
return postEventsCounter, counter, nil
case PostEventsLatency:
return postEventsLatency, localPostEventsLatency, nil
case MySegmentsCounter:
return mySegmentsCounter, counter, nil
case MySegmentsLatency:
return mySegmentsLatency, localMySegmentsLatency, nil
default:
return "", "", errors.New("Key does not exist")
}
}
// StoreCounters stores counters
func (m *MetricWrapper) StoreCounters(key int, value string) {
common, local, err := m.getKey(key)
if err != nil {
return
}
if m.LocalTelemetry != nil {
m.LocalTelemetry.IncCounter(strings.Replace(local, "{status}", value, 1))
}
if value == "ok" {
value = "200"
}
m.Telemetry.IncCounter(strings.Replace(common, "{status}", value, 1))
}
// StoreLatencies stores counters
func (m *MetricWrapper) StoreLatencies(key int, bucket int) {
common, local, err := m.getKey(key)
if err != nil {
return
}
if m.LocalTelemetry != nil {
m.LocalTelemetry.IncLatency(local, bucket)
}
m.Telemetry.IncLatency(common, bucket)
}

73
vendor/github.com/splitio/go-split-commons/v3/storage/mocks/metric.go сгенерированный поставляемый
Просмотреть файл

@@ -1,73 +0,0 @@
package mocks
import "github.com/splitio/go-split-commons/v3/dtos"
// MockMetricStorage is a mocked implementation of Metric Storage
type MockMetricStorage struct {
IncCounterCall func(key string)
IncLatencyCall func(metricName string, index int)
PutGaugeCall func(key string, gauge float64)
PopGaugesCall func() []dtos.GaugeDTO
PopLatenciesCall func() []dtos.LatenciesDTO
PopCountersCall func() []dtos.CounterDTO
PeekCountersCall func() map[string]int64
PeekLatenciesCall func() map[string][]int64
PopGaugesWithMetadataCall func() (*dtos.GaugeDataBulk, error)
PopCountersWithMetadataCall func() (*dtos.CounterDataBulk, error)
PopLatenciesWithMetadataCall func() (*dtos.LatencyDataBulk, error)
}
// IncCounter mock
func (m MockMetricStorage) IncCounter(key string) {
m.IncCounterCall(key)
}
// IncLatency mock
func (m MockMetricStorage) IncLatency(metricName string, index int) {
m.IncLatencyCall(metricName, index)
}
// PutGauge mock
func (m MockMetricStorage) PutGauge(key string, gauge float64) {
m.PutGaugeCall(key, gauge)
}
// PopGauges mock
func (m MockMetricStorage) PopGauges() []dtos.GaugeDTO {
return m.PopGaugesCall()
}
// PopLatencies mock
func (m MockMetricStorage) PopLatencies() []dtos.LatenciesDTO {
return m.PopLatenciesCall()
}
// PopCounters mock
func (m MockMetricStorage) PopCounters() []dtos.CounterDTO {
return m.PopCountersCall()
}
// PeekCounters mock
func (m MockMetricStorage) PeekCounters() map[string]int64 {
return m.PeekCountersCall()
}
// PeekLatencies mock
func (m MockMetricStorage) PeekLatencies() map[string][]int64 {
return m.PeekLatenciesCall()
}
// PopGaugesWithMetadata mock
func (m MockMetricStorage) PopGaugesWithMetadata() (*dtos.GaugeDataBulk, error) {
return m.PopGaugesWithMetadataCall()
}
// PopCountersWithMetadata mock
func (m MockMetricStorage) PopCountersWithMetadata() (*dtos.CounterDataBulk, error) {
return m.PopCountersWithMetadataCall()
}
// PopLatenciesWithMetadata mock
func (m MockMetricStorage) PopLatenciesWithMetadata() (*dtos.LatencyDataBulk, error) {
return m.PopLatenciesWithMetadataCall()
}

5
vendor/github.com/splitio/go-split-commons/v3/storage/mocks/segment.go сгенерированный поставляемый
Просмотреть файл

@@ -10,6 +10,7 @@ type MockSegmentStorage struct {
SegmentContainsKeyCall func(segmentName string, key string) (bool, error)
SetChangeNumberCall func(segmentName string, till int64) error
CountRemovedKeysCall func(segmentName string) int64
SegmentKeysCountCall func() int64
}
// ChangeNumber mock
@@ -41,3 +42,7 @@ func (m MockSegmentStorage) SetChangeNumber(segmentName string, till int64) erro
func (m MockSegmentStorage) CountRemovedKeys(segmentName string) int64 {
return m.CountRemovedKeysCall(segmentName)
}
func (m MockSegmentStorage) SegmentKeysCount() int64 {
return m.SegmentKeysCountCall()
}

179
vendor/github.com/splitio/go-split-commons/v3/storage/mocks/telemetry.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,179 @@
package mocks
import "github.com/splitio/go-split-commons/v3/dtos"
// MockTelemetryStorage is a mocked implementation of Telemetry Storage
type MockTelemetryStorage struct {
RecordConfigDataCall func(configData dtos.Config) error
RecordLatencyCall func(method string, latency int64)
RecordExceptionCall func(method string)
RecordImpressionsStatsCall func(dataType int, count int64)
RecordEventsStatsCall func(dataType int, count int64)
RecordSuccessfulSyncCall func(resource int, time int64)
RecordSyncErrorCall func(resource int, status int)
RecordSyncLatencyCall func(resource int, latency int64)
RecordAuthRejectionsCall func()
RecordTokenRefreshesCall func()
RecordStreamingEventCall func(streamingEvent *dtos.StreamingEvent)
AddTagCall func(tag string)
RecordSessionLengthCall func(session int64)
RecordNonReadyUsageCall func()
RecordBURTimeoutCall func()
PopLatenciesCall func() dtos.MethodLatencies
PopExceptionsCall func() dtos.MethodExceptions
GetImpressionsStatsCall func(dataType int) int64
GetEventsStatsCall func(dataType int) int64
GetLastSynchronizationCall func() dtos.LastSynchronization
PopHTTPErrorsCall func() dtos.HTTPErrors
PopHTTPLatenciesCall func() dtos.HTTPLatencies
PopAuthRejectionsCall func() int64
PopTokenRefreshesCall func() int64
PopStreamingEventsCall func() []dtos.StreamingEvent
PopTagsCall func() []string
GetSessionLengthCall func() int64
GetNonReadyUsagesCall func() int64
GetBURTimeoutsCall func() int64
}
// RecordConfig mock
func (m MockTelemetryStorage) RecordConfigData(configData dtos.Config) error {
return m.RecordConfigDataCall(configData)
}
// RecordLatency mock
func (m MockTelemetryStorage) RecordLatency(method string, latency int64) {
m.RecordLatencyCall(method, latency)
}
// RecordException mock
func (m MockTelemetryStorage) RecordException(method string) { m.RecordExceptionCall(method) }
// RecordImpressionsStats mock
func (m MockTelemetryStorage) RecordImpressionsStats(dataType int, count int64) {
m.RecordImpressionsStatsCall(dataType, count)
}
// RecordEventsStats mock
func (m MockTelemetryStorage) RecordEventsStats(dataType int, count int64) {
m.RecordEventsStatsCall(dataType, count)
}
// RecordSuccessfulSync mock
func (m MockTelemetryStorage) RecordSuccessfulSync(resource int, time int64) {
m.RecordSuccessfulSyncCall(resource, time)
}
// RecordSyncError mock
func (m MockTelemetryStorage) RecordSyncError(resource int, status int) {
m.RecordSyncErrorCall(resource, status)
}
// RecordSyncLatency mock
func (m MockTelemetryStorage) RecordSyncLatency(resource int, latency int64) {
m.RecordSyncLatencyCall(resource, latency)
}
// RecordAuthRejections mock
func (m MockTelemetryStorage) RecordAuthRejections() {
m.RecordAuthRejectionsCall()
}
// RecordTokenRefreshes mock
func (m MockTelemetryStorage) RecordTokenRefreshes() {
m.RecordTokenRefreshesCall()
}
// RecordStreamingEvent mock
func (m MockTelemetryStorage) RecordStreamingEvent(streamingEvent *dtos.StreamingEvent) {
m.RecordStreamingEventCall(streamingEvent)
}
// AddTag mock
func (m MockTelemetryStorage) AddTag(tag string) {
m.AddTagCall(tag)
}
// RecordSessionLength mock
func (m MockTelemetryStorage) RecordSessionLength(session int64) {
m.RecordSessionLengthCall(session)
}
// RecordNonReadyUsage mock
func (m MockTelemetryStorage) RecordNonReadyUsage() {
m.RecordNonReadyUsageCall()
}
// RecordBURTimeout mock
func (m MockTelemetryStorage) RecordBURTimeout() {
m.RecordBURTimeoutCall()
}
// PopLatencies mock
func (m MockTelemetryStorage) PopLatencies() dtos.MethodLatencies {
return m.PopLatenciesCall()
}
//PopExceptions mock
func (m MockTelemetryStorage) PopExceptions() dtos.MethodExceptions {
return m.PopExceptionsCall()
}
// GetImpressionsStats mock
func (m MockTelemetryStorage) GetImpressionsStats(dataType int) int64 {
return m.GetImpressionsStatsCall(dataType)
}
// GetEventsStats mock
func (m MockTelemetryStorage) GetEventsStats(dataType int) int64 {
return m.GetEventsStatsCall(dataType)
}
// GetLastSynchronization mock
func (m MockTelemetryStorage) GetLastSynchronization() dtos.LastSynchronization {
return m.GetLastSynchronizationCall()
}
// PopHTTPErrors mock
func (m MockTelemetryStorage) PopHTTPErrors() dtos.HTTPErrors {
return m.PopHTTPErrorsCall()
}
// PopHTTPLatencies mock
func (m MockTelemetryStorage) PopHTTPLatencies() dtos.HTTPLatencies {
return m.PopHTTPLatenciesCall()
}
// PopAuthRejections mock
func (m MockTelemetryStorage) PopAuthRejections() int64 {
return m.PopAuthRejectionsCall()
}
// PopTokenRefreshes mock
func (m MockTelemetryStorage) PopTokenRefreshes() int64 {
return m.PopTokenRefreshesCall()
}
// PopStreamingEvents mock
func (m MockTelemetryStorage) PopStreamingEvents() []dtos.StreamingEvent {
return m.PopStreamingEventsCall()
}
// PopTags mock
func (m MockTelemetryStorage) PopTags() []string {
return m.PopTagsCall()
}
// GetSessionLength mock
func (m MockTelemetryStorage) GetSessionLength() int64 {
return m.GetSessionLengthCall()
}
// GetNonReadyUsages mock
func (m MockTelemetryStorage) GetNonReadyUsages() int64 {
return m.GetNonReadyUsagesCall()
}
// GetBURTimeouts mock
func (m MockTelemetryStorage) GetBURTimeouts() int64 {
return m.GetBURTimeoutsCall()
}

149
vendor/github.com/splitio/go-split-commons/v3/storage/mutexmap/metrics.go сгенерированный поставляемый
Просмотреть файл

@@ -1,149 +0,0 @@
package mutexmap
import (
"sync"
"github.com/splitio/go-split-commons/v3/dtos"
)
// MMMetricsStorage contains an in-memory implementation of Metrics storage
type MMMetricsStorage struct {
gaugeData map[string]float64
gaugeMutex *sync.Mutex
counterData map[string]int64
countersMutex *sync.RWMutex
latenciesData map[string][]int64
latenciesMutex *sync.RWMutex
}
// NewMMMetricsStorage instantiates a new MMMetricsStorage
func NewMMMetricsStorage() *MMMetricsStorage {
return &MMMetricsStorage{
counterData: make(map[string]int64),
countersMutex: &sync.RWMutex{},
gaugeData: make(map[string]float64),
gaugeMutex: &sync.Mutex{},
latenciesData: make(map[string][]int64),
latenciesMutex: &sync.RWMutex{},
}
}
// PutGauge stores a new gauge value for a specific key
func (m *MMMetricsStorage) PutGauge(key string, gauge float64) {
m.gaugeMutex.Lock()
defer m.gaugeMutex.Unlock()
m.gaugeData[key] = gauge
}
// PopGauges returns and deletes all gauges currently stored
func (m *MMMetricsStorage) PopGauges() []dtos.GaugeDTO {
m.gaugeMutex.Lock()
defer func() {
m.gaugeData = make(map[string]float64)
m.gaugeMutex.Unlock()
}()
gauges := make([]dtos.GaugeDTO, 0)
for key, gauge := range m.gaugeData {
gauges = append(gauges, dtos.GaugeDTO{
MetricName: key,
Gauge: gauge,
})
}
return gauges
}
// IncCounter increments the counter for a specific key. It initializes it in 1 if it doesn't exist when this function
// is called.
func (m *MMMetricsStorage) IncCounter(key string) {
m.countersMutex.Lock()
defer m.countersMutex.Unlock()
_, exists := m.counterData[key]
if !exists {
m.counterData[key] = 1
} else {
m.counterData[key]++
}
}
// PopCounters returns and deletes all the counters stored
func (m *MMMetricsStorage) PopCounters() []dtos.CounterDTO {
m.countersMutex.Lock()
defer func() {
m.counterData = make(map[string]int64)
m.countersMutex.Unlock()
}()
counters := make([]dtos.CounterDTO, 0)
for key, counter := range m.counterData {
counters = append(counters, dtos.CounterDTO{
MetricName: key,
Count: counter,
})
}
return counters
}
// PeekCounters returns Counters
func (m *MMMetricsStorage) PeekCounters() map[string]int64 {
m.countersMutex.RLock()
defer m.countersMutex.RUnlock()
return m.counterData
}
// PeekLatencies returns Latencies
func (m *MMMetricsStorage) PeekLatencies() map[string][]int64 {
m.latenciesMutex.RLock()
defer m.latenciesMutex.RUnlock()
return m.latenciesData
}
// IncLatency increments the latency for a specific key and bucket. If the key doesn't exist it's initialized to
// an empty array of 23 items.
func (m *MMMetricsStorage) IncLatency(metricName string, index int) {
if index < 0 || index > 22 {
return
}
m.latenciesMutex.Lock()
defer m.latenciesMutex.Unlock()
_, exists := m.latenciesData[metricName]
if !exists {
m.latenciesData[metricName] = make([]int64, 23)
m.latenciesData[metricName][index] = 1
} else {
m.latenciesData[metricName][index]++
}
}
// PopLatencies Returns and delete all the latencies currently stored
func (m *MMMetricsStorage) PopLatencies() []dtos.LatenciesDTO {
m.latenciesMutex.Lock()
defer func() {
m.latenciesData = make(map[string][]int64)
m.latenciesMutex.Unlock()
}()
latencies := make([]dtos.LatenciesDTO, 0)
for key, latency := range m.latenciesData {
latencies = append(latencies, dtos.LatenciesDTO{
Latencies: latency,
MetricName: key,
})
}
return latencies
}
// PopGaugesWithMetadata mock
func (m *MMMetricsStorage) PopGaugesWithMetadata() (*dtos.GaugeDataBulk, error) {
panic("Not implemented for inmemory")
}
// PopLatenciesWithMetadata mock
func (m *MMMetricsStorage) PopLatenciesWithMetadata() (*dtos.LatencyDataBulk, error) {
panic("Not implemented for inmemory")
}
// PopCountersWithMetadata mock
func (m *MMMetricsStorage) PopCountersWithMetadata() (*dtos.CounterDataBulk, error) {
panic("Not implemented for inmemory")
}

28
vendor/github.com/splitio/go-split-commons/v3/storage/redis/constants.go сгенерированный поставляемый
Просмотреть файл

@@ -1,17 +1,19 @@
package redis
const (
redisSplit = "SPLITIO.split.{split}" // split object
redisSplitTill = "SPLITIO.splits.till" // last split fetch
redisSegment = "SPLITIO.segment.{segment}" // segment object
redisSegmentTill = "SPLITIO.segment.{segment}.till" // last segment fetch
redisImpressions = "SPLITIO/{sdkVersion}/{instanceId}/impressions.{feature}" // impressions for a feature
redisLatency = "SPLITIO/{sdkVersion}/{instanceId}/latency.{metric}.bucket.{bucket}" // latency bucket
redisCounter = "SPLITIO/{sdkVersion}/{instanceId}/count.{metric}" // counter
redisGauge = "SPLITIO/{sdkVersion}/{instanceId}/gauge.{metric}" // gauge
redisEvents = "SPLITIO.events" // events LIST key
redisImpressionsQueue = "SPLITIO.impressions" // impressions LIST key
redisImpressionsTTL = 60 // impressions default TTL
redisTrafficType = "SPLITIO.trafficType.{trafficType}" // traffic Type fetch
redisHash = "SPLITIO.hash"
redisSplit = "SPLITIO.split.{split}" // split object
redisSplitTill = "SPLITIO.splits.till" // last split fetch
redisSegment = "SPLITIO.segment.{segment}" // segment object
redisSegmentTill = "SPLITIO.segment.{segment}.till" // last segment fetch
redisEvents = "SPLITIO.events" // events LIST key
redisImpressionsQueue = "SPLITIO.impressions" // impressions LIST key
redisImpressionsTTL = 3600 // impressions default TTL
redisTrafficType = "SPLITIO.trafficType.{trafficType}" // traffic Type fetch
redisHash = "SPLITIO.hash" // hash key
redisConfig = "SPLITIO.telemetry.config" // config Key
redisConfigTTL = 3600 // config TTL
redisLatency = "SPLITIO.telemetry.latencies" // latency Key
redisExceptionField = "{sdkVersion}/{machineName}/{machineIP}/{method}" // exception field template
redisException = "SPLITIO.telemetry.exceptions" // exception Key
redisLatencyField = "{sdkVersion}/{machineName}/{machineIP}/{method}/{bucket}" // latency field template
)

24
vendor/github.com/splitio/go-split-commons/v3/storage/redis/impressions.go сгенерированный поставляемый
Просмотреть файл

@@ -14,23 +14,21 @@ const impressionsTTLRefresh = time.Duration(3600) * time.Second
// ImpressionStorage is a redis-based implementation of split storage
type ImpressionStorage struct {
client *redis.PrefixedRedisClient
mutex *sync.Mutex
logger logging.LoggerInterface
redisKey string
impressionsTTL time.Duration
metadata dtos.Metadata
client *redis.PrefixedRedisClient
mutex *sync.Mutex
logger logging.LoggerInterface
redisKey string
metadata dtos.Metadata
}
// NewImpressionStorage creates a new RedisSplitStorage and returns a reference to it
func NewImpressionStorage(client *redis.PrefixedRedisClient, metadata dtos.Metadata, logger logging.LoggerInterface) *ImpressionStorage {
return &ImpressionStorage{
client: client,
mutex: &sync.Mutex{},
logger: logger,
redisKey: redisImpressionsQueue,
impressionsTTL: redisImpressionsTTL,
metadata: metadata,
client: client,
mutex: &sync.Mutex{},
logger: logger,
redisKey: redisImpressionsQueue,
metadata: metadata,
}
}
@@ -83,7 +81,7 @@ func (r *ImpressionStorage) push(impressions []dtos.ImpressionQueueObject) error
// Checks if expiration needs to be set
if inserted == int64(len(impressionsJSON)) {
r.logger.Debug("Proceeding to set expiration for: ", r.redisKey)
result := r.client.Expire(r.redisKey, time.Duration(r.impressionsTTL)*time.Minute)
result := r.client.Expire(r.redisKey, time.Duration(redisImpressionsTTL)*time.Second)
if result == false {
r.logger.Error("Something were wrong setting expiration", errPush)
}

322
vendor/github.com/splitio/go-split-commons/v3/storage/redis/metrics.go сгенерированный поставляемый
Просмотреть файл

@@ -1,322 +0,0 @@
package redis
import (
"fmt"
"regexp"
"strconv"
"strings"
"sync"
"github.com/splitio/go-split-commons/v3/dtos"
"github.com/splitio/go-toolkit/v4/logging"
"github.com/splitio/go-toolkit/v4/redis"
)
// MetricsStorage is a redis-based implementation of split storage
type MetricsStorage struct {
client redis.PrefixedRedisClient
logger logging.LoggerInterface
gaugeSingleTemplate string
countersSingleTemplate string
latenciesSingleTemplate string
gaugeMultiTemplate string
countersMultiTemplate string
latenciesMultiTemplate string
mutex *sync.RWMutex
}
// NewMetricsStorage creates a new RedisSplitStorage and returns a reference to it
func NewMetricsStorage(redisClient *redis.PrefixedRedisClient, metadata dtos.Metadata, logger logging.LoggerInterface) *MetricsStorage {
// @Todo Split Storages between Go-Client and Redis
gaugeSingleTemplate := strings.Replace(redisGauge, "{sdkVersion}", metadata.SDKVersion, 1)
gaugeSingleTemplate = strings.Replace(gaugeSingleTemplate, "{instanceId}", metadata.MachineName, 1)
countersSingleTemplate := strings.Replace(redisCounter, "{sdkVersion}", metadata.SDKVersion, 1)
countersSingleTemplate = strings.Replace(countersSingleTemplate, "{instanceId}", metadata.MachineName, 1)
latenciesSingleTemplate := strings.Replace(redisLatency, "{sdkVersion}", metadata.SDKVersion, 1)
latenciesSingleTemplate = strings.Replace(latenciesSingleTemplate, "{instanceId}", metadata.MachineName, 1)
gaugeMultiTemplate := strings.Replace(redisGauge, "{sdkVersion}", "*", 1)
gaugeMultiTemplate = strings.Replace(gaugeMultiTemplate, "{instanceId}", "*", 1)
gaugeMultiTemplate = strings.Replace(gaugeMultiTemplate, "{metric}", "*", 1)
countersMultiTemplate := strings.Replace(redisCounter, "{sdkVersion}", "*", 1)
countersMultiTemplate = strings.Replace(countersMultiTemplate, "{instanceId}", "*", 1)
countersMultiTemplate = strings.Replace(countersMultiTemplate, "{metric}", "*", 1)
latenciesMultiTemplate := strings.Replace(redisLatency, "{sdkVersion}", "*", 1)
latenciesMultiTemplate = strings.Replace(latenciesMultiTemplate, "{instanceId}", "*", 1)
latenciesMultiTemplate = strings.Replace(latenciesMultiTemplate, "{metric}", "*", 1)
latenciesMultiTemplate = strings.Replace(latenciesMultiTemplate, "{bucket}", "*", 1)
return &MetricsStorage{
client: *redisClient,
logger: logger,
gaugeSingleTemplate: gaugeSingleTemplate,
countersSingleTemplate: countersSingleTemplate,
latenciesSingleTemplate: latenciesSingleTemplate,
gaugeMultiTemplate: gaugeMultiTemplate,
countersMultiTemplate: countersMultiTemplate,
latenciesMultiTemplate: latenciesMultiTemplate,
mutex: &sync.RWMutex{},
}
}
// IncCounter incraeses the count for a specific metric
func (r *MetricsStorage) IncCounter(metric string) {
keyToIncr := strings.Replace(r.countersSingleTemplate, "{metric}", metric, 1)
_, err := r.client.Incr(keyToIncr)
if err != nil {
r.logger.Error(fmt.Sprintf("Error incrementing counterfor metric \"%s\" in redis: %s", metric, err.Error()))
}
}
// IncLatency incraeses the latency of a bucket for a specific metric
func (r *MetricsStorage) IncLatency(metric string, index int) {
keyToIncr := strings.Replace(r.latenciesSingleTemplate, "{metric}", metric, 1)
keyToIncr = strings.Replace(keyToIncr, "{bucket}", strconv.FormatInt(int64(index), 10), 1)
_, err := r.client.Incr(keyToIncr)
if err != nil {
r.logger.Error(fmt.Sprintf(
"Error incrementing latency bucket %d for metric \"%s\" in redis: %s", index, metric, err.Error(),
))
}
}
// PutGauge stores a gauge in redis
func (r *MetricsStorage) PutGauge(key string, gauge float64) {
keyToStore := strings.Replace(r.gaugeSingleTemplate, "{metric}", key, 1)
err := r.client.Set(keyToStore, gauge, 0)
if err != nil {
r.logger.Error(fmt.Sprintf("Error storing gauge \"%s\" in redis: %s\n", key, err))
}
}
// PopCounters some
func (r *MetricsStorage) PopCounters() []dtos.CounterDTO {
panic("Not implemented for redis")
}
// PopGauges some
func (r *MetricsStorage) PopGauges() []dtos.GaugeDTO {
panic("Not implemented for redis")
}
// PopLatencies some
func (r *MetricsStorage) PopLatencies() []dtos.LatenciesDTO {
panic("Not implemented for redis")
}
func (r *MetricsStorage) popByPattern(pattern string, useTransaction bool) (map[string]interface{}, error) {
r.mutex.Lock()
defer r.mutex.Unlock()
keys, err := r.client.Keys(pattern)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
if len(keys) == 0 {
return map[string]interface{}{}, nil
}
values, err := r.client.MGet(keys)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
_, err = r.client.Del(keys...)
if err != nil {
// if we failed to delete the keys, log an error and continue working.
r.logger.Error(err.Error())
}
toReturn := make(map[string]interface{})
for index := range keys {
if index >= len(keys) || index >= len(values) {
break
}
toReturn[keys[index]] = values[index]
}
return toReturn, nil
}
func parseIntRedisValue(s interface{}) (int64, error) {
asStr, ok := s.(string)
if !ok {
return 0, fmt.Errorf("%+v is not a string", s)
}
asInt64, err := strconv.ParseInt(asStr, 10, 64)
if err != nil {
return 0, err
}
return asInt64, nil
}
func parseFloatRedisValue(s interface{}) (float64, error) {
asStr, ok := s.(string)
if !ok {
return 0, fmt.Errorf("%+v is not a string", s)
}
asFloat64, err := strconv.ParseFloat(asStr, 64)
if err != nil {
return 0, err
}
return asFloat64, nil
}
func (r *MetricsStorage) parseLatencyKey(key string) (string, string, string, int, error) {
re := regexp.MustCompile(`(\w+.)?SPLITIO\/([^\/]+)\/([^\/]+)\/latency.([^\/]+).bucket.([0-9]*)`)
match := re.FindStringSubmatch(key)
if len(match) < 6 {
return "", "", "", 0, fmt.Errorf("Error parsing key %s", key)
}
sdkNameAndVersion := match[2]
if sdkNameAndVersion == "" {
return "", "", "", 0, fmt.Errorf("Invalid sdk name/version")
}
machineIP := match[3]
if machineIP == "" {
return "", "", "", 0, fmt.Errorf("Invalid machine IP")
}
metricName := match[4]
if metricName == "" {
return "", "", "", 0, fmt.Errorf("Invalid feature name")
}
bucketNumber, err := strconv.Atoi(match[5])
if err != nil {
return "", "", "", 0, fmt.Errorf("Error parsing bucket number: %s", err.Error())
}
r.logger.Verbose("Impression parsed key", match)
return sdkNameAndVersion, machineIP, metricName, bucketNumber, nil
}
func (r *MetricsStorage) parseMetricKey(metricType string, key string) (string, string, string, error) {
var re = regexp.MustCompile(strings.Replace(
`(\w+.)?SPLITIO\/([^\/]+)\/([^\/]+)\/{metricType}.([\s\S]*)`,
"{metricType}",
metricType,
1,
))
match := re.FindStringSubmatch(key)
if len(match) < 5 {
return "", "", "", fmt.Errorf("Error parsing key %s", key)
}
sdkNameAndVersion := match[2]
if sdkNameAndVersion == "" {
return "", "", "", fmt.Errorf("Invalid sdk name/version")
}
machineIP := match[3]
if machineIP == "" {
return "", "", "", fmt.Errorf("Invalid machine IP")
}
metricName := match[4]
if metricName == "" {
return "", "", "", fmt.Errorf("Invalid feature name")
}
r.logger.Verbose("Impression parsed key", match)
return sdkNameAndVersion, machineIP, metricName, nil
}
// PopGaugesWithMetadata returns gauges values saved in Redis by SDKs
func (r *MetricsStorage) PopGaugesWithMetadata() (*dtos.GaugeDataBulk, error) {
data, err := r.popByPattern(r.gaugeMultiTemplate, false)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
gaugesToReturn := dtos.NewGaugeDataBulk()
for key, value := range data {
sdkNameAndVersion, machineIP, metricName, err := r.parseMetricKey("gauge", key)
if err != nil {
r.logger.Error(fmt.Sprintf("Unable to parse key %s. Skipping", key))
continue
}
asFloat, err := parseFloatRedisValue(value)
if err != nil {
r.logger.Error(fmt.Sprintf("Unable to parse value %+v. Skipping", value))
continue
}
gaugesToReturn.PutGauge(sdkNameAndVersion, machineIP, metricName, asFloat)
}
return gaugesToReturn, nil
}
// PopCountersWithMetadata returns counter values saved in Redis by SDKs
func (r *MetricsStorage) PopCountersWithMetadata() (*dtos.CounterDataBulk, error) {
data, err := r.popByPattern(r.countersMultiTemplate, false)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
countersToReturn := dtos.NewCounterDataBulk()
for key, value := range data {
sdkNameAndVersion, machineIP, metricName, err := r.parseMetricKey("count", key)
if err != nil {
r.logger.Error("Unable to parse key %s. Skipping", key)
continue
}
asInt, err := parseIntRedisValue(value)
if err != nil {
r.logger.Error(err.Error())
continue
}
countersToReturn.PutCounter(sdkNameAndVersion, machineIP, metricName, asInt)
}
return countersToReturn, nil
}
// PopLatenciesWithMetadata returns latency values saved in Redis by SDKs
func (r *MetricsStorage) PopLatenciesWithMetadata() (*dtos.LatencyDataBulk, error) {
data, err := r.popByPattern(r.latenciesMultiTemplate, false)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
latenciesToReturn := dtos.NewLatencyDataBulk()
for key, value := range data {
value, err := parseIntRedisValue(value)
if err != nil {
r.logger.Warning(fmt.Sprintf("Unable to parse value of key %s. Skipping", key))
continue
}
sdkNameAndVersion, machineIP, metricName, bucketNumber, err := r.parseLatencyKey(key)
if err != nil {
r.logger.Warning(fmt.Sprintf("Unable to parse key %s. Skipping", key))
continue
}
latenciesToReturn.PutLatency(sdkNameAndVersion, machineIP, metricName, bucketNumber, value)
}
r.logger.Verbose(latenciesToReturn)
return latenciesToReturn, nil
}
// PeekCounters returns Counters
func (r *MetricsStorage) PeekCounters() map[string]int64 {
return make(map[string]int64, 0)
}
// PeekLatencies returns Latencies
func (r *MetricsStorage) PeekLatencies() map[string][]int64 {
return make(map[string][]int64, 0)
}

6
vendor/github.com/splitio/go-split-commons/v3/storage/redis/segments.go сгенерированный поставляемый
Просмотреть файл

@@ -6,6 +6,7 @@ import (
"strings"
"sync"
"github.com/splitio/go-split-commons/v3/storage"
"github.com/splitio/go-toolkit/v4/datastructures/set"
"github.com/splitio/go-toolkit/v4/logging"
"github.com/splitio/go-toolkit/v4/redis"
@@ -19,7 +20,7 @@ type SegmentStorage struct {
}
// NewSegmentStorage creates a new RedisSegmentStorage and returns a reference to it
func NewSegmentStorage(redisClient *redis.PrefixedRedisClient, logger logging.LoggerInterface) *SegmentStorage {
func NewSegmentStorage(redisClient *redis.PrefixedRedisClient, logger logging.LoggerInterface) storage.SegmentStorage {
return &SegmentStorage{
client: *redisClient,
logger: logger,
@@ -98,3 +99,6 @@ func (r *SegmentStorage) SegmentContainsKey(segmentName string, key string) (boo
// CountRemovedKeys method
func (r *SegmentStorage) CountRemovedKeys(segmentName string) int64 { return 0 }
// SegmentKeysCount method
func (r *SegmentStorage) SegmentKeysCount() int64 { return 0 }

105
vendor/github.com/splitio/go-split-commons/v3/storage/redis/telemetry.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,105 @@
package redis
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/splitio/go-split-commons/v3/dtos"
"github.com/splitio/go-split-commons/v3/storage"
"github.com/splitio/go-split-commons/v3/telemetry"
"github.com/splitio/go-toolkit/v4/logging"
"github.com/splitio/go-toolkit/v4/redis"
)
const (
sdkVersion = "{sdkVersion}"
machineIP = "{machineIP}"
machineName = "{machineName}"
name = "{method}"
bucketName = "{bucket}"
)
// TelemetryStorage is a redis-based implementation of telemetry storage
type TelemetryStorage struct {
client *redis.PrefixedRedisClient
exceptionTemplate string
latencyTemplate string
logger logging.LoggerInterface
metadata dtos.Metadata
}
// NewTelemetryStorage creates a new RedisTelemetryStorage and returns a reference to it
func NewTelemetryStorage(redisClient *redis.PrefixedRedisClient, logger logging.LoggerInterface, metadata dtos.Metadata) storage.TelemetryRedisProducer {
replacer := strings.NewReplacer(sdkVersion, metadata.SDKVersion, machineName, metadata.MachineName, machineIP, metadata.MachineIP)
exceptionTemplate := replacer.Replace(redisExceptionField)
latencyTemplate := replacer.Replace(redisLatencyField)
return &TelemetryStorage{
client: redisClient,
exceptionTemplate: exceptionTemplate,
latencyTemplate: latencyTemplate,
logger: logger,
metadata: metadata,
}
}
// TELEMETRY STORAGE PRODUCER
// RecordConfigData push config into queue
func (t *TelemetryStorage) RecordConfigData(configData dtos.Config) error {
jsonData, err := json.Marshal(dtos.TelemetryQueueObject{
Metadata: t.metadata,
Config: configData,
})
if err != nil {
t.logger.Error("Error encoding impression in json", err.Error())
}
inserted, errPush := t.client.RPush(redisConfig, jsonData)
if errPush != nil {
t.logger.Error("Something were wrong pushing config data to redis", errPush)
return errPush
}
// Checks if expiration needs to be set
if inserted == 1 {
t.logger.Debug("Proceeding to set expiration for: ", redisConfig)
result := t.client.Expire(redisConfig, time.Duration(redisConfigTTL)*time.Second)
if !result {
t.logger.Error("Something were wrong setting expiration", errPush)
}
}
return nil
}
// RecordLatency stores latency for method
func (t *TelemetryStorage) RecordLatency(method string, latency int64) {
bucket := telemetry.Bucket(latency)
field := strings.Replace(t.latencyTemplate, name, method, 1)
field = strings.Replace(field, bucketName, fmt.Sprintf("%d", bucket), 1)
_, err := t.client.HIncrBy(redisLatency, field, 1)
if err != nil {
t.logger.Error("Error recording in redis.", err.Error())
}
}
// RecordException stores exceptions for method
func (t *TelemetryStorage) RecordException(method string) {
field := strings.Replace(t.exceptionTemplate, name, method, 1)
_, err := t.client.HIncrBy(redisException, field, 1)
if err != nil {
t.logger.Error("Error recording in redis.", err.Error())
}
}
// RecordNonReadyUsage records non ready usage
func (t *TelemetryStorage) RecordNonReadyUsage() {
// No-Op. Redis is implicitly ready and does not need to wait for anything. Tracking not required.
}
// RecordBURTimeout records bur timeodout
func (t *TelemetryStorage) RecordBURTimeout() {
// No-Op. Redis is implicitly ready and does not need to block for anything. Tracking not required.
}