MM-28859 Add feature flag managment system using split.io and remove viper. (#15954)

* Add feature flag managment system using split.io and remove viper.

* Fixing tests.

* Attempt to fix postgres tests.

* Fix watch filepath for advanced logging.

* Review fixes.

* Some error wrapping.

* Remove unessisary store interface.

* Desanitize SplitKey

* Simplify.

* Review feedback.

* Rename split mlog adatper to split logger.

* fsInner

* Style.

* Restore oldcfg test.

* Downgrading non-actionable feature flag errors to warnings.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>
Этот коммит содержится в:
Christopher Speller
2020-10-29 15:54:39 -07:00
коммит произвёл GitHub
родитель 8bb772638c
Коммит 1aadd36644
423 изменённых файлов: 37646 добавлений и 20257 удалений

13
vendor/github.com/splitio/go-split-commons/v2/LICENSE сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,13 @@
Copyright © 2020 Split Software, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

89
vendor/github.com/splitio/go-split-commons/v2/conf/conf.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,89 @@
package conf
import (
"crypto/tls"
)
// RedisConfig struct is used to cofigure the redis parameters
type RedisConfig struct {
Host string
Port int
Database int
Password string
Prefix string
// The network type, either tcp or unix.
// Default is tcp.
Network string
// Maximum number of retries before giving up.
// Default is to not retry failed commands.
MaxRetries int
// Dial timeout for establishing new connections.
// Default is 5 seconds.
DialTimeout int
// Timeout for socket reads. If reached, commands will fail
// with a timeout instead of blocking.
// Default is 10 seconds.
ReadTimeout int
// Timeout for socket writes. If reached, commands will fail
// with a timeout instead of blocking.
// Default is 3 seconds.
WriteTimeout int
// Maximum number of socket connections.
// Default is 10 connections.
PoolSize int
// Redis sentinel replication support
SentinelAddresses []string
SentinelMaster string
// Redis cluster replication support
ClusterNodes []string
ClusterKeyHashTag string
TLSConfig *tls.Config
}
// TaskPeriods struct is used to configure the period for each synchronization task
type TaskPeriods struct {
SplitSync int
SegmentSync int
ImpressionSync int
GaugeSync int
CounterSync int
LatencySync int
EventsSync int
}
// AdvancedConfig exposes more configurable parameters that can be used to further tailor the sdk to the user's needs
// - HTTPTimeout - Timeout for HTTP requests when doing synchronization
// - SegmentQueueSize - How many segments can be queued for updating (should be >= # segments the user has)
// - SegmentWorkers - How many workers will be used when performing segments sync.
type AdvancedConfig struct {
HTTPTimeout int
SegmentQueueSize int
SegmentWorkers int
SdkURL string
EventsURL string
EventsBulkSize int64
EventsQueueSize int
ImpressionsQueueSize int
ImpressionsBulkSize int64
StreamingEnabled bool
AuthServiceURL string
StreamingServiceURL string
SplitUpdateQueueSize int64
SegmentUpdateQueueSize int64
}
// ManagerConfig exposes configurable parameters for ImpressionManager
type ManagerConfig struct {
OperationMode string
ImpressionsMode string
ListenerEnabled bool
}

52
vendor/github.com/splitio/go-split-commons/v2/conf/defaults.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,52 @@
package conf
const (
defaultHTTPTimeout = 30
defaultSegmentQueueSize = 500
defaultSegmentWorkers = 10
defaultEventsBulkSize = 5000
defaultEventsQueueSize = 10000
defaultImpressionsQueueSize = 10000
defaultImpressionsBulkSize = 5000
defaultStreamingEnabled = true
defaultSplitUpdateQueueSize = 5000
defaultSegmentUpdateQueueSize = 5000
defaultAuthServiceURL = "https://auth.split.io"
defaultEventsURL = "https://events.split.io/api"
defaultSdkURL = "https://sdk.split.io/api"
defaultStreamingServiceURL = "https://streaming.split.io/sse"
)
const (
// ImpressionsModeOptimized will avoid sending duplicated events
ImpressionsModeOptimized = "optimized"
// ImpressionsModeDebug will send all the impressions generated
ImpressionsModeDebug = "debug"
)
const (
// Standalone mode
Standalone = "inmemory-standalone"
// ProducerSync mode
ProducerSync = "producer-sync"
)
// GetDefaultAdvancedConfig returns default conf
func GetDefaultAdvancedConfig() AdvancedConfig {
return AdvancedConfig{
EventsQueueSize: defaultEventsQueueSize,
HTTPTimeout: defaultHTTPTimeout,
EventsBulkSize: defaultEventsBulkSize,
ImpressionsBulkSize: defaultImpressionsBulkSize,
ImpressionsQueueSize: defaultImpressionsQueueSize,
SegmentQueueSize: defaultSegmentQueueSize,
SegmentUpdateQueueSize: defaultSegmentUpdateQueueSize,
SegmentWorkers: defaultSegmentWorkers,
SplitUpdateQueueSize: defaultSplitUpdateQueueSize,
StreamingEnabled: defaultStreamingEnabled,
AuthServiceURL: defaultAuthServiceURL,
EventsURL: defaultEventsURL,
SdkURL: defaultSdkURL,
StreamingServiceURL: defaultStreamingServiceURL,
}
}

35
vendor/github.com/splitio/go-split-commons/v2/dtos/event.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,35 @@
package dtos
// EventDTO struct mapping events json
type EventDTO struct {
Key string `json:"key"`
TrafficTypeName string `json:"trafficTypeName"`
EventTypeID string `json:"eventTypeId"`
Value interface{} `json:"value"`
Timestamp int64 `json:"timestamp"`
Properties map[string]interface{} `json:"properties,omitempty"`
}
// Size returns a relatively accurate estimation of the size of the event
func (e *EventDTO) Size() int {
size := 1024
if e.Properties == nil {
return size
}
for key, value := range e.Properties {
size += len(key)
switch typedValue := value.(type) {
case string:
size += len(typedValue)
default:
}
}
return size
}
// QueueStoredEventDTO maps the stored JSON object in redis by SDKs
type QueueStoredEventDTO struct {
Metadata Metadata `json:"m"`
Event EventDTO `json:"e"`
}

12
vendor/github.com/splitio/go-split-commons/v2/dtos/http.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,12 @@
package dtos
// HTTPError represents a http error
type HTTPError struct {
Code int
Message string
}
// Error implements golang error interface
func (h HTTPError) Error() string {
return h.Message
}

48
vendor/github.com/splitio/go-split-commons/v2/dtos/impression.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,48 @@
package dtos
// Impression struct to map an impression
type Impression struct {
KeyName string `json:"k"`
BucketingKey string `json:"b"`
FeatureName string `json:"f"`
Treatment string `json:"t"`
Label string `json:"r"`
ChangeNumber int64 `json:"c"`
Time int64 `json:"m"`
Pt int64 `json:"pt,omitempty"`
}
// ImpressionQueueObject struct mapping impressions
type ImpressionQueueObject struct {
Metadata Metadata `json:"m"`
Impression Impression `json:"i"`
}
// ImpressionDTO struct to map an impression
type ImpressionDTO struct {
KeyName string `json:"k"`
Treatment string `json:"t"`
Time int64 `json:"m"`
ChangeNumber int64 `json:"c"`
Label string `json:"r"`
BucketingKey string `json:"b,omitempty"`
Pt int64 `json:"pt,omitempty"`
}
// ImpressionsDTO struct mapping impressions to post
type ImpressionsDTO struct {
TestName string `json:"f"`
KeyImpressions []ImpressionDTO `json:"i"`
}
// ImpressionsInTimeFrameDTO struct mapping impressionsCount in time window
type ImpressionsInTimeFrameDTO struct {
FeatureName string `json:"f"`
TimeFrame int64 `json:"m"`
RawCount int64 `json:"rc"`
}
// ImpressionsCountDTO struct mapping impressions count to post
type ImpressionsCountDTO struct {
PerFeature []ImpressionsInTimeFrameDTO `json:"pf"`
}

8
vendor/github.com/splitio/go-split-commons/v2/dtos/metadata.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,8 @@
package dtos
// Metadata is used to store sdk metadata
type Metadata struct {
SDKVersion string `json:"s"`
MachineIP string `json:"i"`
MachineName string `json:"n"`
}

132
vendor/github.com/splitio/go-split-commons/v2/dtos/metrics.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,132 @@
package dtos
// LatenciesDTO struct mapping latencies post
type LatenciesDTO struct {
MetricName string `json:"name"`
Latencies []int64 `json:"latencies"`
}
// CounterDTO struct mapping counts post
type CounterDTO struct {
MetricName string `json:"name"`
Count int64 `json:"delta"`
}
// GaugeDTO struct mapping gauges post
type GaugeDTO struct {
MetricName string `json:"name"`
Gauge float64 `json:"value"`
}
const maxBuckets = 23
// LatencyDataBulk holds all latencies fetched from storage sorted properly.
type LatencyDataBulk struct {
data map[string]map[string]map[string][]int64
}
// PutLatency adds a new latency to the structure
func (l *LatencyDataBulk) PutLatency(sdk string, machineIP string, metricName string, bucketNumber int, value int64) {
if _, ok := l.data[sdk]; !ok {
l.data[sdk] = make(map[string]map[string][]int64)
}
if _, ok := l.data[sdk][machineIP]; !ok {
l.data[sdk][machineIP] = make(map[string][]int64)
}
if _, ok := l.data[sdk][machineIP][metricName]; !ok {
l.data[sdk][machineIP][metricName] = make([]int64, maxBuckets)
}
l.data[sdk][machineIP][metricName][bucketNumber] = value
}
// ForEach iterates thru all latencies
func (l *LatencyDataBulk) ForEach(callback func(string, string, map[string][]int64)) {
for sdk, byIP := range l.data {
for ip, byName := range byIP {
callback(sdk, ip, byName)
}
}
}
// NewLatencyDataBulk creates a new Latency holding structure
func NewLatencyDataBulk() *LatencyDataBulk {
return &LatencyDataBulk{
data: make(map[string]map[string]map[string][]int64),
}
}
// CounterDataBulk holds all counters fetched from storage sorted properly.
type CounterDataBulk struct {
data map[string]map[string]map[string]int64
}
// PutCounter adds a counter to the structure
func (l *CounterDataBulk) PutCounter(sdk string, machineIP string, metricName string, value int64) {
if _, ok := l.data[sdk]; !ok {
l.data[sdk] = make(map[string]map[string]int64)
}
if _, ok := l.data[sdk][machineIP]; !ok {
l.data[sdk][machineIP] = make(map[string]int64)
}
l.data[sdk][machineIP][metricName] = value
}
// ForEach iterates thru all counters
func (l *CounterDataBulk) ForEach(callback func(string, string, map[string]int64)) {
for sdk, byIP := range l.data {
for ip, byName := range byIP {
callback(sdk, ip, byName)
}
}
}
// NewCounterDataBulk creates a new Counter holding structure
func NewCounterDataBulk() *CounterDataBulk {
return &CounterDataBulk{
data: make(map[string]map[string]map[string]int64),
}
}
// GaugeDataBulk holds all gauges fetched from storage sorted properly.
type GaugeDataBulk struct {
data map[string]map[string]map[string]float64
}
// PutGauge adds a gauge to the structure
func (l *GaugeDataBulk) PutGauge(sdk string, machineIP string, metricName string, value float64) {
if _, ok := l.data[sdk]; !ok {
l.data[sdk] = make(map[string]map[string]float64)
}
if _, ok := l.data[sdk][machineIP]; !ok {
l.data[sdk][machineIP] = make(map[string]float64)
}
l.data[sdk][machineIP][metricName] = value
}
// ForEach iterates thru all gauges
func (l *GaugeDataBulk) ForEach(callback func(string, string, string, float64)) {
for sdk, byIP := range l.data {
for ip, byName := range byIP {
for name, value := range byName {
callback(sdk, ip, name, value)
}
}
}
}
// NewGaugeDataBulk creates a new Gauge holding structure
func NewGaugeDataBulk() *GaugeDataBulk {
return &GaugeDataBulk{
data: make(map[string]map[string]map[string]float64),
}
}

149
vendor/github.com/splitio/go-split-commons/v2/dtos/notification.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,149 @@
package dtos
const (
// SplitUpdate used when split is updated
SplitUpdate = "SPLIT_UPDATE"
// SplitKill used when split is killed
SplitKill = "SPLIT_KILL"
// SegmentUpdate used when segment is updated
SegmentUpdate = "SEGMENT_UPDATE"
// MySegmentsUpdate used when mySegment is updated
MySegmentsUpdate = "MY_SEGMENTS_UPDATE"
// Control for control
Control = "CONTROL"
// StreamingPause for controlType
StreamingPause = "STREAMING_PAUSED"
// StreamingResumed for controlType
StreamingResumed = "STREAMING_RESUMED"
// StreamingDisabled for controlType
StreamingDisabled = "STREAMING_DISABLED"
)
// IncomingNotification struct for incoming notification from streaming
type IncomingNotification struct {
Channel string `json:"channel"`
ChangeNumber *int64 `json:"changeNumber,omitempty"`
ControlType *string `json:"controlType,omitempty"`
DefaultTreatment *string `json:"defaultTreatment,omitempty"`
SegmentName *string `json:"segmentName,omitempty"`
SplitName *string `json:"splitName,omitempty"`
Timestamp *int64 `json:"timestamp,omitempty"`
Type string `json:"type"`
}
// Notification should be implemented by all notification types
type Notification interface {
ChannelName() string
NotificationType() string
}
// base struct with added logic that wraps around a DTO
type base struct {
channelName string
notificationType string
}
// ChannelName returns channel name
func (b base) ChannelName() string {
return b.channelName
}
// NotificationType returns the type of the notification
func (b base) NotificationType() string {
return b.notificationType
}
// ControlNotification notification for control channels
type ControlNotification struct {
base
ControlType string
}
// NewControlNotification builds a notification for controlling connection
func NewControlNotification(channelName string, controlType string) ControlNotification {
return ControlNotification{
base: base{
channelName: channelName,
notificationType: Control,
},
ControlType: controlType,
}
}
// MySegmentsNotification notification when MySegments is updated
type MySegmentsNotification struct {
base
IncludesPayload bool
Payload []string
ChangeNumber int64
}
// NewMySegmentsNotification builds a MySegments notification
func NewMySegmentsNotification(channelName string, includesPayload bool, payload []string, changeNumber int64) MySegmentsNotification {
return MySegmentsNotification{
base: base{
channelName: channelName,
notificationType: MySegmentsUpdate,
},
IncludesPayload: includesPayload,
Payload: payload,
ChangeNumber: changeNumber,
}
}
// SegmentChangeNotification notification when a Segment is updated
type SegmentChangeNotification struct {
base
ChangeNumber int64
SegmentName string
}
// NewSegmentChangeNotification builds a segment change notification
func NewSegmentChangeNotification(channelName string, changeNumber int64, segmentName string) SegmentChangeNotification {
return SegmentChangeNotification{
base: base{
channelName: channelName,
notificationType: SegmentUpdate,
},
ChangeNumber: changeNumber,
SegmentName: segmentName,
}
}
// SplitChangeNotification notification to send a fetch to splitChanges
type SplitChangeNotification struct {
base
ChangeNumber int64
}
// NewSplitChangeNotification builds a split change notification
func NewSplitChangeNotification(channelName string, changeNumber int64) SplitChangeNotification {
return SplitChangeNotification{
base: base{
channelName: channelName,
notificationType: SplitUpdate,
},
ChangeNumber: changeNumber,
}
}
// SplitKillNotification notification when Split is killed
type SplitKillNotification struct {
base
ChangeNumber int64
DefaultTreatment string
SplitName string
}
// NewSplitKillNotification builds a killed split notification
func NewSplitKillNotification(channelName string, changeNumber int64, defaultTreatment string, splitName string) SplitKillNotification {
return SplitKillNotification{
base: base{
channelName: channelName,
notificationType: SplitKill,
},
ChangeNumber: changeNumber,
DefaultTreatment: defaultTreatment,
SplitName: splitName,
}
}

22
vendor/github.com/splitio/go-split-commons/v2/dtos/segment.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,22 @@
package dtos
// SegmentChangesDTO struct to map a segment change message.
type SegmentChangesDTO struct {
Name string `json:"name"`
Added []string `json:"added"`
Removed []string `json:"removed"`
Since int64 `json:"since"`
Till int64 `json:"till"`
}
// SegmentKeyDTO maps key data
type SegmentKeyDTO struct {
Name string `json:"name"`
LastModified int64 `json:"lastModified"`
Removed bool `json:"removed"`
}
// MySegmentDTO struct mapping segment data for mySegments endpoint
type MySegmentDTO struct {
Name string `json:"name"`
}

100
vendor/github.com/splitio/go-split-commons/v2/dtos/split.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,100 @@
package dtos
import "encoding/json"
// SplitChangesDTO structure to map JSON message sent by Split servers.
type SplitChangesDTO struct {
Till int64 `json:"till"`
Since int64 `json:"since"`
Splits []SplitDTO `json:"splits"`
}
// SplitDTO structure to map an Split definition fetched from JSON message.
type SplitDTO struct {
ChangeNumber int64 `json:"changeNumber"`
TrafficTypeName string `json:"trafficTypeName"`
Name string `json:"name"`
TrafficAllocation int `json:"trafficAllocation"`
TrafficAllocationSeed int64 `json:"trafficAllocationSeed"`
Seed int64 `json:"seed"`
Status string `json:"status"`
Killed bool `json:"killed"`
DefaultTreatment string `json:"defaultTreatment"`
Algo int `json:"algo"`
Conditions []ConditionDTO `json:"conditions"`
Configurations map[string]string `json:"configurations"`
}
// MarshalBinary exports SplitDTO to JSON string
func (s SplitDTO) MarshalBinary() (data []byte, err error) {
return json.Marshal(s)
}
// ConditionDTO structure to map a Condition fetched from JSON message.
type ConditionDTO struct {
ConditionType string `json:"conditionType"`
MatcherGroup MatcherGroupDTO `json:"matcherGroup"`
Partitions []PartitionDTO `json:"partitions"`
Label string `json:"label"`
}
// PartitionDTO structure to map a Partition definition fetched from JSON message.
type PartitionDTO struct {
Treatment string `json:"treatment"`
Size int `json:"size"`
}
// MatcherGroupDTO structure to map a Matcher Group definition fetched from JSON message.
type MatcherGroupDTO struct {
Combiner string `json:"combiner"`
Matchers []MatcherDTO `json:"matchers"`
}
// MatcherDTO structure to map a Matcher definition fetched from JSON message.
type MatcherDTO struct {
KeySelector *KeySelectorDTO `json:"keySelector"`
MatcherType string `json:"matcherType"`
Negate bool `json:"negate"`
UserDefinedSegment *UserDefinedSegmentMatcherDataDTO `json:"userDefinedSegmentMatcherData"`
Whitelist *WhitelistMatcherDataDTO `json:"whitelistMatcherData"`
UnaryNumeric *UnaryNumericMatcherDataDTO `json:"unaryNumericMatcherData"`
Between *BetweenMatcherDataDTO `json:"betweenMatcherData"`
Dependency *DependencyMatcherDataDTO `json:"dependencyMatcherData"`
Boolean *bool `json:"booleanMatcherData"`
String *string `json:"stringMatcherData"`
}
// UserDefinedSegmentMatcherDataDTO structure to map a Matcher definition fetched from JSON message.
type UserDefinedSegmentMatcherDataDTO struct {
SegmentName string `json:"segmentName"`
}
// BetweenMatcherDataDTO structure to map a Matcher definition fetched from JSON message.
type BetweenMatcherDataDTO struct {
DataType string `json:"dataType"` //NUMBER or DATETIME
Start int64 `json:"start"`
End int64 `json:"end"`
}
// UnaryNumericMatcherDataDTO structure to map a Matcher definition fetched from JSON message.
type UnaryNumericMatcherDataDTO struct {
DataType string `json:"dataType"` //NUMBER or DATETIME
Value int64 `json:"value"`
}
// WhitelistMatcherDataDTO structure to map a Matcher definition fetched from JSON message.
type WhitelistMatcherDataDTO struct {
Whitelist []string `json:"whitelist"`
}
// DependencyMatcherDataDTO structure to map matcher definition fetched from JSON message.
type DependencyMatcherDataDTO struct {
Split string `json:"split"`
Treatments []string `json:"treatments"`
}
// KeySelectorDTO structure to map a Key slector definition fetched from JSON message.
type KeySelectorDTO struct {
TrafficType string `json:"trafficType"`
Attribute *string `json:"attribute"`
}

103
vendor/github.com/splitio/go-split-commons/v2/dtos/token.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,103 @@
package dtos
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
)
const gracePeriod = 10 * time.Minute
const metadataPlaceHolder = "channel-metadata:publishers"
const occupancy = "[?occupancy=metrics.publishers]"
// Token dto
type Token struct {
Token string `json:"token"`
PushEnabled bool `json:"pushEnabled"`
}
// TokenPayload payload dto
type TokenPayload struct {
Capabilitites string `json:"x-ably-capability"`
Exp int64 `json:"exp"`
Iat int64 `json:"iat"`
}
// ParsedCapabilities capabilities
type ParsedCapabilities map[string][]string
func isMetadataType(capabilities []string) bool {
for _, capability := range capabilities {
if capability == metadataPlaceHolder {
return true
}
}
return false
}
// ChannelList grabs the channel list from capabilities
func (t *Token) ChannelList() ([]string, error) {
if !t.PushEnabled || t.Token == "" {
return nil, errors.New("Push disabled or no token set")
}
tokenParts := strings.Split(t.Token, ".")
if len(tokenParts) < 2 {
return nil, errors.New("Cannot decode token")
}
decodedPayload, err := base64.RawURLEncoding.DecodeString(tokenParts[1])
if err != nil {
return nil, err
}
var parsedPayload TokenPayload
err = json.Unmarshal(decodedPayload, &parsedPayload)
if err != nil {
return nil, err
}
var parsedCapabilities ParsedCapabilities
err = json.Unmarshal([]byte(parsedPayload.Capabilitites), &parsedCapabilities)
if err != nil {
return nil, err
}
channelList := make([]string, 0, len(parsedCapabilities))
for channelName := range parsedCapabilities {
if isMetadataType(parsedCapabilities[channelName]) {
channelList = append(channelList, fmt.Sprintf("%s%s", occupancy, channelName))
} else {
channelList = append(channelList, channelName)
}
}
return channelList, nil
}
// CalculateNextTokenExpiration calculates next token expiration
func (t *Token) CalculateNextTokenExpiration() (time.Duration, error) {
if !t.PushEnabled || t.Token == "" {
return 0, errors.New("Push disabled or no token set")
}
tokenParts := strings.Split(t.Token, ".")
if len(tokenParts) < 2 {
return 0, errors.New("Cannot decode token")
}
decodedPayload, err := base64.RawURLEncoding.DecodeString(tokenParts[1])
if err != nil {
return 0, err
}
var parsedPayload TokenPayload
err = json.Unmarshal(decodedPayload, &parsedPayload)
if err != nil {
return 0, err
}
tokenDuration := parsedPayload.Exp - parsedPayload.Iat
return time.Duration(tokenDuration)*time.Second - gracePeriod, nil
}

59
vendor/github.com/splitio/go-split-commons/v2/provisional/impcounter.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,59 @@
package provisional
import (
"sync"
"github.com/splitio/go-split-commons/v2/util"
)
// Key struct for mapping each key to an amount
type Key struct {
FeatureName string
TimeFrame int64
}
// ImpressionsCounter struct for storing generated impressions counts
type ImpressionsCounter struct {
impressionsCounts map[Key]int64
mutex *sync.RWMutex
}
// NewImpressionsCounter creates new ImpressionsCounter
func NewImpressionsCounter() *ImpressionsCounter {
return &ImpressionsCounter{
impressionsCounts: make(map[Key]int64),
mutex: &sync.RWMutex{},
}
}
func makeKey(splitName string, timeFrame int64) Key {
return Key{
FeatureName: splitName,
TimeFrame: util.TruncateTimeFrame(timeFrame),
}
}
// Inc increments the quantity of impressions with the passed splitName and timeFrame
func (i *ImpressionsCounter) Inc(splitName string, timeFrame int64, amount int64) {
i.mutex.Lock()
defer i.mutex.Unlock()
key := makeKey(splitName, timeFrame)
currentAmount, _ := i.impressionsCounts[key]
i.impressionsCounts[key] = currentAmount + amount
}
// PopAll returns all the elements stored in the cache and resets the cache
func (i *ImpressionsCounter) PopAll() map[Key]int64 {
i.mutex.Lock()
defer i.mutex.Unlock()
toReturn := i.impressionsCounts
i.impressionsCounts = make(map[Key]int64)
return toReturn
}
// Size returns how many keys are stored in cache
func (i *ImpressionsCounter) Size() int {
i.mutex.RLock()
defer i.mutex.RUnlock()
return len(i.impressionsCounts)
}

43
vendor/github.com/splitio/go-split-commons/v2/provisional/imphasher.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,43 @@
package provisional
import (
"fmt"
"strings"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/provisional/hashing"
)
const hashKeyTemplate = "%s:%s:%s:%s:%d"
func unknownIfEmpty(s string) string {
if len(strings.TrimSpace(s)) == 0 {
return "UNKNOWN"
}
return s
}
// ImpressionHasher interface
type ImpressionHasher interface {
Process(featureName string, impression *dtos.Impression) (int64, error)
}
// ImpressionHasherImpl implements the hasher interface, mapping certain fields to an int64
type ImpressionHasherImpl struct{}
// Process an impression and return the 64 LSBs of a murmur3-128 digest
func (h *ImpressionHasherImpl) Process(featureName string, impression *dtos.Impression) (int64, error) {
if impression == nil {
return 0, fmt.Errorf("keyImpression cannot be nil")
}
toHash := fmt.Sprintf(hashKeyTemplate,
unknownIfEmpty(impression.KeyName),
unknownIfEmpty(featureName),
unknownIfEmpty(impression.Treatment),
unknownIfEmpty(impression.Label),
impression.ChangeNumber)
h1, _ := hashing.Sum128([]byte(toHash))
return int64(h1), nil
}

76
vendor/github.com/splitio/go-split-commons/v2/provisional/impmanager.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,76 @@
package provisional
import (
"time"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/util"
)
const lastSeenCacheSize = 500000 // cache up to 500k impression hashes
// ImpressionManager interface
type ImpressionManager interface {
ProcessImpressions(impressions []dtos.Impression) ([]dtos.Impression, []dtos.Impression)
}
// ImpressionManagerImpl implements
type ImpressionManagerImpl struct {
impressionObserver ImpressionObserver
impressionsCounter *ImpressionsCounter
shouldAddPreviousTime bool
isOptimized bool
listenerEnabled bool
}
// NewImpressionManager creates new ImpManager
func NewImpressionManager(managerConfig conf.ManagerConfig, impressionCounter *ImpressionsCounter) (ImpressionManager, error) {
impressionObserver, err := NewImpressionObserver(lastSeenCacheSize)
if err != nil {
return nil, err
}
impManager := &ImpressionManagerImpl{
impressionObserver: impressionObserver,
impressionsCounter: impressionCounter,
shouldAddPreviousTime: util.ShouldAddPreviousTime(managerConfig),
isOptimized: impressionCounter != nil && util.ShouldBeOptimized(managerConfig),
listenerEnabled: managerConfig.ListenerEnabled,
}
return impManager, nil
}
func (i *ImpressionManagerImpl) processImpression(impression dtos.Impression, forLog []dtos.Impression, forListener []dtos.Impression) ([]dtos.Impression, []dtos.Impression) {
if i.shouldAddPreviousTime {
impression.Pt, _ = i.impressionObserver.TestAndSet(impression.FeatureName, &impression) // Adds previous time if it is enabled
}
now := time.Now().UTC().UnixNano()
if i.isOptimized { // isOptimized
i.impressionsCounter.Inc(impression.FeatureName, now, 1) // Increments impression counter per featureName
}
if !i.isOptimized || impression.Pt == 0 || impression.Pt < util.TruncateTimeFrame(now) {
forLog = append(forLog, impression)
}
if i.listenerEnabled {
forListener = append(forListener, impression)
}
return forLog, forListener
}
// ProcessImpressions bulk processes
func (i *ImpressionManagerImpl) ProcessImpressions(impressions []dtos.Impression) ([]dtos.Impression, []dtos.Impression) {
forLog := make([]dtos.Impression, 0, len(impressions))
forListener := make([]dtos.Impression, 0, len(impressions))
for _, impression := range impressions {
forLog, forListener = i.processImpression(impression, forLog, forListener)
}
return forLog, forListener
}

61
vendor/github.com/splitio/go-split-commons/v2/provisional/impobserver.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,61 @@
package provisional
import (
"fmt"
"sync"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/provisional/int64cache"
)
// ImpressionObserver is used to check wether an impression has been previously seen
type ImpressionObserver interface {
TestAndSet(featureName string, impression *dtos.Impression) (int64, error)
}
// ImpressionObserverImpl is an implementation of the ImpressionObserver interface
type ImpressionObserverImpl struct {
cache int64cache.Int64Cache
hasher ImpressionHasher
mutex sync.Mutex
}
// Atomically fetch cache data and update it
func (o *ImpressionObserverImpl) testAndSet(key int64, newValue int64) (int64, error) {
o.mutex.Lock()
defer o.mutex.Unlock()
old, err := o.cache.Get(key)
o.cache.Set(key, newValue)
return old, err
}
// TestAndSet hashes the impression, updates the cache and returns the previous value
func (o *ImpressionObserverImpl) TestAndSet(featureName string, impression *dtos.Impression) (int64, error) {
hash, err := o.hasher.Process(featureName, impression)
if err != nil {
return 0, fmt.Errorf("error hashing impression: %s", err.Error())
}
return o.testAndSet(hash, impression.Time)
}
// NewImpressionObserver constructs a new ImpressionObserver
func NewImpressionObserver(size int) (*ImpressionObserverImpl, error) {
cache, err := int64cache.NewInt64Cache(size)
if err != nil {
return nil, fmt.Errorf("error building cache: %s", err.Error())
}
return &ImpressionObserverImpl{
cache: cache,
hasher: &ImpressionHasherImpl{},
mutex: sync.Mutex{},
}, nil
}
// ImpressionObserverNoOp is an implementation of the ImpressionObserver interface
type ImpressionObserverNoOp struct{}
// TestAndSet that does nothing
func (o *ImpressionObserverNoOp) TestAndSet(featureName string, impression *dtos.Impression) (int64, error) {
return 0, nil
}

27
vendor/github.com/splitio/go-split-commons/v2/push/dtos.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,27 @@
package push
// IncomingEvent struct to process every kind of notification that comes from streaming
type IncomingEvent struct {
id *string
timestamp *int64
encoding *string
data *string
name *string
clientID *string
event string
channel *string
message *string
code *int
statusCode *int
href *string
}
// Metrics dto
type Metrics struct {
Publishers int `json:"publishers"`
}
// Occupancy dto
type Occupancy struct {
Data Metrics `json:"metrics"`
}

89
vendor/github.com/splitio/go-split-commons/v2/push/eventhandler.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,89 @@
package push
import (
"encoding/json"
"fmt"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
// EventHandler struct
type EventHandler struct {
keeper *Keeper
parser *NotificationParser
processor *Processor
logger logging.LoggerInterface
}
// NewEventHandler builds new EventHandler
func NewEventHandler(keeper *Keeper, parser *NotificationParser, processor *Processor, logger logging.LoggerInterface) *EventHandler {
return &EventHandler{
keeper: keeper,
parser: parser,
processor: processor,
logger: logger,
}
}
func (e *EventHandler) wrapOccupancy(incomingEvent IncomingEvent) *Occupancy {
if incomingEvent.data == nil {
return nil
}
var occupancy *Occupancy
err := json.Unmarshal([]byte(*incomingEvent.data), &occupancy)
if err != nil {
return nil
}
return occupancy
}
func (e *EventHandler) wrapUpdateEvent(incomingEvent IncomingEvent) *dtos.IncomingNotification {
if incomingEvent.data == nil {
return nil
}
var incomingNotification *dtos.IncomingNotification
err := json.Unmarshal([]byte(*incomingEvent.data), &incomingNotification)
if err != nil {
e.logger.Error("cannot parse data as IncomingNotification type")
return nil
}
incomingNotification.Channel = *incomingEvent.channel
return incomingNotification
}
// HandleIncomingMessage handles incoming message from streaming
func (e *EventHandler) HandleIncomingMessage(event map[string]interface{}) {
incomingEvent := e.parser.Parse(event)
switch incomingEvent.event {
case update:
e.logger.Debug("Update event received")
incomingNotification := e.wrapUpdateEvent(incomingEvent)
if incomingNotification == nil {
e.logger.Debug("Skipping incoming notification...")
return
}
e.logger.Debug("Incoming Notification:", incomingNotification)
err := e.processor.Process(*incomingNotification)
if err != nil {
e.logger.Debug("Could not process notification", err.Error())
return
}
case occupancy:
e.logger.Debug("Presence event received")
occupancy := e.wrapOccupancy(incomingEvent)
if occupancy == nil || incomingEvent.channel == nil {
e.logger.Debug("Skipping occupancy...")
return
}
e.keeper.UpdateManagers(*incomingEvent.channel, occupancy.Data.Publishers)
return
case errorType: // TODO: Update this when logic is fully defined
e.logger.Error(fmt.Sprintf("Error received: %+v", incomingEvent))
default:
e.logger.Debug(fmt.Sprintf("Unexpected incomingEvent: %+v", incomingEvent))
e.logger.Error("Unexpected type of event received")
}
}

10
vendor/github.com/splitio/go-split-commons/v2/push/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,10 @@
package push
// Manager interface for Push Manager
type Manager interface {
Start()
Stop()
StartWorkers()
StopWorkers()
IsRunning() bool
}

98
vendor/github.com/splitio/go-split-commons/v2/push/keeper.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,98 @@
package push
import (
"strings"
"sync"
)
const (
// PublisherNotPresent there are no publishers sending data
PublisherNotPresent = iota
// PublisherAvailable there are publishers running
PublisherAvailable
)
const (
prefix = "[?occupancy=metrics.publishers]"
)
// last struct for storing the last notification
type last struct {
manager string
timestamp int64
mutex *sync.RWMutex
}
// Keeper struct
type Keeper struct {
managers map[string]int
activeRegion string
last last
publishers chan<- int
mutex *sync.RWMutex
}
// NewKeeper creates new keeper
func NewKeeper(publishers chan int) *Keeper {
last := last{
mutex: &sync.RWMutex{},
}
return &Keeper{
managers: make(map[string]int),
activeRegion: "us-east-1",
mutex: &sync.RWMutex{},
publishers: publishers,
last: last,
}
}
func (k *Keeper) cleanManagerPrefix(manager string) string {
return strings.Replace(manager, prefix, "", -1)
}
// Publishers returns the quantity of publishers for a particular manager
func (k *Keeper) Publishers(manager string) *int {
k.mutex.RLock()
defer k.mutex.RUnlock()
publisher, ok := k.managers[manager]
if ok {
return &publisher
}
return nil
}
// UpdateManagers updates current manager count
func (k *Keeper) UpdateManagers(manager string, publishers int) {
parsedManager := k.cleanManagerPrefix(manager)
k.mutex.Lock()
defer k.mutex.Unlock()
k.managers[parsedManager] = publishers
isAvailable := false
for _, publishers := range k.managers {
if publishers > 0 {
isAvailable = true
break
}
}
if !isAvailable {
k.publishers <- PublisherNotPresent
return
}
k.publishers <- PublisherAvailable
}
// LastNotification return the latest notification saved
func (k *Keeper) LastNotification() (string, int64) {
k.last.mutex.RLock()
defer k.last.mutex.RUnlock()
return k.last.manager, k.last.timestamp
}
// UpdateLastNotification updates last message received
func (k *Keeper) UpdateLastNotification(manager string, timestamp int64) {
k.last.mutex.Lock()
defer k.last.mutex.Unlock()
k.last.manager = k.cleanManagerPrefix(manager)
k.last.timestamp = timestamp
}

64
vendor/github.com/splitio/go-split-commons/v2/push/parser.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,64 @@
package push
import (
"github.com/splitio/go-toolkit/v3/common"
"github.com/splitio/go-toolkit/v3/logging"
)
const (
update = "update"
errorType = "error"
occupancy = "[meta]occupancy"
)
// NotificationParser struct
type NotificationParser struct {
logger logging.LoggerInterface
}
// NewNotificationParser creates notifcation parser
func NewNotificationParser(logger logging.LoggerInterface) *NotificationParser {
return &NotificationParser{
logger: logger,
}
}
// Parse parses incoming event from streaming
func (n *NotificationParser) Parse(event map[string]interface{}) IncomingEvent {
incomingEvent := IncomingEvent{
id: common.AsStringOrNil(event["id"]),
encoding: common.AsStringOrNil(event["encoding"]),
data: common.AsStringOrNil(event["data"]),
name: common.AsStringOrNil(event["name"]),
clientID: common.AsStringOrNil(event["clientId"]),
channel: common.AsStringOrNil(event["channel"]),
message: common.AsStringOrNil(event["message"]),
href: common.AsStringOrNil(event["href"]),
}
timestamp := common.AsFloat64OrNil(event["timestamp"])
if timestamp != nil {
incomingEvent.timestamp = common.Int64Ref(int64(*timestamp))
}
code := common.AsFloat64OrNil(event["code"])
if code != nil {
incomingEvent.code = common.IntRef(int(*code))
}
statusCode := common.AsFloat64OrNil(event["statusCode"])
if statusCode != nil {
incomingEvent.statusCode = common.IntRef(int(*statusCode))
}
if incomingEvent.code != nil && incomingEvent.statusCode != nil {
incomingEvent.event = errorType
return incomingEvent
}
if incomingEvent.name != nil && *incomingEvent.name == occupancy {
incomingEvent.event = occupancy
return incomingEvent
}
incomingEvent.event = update
return incomingEvent
}

112
vendor/github.com/splitio/go-split-commons/v2/push/processor.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,112 @@
package push
import (
"errors"
"fmt"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/storage"
"github.com/splitio/go-toolkit/v3/logging"
)
const (
segmentQueueCheck = 5000
splitQueueCheck = 5000
streamingPausedType = "STREAMING_PAUSED"
streamingResumedType = "STREAMING_RESUMED"
streamingDisabledType = "STREAMING_DISABLED"
)
const (
// StreamingPaused The SDK should stop processing incoming UPDATE-type events
streamingPaused = iota
// StreamingResumed The SDK should resume processing UPDATE-type events (if not already)
streamingResumed
// StreamingDisabled The SDK should disable streaming completely and dont try to reconnect until the SDK is re-instantiated
streamingDisabled
)
// Processor struct for notification processor
type Processor struct {
segmentQueue chan dtos.SegmentChangeNotification
splitQueue chan dtos.SplitChangeNotification
splitStorage storage.SplitStorageProducer
controlStatus chan<- int
logger logging.LoggerInterface
}
// NewProcessor creates new processor
func NewProcessor(segmentQueue chan dtos.SegmentChangeNotification, splitQueue chan dtos.SplitChangeNotification, splitStorage storage.SplitStorageProducer, logger logging.LoggerInterface, controlStatus chan int) (*Processor, error) {
if cap(segmentQueue) < segmentQueueCheck {
return nil, errors.New("Small size of segmentQueue")
}
if cap(splitQueue) < splitQueueCheck {
return nil, errors.New("Small size of splitQueue")
}
if cap(controlStatus) < 1 {
return nil, errors.New("Small size for control chan")
}
return &Processor{
segmentQueue: segmentQueue,
splitQueue: splitQueue,
splitStorage: splitStorage,
controlStatus: controlStatus,
logger: logger,
}, nil
}
// Process takes an incoming notification and generates appropriate notifications for it.
func (p *Processor) Process(i dtos.IncomingNotification) error {
switch i.Type {
case dtos.SplitUpdate:
if i.ChangeNumber == nil {
return errors.New("ChangeNumber could not be nil, discarded")
}
splitUpdate := dtos.NewSplitChangeNotification(i.Channel, *i.ChangeNumber)
p.splitQueue <- splitUpdate
case dtos.SegmentUpdate:
if i.ChangeNumber == nil {
return errors.New("ChangeNumber could not be nil, discarded")
}
if i.SegmentName == nil {
return errors.New("SegmentName could not be nil, discarded")
}
segmentUpdate := dtos.NewSegmentChangeNotification(i.Channel, *i.ChangeNumber, *i.SegmentName)
p.segmentQueue <- segmentUpdate
case dtos.SplitKill:
if i.ChangeNumber == nil {
return errors.New("ChangeNumber could not be nil, discarded")
}
if i.SplitName == nil {
return errors.New("SplitName could not be nil, discarded")
}
if i.DefaultTreatment == nil {
return errors.New("DefaultTreatment could not be nil, discarded")
}
splitUpdate := dtos.NewSplitChangeNotification(i.Channel, *i.ChangeNumber)
p.splitStorage.KillLocally(*i.SplitName, *i.DefaultTreatment, *i.ChangeNumber)
p.splitQueue <- splitUpdate
case dtos.Control:
if i.ControlType == nil {
return errors.New("ControlType could not be nil, discarded")
}
control := dtos.NewControlNotification(i.Channel, *i.ControlType)
switch control.ControlType {
case streamingDisabledType:
p.logger.Debug("Received notification for disabling streaming")
p.controlStatus <- streamingDisabled
case streamingPausedType:
p.logger.Debug("Received notification for pausing streaming")
p.controlStatus <- streamingPaused
case streamingResumedType:
p.logger.Debug("Received notification for resuming streaming")
p.controlStatus <- streamingResumed
default:
p.logger.Debug(fmt.Sprintf("%s Unexpected type of Control Notification", control.ControlType))
}
default:
return fmt.Errorf("Unknown IncomingNotification type: %T", i)
}
return nil
}

370
vendor/github.com/splitio/go-split-commons/v2/push/pushmanager.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,370 @@
package push
import (
"errors"
"fmt"
"net/http"
"sync/atomic"
"time"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-split-commons/v2/service/api/sse"
"github.com/splitio/go-split-commons/v2/storage"
"github.com/splitio/go-toolkit/v3/common"
"github.com/splitio/go-toolkit/v3/logging"
sseStatus "github.com/splitio/go-toolkit/v3/sse"
)
const (
resetTimer = 120
maxPeriod = 30 * time.Minute
)
const (
// Ready represents ready
Ready = iota
// PushIsDown there are no publishers for streaming
PushIsDown
// PushIsUp there are publishers presents
PushIsUp
// BackoffAuth backoff is running for authentication
BackoffAuth
// BackoffSSE backoff is running for connecting to stream
BackoffSSE
// TokenExpiration flag to restart push services
TokenExpiration
// StreamingPaused flag for pausing streaming
StreamingPaused
// StreamingResumed flag for resuming streaming
StreamingResumed
// StreamingDisabled flag for disabling streaming
StreamingDisabled
// Reconnect flag to reconnect
Reconnect
// NonRetriableError represents an error that will force switching to polling
NonRetriableError
)
// PushManager struct for managing push services
type PushManager struct {
authClient service.AuthClient
sseClient *sse.StreamingClient
segmentWorker *SegmentUpdateWorker
splitWorker *SplitUpdateWorker
eventHandler *EventHandler
managerStatus chan<- int
streamingStatus chan int
publishers chan int
logger logging.LoggerInterface
cancelAuthBackoff chan struct{}
cancelSSEBackoff chan struct{}
cancelTokenExpiration chan struct{}
cancelStreamingWatcher chan struct{}
control chan int
status atomic.Value
}
// NewPushManager creates new PushManager
func NewPushManager(
logger logging.LoggerInterface,
synchronizeSegmentHandler func(segmentName string, till *int64) error,
synchronizeSplitsHandler func(till *int64) error,
splitStorage storage.SplitStorage,
config *conf.AdvancedConfig,
managerStatus chan int,
authClient service.AuthClient,
) (Manager, error) {
splitQueue := make(chan dtos.SplitChangeNotification, config.SplitUpdateQueueSize)
segmentQueue := make(chan dtos.SegmentChangeNotification, config.SegmentUpdateQueueSize)
control := make(chan int, 1)
processor, err := NewProcessor(segmentQueue, splitQueue, splitStorage, logger, control)
if err != nil {
return nil, err
}
parser := NewNotificationParser(logger)
if parser == nil {
return nil, errors.New("Could not instantiate NotificationParser")
}
publishers := make(chan int, 1000)
keeper := NewKeeper(publishers)
if keeper == nil {
return nil, errors.New("Could not instantiate Keeper")
}
eventHandler := NewEventHandler(keeper, parser, processor, logger)
segmentWorker, err := NewSegmentUpdateWorker(segmentQueue, synchronizeSegmentHandler, logger)
if err != nil {
return nil, err
}
splitWorker, err := NewSplitUpdateWorker(splitQueue, synchronizeSplitsHandler, logger)
if err != nil {
return nil, err
}
streamingStatus := make(chan int, 1000)
status := atomic.Value{}
status.Store(Ready)
return &PushManager{
authClient: authClient,
sseClient: sse.NewStreamingClient(config, streamingStatus, logger),
segmentWorker: segmentWorker,
splitWorker: splitWorker,
managerStatus: managerStatus,
streamingStatus: streamingStatus,
eventHandler: eventHandler,
publishers: publishers,
logger: logger,
cancelAuthBackoff: make(chan struct{}, 1),
cancelSSEBackoff: make(chan struct{}, 1),
cancelTokenExpiration: make(chan struct{}, 1),
cancelStreamingWatcher: make(chan struct{}, 1),
control: control,
status: status,
}, nil
}
func (p *PushManager) cancelStreaming() {
p.logger.Error("Error, switching to polling")
p.managerStatus <- NonRetriableError
}
func (p *PushManager) performAuthentication(errResult chan error) *dtos.Token {
select {
case <-p.cancelAuthBackoff:
// Discarding previous msg
default:
}
tokenResult := make(chan *dtos.Token, 1)
cancelAuthBackoff := common.WithBackoffCancelling(1*time.Second, maxPeriod, func() bool {
token, err := p.authClient.Authenticate()
if err != nil {
errType, ok := err.(dtos.HTTPError)
if ok && errType.Code >= http.StatusInternalServerError {
p.managerStatus <- BackoffAuth
return false // It will continue retrying
}
errResult <- errors.New("Error authenticating")
return true
}
tokenResult <- token
return true // Result is OK, Stopping Here, no more backoff
})
defer cancelAuthBackoff()
select {
case token := <-tokenResult:
if !token.PushEnabled {
return nil
}
return token
case err := <-errResult:
p.logger.Error(err.Error())
return nil
case <-p.cancelAuthBackoff:
return nil
}
}
func (p *PushManager) connectToStreaming(errResult chan error, token string, channels []string) error {
select {
case <-p.cancelSSEBackoff:
// Discarding previous msg
default:
}
sseResult := make(chan struct{}, 1)
cancelSSEBackoff := common.WithBackoffCancelling(1*time.Second, maxPeriod, func() bool {
p.sseClient.ConnectStreaming(token, channels, p.eventHandler.HandleIncomingMessage)
status := <-p.streamingStatus
switch status {
case sseStatus.OK:
sseResult <- struct{}{}
return true
case sseStatus.ErrorInternal:
p.managerStatus <- BackoffSSE
return false // It will continue retrying
default:
errResult <- errors.New("Error connecting streaming")
return true
}
})
defer cancelSSEBackoff()
select {
case <-sseResult:
return nil
case err := <-errResult:
p.logger.Error(err.Error())
return err
case <-p.cancelSSEBackoff:
return nil
}
}
func (p *PushManager) fetchStreamingToken(errResult chan error) (string, []string, error) {
token := p.performAuthentication(errResult)
if token == nil {
return "", []string{}, errors.New("Could not perform authentication")
}
channels, err := token.ChannelList()
if err != nil {
return "", []string{}, errors.New("Could not perform authentication")
}
nextTokenExpiration, err := token.CalculateNextTokenExpiration()
if err != nil {
return "", []string{}, errors.New("Could not perform authentication")
}
go func() {
// Create timeout timer for calculating next token expiration
idleDuration := nextTokenExpiration
tokenExpirationTimer := time.NewTimer(idleDuration)
defer tokenExpirationTimer.Stop()
select {
case <-tokenExpirationTimer.C: // Timedout
p.logger.Info("Token expired")
p.managerStatus <- TokenExpiration
return
case <-p.cancelTokenExpiration:
return
}
}()
return token.Token, channels, nil
}
func (p *PushManager) streamingStatusWatcher() {
for {
select {
case status := <-p.streamingStatus: // Streaming SSE Status
switch status {
case sseStatus.ErrorKeepAlive: // On ConnectionTimedOut -> Reconnect
fallthrough
case sseStatus.ErrorInternal: // On Error >= 500 -> Reconnect
fallthrough
case sseStatus.ErrorReadingStream: // On IOF -> Reconnect
p.managerStatus <- Reconnect
default: // Whatever other errors -> Send Error to disconnect
p.cancelStreaming()
}
case publisherStatus := <-p.publishers: // Publisher Available/Not Available
switch publisherStatus {
case PublisherNotPresent:
if p.status.Load().(int) != StreamingPaused {
p.managerStatus <- PushIsDown
}
case PublisherAvailable:
if p.status.Load().(int) != StreamingPaused {
p.managerStatus <- PushIsUp
}
default:
p.logger.Debug(fmt.Sprintf("Unexpected publisher status received %d", publisherStatus))
}
case controlStatus := <-p.control:
switch controlStatus {
case streamingPaused:
p.logger.Debug("Received Pause Streaming Notification")
if p.status.Load().(int) != StreamingPaused {
p.logger.Info("Sending Pause Streaming")
p.status.Store(StreamingPaused)
p.managerStatus <- PushIsDown
}
case streamingResumed:
p.logger.Debug("Received Resume Streaming Notification")
if p.status.Load().(int) == StreamingPaused {
p.status.Store(StreamingResumed)
publishersAvailable := p.eventHandler.keeper.Publishers("control_pri")
if publishersAvailable != nil && *publishersAvailable > 0 {
p.logger.Info("Sending Resume Streaming")
p.managerStatus <- PushIsUp
}
}
case streamingDisabled:
p.logger.Info("Received Streaming Disabled Notification")
p.managerStatus <- StreamingDisabled
default:
p.logger.Debug(fmt.Sprintf("Unexpected control status received %d", controlStatus))
}
case <-p.cancelStreamingWatcher: // Stopping Watcher
return
}
}
}
func (p *PushManager) drainStatus() {
select {
case <-p.cancelStreamingWatcher: // Discarding previous msg
default:
}
select {
case <-p.cancelTokenExpiration: // Discarding previous token expiration
default:
}
}
// Start push services
func (p *PushManager) Start() {
if p.IsRunning() {
p.logger.Info("PushManager is already running, skipping Start")
return
}
p.drainStatus()
// errResult listener for fetching token and connecting to SSE
errResult := make(chan error, 1)
token, channels, err := p.fetchStreamingToken(errResult)
if err != nil {
p.cancelStreaming()
return
}
err = p.connectToStreaming(errResult, token, channels)
if err != nil {
p.cancelStreaming()
return
}
// Everything is good, starting workers
p.splitWorker.Start()
p.segmentWorker.Start()
// Sending Ready
p.managerStatus <- Ready
// Starting streaming status watcher, it will listen 1) errors in SSE, 2) publishers changes, 3) stop
go p.streamingStatusWatcher()
}
// Stop push services
func (p *PushManager) Stop() {
p.logger.Info("Stopping Push Services")
p.cancelAuthBackoff <- struct{}{}
p.cancelSSEBackoff <- struct{}{}
p.cancelTokenExpiration <- struct{}{}
p.cancelStreamingWatcher <- struct{}{}
if p.sseClient.IsRunning() {
p.sseClient.StopStreaming(true)
}
p.StopWorkers()
}
// IsRunning returns true if the services are running
func (p *PushManager) IsRunning() bool {
return p.sseClient.IsRunning() || p.splitWorker.IsRunning() || p.segmentWorker.IsRunning()
}
// StopWorkers stops workers
func (p *PushManager) StopWorkers() {
if p.splitWorker.IsRunning() {
p.splitWorker.Stop()
}
if p.segmentWorker.IsRunning() {
p.segmentWorker.Stop()
}
}
// StartWorkers starts workers
func (p *PushManager) StartWorkers() {
if !p.splitWorker.IsRunning() {
p.splitWorker.Start()
}
if !p.segmentWorker.IsRunning() {
p.segmentWorker.Start()
}
}

76
vendor/github.com/splitio/go-split-commons/v2/push/segment.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,76 @@
package push
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
// SegmentUpdateWorker struct
type SegmentUpdateWorker struct {
activeGoroutines *sync.WaitGroup
segmentQueue chan dtos.SegmentChangeNotification
handler func(segmentName string, till *int64) error
logger logging.LoggerInterface
stop chan struct{}
running atomic.Value
}
// NewSegmentUpdateWorker creates SegmentUpdateWorker
func NewSegmentUpdateWorker(segmentQueue chan dtos.SegmentChangeNotification, handler func(segmentName string, till *int64) error, logger logging.LoggerInterface) (*SegmentUpdateWorker, error) {
if cap(segmentQueue) < 5000 {
return nil, errors.New("")
}
running := atomic.Value{}
running.Store(false)
return &SegmentUpdateWorker{
segmentQueue: segmentQueue,
handler: handler,
logger: logger,
stop: make(chan struct{}, 1),
running: running,
}, nil
}
// Start starts worker
func (s *SegmentUpdateWorker) Start() {
s.logger.Debug("Started SegmentUpdateWorker")
if s.IsRunning() {
s.logger.Debug("Segment worker is already running")
return
}
s.running.Store(true)
go func() {
for {
select {
case segmentUpdate := <-s.segmentQueue:
s.logger.Debug("Received Segment update and proceding to perform fetch")
s.logger.Debug(fmt.Sprintf("SegmentName: %s\nChangeNumber: %d", segmentUpdate.SegmentName, &segmentUpdate.ChangeNumber))
err := s.handler(segmentUpdate.SegmentName, &segmentUpdate.ChangeNumber)
if err != nil {
s.logger.Error(err)
}
case <-s.stop:
return
}
}
}()
}
// Stop stops worker
func (s *SegmentUpdateWorker) Stop() {
if s.IsRunning() {
s.stop <- struct{}{}
s.running.Store(false)
}
}
// IsRunning indicates if worker is running or not
func (s *SegmentUpdateWorker) IsRunning() bool {
return s.running.Load().(bool)
}

77
vendor/github.com/splitio/go-split-commons/v2/push/split.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,77 @@
package push
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
// SplitUpdateWorker struct
type SplitUpdateWorker struct {
activeGoroutines *sync.WaitGroup
splitQueue chan dtos.SplitChangeNotification
handler func(till *int64) error
logger logging.LoggerInterface
stop chan struct{}
running atomic.Value
}
// NewSplitUpdateWorker creates SplitUpdateWorker
func NewSplitUpdateWorker(splitQueue chan dtos.SplitChangeNotification, handler func(till *int64) error, logger logging.LoggerInterface) (*SplitUpdateWorker, error) {
if cap(splitQueue) < 5000 {
return nil, errors.New("")
}
running := atomic.Value{}
running.Store(false)
return &SplitUpdateWorker{
activeGoroutines: &sync.WaitGroup{},
splitQueue: splitQueue,
handler: handler,
logger: logger,
running: running,
stop: make(chan struct{}, 1),
}, nil
}
// Start starts worker
func (s *SplitUpdateWorker) Start() {
s.logger.Debug("Started SplitUpdateWorker")
if s.IsRunning() {
s.logger.Info("Split worker is already running")
return
}
s.running.Store(true)
go func() {
for {
select {
case splitUpdate := <-s.splitQueue:
s.logger.Debug("Received Split update and proceding to perform fetch")
s.logger.Debug(fmt.Sprintf("ChangeNumber: %d", splitUpdate.ChangeNumber))
err := s.handler(&splitUpdate.ChangeNumber)
if err != nil {
s.logger.Error(err)
}
case <-s.stop:
return
}
}
}()
}
// Stop stops worker
func (s *SplitUpdateWorker) Stop() {
if s.IsRunning() {
s.stop <- struct{}{}
s.running.Store(false)
}
}
// IsRunning indicates if worker is running or not
func (s *SplitUpdateWorker) IsRunning() bool {
return s.running.Load().(bool)
}

39
vendor/github.com/splitio/go-split-commons/v2/service/api/auth.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,39 @@
package api
import (
"encoding/json"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
// AuthAPIClient struct is responsible for authenticating client for push services
type AuthAPIClient struct {
client Client
logger logging.LoggerInterface
}
// NewAuthAPIClient instantiates and return an AuthAPIClient
func NewAuthAPIClient(apikey string, cfg conf.AdvancedConfig, logger logging.LoggerInterface, metadata dtos.Metadata) *AuthAPIClient {
return &AuthAPIClient{
client: NewHTTPClient(apikey, cfg, cfg.AuthServiceURL, logger, metadata),
logger: logger,
}
}
// Authenticate performs authentication for push services
func (a *AuthAPIClient) Authenticate() (*dtos.Token, error) {
raw, err := a.client.Get("/api/auth")
if err != nil {
a.logger.Error("Error while authenticating for streaming", err)
return nil, err
}
token := dtos.Token{}
err = json.Unmarshal(raw, &token)
if err != nil {
return nil, err
}
return &token, nil
}

157
vendor/github.com/splitio/go-split-commons/v2/service/api/client.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,157 @@
package api
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"net/http"
"time"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
// Client interface for HTTPClient
type Client interface {
Get(service string) ([]byte, error)
Post(service string, body []byte, headers map[string]string) error
}
// HTTPClient structure to wrap up the net/http.Client
type HTTPClient struct {
url string
httpClient *http.Client
headers map[string]string
logger logging.LoggerInterface
apikey string
metadata dtos.Metadata
}
// NewHTTPClient instance of HttpClient
func NewHTTPClient(
apikey string,
cfg conf.AdvancedConfig,
endpoint string,
logger logging.LoggerInterface,
metadata dtos.Metadata,
) Client {
var timeout int
timeout = cfg.HTTPTimeout
client := &http.Client{Timeout: time.Duration(timeout) * time.Second}
return &HTTPClient{
url: endpoint,
httpClient: client,
logger: logger,
apikey: apikey,
metadata: metadata,
}
}
// Get method is a get call to an url
func (c *HTTPClient) Get(service string) ([]byte, error) {
serviceURL := c.url + service
c.logger.Debug("[GET] ", serviceURL)
req, _ := http.NewRequest("GET", serviceURL, nil)
authorization := c.apikey
c.logger.Debug("Authorization [ApiKey]: ", logging.ObfuscateAPIKey(authorization))
req.Header.Add("Accept-Encoding", "gzip")
req.Header.Add("Content-Type", "application/json")
c.logger.Debug(fmt.Sprintf("Headers: %v", req.Header))
req.Header.Add("Authorization", "Bearer "+authorization)
req.Header.Add("SplitSDKVersion", c.metadata.SDKVersion)
req.Header.Add("SplitSDKMachineName", c.metadata.MachineName)
req.Header.Add("SplitSDKMachineIP", c.metadata.MachineIP)
resp, err := c.httpClient.Do(req)
if err != nil {
c.logger.Error("Error requesting data to API: ", req.URL.String(), err.Error())
return nil, err
}
defer resp.Body.Close()
// Check that the server actually sent compressed data
var reader io.ReadCloser
switch resp.Header.Get("Content-Encoding") {
case "gzip":
reader, _ = gzip.NewReader(resp.Body)
defer reader.Close()
default:
reader = resp.Body
}
body, err := ioutil.ReadAll(reader)
if err != nil {
c.logger.Error(err.Error())
return nil, err
}
c.logger.Verbose("[RESPONSE_BODY]", string(body), "[END_RESPONSE_BODY]")
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return body, nil
}
c.logger.Error(fmt.Sprintf("GET method: Status Code: %d - %s", resp.StatusCode, resp.Status))
return nil, &dtos.HTTPError{
Code: resp.StatusCode,
Message: resp.Status,
}
}
// Post performs a HTTP POST request
func (c *HTTPClient) Post(service string, body []byte, headers map[string]string) error {
serviceURL := c.url + service
c.logger.Debug("[POST] ", serviceURL)
req, _ := http.NewRequest("POST", serviceURL, bytes.NewBuffer(body))
//****************
req.Close = true // To prevent EOF error when connection is closed
//****************
authorization := c.apikey
c.logger.Debug("Authorization [ApiKey]: ", logging.ObfuscateAPIKey(authorization))
req.Header.Add("Accept-Encoding", "gzip")
req.Header.Add("Content-Type", "application/json")
for headerName, headerValue := range headers {
req.Header.Add(headerName, headerValue)
}
c.logger.Debug(fmt.Sprintf("Headers: %v", req.Header))
req.Header.Add("Authorization", "Bearer "+authorization)
c.logger.Verbose("[REQUEST_BODY]", string(body), "[END_REQUEST_BODY]")
resp, err := c.httpClient.Do(req)
if err != nil {
c.logger.Error("Error posting data to API: ", req.URL.String(), err.Error())
return err
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
c.logger.Error(err.Error())
return err
}
c.logger.Verbose("[RESPONSE_BODY]", string(respBody), "[END_RESPONSE_BODY]")
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return nil
}
c.logger.Error(fmt.Sprintf("POST method: Status Code: %d - %s", resp.StatusCode, resp.Status))
return &dtos.HTTPError{
Code: resp.StatusCode,
Message: resp.Status,
}
}

110
vendor/github.com/splitio/go-split-commons/v2/service/api/http_fetchers.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,110 @@
package api
import (
"bytes"
"encoding/json"
"strconv"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
type httpFetcherBase struct {
client Client
logger logging.LoggerInterface
}
func (h *httpFetcherBase) fetchRaw(url string, since int64) ([]byte, error) {
var bufferQuery bytes.Buffer
bufferQuery.WriteString(url)
if since >= -1 {
bufferQuery.WriteString("?since=")
bufferQuery.WriteString(strconv.FormatInt(since, 10))
}
data, err := h.client.Get(bufferQuery.String())
if err != nil {
return nil, err
}
return data, nil
}
// HTTPSplitFetcher struct is responsible for fetching splits from the backend via HTTP protocol
type HTTPSplitFetcher struct {
httpFetcherBase
}
// NewHTTPSplitFetcher instantiates and return an HTTPSplitFetcher
func NewHTTPSplitFetcher(
apikey string,
cfg conf.AdvancedConfig,
logger logging.LoggerInterface,
metadata dtos.Metadata,
) *HTTPSplitFetcher {
return &HTTPSplitFetcher{
httpFetcherBase: httpFetcherBase{
client: NewHTTPClient(apikey, cfg, cfg.SdkURL, logger, metadata),
logger: logger,
},
}
}
// Fetch makes an http call to the split backend and returns the list of updated splits
func (f *HTTPSplitFetcher) Fetch(since int64) (*dtos.SplitChangesDTO, error) {
data, err := f.fetchRaw("/splitChanges", since)
if err != nil {
f.logger.Error("Error fetching split changes ", err)
return nil, err
}
var splitChangesDto dtos.SplitChangesDTO
err = json.Unmarshal(data, &splitChangesDto)
if err != nil {
f.logger.Error("Error parsing split changes JSON ", err)
return nil, err
}
return &splitChangesDto, nil
}
// HTTPSegmentFetcher struct is responsible for fetching segment by name from the API via HTTP method
type HTTPSegmentFetcher struct {
httpFetcherBase
}
// NewHTTPSegmentFetcher instantiates and returns a new HTTPSegmentFetcher.
func NewHTTPSegmentFetcher(
apikey string,
cfg conf.AdvancedConfig,
logger logging.LoggerInterface,
metadata dtos.Metadata,
) *HTTPSegmentFetcher {
return &HTTPSegmentFetcher{
httpFetcherBase: httpFetcherBase{
client: NewHTTPClient(apikey, cfg, cfg.SdkURL, logger, metadata),
logger: logger,
},
}
}
// Fetch issues a GET request to the split backend and returns the contents of a particular segment
func (f *HTTPSegmentFetcher) Fetch(segmentName string, since int64) (*dtos.SegmentChangesDTO, error) {
var bufferQuery bytes.Buffer
bufferQuery.WriteString("/segmentChanges/")
bufferQuery.WriteString(segmentName)
data, err := f.fetchRaw(bufferQuery.String(), since)
if err != nil {
f.logger.Error(err.Error())
return nil, err
}
var segmentChangesDto dtos.SegmentChangesDTO
err = json.Unmarshal(data, &segmentChangesDto)
if err != nil {
f.logger.Error("Error parsing segment changes JSON for segment ", segmentName, err)
return nil, err
}
return &segmentChangesDto, nil
}

197
vendor/github.com/splitio/go-split-commons/v2/service/api/http_recorders.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,197 @@
package api
import (
"encoding/json"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
type httpRecorderBase struct {
client Client
logger logging.LoggerInterface
}
// RecordRaw records raw data
func (h *httpRecorderBase) RecordRaw(url string, data []byte, metadata dtos.Metadata, extraHeaders map[string]string) error {
headers := make(map[string]string)
headers["SplitSDKVersion"] = metadata.SDKVersion
if metadata.MachineName != "NA" && metadata.MachineName != "unknown" {
headers["SplitSDKMachineName"] = metadata.MachineName
}
if metadata.MachineIP != "NA" && metadata.MachineIP != "unknown" {
headers["SplitSDKMachineIP"] = metadata.MachineIP
}
if extraHeaders != nil {
for header, value := range extraHeaders {
headers[header] = value
}
}
return h.client.Post(url, data, headers)
}
// HTTPImpressionRecorder is a struct responsible for submitting impression bulks to the backend
type HTTPImpressionRecorder struct {
httpRecorderBase
}
// Record sends an array (or slice) of impressionsRecord to the backend
func (i *HTTPImpressionRecorder) Record(impressions []dtos.ImpressionsDTO, metadata dtos.Metadata, extraHeaders map[string]string) error {
data, err := json.Marshal(impressions)
if err != nil {
i.logger.Error("Error marshaling JSON", err.Error())
return err
}
err = i.RecordRaw("/testImpressions/bulk", data, metadata, extraHeaders)
if err != nil {
i.logger.Error("Error posting impressions", err.Error())
return err
}
return nil
}
// RecordImpressionsCount sens impressionsCount
func (i *HTTPImpressionRecorder) RecordImpressionsCount(pf dtos.ImpressionsCountDTO, metadata dtos.Metadata) error {
if len(pf.PerFeature) == 0 {
return nil
}
data, err := json.Marshal(pf)
if err != nil {
i.logger.Error("Error marshaling JSON", err.Error())
return err
}
err = i.RecordRaw("/testImpressions/count", data, metadata, nil)
if err != nil {
i.logger.Error("Error posting impressionsCount", err.Error())
return err
}
return nil
}
// NewHTTPImpressionRecorder instantiates an HTTPImpressionRecorder
func NewHTTPImpressionRecorder(
apikey string,
cfg conf.AdvancedConfig,
logger logging.LoggerInterface,
) *HTTPImpressionRecorder {
client := NewHTTPClient(apikey, cfg, cfg.EventsURL, logger, dtos.Metadata{})
return &HTTPImpressionRecorder{
httpRecorderBase: httpRecorderBase{
client: client,
logger: logger,
},
}
}
// HTTPMetricsRecorder is a struct responsible for submitting metrics (latency, gauge, counters) to the backend
type HTTPMetricsRecorder struct {
httpRecorderBase
}
// RecordCounters method submits counter metrics to the backend
func (m *HTTPMetricsRecorder) RecordCounters(counters []dtos.CounterDTO, metadata dtos.Metadata) error {
data, err := json.Marshal(counters)
if err != nil {
m.logger.Error("Error marshaling JSON", err.Error())
return err
}
err = m.RecordRaw("/metrics/counters", data, metadata, nil)
if err != nil {
m.logger.Error("Error posting counters", err.Error())
return err
}
return nil
}
// RecordLatencies method submits latency metrics to the backend
func (m *HTTPMetricsRecorder) RecordLatencies(latencies []dtos.LatenciesDTO, metadata dtos.Metadata) error {
data, err := json.Marshal(latencies)
if err != nil {
m.logger.Error("Error marshaling JSON", err.Error())
return err
}
err = m.RecordRaw("/metrics/times", data, metadata, nil)
if err != nil {
m.logger.Error("Error posting latencies", err.Error())
return err
}
return nil
}
// RecordGauge method submits gauge metrics to the backend
func (m *HTTPMetricsRecorder) RecordGauge(gauge dtos.GaugeDTO, metadata dtos.Metadata) error {
data, err := json.Marshal(gauge)
if err != nil {
m.logger.Error("Error marshaling JSON", err.Error())
return err
}
err = m.RecordRaw("/metrics/gauge", data, metadata, nil)
if err != nil {
m.logger.Error("Error posting gauges", err.Error())
return err
}
return nil
}
// NewHTTPMetricsRecorder instantiates an HTTPMetricsRecorder
func NewHTTPMetricsRecorder(
apikey string,
cfg conf.AdvancedConfig,
logger logging.LoggerInterface,
) *HTTPMetricsRecorder {
client := NewHTTPClient(apikey, cfg, cfg.EventsURL, logger, dtos.Metadata{})
return &HTTPMetricsRecorder{
httpRecorderBase: httpRecorderBase{
client: client,
logger: logger,
},
}
}
// HTTPEventsRecorder is a struct responsible for submitting events bulks to the backend
type HTTPEventsRecorder struct {
httpRecorderBase
}
// Record sends an array (or slice) of dtos.EventDTO to the backend
func (i *HTTPEventsRecorder) Record(events []dtos.EventDTO, metadata dtos.Metadata) error {
data, err := json.Marshal(events)
if err != nil {
i.logger.Error("Error marshaling JSON", err.Error())
return err
}
err = i.RecordRaw("/events/bulk", data, metadata, nil)
if err != nil {
i.logger.Error("Error posting events", err.Error())
return err
}
return nil
}
// NewHTTPEventsRecorder instantiates an HTTPEventsRecorder
func NewHTTPEventsRecorder(
apikey string,
cfg conf.AdvancedConfig,
logger logging.LoggerInterface,
) *HTTPEventsRecorder {
client := NewHTTPClient(apikey, cfg, cfg.EventsURL, logger, dtos.Metadata{})
return &HTTPEventsRecorder{
httpRecorderBase: httpRecorderBase{
client: client,
logger: logger,
},
}
}

127
vendor/github.com/splitio/go-split-commons/v2/service/api/sse/client.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,127 @@
package sse
import (
"strings"
"sync"
"sync/atomic"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/sse"
)
const (
version = "1.1"
keepAlive = 120
)
// StreamingClient struct
type StreamingClient struct {
mutex *sync.RWMutex
sseClient *sse.SSEClient
sseStatus chan int
streamingStatus chan<- int
running atomic.Value
logger logging.LoggerInterface
stopped chan struct{}
}
// NewStreamingClient creates new SSE Client
func NewStreamingClient(cfg *conf.AdvancedConfig, streamingStatus chan int, logger logging.LoggerInterface) *StreamingClient {
sseStatus := make(chan int, 1)
sseClient, _ := sse.NewSSEClient(cfg.StreamingServiceURL, sseStatus, keepAlive, logger)
running := atomic.Value{}
running.Store(false)
return &StreamingClient{
mutex: &sync.RWMutex{},
sseClient: sseClient,
sseStatus: sseStatus,
streamingStatus: streamingStatus,
logger: logger,
running: running,
stopped: make(chan struct{}, 1),
}
}
// ConnectStreaming connects to streaming
func (s *StreamingClient) ConnectStreaming(token string, channelList []string, handleIncomingMessage func(e map[string]interface{})) {
params := make(map[string]string)
params["channels"] = strings.Join(append(channelList), ",")
params["accessToken"] = token
params["v"] = version
httpHandlerExited := make(chan struct{}, 1)
go func() {
s.sseClient.Do(params, handleIncomingMessage)
httpHandlerExited <- struct{}{}
}()
// Consume remaining message in completion signaling channel if any:
select {
case <-s.stopped:
default:
}
select {
case <-s.sseStatus:
default:
}
go func() {
defer func() { // When this goroutine exits, StopStreaming is freed
select {
case s.stopped <- struct{}{}:
default:
}
}()
for {
select {
case <-httpHandlerExited:
return
case status := <-s.sseStatus:
switch status {
case sse.OK:
s.logger.Info("SSE OK")
s.running.Store(true)
s.streamingStatus <- sse.OK
case sse.ErrorConnectToStreaming:
s.logger.Error("Error connecting to streaming")
s.streamingStatus <- sse.ErrorConnectToStreaming
case sse.ErrorKeepAlive:
s.logger.Error("Connection timed out")
s.streamingStatus <- sse.ErrorKeepAlive
case sse.ErrorOnClientCreation:
s.logger.Error("Could not create client for streaming")
s.streamingStatus <- sse.ErrorOnClientCreation
case sse.ErrorReadingStream:
s.logger.Error("Error reading streaming buffer")
s.streamingStatus <- sse.ErrorReadingStream
case sse.ErrorRequestPerformed:
s.logger.Error("Error performing request when connect to stream service")
s.streamingStatus <- sse.ErrorRequestPerformed
case sse.ErrorInternal:
s.logger.Error("Internal Error when connect to stream service")
s.streamingStatus <- sse.ErrorInternal
default:
s.logger.Error("Unexpected error occured with streaming")
s.streamingStatus <- sse.ErrorUnexpected
}
}
}
}()
}
// StopStreaming stops streaming
func (s *StreamingClient) StopStreaming(blocking bool) {
s.sseClient.Shutdown()
s.logger.Info("Stopped streaming")
s.running.Store(false)
if blocking {
<-s.stopped
}
}
// IsRunning returns true if it's running
func (s *StreamingClient) IsRunning() bool {
return s.running.Load().(bool)
}

38
vendor/github.com/splitio/go-split-commons/v2/service/interfaces.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,38 @@
package service
import (
"github.com/splitio/go-split-commons/v2/dtos"
)
// AuthClient inteface to be implemneted by AuthClient
type AuthClient interface {
Authenticate() (*dtos.Token, error)
}
// SplitFetcher interface to be implemented by Split Fetchers
type SplitFetcher interface {
Fetch(changeNumber int64) (*dtos.SplitChangesDTO, error)
}
// SegmentFetcher interface to be implemented by Split Fetchers
type SegmentFetcher interface {
Fetch(name string, changeNumber int64) (*dtos.SegmentChangesDTO, error)
}
// ImpressionsRecorder interface to be implemented by Impressions loggers
type ImpressionsRecorder interface {
Record(impressions []dtos.ImpressionsDTO, metadata dtos.Metadata, extraHeaders map[string]string) error
RecordImpressionsCount(pf dtos.ImpressionsCountDTO, metadata dtos.Metadata) error
}
// MetricsRecorder interface to be implemented by Metrics loggers
type MetricsRecorder interface {
RecordLatencies(latencies []dtos.LatenciesDTO, metadata dtos.Metadata) error
RecordCounters(counters []dtos.CounterDTO, metadata dtos.Metadata) error
RecordGauge(gauge dtos.GaugeDTO, metadata dtos.Metadata) error
}
// EventsRecorder interface to post events
type EventsRecorder interface {
Record(events []dtos.EventDTO, metadata dtos.Metadata) error
}

258
vendor/github.com/splitio/go-split-commons/v2/service/local/splitFetcher.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,258 @@
package local
import (
"fmt"
"io/ioutil"
"log"
"regexp"
"runtime/debug"
"strings"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
yaml "gopkg.in/yaml.v2"
)
const (
// SplitFileFormatClassic represents the file format of the standard split definition file <feature treatment>
SplitFileFormatClassic = iota
// SplitFileFormatJSON represents the file format of a JSON representation of split dtos
SplitFileFormatJSON
// SplitFileFormatYAML represents the file format of a YAML representation of split dtos
SplitFileFormatYAML
)
// FileSplitFetcher struct fetches splits from a file
type FileSplitFetcher struct {
splitFile string
fileFormat int
lastChangeNumber int64
}
// NewFileSplitFetcher returns a new instance of LocalFileSplitFetcher
func NewFileSplitFetcher(splitFile string, logger logging.LoggerInterface) *FileSplitFetcher {
var r = regexp.MustCompile("(?i)(.yml$|.yaml$)")
if r.MatchString(splitFile) {
return &FileSplitFetcher{
splitFile: splitFile,
fileFormat: SplitFileFormatYAML,
}
}
logger.Warning("Localhost mode: .split mocks will be deprecated soon in favor of YAML files, which provide more targeting power. Take a look in our documentation.")
return &FileSplitFetcher{
splitFile: splitFile,
fileFormat: SplitFileFormatClassic,
}
}
func parseSplitsClassic(data string) []dtos.SplitDTO {
splits := make([]dtos.SplitDTO, 0)
lines := strings.Split(data, "\n")
for _, line := range lines {
words := strings.Fields(line)
if len(words) < 2 || len(words[0]) < 1 || words[0][0] == '#' {
// Skip the line if it has less than two words, the words are empty strings or
// it begins with '#' character
continue
}
splitName := words[0]
treatment := words[1]
splits = append(splits, createSplit(
splitName,
treatment,
createRolloutCondition(treatment),
make(map[string]string),
))
}
return splits
}
func createSplit(splitName string, treatment string, condition dtos.ConditionDTO, configurations map[string]string) dtos.SplitDTO {
split := dtos.SplitDTO{
Name: splitName,
TrafficAllocation: 100,
Conditions: []dtos.ConditionDTO{condition},
Status: "ACTIVE",
DefaultTreatment: "control",
Configurations: configurations,
}
return split
}
func createWhitelistedCondition(treatment string, keys interface{}) dtos.ConditionDTO {
var whitelist []string
switch keys := keys.(type) {
case string:
whitelist = []string{keys}
case []string:
whitelist = keys
case []interface{}:
whitelist = make([]string, 0)
for _, key := range keys {
k, ok := key.(string)
if ok {
whitelist = append(whitelist, k)
}
}
default:
whitelist = make([]string, 0)
}
return dtos.ConditionDTO{
ConditionType: "WHITELIST",
Label: "LOCAL_",
MatcherGroup: dtos.MatcherGroupDTO{
Combiner: "AND",
Matchers: []dtos.MatcherDTO{
{
MatcherType: "WHITELIST",
Negate: false,
Whitelist: &dtos.WhitelistMatcherDataDTO{
Whitelist: whitelist,
},
},
},
},
Partitions: []dtos.PartitionDTO{
{
Size: 100,
Treatment: treatment,
},
},
}
}
func createRolloutCondition(treatment string) dtos.ConditionDTO {
return dtos.ConditionDTO{
ConditionType: "ROLLOUT",
Label: "LOCAL_ROLLOUT",
MatcherGroup: dtos.MatcherGroupDTO{
Combiner: "AND",
Matchers: []dtos.MatcherDTO{
{
MatcherType: "ALL_KEYS",
Negate: false,
},
},
},
Partitions: []dtos.PartitionDTO{
{
Size: 100,
Treatment: treatment,
},
{
Size: 0,
Treatment: "_",
},
},
}
}
func createCondition(keys interface{}, treatment string) dtos.ConditionDTO {
if keys != nil {
return createWhitelistedCondition(treatment, keys)
}
return createRolloutCondition(treatment)
}
func parseSplitsYAML(data string) (d []dtos.SplitDTO) {
// Set up a guard deferred function to recover if some error occurs during parsing
defer func() {
if r := recover(); r != nil {
// At this point we'll only trust that the logger isn't panicking trust
// that the logger isn't panicking
log.Fatalf("Localhost Parsing: %v", string(debug.Stack()))
d = make([]dtos.SplitDTO, 0)
}
}()
splits := make([]dtos.SplitDTO, 0)
var splitsFromYAML []map[string]map[string]interface{}
err := yaml.Unmarshal([]byte(data), &splitsFromYAML)
if err != nil {
log.Fatalf("error: %v", err)
return splits
}
splitsToParse := make(map[string]dtos.SplitDTO, 0)
for _, splitMap := range splitsFromYAML {
for splitName, splitParsed := range splitMap {
split, ok := splitsToParse[splitName]
treatment, isString := splitParsed["treatment"].(string)
if !isString {
break
}
config, isValidConfig := splitParsed["config"].(string)
if !ok {
configurations := make(map[string]string)
if isValidConfig {
configurations[treatment] = config
}
splitsToParse[splitName] = createSplit(
splitName,
treatment,
createCondition(splitParsed["keys"], treatment),
configurations,
)
} else {
newCondition := createCondition(splitParsed["keys"], treatment)
if newCondition.ConditionType == "ROLLOUT" {
split.Conditions = append(split.Conditions, newCondition)
} else {
split.Conditions = append([]dtos.ConditionDTO{newCondition}, split.Conditions...)
}
configurations := split.Configurations
if isValidConfig {
configurations[treatment] = config
}
split.Configurations = configurations
splitsToParse[splitName] = split
}
}
}
for _, split := range splitsToParse {
splits = append(splits, split)
}
return splits
}
// Fetch parses the file and returns the appropriate structures
func (s *FileSplitFetcher) Fetch(changeNumber int64) (*dtos.SplitChangesDTO, error) {
fileContents, err := ioutil.ReadFile(s.splitFile)
if err != nil {
return nil, err
}
var splits []dtos.SplitDTO
var till int64
since := s.lastChangeNumber
if s.lastChangeNumber != 0 {
//The first time we should return since == till
till = since + 1
}
data := string(fileContents)
switch s.fileFormat {
case SplitFileFormatClassic:
splits = parseSplitsClassic(data)
case SplitFileFormatYAML:
splits = parseSplitsYAML(data)
case SplitFileFormatJSON:
return nil, fmt.Errorf("JSON is not yet supported")
default:
return nil, fmt.Errorf("Unsupported file format")
}
s.lastChangeNumber++
return &dtos.SplitChangesDTO{
Splits: splits,
Since: since,
Till: till,
}, nil
}

35
vendor/github.com/splitio/go-split-commons/v2/service/split.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,35 @@
package service
import (
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/service/api"
"github.com/splitio/go-toolkit/v3/logging"
)
// SplitAPI struct for fetchers and recorders
type SplitAPI struct {
AuthClient AuthClient
SplitFetcher SplitFetcher
SegmentFetcher SegmentFetcher
ImpressionRecorder ImpressionsRecorder
EventRecorder EventsRecorder
MetricRecorder MetricsRecorder
}
// NewSplitAPI creates new splitAPI
func NewSplitAPI(
apikey string,
conf conf.AdvancedConfig,
logger logging.LoggerInterface,
metadata dtos.Metadata,
) *SplitAPI {
return &SplitAPI{
AuthClient: api.NewAuthAPIClient(apikey, conf, logger, metadata),
SplitFetcher: api.NewHTTPSplitFetcher(apikey, conf, logger, metadata),
SegmentFetcher: api.NewHTTPSegmentFetcher(apikey, conf, logger, metadata),
ImpressionRecorder: api.NewHTTPImpressionRecorder(apikey, conf, logger),
EventRecorder: api.NewHTTPEventsRecorder(apikey, conf, logger),
MetricRecorder: api.NewHTTPMetricsRecorder(apikey, conf, logger),
}
}

118
vendor/github.com/splitio/go-split-commons/v2/storage/interfaces.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,118 @@
package storage
import (
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/datastructures/set"
)
// SplitStorageProducer should be implemented by structs that offer writing splits in storage
type SplitStorageProducer interface {
KillLocally(splitName string, defaultTreatment string, changeNumber int64)
PutMany(splits []dtos.SplitDTO, changeNumber int64)
Remove(splitName string)
SetChangeNumber(changeNumber int64) error
}
// SplitStorageConsumer should be implemented by structs that offer reading splits from storage
type SplitStorageConsumer interface {
All() []dtos.SplitDTO
ChangeNumber() (int64, error)
FetchMany(splitNames []string) map[string]*dtos.SplitDTO
SegmentNames() *set.ThreadUnsafeSet // Not in Spec
Split(splitName string) *dtos.SplitDTO
SplitNames() []string
TrafficTypeExists(trafficType string) bool
}
// SegmentStorageProducer interface should be implemented by all structs that offer writing segments
type SegmentStorageProducer interface {
Update(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, changeNumber int64) error
SetChangeNumber(segmentName string, till int64) error
}
// SegmentStorageConsumer interface should be implemented by all structs that ofer reading segments
type SegmentStorageConsumer interface {
ChangeNumber(segmentName string) (int64, error)
CountRemovedKeys(segmentName string) int64
Keys(segmentName string) *set.ThreadUnsafeSet
SegmentContainsKey(segmentName string, key string) (bool, error)
}
// ImpressionStorageProducer interface should be impemented by structs that accept incoming impressions
type ImpressionStorageProducer interface {
LogImpressions(impressions []dtos.Impression) error
}
// ImpressionStorageConsumer interface should be implemented by structs that offer popping impressions
type ImpressionStorageConsumer interface {
Count() int64
Drop(size *int64) error
Empty() bool
PopN(n int64) ([]dtos.Impression, error)
PopNWithMetadata(n int64) ([]dtos.ImpressionQueueObject, error)
}
// MetricsStorageProducer interface should be impemented by structs that accept incoming metrics
type MetricsStorageProducer interface {
PutGauge(key string, gauge float64)
IncLatency(metricName string, index int)
IncCounter(key string)
}
// MetricsStorageConsumer interface should be implemented by structs that offer popping metrics
type MetricsStorageConsumer interface {
PeekCounters() map[string]int64
PeekLatencies() map[string][]int64
PopGauges() []dtos.GaugeDTO
PopLatencies() []dtos.LatenciesDTO
PopCounters() []dtos.CounterDTO
PopGaugesWithMetadata() (*dtos.GaugeDataBulk, error)
PopLatenciesWithMetadata() (*dtos.LatencyDataBulk, error)
PopCountersWithMetadata() (*dtos.CounterDataBulk, error)
}
// EventStorageProducer interface should be implemented by structs that accept incoming events
type EventStorageProducer interface {
Push(event dtos.EventDTO, size int) error
}
// EventStorageConsumer interface should be implemented by structs that offer popping impressions
type EventStorageConsumer interface {
Count() int64
Drop(size *int64) error
Empty() bool
PopN(n int64) ([]dtos.EventDTO, error)
PopNWithMetadata(n int64) ([]dtos.QueueStoredEventDTO, error)
}
// --- Wide Interfaces
// SplitStorage wraps consumer & producer interfaces
type SplitStorage interface {
SplitStorageProducer
SplitStorageConsumer
}
// SegmentStorage wraps consumer and producer interfaces
type SegmentStorage interface {
SegmentStorageProducer
SegmentStorageConsumer
}
// ImpressionStorage wraps consumer & producer interfaces
type ImpressionStorage interface {
ImpressionStorageConsumer
ImpressionStorageProducer
}
// MetricsStorage wraps consumer and producer interfaces
type MetricsStorage interface {
MetricsStorageConsumer
MetricsStorageProducer
}
// EventsStorage wraps consumer and producer interfaces
type EventsStorage interface {
EventStorageConsumer
EventStorageProducer
}

125
vendor/github.com/splitio/go-split-commons/v2/storage/metricWrapper.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,125 @@
package storage
import (
"errors"
"strings"
"github.com/splitio/go-toolkit/v3/logging"
)
// MetricWrapper struct
type MetricWrapper struct {
Telemetry MetricsStorage
LocalTelemetry MetricsStorage
logger logging.LoggerInterface
}
const (
// SplitChangesCounter counters
SplitChangesCounter = iota
// SplitChangesLatency latencies
SplitChangesLatency
// SegmentChangesCounter counters
SegmentChangesCounter
// SegmentChangesLatency latencies
SegmentChangesLatency
// TestImpressionsCounter counter
TestImpressionsCounter
// TestImpressionsLatency latencies
TestImpressionsLatency
// PostEventsCounter counter
PostEventsCounter
//PostEventsLatency latencies
PostEventsLatency
// MySegmentsCounter counters
MySegmentsCounter
// MySegmentsLatency latencies
MySegmentsLatency
)
const (
counter = "backend::request.{status}"
splitChangesCounter = "splitChangeFetcher.status.{status}"
splitChangesLatency = "splitChangeFetcher.time"
localSplitChangesLatency = "backend::/api/splitChanges"
segmentChangesCounter = "segmentChangeFetcher.status.{status}"
segmentChangesLatency = "segmentChangeFetcher.time"
localSegmentChangesLatency = "backend::/api/segmentChanges"
testImpressionsCounter = "testImpressions.status.{status}"
testImpressionsLatency = "testImpressions.time"
localTestImpressionsLatency = "backend::/api/testImpressions/bulk"
postEventsCounter = "events.status.{status}"
postEventsLatency = "events.time"
localPostEventsLatency = "backend::/api/events/bulk"
mySegmentsCounter = "mySegments.status.{status}"
mySegmentsLatency = "mySegments.time"
localMySegmentsLatency = "backend::/api/mySegments"
)
// NewMetricWrapper builds new wrapper
func NewMetricWrapper(telemetry MetricsStorage, localTelemetry MetricsStorage, logger logging.LoggerInterface) *MetricWrapper {
return &MetricWrapper{
LocalTelemetry: localTelemetry,
logger: logger,
Telemetry: telemetry,
}
}
func (m *MetricWrapper) getKey(key int) (string, string, error) {
switch key {
case SplitChangesCounter:
return splitChangesCounter, counter, nil
case SplitChangesLatency:
return splitChangesLatency, localSplitChangesLatency, nil
case SegmentChangesCounter:
return segmentChangesCounter, counter, nil
case SegmentChangesLatency:
return segmentChangesLatency, localSegmentChangesLatency, nil
case TestImpressionsCounter:
return testImpressionsCounter, counter, nil
case TestImpressionsLatency:
return testImpressionsLatency, localTestImpressionsLatency, nil
case PostEventsCounter:
return postEventsCounter, counter, nil
case PostEventsLatency:
return postEventsLatency, localPostEventsLatency, nil
case MySegmentsCounter:
return mySegmentsCounter, counter, nil
case MySegmentsLatency:
return mySegmentsLatency, localMySegmentsLatency, nil
default:
return "", "", errors.New("Key does not exist")
}
}
// StoreCounters stores counters
func (m *MetricWrapper) StoreCounters(key int, value string) {
common, local, err := m.getKey(key)
if err != nil {
return
}
if m.LocalTelemetry != nil {
m.LocalTelemetry.IncCounter(strings.Replace(local, "{status}", value, 1))
}
if value == "ok" {
value = "200"
}
m.Telemetry.IncCounter(strings.Replace(common, "{status}", value, 1))
}
// StoreLatencies stores counters
func (m *MetricWrapper) StoreLatencies(key int, bucket int) {
common, local, err := m.getKey(key)
if err != nil {
return
}
if m.LocalTelemetry != nil {
m.LocalTelemetry.IncLatency(local, bucket)
}
m.Telemetry.IncLatency(common, bucket)
}

43
vendor/github.com/splitio/go-split-commons/v2/storage/mocks/event.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,43 @@
package mocks
import "github.com/splitio/go-split-commons/v2/dtos"
// MockEventStorage is a mocked implementation of Event Storage
type MockEventStorage struct {
EmptyCall func() bool
CountCall func() int64
PopNCall func(n int64) ([]dtos.EventDTO, error)
PopNWithMetadataCall func(n int64) ([]dtos.QueueStoredEventDTO, error)
PushCall func(event dtos.EventDTO, size int) error
DropCall func(size *int64) error
}
// Empty mock
func (m MockEventStorage) Empty() bool {
return m.EmptyCall()
}
// Count mock
func (m MockEventStorage) Count() int64 {
return m.CountCall()
}
// PopN mock
func (m MockEventStorage) PopN(n int64) ([]dtos.EventDTO, error) {
return m.PopNCall(n)
}
// PopNWithMetadata mock
func (m MockEventStorage) PopNWithMetadata(n int64) ([]dtos.QueueStoredEventDTO, error) {
return m.PopNWithMetadataCall(n)
}
// Push mock
func (m MockEventStorage) Push(event dtos.EventDTO, size int) error {
return m.PushCall(event, size)
}
// Drop mock
func (m MockEventStorage) Drop(size *int64) error {
return m.Drop(size)
}

43
vendor/github.com/splitio/go-split-commons/v2/storage/mocks/impression.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,43 @@
package mocks
import "github.com/splitio/go-split-commons/v2/dtos"
// MockImpressionStorage is a mocked implementation of Impression Storage
type MockImpressionStorage struct {
EmptyCall func() bool
CountCall func() int64
LogImpressionsCall func(impressions []dtos.Impression) error
PopNCall func(n int64) ([]dtos.Impression, error)
PopNWithMetadataCall func(n int64) ([]dtos.ImpressionQueueObject, error)
DropCall func(size *int64) error
}
// Empty mock
func (m MockImpressionStorage) Empty() bool {
return m.EmptyCall()
}
// Count mock
func (m MockImpressionStorage) Count() int64 {
return m.CountCall()
}
// LogImpressions mock
func (m MockImpressionStorage) LogImpressions(impressions []dtos.Impression) error {
return m.LogImpressionsCall(impressions)
}
// PopN mock
func (m MockImpressionStorage) PopN(n int64) ([]dtos.Impression, error) {
return m.PopNCall(n)
}
// PopNWithMetadata mock
func (m MockImpressionStorage) PopNWithMetadata(n int64) ([]dtos.ImpressionQueueObject, error) {
return m.PopNWithMetadataCall(n)
}
// Drop mock
func (m MockImpressionStorage) Drop(size *int64) error {
return m.Drop(size)
}

73
vendor/github.com/splitio/go-split-commons/v2/storage/mocks/metric.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,73 @@
package mocks
import "github.com/splitio/go-split-commons/v2/dtos"
// MockMetricStorage is a mocked implementation of Metric Storage
type MockMetricStorage struct {
IncCounterCall func(key string)
IncLatencyCall func(metricName string, index int)
PutGaugeCall func(key string, gauge float64)
PopGaugesCall func() []dtos.GaugeDTO
PopLatenciesCall func() []dtos.LatenciesDTO
PopCountersCall func() []dtos.CounterDTO
PeekCountersCall func() map[string]int64
PeekLatenciesCall func() map[string][]int64
PopGaugesWithMetadataCall func() (*dtos.GaugeDataBulk, error)
PopCountersWithMetadataCall func() (*dtos.CounterDataBulk, error)
PopLatenciesWithMetadataCall func() (*dtos.LatencyDataBulk, error)
}
// IncCounter mock
func (m MockMetricStorage) IncCounter(key string) {
m.IncCounterCall(key)
}
// IncLatency mock
func (m MockMetricStorage) IncLatency(metricName string, index int) {
m.IncLatencyCall(metricName, index)
}
// PutGauge mock
func (m MockMetricStorage) PutGauge(key string, gauge float64) {
m.PutGaugeCall(key, gauge)
}
// PopGauges mock
func (m MockMetricStorage) PopGauges() []dtos.GaugeDTO {
return m.PopGaugesCall()
}
// PopLatencies mock
func (m MockMetricStorage) PopLatencies() []dtos.LatenciesDTO {
return m.PopLatenciesCall()
}
// PopCounters mock
func (m MockMetricStorage) PopCounters() []dtos.CounterDTO {
return m.PopCountersCall()
}
// PeekCounters mock
func (m MockMetricStorage) PeekCounters() map[string]int64 {
return m.PeekCountersCall()
}
// PeekLatencies mock
func (m MockMetricStorage) PeekLatencies() map[string][]int64 {
return m.PeekLatenciesCall()
}
// PopGaugesWithMetadata mock
func (m MockMetricStorage) PopGaugesWithMetadata() (*dtos.GaugeDataBulk, error) {
return m.PopGaugesWithMetadataCall()
}
// PopCountersWithMetadata mock
func (m MockMetricStorage) PopCountersWithMetadata() (*dtos.CounterDataBulk, error) {
return m.PopCountersWithMetadataCall()
}
// PopLatenciesWithMetadata mock
func (m MockMetricStorage) PopLatenciesWithMetadata() (*dtos.LatencyDataBulk, error) {
return m.PopLatenciesWithMetadataCall()
}

43
vendor/github.com/splitio/go-split-commons/v2/storage/mocks/segment.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,43 @@
package mocks
import "github.com/splitio/go-toolkit/v3/datastructures/set"
// MockSegmentStorage is a mocked implementation of Segment Storage
type MockSegmentStorage struct {
ChangeNumberCall func(segmentName string) (int64, error)
KeysCall func(segmentName string) *set.ThreadUnsafeSet
UpdateCall func(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, changeNumber int64) error
SegmentContainsKeyCall func(segmentName string, key string) (bool, error)
SetChangeNumberCall func(segmentName string, till int64) error
CountRemovedKeysCall func(segmentName string) int64
}
// ChangeNumber mock
func (m MockSegmentStorage) ChangeNumber(segmentName string) (int64, error) {
return m.ChangeNumberCall(segmentName)
}
// Keys mock
func (m MockSegmentStorage) Keys(segmentName string) *set.ThreadUnsafeSet {
return m.KeysCall(segmentName)
}
// Update mock
func (m MockSegmentStorage) Update(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, changeNumber int64) error {
return m.UpdateCall(name, toAdd, toRemove, changeNumber)
}
// SegmentContainsKey mock
func (m MockSegmentStorage) SegmentContainsKey(segmentName string, key string) (bool, error) {
return m.SegmentContainsKeyCall(segmentName, key)
}
// SetChangeNumber mock
func (m MockSegmentStorage) SetChangeNumber(segmentName string, till int64) error {
return m.SetChangeNumberCall(segmentName, till)
}
// CountRemovedKeys mock
func (m MockSegmentStorage) CountRemovedKeys(segmentName string) int64 {
return m.CountRemovedKeysCall(segmentName)
}

76
vendor/github.com/splitio/go-split-commons/v2/storage/mocks/split.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,76 @@
package mocks
import (
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/datastructures/set"
)
// MockSplitStorage is a mocked implementation of Split Storage
type MockSplitStorage struct {
AllCall func() []dtos.SplitDTO
ChangeNumberCall func() (int64, error)
FetchManyCall func(splitNames []string) map[string]*dtos.SplitDTO
KillLocallyCall func(splitName string, defaultTreatment string, changeNumber int64)
PutManyCall func(splits []dtos.SplitDTO, changeNumber int64)
RemoveCall func(splitName string)
SegmentNamesCall func() *set.ThreadUnsafeSet
SetChangeNumberCall func(changeNumber int64) error
SplitCall func(splitName string) *dtos.SplitDTO
SplitNamesCall func() []string
TrafficTypeExistsCall func(trafficType string) bool
}
// All mock
func (m MockSplitStorage) All() []dtos.SplitDTO {
return m.AllCall()
}
// ChangeNumber mock
func (m MockSplitStorage) ChangeNumber() (int64, error) {
return m.ChangeNumberCall()
}
// FetchMany mock
func (m MockSplitStorage) FetchMany(splitNames []string) map[string]*dtos.SplitDTO {
return m.FetchManyCall(splitNames)
}
// KillLocally mock
func (m MockSplitStorage) KillLocally(splitName string, defaultTreatment string, changeNumber int64) {
m.KillLocallyCall(splitName, defaultTreatment, changeNumber)
}
// PutMany mock
func (m MockSplitStorage) PutMany(splits []dtos.SplitDTO, changeNumber int64) {
m.PutManyCall(splits, changeNumber)
}
// Remove mock
func (m MockSplitStorage) Remove(splitname string) {
m.RemoveCall(splitname)
}
// SegmentNames mock
func (m MockSplitStorage) SegmentNames() *set.ThreadUnsafeSet {
return m.SegmentNamesCall()
}
// SetChangeNumber mock
func (m MockSplitStorage) SetChangeNumber(changeNumber int64) error {
return m.SetChangeNumberCall(changeNumber)
}
// Split mock
func (m MockSplitStorage) Split(splitName string) *dtos.SplitDTO {
return m.SplitCall(splitName)
}
// SplitNames mock
func (m MockSplitStorage) SplitNames() []string {
return m.SplitNamesCall()
}
// TrafficTypeExists mock
func (m MockSplitStorage) TrafficTypeExists(trafficType string) bool {
return m.TrafficTypeExistsCall(trafficType)
}

149
vendor/github.com/splitio/go-split-commons/v2/storage/mutexmap/metrics.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,149 @@
package mutexmap
import (
"sync"
"github.com/splitio/go-split-commons/v2/dtos"
)
// MMMetricsStorage contains an in-memory implementation of Metrics storage
type MMMetricsStorage struct {
gaugeData map[string]float64
gaugeMutex *sync.Mutex
counterData map[string]int64
countersMutex *sync.RWMutex
latenciesData map[string][]int64
latenciesMutex *sync.RWMutex
}
// NewMMMetricsStorage instantiates a new MMMetricsStorage
func NewMMMetricsStorage() *MMMetricsStorage {
return &MMMetricsStorage{
counterData: make(map[string]int64),
countersMutex: &sync.RWMutex{},
gaugeData: make(map[string]float64),
gaugeMutex: &sync.Mutex{},
latenciesData: make(map[string][]int64),
latenciesMutex: &sync.RWMutex{},
}
}
// PutGauge stores a new gauge value for a specific key
func (m *MMMetricsStorage) PutGauge(key string, gauge float64) {
m.gaugeMutex.Lock()
defer m.gaugeMutex.Unlock()
m.gaugeData[key] = gauge
}
// PopGauges returns and deletes all gauges currently stored
func (m *MMMetricsStorage) PopGauges() []dtos.GaugeDTO {
m.gaugeMutex.Lock()
defer func() {
m.gaugeData = make(map[string]float64)
m.gaugeMutex.Unlock()
}()
gauges := make([]dtos.GaugeDTO, 0)
for key, gauge := range m.gaugeData {
gauges = append(gauges, dtos.GaugeDTO{
MetricName: key,
Gauge: gauge,
})
}
return gauges
}
// IncCounter increments the counter for a specific key. It initializes it in 1 if it doesn't exist when this function
// is called.
func (m *MMMetricsStorage) IncCounter(key string) {
m.countersMutex.Lock()
defer m.countersMutex.Unlock()
_, exists := m.counterData[key]
if !exists {
m.counterData[key] = 1
} else {
m.counterData[key]++
}
}
// PopCounters returns and deletes all the counters stored
func (m *MMMetricsStorage) PopCounters() []dtos.CounterDTO {
m.countersMutex.Lock()
defer func() {
m.counterData = make(map[string]int64)
m.countersMutex.Unlock()
}()
counters := make([]dtos.CounterDTO, 0)
for key, counter := range m.counterData {
counters = append(counters, dtos.CounterDTO{
MetricName: key,
Count: counter,
})
}
return counters
}
// PeekCounters returns Counters
func (m *MMMetricsStorage) PeekCounters() map[string]int64 {
m.countersMutex.RLock()
defer m.countersMutex.RUnlock()
return m.counterData
}
// PeekLatencies returns Latencies
func (m *MMMetricsStorage) PeekLatencies() map[string][]int64 {
m.latenciesMutex.RLock()
defer m.latenciesMutex.RUnlock()
return m.latenciesData
}
// IncLatency increments the latency for a specific key and bucket. If the key doesn't exist it's initialized to
// an empty array of 23 items.
func (m *MMMetricsStorage) IncLatency(metricName string, index int) {
if index < 0 || index > 22 {
return
}
m.latenciesMutex.Lock()
defer m.latenciesMutex.Unlock()
_, exists := m.latenciesData[metricName]
if !exists {
m.latenciesData[metricName] = make([]int64, 23)
m.latenciesData[metricName][index] = 1
} else {
m.latenciesData[metricName][index]++
}
}
// PopLatencies Returns and delete all the latencies currently stored
func (m *MMMetricsStorage) PopLatencies() []dtos.LatenciesDTO {
m.latenciesMutex.Lock()
defer func() {
m.latenciesData = make(map[string][]int64)
m.latenciesMutex.Unlock()
}()
latencies := make([]dtos.LatenciesDTO, 0)
for key, latency := range m.latenciesData {
latencies = append(latencies, dtos.LatenciesDTO{
Latencies: latency,
MetricName: key,
})
}
return latencies
}
// PopGaugesWithMetadata mock
func (m *MMMetricsStorage) PopGaugesWithMetadata() (*dtos.GaugeDataBulk, error) {
panic("Not implemented for inmemory")
}
// PopLatenciesWithMetadata mock
func (m *MMMetricsStorage) PopLatenciesWithMetadata() (*dtos.LatencyDataBulk, error) {
panic("Not implemented for inmemory")
}
// PopCountersWithMetadata mock
func (m *MMMetricsStorage) PopCountersWithMetadata() (*dtos.CounterDataBulk, error) {
panic("Not implemented for inmemory")
}

88
vendor/github.com/splitio/go-split-commons/v2/storage/mutexmap/segments.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,88 @@
package mutexmap
import (
"fmt"
"sync"
"github.com/splitio/go-toolkit/v3/datastructures/set"
)
// MMSegmentStorage contains is an in-memory implementation of segment storage
type MMSegmentStorage struct {
data map[string]*set.ThreadUnsafeSet
till map[string]int64
mutex *sync.RWMutex
tillMutex *sync.RWMutex
}
// NewMMSegmentStorage instantiates a new MMSegmentStorage
func NewMMSegmentStorage() *MMSegmentStorage {
return &MMSegmentStorage{
data: make(map[string]*set.ThreadUnsafeSet),
till: make(map[string]int64),
mutex: &sync.RWMutex{},
tillMutex: &sync.RWMutex{},
}
}
// ChangeNumber returns the latest timestamp the segment was fetched
func (m *MMSegmentStorage) ChangeNumber(segmentName string) (int64, error) {
m.tillMutex.RLock()
defer m.tillMutex.RUnlock()
return m.till[segmentName], nil
}
// Keys retrieves a segment from the in-memory storage
// NOTE: A pointer TO A COPY is returned, in order to avoid race conditions between
// evaluations and sdk <-> backend sync
func (m *MMSegmentStorage) Keys(segmentName string) *set.ThreadUnsafeSet {
// @TODO replace to IsInSegment
m.mutex.RLock()
defer m.mutex.RUnlock()
item, exists := m.data[segmentName]
if !exists {
return nil
}
s := item.Copy().(*set.ThreadUnsafeSet)
return s
}
// SegmentContainsKey returns true if the segment contains a specific key
func (m *MMSegmentStorage) SegmentContainsKey(segmentName string, key string) (bool, error) {
m.mutex.RLock()
defer m.mutex.RUnlock()
item, exists := m.data[segmentName]
if !exists {
return false, fmt.Errorf("segment %s not found in storage", segmentName)
}
return item.Has(key), nil
}
// SetChangeNumber sets the till value belong to segmentName
func (m *MMSegmentStorage) SetChangeNumber(name string, till int64) error {
m.tillMutex.Lock()
defer m.tillMutex.Unlock()
m.till[name] = till
return nil
}
// Update adds a new segment to the in-memory storage
func (m *MMSegmentStorage) Update(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, till int64) error {
m.mutex.Lock()
defer m.mutex.Unlock()
_, ok := m.data[name]
if !ok {
m.data[name] = set.NewSet()
}
if !toRemove.IsEmpty() {
m.data[name].Remove(toRemove.List()...)
}
if !toAdd.IsEmpty() {
m.data[name].Add(toAdd.List()...)
}
m.SetChangeNumber(name, till)
return nil
}
// CountRemovedKeys method
func (m *MMSegmentStorage) CountRemovedKeys(segmentName string) int64 { return 0 }

194
vendor/github.com/splitio/go-split-commons/v2/storage/mutexmap/splits.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,194 @@
package mutexmap
import (
"sync"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/datastructures/set"
)
// MMSplitStorage struct contains is an in-memory implementation of split storage
type MMSplitStorage struct {
data map[string]dtos.SplitDTO
trafficTypes map[string]int64
till int64
mutex *sync.RWMutex
ttMutex *sync.RWMutex
tillMutex *sync.RWMutex
}
// NewMMSplitStorage instantiates a new MMSplitStorage
func NewMMSplitStorage() *MMSplitStorage {
return &MMSplitStorage{
data: make(map[string]dtos.SplitDTO),
trafficTypes: make(map[string]int64),
till: 0,
mutex: &sync.RWMutex{},
ttMutex: &sync.RWMutex{},
tillMutex: &sync.RWMutex{},
}
}
// All returns a list with a copy of each split.
// NOTE: This method will block any further operations regarding splits. Use with caution
func (m *MMSplitStorage) All() []dtos.SplitDTO {
m.mutex.RLock()
defer m.mutex.RUnlock()
splitList := make([]dtos.SplitDTO, 0)
for _, split := range m.data {
splitList = append(splitList, split)
}
return splitList
}
// ChangeNumber returns the last timestamp the split was fetched
func (m *MMSplitStorage) ChangeNumber() (int64, error) {
m.tillMutex.RLock()
defer m.tillMutex.RUnlock()
return m.till, nil
}
func (m *MMSplitStorage) _get(splitName string) *dtos.SplitDTO {
item, exists := m.data[splitName]
if !exists {
return nil
}
return &item
}
// FetchMany fetches features in redis and returns an array of split dtos
func (m *MMSplitStorage) FetchMany(splitNames []string) map[string]*dtos.SplitDTO {
m.mutex.RLock()
defer m.mutex.RUnlock()
splits := make(map[string]*dtos.SplitDTO)
for _, splitName := range splitNames {
splits[splitName] = m._get(splitName)
}
return splits
}
// KillLocally kills the split locally
func (m *MMSplitStorage) KillLocally(splitName string, defaultTreatment string, changeNumber int64) {
m.mutex.Lock()
defer m.mutex.Unlock()
split := m._get(splitName)
till, err := m.ChangeNumber()
if err != nil {
return
}
if split != nil && till < changeNumber {
split.DefaultTreatment = defaultTreatment
split.Killed = true
split.ChangeNumber = changeNumber
m.data[split.Name] = *split
}
}
// increaseTrafficTypeCount increases value for a traffic type
func (m *MMSplitStorage) increaseTrafficTypeCount(trafficType string) {
m.ttMutex.Lock()
defer m.ttMutex.Unlock()
_, exists := m.trafficTypes[trafficType]
if !exists {
m.trafficTypes[trafficType] = 1
} else {
m.trafficTypes[trafficType]++
}
}
// decreaseTrafficTypeCount decreases value for a traffic type
func (m *MMSplitStorage) decreaseTrafficTypeCount(trafficType string) {
m.ttMutex.Lock()
defer m.ttMutex.Unlock()
value, exists := m.trafficTypes[trafficType]
if exists {
if value > 0 {
m.trafficTypes[trafficType]--
} else {
delete(m.trafficTypes, trafficType)
}
}
}
// PutMany bulk inserts splits into the in-memory storage
func (m *MMSplitStorage) PutMany(splits []dtos.SplitDTO, till int64) {
m.mutex.Lock()
defer m.mutex.Unlock()
for _, split := range splits {
existing, thisIsAnUpdate := m.data[split.Name]
if thisIsAnUpdate {
// If it's an update, we decrement the traffic type count of the existing split,
// and then add the updated one (as part of the normal flow), in case it's different.
m.decreaseTrafficTypeCount(existing.TrafficTypeName)
}
m.data[split.Name] = split
m.increaseTrafficTypeCount(split.TrafficTypeName)
}
m.SetChangeNumber(till)
}
// Remove deletes a split from the in-memory storage
func (m *MMSplitStorage) Remove(splitName string) {
m.mutex.Lock()
defer m.mutex.Unlock()
split, exists := m.data[splitName]
if exists {
delete(m.data, splitName)
m.decreaseTrafficTypeCount(split.TrafficTypeName)
}
}
// SegmentNames returns a slice with the names of all segments referenced in splits
func (m *MMSplitStorage) SegmentNames() *set.ThreadUnsafeSet {
segments := set.NewSet()
m.mutex.RLock()
defer m.mutex.RUnlock()
for _, split := range m.data {
for _, condition := range split.Conditions {
for _, matcher := range condition.MatcherGroup.Matchers {
if matcher.UserDefinedSegment != nil {
segments.Add(matcher.UserDefinedSegment.SegmentName)
}
}
}
}
return segments
}
// SetChangeNumber sets the till value belong to split
func (m *MMSplitStorage) SetChangeNumber(till int64) error {
m.tillMutex.Lock()
defer m.tillMutex.Unlock()
m.till = till
return nil
}
// Split retrieves a split from the MMSplitStorage
// NOTE: A pointer TO A COPY is returned, in order to avoid race conditions between
// evaluations and sdk <-> backend sync
func (m *MMSplitStorage) Split(splitName string) *dtos.SplitDTO {
m.mutex.RLock()
defer m.mutex.RUnlock()
return m._get(splitName)
}
// SplitNames returns a slice with the names of all the current splits
func (m *MMSplitStorage) SplitNames() []string {
m.mutex.RLock()
defer m.mutex.RUnlock()
splitNames := make([]string, 0)
for key := range m.data {
splitNames = append(splitNames, key)
}
return splitNames
}
// TrafficTypeExists returns true or false depending on existence and counter
// of trafficType
func (m *MMSplitStorage) TrafficTypeExists(trafficType string) bool {
m.ttMutex.RLock()
defer m.ttMutex.RUnlock()
value, exists := m.trafficTypes[trafficType]
return exists && value > 0
}

6
vendor/github.com/splitio/go-split-commons/v2/storage/mutexqueue/constants.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,6 @@
package mutexqueue
import "errors"
// ErrorMaxSizeReached queue max size error
var ErrorMaxSizeReached = errors.New("Queue max size has been reached")

136
vendor/github.com/splitio/go-split-commons/v2/storage/mutexqueue/events.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,136 @@
package mutexqueue
import (
"container/list"
"fmt"
"sync"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
// MaxAccumulatedBytes is the maximum size to accumulate in events before flush (in bytes)
const MaxAccumulatedBytes = 5 * 1024 * 1024
// NewMQEventsStorage returns an instance of MQEventsStorage
func NewMQEventsStorage(queueSize int, isFull chan string, logger logging.LoggerInterface) *MQEventsStorage {
return &MQEventsStorage{
queue: list.New(),
size: queueSize,
mutexQueue: &sync.Mutex{},
fullChan: isFull,
logger: logger,
}
}
type eventWrapper struct {
event dtos.EventDTO
size int
}
// MQEventsStorage in memory events storage
type MQEventsStorage struct {
queue *list.List
size int
accumulatedBytes int
mutexQueue *sync.Mutex
fullChan chan string //only write channel
logger logging.LoggerInterface
}
func (s *MQEventsStorage) sendSignalIsFull() {
// Nom blocking select
select {
case s.fullChan <- "EVENTS_FULL":
// Send "queue is full" signal
break
default:
s.logger.Debug("Some error occurred on sending signal for events")
}
}
// Push an event into slice
func (s *MQEventsStorage) Push(event dtos.EventDTO, size int) error {
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
if s.queue.Len()+1 > s.size {
s.sendSignalIsFull()
return ErrorMaxSizeReached
}
// Add element
s.queue.PushBack(eventWrapper{event: event, size: size})
s.accumulatedBytes += size
if s.queue.Len() == s.size || s.accumulatedBytes >= MaxAccumulatedBytes {
s.sendSignalIsFull()
}
return nil
}
// PopN pop N elements from queue
func (s *MQEventsStorage) PopN(n int64) ([]dtos.EventDTO, error) {
var toReturn []dtos.EventDTO
var totalItems int
// Mutexing queue
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
if int64(s.queue.Len()) >= n {
totalItems = int(n)
} else {
totalItems = s.queue.Len()
}
toReturn = make([]dtos.EventDTO, 0)
accumulated := 0
errorCount := 0
for i := 0; i < totalItems; i++ {
bundled, ok := s.queue.Remove(s.queue.Front()).(eventWrapper)
if !ok {
errorCount++
continue
}
toReturn = append(toReturn, bundled.event)
accumulated += bundled.size
if accumulated >= MaxAccumulatedBytes {
// If we reached the maximum allowed size, break the loop so that we don't sent huge POST bodies to the BE
break
}
}
s.accumulatedBytes -= accumulated
if errorCount > 0 {
return toReturn, fmt.Errorf("%d elements could not be decoded", errorCount)
}
return toReturn, nil
}
// PopNWithMetadata pop N elements from queue
func (s *MQEventsStorage) PopNWithMetadata(n int64) ([]dtos.QueueStoredEventDTO, error) {
panic("Not implemented for inmemory")
}
// Empty returns if slice len if zero
func (s *MQEventsStorage) Empty() bool {
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
return s.queue.Len() == 0
}
// Count returns the number of events into slice
func (s *MQEventsStorage) Count() int64 {
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
return int64(s.queue.Len())
}
// Drop drops
func (s *MQEventsStorage) Drop(size *int64) error {
panic("Not implemented for inmemory")
}

108
vendor/github.com/splitio/go-split-commons/v2/storage/mutexqueue/impressions.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,108 @@
package mutexqueue
import (
"container/list"
"sync"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
)
// NewMQImpressionsStorage returns an instance of MQEventsStorage
func NewMQImpressionsStorage(queueSize int, isFull chan<- string, logger logging.LoggerInterface) *MQImpressionsStorage {
return &MQImpressionsStorage{
queue: list.New(),
size: queueSize,
mutexQueue: &sync.Mutex{},
fullChan: isFull,
logger: logger,
}
}
// MQImpressionsStorage in memory events storage
type MQImpressionsStorage struct {
queue *list.List
size int
mutexQueue *sync.Mutex
fullChan chan<- string //only write channel
logger logging.LoggerInterface
}
func (s *MQImpressionsStorage) sendSignalIsFull() {
// Nom blocking select
select {
case s.fullChan <- "IMPRESSIONS_FULL":
// Send "queue is full" signal
break
default:
s.logger.Debug("Some error occurred on sending signal for impressions")
break
}
}
// Empty returns if slice len if zero
func (s *MQImpressionsStorage) Empty() bool {
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
return s.queue.Len() == 0
}
// Count returns len
func (s *MQImpressionsStorage) Count() int64 {
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
return int64(s.queue.Len())
}
// LogImpressions inserts impressions into the queue
func (s *MQImpressionsStorage) LogImpressions(impressions []dtos.Impression) error {
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
for _, impression := range impressions {
if s.queue.Len()+1 > s.size {
s.sendSignalIsFull()
return ErrorMaxSizeReached
}
// Add element
s.queue.PushBack(impression)
if s.queue.Len() == s.size {
s.sendSignalIsFull()
}
}
return nil
}
// PopN pop N elements from queue
func (s *MQImpressionsStorage) PopN(n int64) ([]dtos.Impression, error) {
var toReturn []dtos.Impression
var totalItems int
// Mutexing queue
s.mutexQueue.Lock()
defer s.mutexQueue.Unlock()
if int64(s.queue.Len()) >= n {
totalItems = int(n)
} else {
totalItems = s.queue.Len()
}
toReturn = make([]dtos.Impression, totalItems)
for i := 0; i < totalItems; i++ {
toReturn[i] = s.queue.Remove(s.queue.Front()).(dtos.Impression)
}
return toReturn, nil
}
// PopNWithMetadata pop N elements from queue
func (s *MQImpressionsStorage) PopNWithMetadata(n int64) ([]dtos.ImpressionQueueObject, error) {
panic("Not implemented for inmemory")
}
// Drop drops
func (s *MQImpressionsStorage) Drop(size *int64) error {
panic("Not implemented for inmemory")
}

17
vendor/github.com/splitio/go-split-commons/v2/storage/redis/constants.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,17 @@
package redis
const (
redisSplit = "SPLITIO.split.{split}" // split object
redisSplitTill = "SPLITIO.splits.till" // last split fetch
redisSegment = "SPLITIO.segment.{segment}" // segment object
redisSegmentTill = "SPLITIO.segment.{segment}.till" // last segment fetch
redisImpressions = "SPLITIO/{sdkVersion}/{instanceId}/impressions.{feature}" // impressions for a feature
redisLatency = "SPLITIO/{sdkVersion}/{instanceId}/latency.{metric}.bucket.{bucket}" // latency bucket
redisCounter = "SPLITIO/{sdkVersion}/{instanceId}/count.{metric}" // counter
redisGauge = "SPLITIO/{sdkVersion}/{instanceId}/gauge.{metric}" // gauge
redisEvents = "SPLITIO.events" // events LIST key
redisImpressionsQueue = "SPLITIO.impressions" // impressions LIST key
redisImpressionsTTL = 60 // impressions default TTL
redisTrafficType = "SPLITIO.trafficType.{trafficType}" // traffic Type fetch
redisHash = "SPLITIO.hash"
)

177
vendor/github.com/splitio/go-split-commons/v2/storage/redis/events.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,177 @@
package redis
import (
"encoding/json"
"math"
"sync"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/queuecache"
"github.com/splitio/go-toolkit/v3/redis"
)
// EventsStorage redis implementation of EventsStorage interface
type EventsStorage struct {
cache queuecache.InMemoryQueueCacheOverlay
client *redis.PrefixedRedisClient
logger logging.LoggerInterface
metadata dtos.Metadata
redisKey string
refillMutex *sync.RWMutex
mutex *sync.RWMutex
}
// maxAccumulatedSize is the maximum number of bytes to be fetched from cache before posting to the backend
const maxAccumulatedSize = 5 * 1024 * 1024
// maxEventSize is the maximum allowed event size
const maxEventSize = 32 * 1024
// NewEventStorageConsumer storage for consumer
func NewEventStorageConsumer(redisClient *redis.PrefixedRedisClient, metadata dtos.Metadata, logger logging.LoggerInterface) *EventsStorage {
return &EventsStorage{
cache: queuecache.InMemoryQueueCacheOverlay{},
client: redisClient,
logger: logger,
metadata: metadata,
redisKey: redisEvents,
refillMutex: &sync.RWMutex{},
mutex: &sync.RWMutex{},
}
}
// NewEventsStorage returns an instance of RedisEventsStorage
func NewEventsStorage(redisClient *redis.PrefixedRedisClient, metadata dtos.Metadata, logger logging.LoggerInterface) *EventsStorage {
refillMutex := &sync.RWMutex{}
refillFunc := func(count int) ([]interface{}, error) {
refillMutex.Lock()
defer refillMutex.Unlock()
lrange, err := redisClient.LRange(redisEvents, 0, int64(count-1))
if err != nil {
logger.Error("Fetching events", err)
return nil, err
}
totalFetchedEvents := len(lrange)
idxFrom := count
if totalFetchedEvents < count {
idxFrom = totalFetchedEvents
}
err = redisClient.LTrim(redisEvents, int64(idxFrom), -1)
if err != nil {
logger.Error("Trim events", err)
return nil, err
}
toReturn := make([]interface{}, len(lrange))
for index, item := range lrange {
toReturn[index] = item
}
return toReturn, nil
}
return &EventsStorage{
cache: *queuecache.New(10000, refillFunc),
client: redisClient,
logger: logger,
metadata: metadata,
redisKey: redisEvents,
refillMutex: refillMutex,
mutex: &sync.RWMutex{},
}
}
// Push events into Redis LIST data type with RPUSH command
func (r *EventsStorage) Push(event dtos.EventDTO, _ int) error {
var queueMessage = dtos.QueueStoredEventDTO{Metadata: r.metadata, Event: event}
eventJSON, err := json.Marshal(queueMessage)
if err != nil {
r.logger.Error("Something were wrong marshaling provided event to JSON", err.Error())
return err
}
r.logger.Debug("Pushing events to:", r.redisKey, string(eventJSON))
_, errPush := r.client.RPush(r.redisKey, eventJSON)
if errPush != nil {
r.logger.Error("Something were wrong pushing event to redis", errPush)
return errPush
}
return nil
}
// PopN return N elements from 0 to N
func (r *EventsStorage) PopN(n int64) ([]dtos.EventDTO, error) {
panic("Not implemented for redis")
}
// PopNWithMetadata pop N elements from queue
func (r *EventsStorage) PopNWithMetadata(n int64) ([]dtos.QueueStoredEventDTO, error) {
r.mutex.Lock()
defer r.mutex.Unlock()
toReturn := make([]dtos.QueueStoredEventDTO, n)
var err error
fetchedCount := 0
accumulatedSize := 0
writeIndex := 0
for r.Count() > 0 && int64(fetchedCount) < n && accumulatedSize+maxEventSize < maxAccumulatedSize && err == nil {
numberOfItemsToFetch := int(math.Min(
float64((maxAccumulatedSize-accumulatedSize)/maxEventSize),
float64(n-int64(fetchedCount)),
))
elems, err := r.cache.Fetch(numberOfItemsToFetch)
if err != nil {
r.logger.Error("Error fetching events", err.Error())
break
}
for _, elem := range elems {
asStr, ok := elem.(string)
if !ok {
r.logger.Error("Error type-asserting event as string", err.Error())
continue
}
storedEventDTO := dtos.QueueStoredEventDTO{}
err = json.Unmarshal([]byte(asStr), &storedEventDTO)
if err != nil {
r.logger.Error("Error decoding event JSON", err.Error())
continue
}
accumulatedSize += storedEventDTO.Event.Size()
toReturn[writeIndex] = storedEventDTO
writeIndex++
}
fetchedCount += len(elems)
}
return toReturn[0:writeIndex], nil
}
// Count returns the number of items in the redis list
func (r *EventsStorage) Count() int64 {
val, err := r.client.LLen(r.redisKey)
if err != nil {
return 0
}
return val
}
// Empty returns true if redis list is zero length
func (r *EventsStorage) Empty() bool {
return r.Count() == 0
}
// Drop drops events from queue
func (r *EventsStorage) Drop(size *int64) error {
r.mutex.Lock()
defer r.mutex.Unlock()
if size == nil {
_, err := r.client.Del(r.redisKey)
return err
}
return r.client.LTrim(r.redisKey, *size, -1)
}

149
vendor/github.com/splitio/go-split-commons/v2/storage/redis/impressions.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,149 @@
package redis
import (
"encoding/json"
"sync"
"time"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/redis"
)
const impressionsTTLRefresh = time.Duration(3600) * time.Second
// ImpressionStorage is a redis-based implementation of split storage
type ImpressionStorage struct {
client *redis.PrefixedRedisClient
mutex *sync.Mutex
logger logging.LoggerInterface
redisKey string
impressionsTTL time.Duration
metadata dtos.Metadata
}
// NewImpressionStorage creates a new RedisSplitStorage and returns a reference to it
func NewImpressionStorage(client *redis.PrefixedRedisClient, metadata dtos.Metadata, logger logging.LoggerInterface) *ImpressionStorage {
return &ImpressionStorage{
client: client,
mutex: &sync.Mutex{},
logger: logger,
redisKey: redisImpressionsQueue,
impressionsTTL: redisImpressionsTTL,
metadata: metadata,
}
}
// Count returns the size of the impressions queue
func (r *ImpressionStorage) Count() int64 {
val, err := r.client.LLen(r.redisKey)
if err != nil {
return 0
}
return val
}
// Drop drops impressions from queue
func (r *ImpressionStorage) Drop(size *int64) error {
r.mutex.Lock()
defer r.mutex.Unlock()
if size == nil {
_, err := r.client.Del(r.redisKey)
return err
}
return r.client.LTrim(r.redisKey, *size, -1)
}
// Empty returns true if redis list is zero length
func (r *ImpressionStorage) Empty() bool {
return r.Count() == 0
}
// push stores impressions in redis
func (r *ImpressionStorage) push(impressions []dtos.ImpressionQueueObject) error {
var impressionsJSON []interface{}
for _, impression := range impressions {
iJSON, err := json.Marshal(impression)
if err != nil {
r.logger.Error("Error encoding impression in json")
r.logger.Error(err)
} else {
impressionsJSON = append(impressionsJSON, iJSON)
}
}
r.logger.Debug("Pushing impressions to: ", r.redisKey, len(impressionsJSON))
inserted, errPush := r.client.RPush(r.redisKey, impressionsJSON...)
if errPush != nil {
r.logger.Error("Something were wrong pushing impressions to redis", errPush)
return errPush
}
// Checks if expiration needs to be set
if inserted == int64(len(impressionsJSON)) {
r.logger.Debug("Proceeding to set expiration for: ", r.redisKey)
result := r.client.Expire(r.redisKey, time.Duration(r.impressionsTTL)*time.Minute)
if result == false {
r.logger.Error("Something were wrong setting expiration", errPush)
}
}
return nil
}
// LogImpressions stores impressions in redis as Queue
func (r *ImpressionStorage) LogImpressions(impressions []dtos.Impression) error {
var impressionsToStore []dtos.ImpressionQueueObject
for _, i := range impressions {
var impression = dtos.ImpressionQueueObject{Metadata: r.metadata, Impression: i}
impressionsToStore = append(impressionsToStore, impression)
}
if len(impressionsToStore) > 0 {
return r.push(impressionsToStore)
}
return nil
}
// PopN return N elements from 0 to N
func (r *ImpressionStorage) PopN(n int64) ([]dtos.Impression, error) {
panic("Not implemented for redis")
}
// PopNWithMetadata pop N elements from queue
func (r *ImpressionStorage) PopNWithMetadata(n int64) ([]dtos.ImpressionQueueObject, error) {
r.mutex.Lock()
defer r.mutex.Unlock()
toReturn := make([]dtos.ImpressionQueueObject, 0, n)
lrange, err := r.client.LRange(r.redisKey, 0, n-1)
if err != nil {
r.logger.Error("Error fetching impressions")
return toReturn, err
}
fetchedCount := int64(len(lrange))
err = r.client.LTrim(r.redisKey, fetchedCount, int64(-1))
if err != nil {
r.logger.Error("Error trimming impressions")
return toReturn, err
}
// This operation will simply do nothing if the key no longer exists (queue is empty)
// It's only done in the "successful" exit path so that the TTL is not overriden if impressons weren't
// popped correctly. This will result in impressions getting lost but will prevent the queue from taking
// a huge amount of memory.
r.client.Expire(r.redisKey, impressionsTTLRefresh)
for _, asStr := range lrange {
storedImpressionDTO := dtos.ImpressionQueueObject{}
err = json.Unmarshal([]byte(asStr), &storedImpressionDTO)
if err != nil {
r.logger.Error("Error decoding event JSON", err.Error())
continue
}
toReturn = append(toReturn, storedImpressionDTO)
}
return toReturn, nil
}

322
vendor/github.com/splitio/go-split-commons/v2/storage/redis/metrics.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,322 @@
package redis
import (
"fmt"
"regexp"
"strconv"
"strings"
"sync"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/redis"
)
// MetricsStorage is a redis-based implementation of split storage
type MetricsStorage struct {
client redis.PrefixedRedisClient
logger logging.LoggerInterface
gaugeSingleTemplate string
countersSingleTemplate string
latenciesSingleTemplate string
gaugeMultiTemplate string
countersMultiTemplate string
latenciesMultiTemplate string
mutex *sync.RWMutex
}
// NewMetricsStorage creates a new RedisSplitStorage and returns a reference to it
func NewMetricsStorage(redisClient *redis.PrefixedRedisClient, metadata dtos.Metadata, logger logging.LoggerInterface) *MetricsStorage {
// @Todo Split Storages between Go-Client and Redis
gaugeSingleTemplate := strings.Replace(redisGauge, "{sdkVersion}", metadata.SDKVersion, 1)
gaugeSingleTemplate = strings.Replace(gaugeSingleTemplate, "{instanceId}", metadata.MachineName, 1)
countersSingleTemplate := strings.Replace(redisCounter, "{sdkVersion}", metadata.SDKVersion, 1)
countersSingleTemplate = strings.Replace(countersSingleTemplate, "{instanceId}", metadata.MachineName, 1)
latenciesSingleTemplate := strings.Replace(redisLatency, "{sdkVersion}", metadata.SDKVersion, 1)
latenciesSingleTemplate = strings.Replace(latenciesSingleTemplate, "{instanceId}", metadata.MachineName, 1)
gaugeMultiTemplate := strings.Replace(redisGauge, "{sdkVersion}", "*", 1)
gaugeMultiTemplate = strings.Replace(gaugeMultiTemplate, "{instanceId}", "*", 1)
gaugeMultiTemplate = strings.Replace(gaugeMultiTemplate, "{metric}", "*", 1)
countersMultiTemplate := strings.Replace(redisCounter, "{sdkVersion}", "*", 1)
countersMultiTemplate = strings.Replace(countersMultiTemplate, "{instanceId}", "*", 1)
countersMultiTemplate = strings.Replace(countersMultiTemplate, "{metric}", "*", 1)
latenciesMultiTemplate := strings.Replace(redisLatency, "{sdkVersion}", "*", 1)
latenciesMultiTemplate = strings.Replace(latenciesMultiTemplate, "{instanceId}", "*", 1)
latenciesMultiTemplate = strings.Replace(latenciesMultiTemplate, "{metric}", "*", 1)
latenciesMultiTemplate = strings.Replace(latenciesMultiTemplate, "{bucket}", "*", 1)
return &MetricsStorage{
client: *redisClient,
logger: logger,
gaugeSingleTemplate: gaugeSingleTemplate,
countersSingleTemplate: countersSingleTemplate,
latenciesSingleTemplate: latenciesSingleTemplate,
gaugeMultiTemplate: gaugeMultiTemplate,
countersMultiTemplate: countersMultiTemplate,
latenciesMultiTemplate: latenciesMultiTemplate,
mutex: &sync.RWMutex{},
}
}
// IncCounter incraeses the count for a specific metric
func (r *MetricsStorage) IncCounter(metric string) {
keyToIncr := strings.Replace(r.countersSingleTemplate, "{metric}", metric, 1)
_, err := r.client.Incr(keyToIncr)
if err != nil {
r.logger.Error(fmt.Sprintf("Error incrementing counterfor metric \"%s\" in redis: %s", metric, err.Error()))
}
}
// IncLatency incraeses the latency of a bucket for a specific metric
func (r *MetricsStorage) IncLatency(metric string, index int) {
keyToIncr := strings.Replace(r.latenciesSingleTemplate, "{metric}", metric, 1)
keyToIncr = strings.Replace(keyToIncr, "{bucket}", strconv.FormatInt(int64(index), 10), 1)
_, err := r.client.Incr(keyToIncr)
if err != nil {
r.logger.Error(fmt.Sprintf(
"Error incrementing latency bucket %d for metric \"%s\" in redis: %s", index, metric, err.Error(),
))
}
}
// PutGauge stores a gauge in redis
func (r *MetricsStorage) PutGauge(key string, gauge float64) {
keyToStore := strings.Replace(r.gaugeSingleTemplate, "{metric}", key, 1)
err := r.client.Set(keyToStore, gauge, 0)
if err != nil {
r.logger.Error(fmt.Sprintf("Error storing gauge \"%s\" in redis: %s\n", key, err))
}
}
// PopCounters some
func (r *MetricsStorage) PopCounters() []dtos.CounterDTO {
panic("Not implemented for redis")
}
// PopGauges some
func (r *MetricsStorage) PopGauges() []dtos.GaugeDTO {
panic("Not implemented for redis")
}
// PopLatencies some
func (r *MetricsStorage) PopLatencies() []dtos.LatenciesDTO {
panic("Not implemented for redis")
}
func (r *MetricsStorage) popByPattern(pattern string, useTransaction bool) (map[string]interface{}, error) {
r.mutex.Lock()
defer r.mutex.Unlock()
keys, err := r.client.Keys(pattern)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
if len(keys) == 0 {
return map[string]interface{}{}, nil
}
values, err := r.client.MGet(keys)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
_, err = r.client.Del(keys...)
if err != nil {
// if we failed to delete the keys, log an error and continue working.
r.logger.Error(err.Error())
}
toReturn := make(map[string]interface{})
for index := range keys {
if index >= len(keys) || index >= len(values) {
break
}
toReturn[keys[index]] = values[index]
}
return toReturn, nil
}
func parseIntRedisValue(s interface{}) (int64, error) {
asStr, ok := s.(string)
if !ok {
return 0, fmt.Errorf("%+v is not a string", s)
}
asInt64, err := strconv.ParseInt(asStr, 10, 64)
if err != nil {
return 0, err
}
return asInt64, nil
}
func parseFloatRedisValue(s interface{}) (float64, error) {
asStr, ok := s.(string)
if !ok {
return 0, fmt.Errorf("%+v is not a string", s)
}
asFloat64, err := strconv.ParseFloat(asStr, 64)
if err != nil {
return 0, err
}
return asFloat64, nil
}
func (r *MetricsStorage) parseLatencyKey(key string) (string, string, string, int, error) {
re := regexp.MustCompile(`(\w+.)?SPLITIO\/([^\/]+)\/([^\/]+)\/latency.([^\/]+).bucket.([0-9]*)`)
match := re.FindStringSubmatch(key)
if len(match) < 6 {
return "", "", "", 0, fmt.Errorf("Error parsing key %s", key)
}
sdkNameAndVersion := match[2]
if sdkNameAndVersion == "" {
return "", "", "", 0, fmt.Errorf("Invalid sdk name/version")
}
machineIP := match[3]
if machineIP == "" {
return "", "", "", 0, fmt.Errorf("Invalid machine IP")
}
metricName := match[4]
if metricName == "" {
return "", "", "", 0, fmt.Errorf("Invalid feature name")
}
bucketNumber, err := strconv.Atoi(match[5])
if err != nil {
return "", "", "", 0, fmt.Errorf("Error parsing bucket number: %s", err.Error())
}
r.logger.Verbose("Impression parsed key", match)
return sdkNameAndVersion, machineIP, metricName, bucketNumber, nil
}
func (r *MetricsStorage) parseMetricKey(metricType string, key string) (string, string, string, error) {
var re = regexp.MustCompile(strings.Replace(
`(\w+.)?SPLITIO\/([^\/]+)\/([^\/]+)\/{metricType}.([\s\S]*)`,
"{metricType}",
metricType,
1,
))
match := re.FindStringSubmatch(key)
if len(match) < 5 {
return "", "", "", fmt.Errorf("Error parsing key %s", key)
}
sdkNameAndVersion := match[2]
if sdkNameAndVersion == "" {
return "", "", "", fmt.Errorf("Invalid sdk name/version")
}
machineIP := match[3]
if machineIP == "" {
return "", "", "", fmt.Errorf("Invalid machine IP")
}
metricName := match[4]
if metricName == "" {
return "", "", "", fmt.Errorf("Invalid feature name")
}
r.logger.Verbose("Impression parsed key", match)
return sdkNameAndVersion, machineIP, metricName, nil
}
// PopGaugesWithMetadata returns gauges values saved in Redis by SDKs
func (r *MetricsStorage) PopGaugesWithMetadata() (*dtos.GaugeDataBulk, error) {
data, err := r.popByPattern(r.gaugeMultiTemplate, false)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
gaugesToReturn := dtos.NewGaugeDataBulk()
for key, value := range data {
sdkNameAndVersion, machineIP, metricName, err := r.parseMetricKey("gauge", key)
if err != nil {
r.logger.Error(fmt.Sprintf("Unable to parse key %s. Skipping", key))
continue
}
asFloat, err := parseFloatRedisValue(value)
if err != nil {
r.logger.Error(fmt.Sprintf("Unable to parse value %+v. Skipping", value))
continue
}
gaugesToReturn.PutGauge(sdkNameAndVersion, machineIP, metricName, asFloat)
}
return gaugesToReturn, nil
}
// PopCountersWithMetadata returns counter values saved in Redis by SDKs
func (r *MetricsStorage) PopCountersWithMetadata() (*dtos.CounterDataBulk, error) {
data, err := r.popByPattern(r.countersMultiTemplate, false)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
countersToReturn := dtos.NewCounterDataBulk()
for key, value := range data {
sdkNameAndVersion, machineIP, metricName, err := r.parseMetricKey("count", key)
if err != nil {
r.logger.Error("Unable to parse key %s. Skipping", key)
continue
}
asInt, err := parseIntRedisValue(value)
if err != nil {
r.logger.Error(err.Error())
continue
}
countersToReturn.PutCounter(sdkNameAndVersion, machineIP, metricName, asInt)
}
return countersToReturn, nil
}
// PopLatenciesWithMetadata returns latency values saved in Redis by SDKs
func (r *MetricsStorage) PopLatenciesWithMetadata() (*dtos.LatencyDataBulk, error) {
data, err := r.popByPattern(r.latenciesMultiTemplate, false)
if err != nil {
r.logger.Error(err.Error())
return nil, err
}
latenciesToReturn := dtos.NewLatencyDataBulk()
for key, value := range data {
value, err := parseIntRedisValue(value)
if err != nil {
r.logger.Warning(fmt.Sprintf("Unable to parse value of key %s. Skipping", key))
continue
}
sdkNameAndVersion, machineIP, metricName, bucketNumber, err := r.parseLatencyKey(key)
if err != nil {
r.logger.Warning(fmt.Sprintf("Unable to parse key %s. Skipping", key))
continue
}
latenciesToReturn.PutLatency(sdkNameAndVersion, machineIP, metricName, bucketNumber, value)
}
r.logger.Verbose(latenciesToReturn)
return latenciesToReturn, nil
}
// PeekCounters returns Counters
func (r *MetricsStorage) PeekCounters() map[string]int64 {
return make(map[string]int64, 0)
}
// PeekLatencies returns Latencies
func (r *MetricsStorage) PeekLatencies() map[string][]int64 {
return make(map[string][]int64, 0)
}

56
vendor/github.com/splitio/go-split-commons/v2/storage/redis/miscstorage.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,56 @@
package redis
import (
"errors"
"strings"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/redis"
)
// ErrorHashNotPresent constant
const ErrorHashNotPresent = "hash-not-present"
const clearAllSCriptTemplate = `
local toDelete = redis.call('KEYS', '{KEY_NAMESPACE}*')
local count = 0
for _, key in ipairs(toDelete) do
redis.call('DEL', key)
count = count + 1
end
return count
`
// MiscStorage provides methods to handle the synchronizer's initialization procedure
type MiscStorage struct {
client *redis.PrefixedRedisClient
logger logging.LoggerInterface
}
// GetApikeyHash gets hashed apikey from redis
func (m *MiscStorage) GetApikeyHash() (string, error) {
res, err := m.client.Get(redisHash)
if err != nil && err.Error() == "redis: nil" {
return "", errors.New(ErrorHashNotPresent)
}
return res, err
}
// SetApikeyHash sets hashed apikey in redis
func (m *MiscStorage) SetApikeyHash(newApikeyHash string) error {
return m.client.Set(redisHash, newApikeyHash, 0)
}
// ClearAll cleans previous used data
func (m *MiscStorage) ClearAll() error {
luaCMD := strings.Replace(clearAllSCriptTemplate, "{KEY_NAMESPACE}", m.client.Prefix, 1)
return m.client.Eval(luaCMD, []string{}, nil)
}
// NewMiscStorage creates a new MiscStorageAdapter and returns a reference to it
func NewMiscStorage(client *redis.PrefixedRedisClient, logger logging.LoggerInterface) *MiscStorage {
return &MiscStorage{
client: client,
logger: logger,
}
}

74
vendor/github.com/splitio/go-split-commons/v2/storage/redis/redis.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,74 @@
package redis
import (
"errors"
"fmt"
"strings"
"time"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/redis"
"github.com/splitio/go-toolkit/v3/redis/helpers"
)
// NewRedisClient returns a new Prefixed Redis Client
func NewRedisClient(config *conf.RedisConfig, logger logging.LoggerInterface) (*redis.PrefixedRedisClient, error) {
prefix := config.Prefix
if len(config.SentinelAddresses) > 0 && len(config.ClusterNodes) > 0 {
return nil, errors.New("Incompatible configuration of redis, Sentinel and Cluster cannot be enabled at the same time")
}
universalOptions := &redis.UniversalOptions{
Password: config.Password,
DB: config.Database,
TLSConfig: config.TLSConfig,
MaxRetries: config.MaxRetries,
PoolSize: config.PoolSize,
DialTimeout: time.Duration(config.DialTimeout) * time.Second,
ReadTimeout: time.Duration(config.ReadTimeout) * time.Second,
WriteTimeout: time.Duration(config.WriteTimeout) * time.Second,
}
if len(config.SentinelAddresses) > 0 {
logger.Info("To start as Sentinel Mode")
if config.SentinelMaster == "" {
return nil, errors.New("Missing redis sentinel master name")
}
universalOptions.MasterName = config.SentinelMaster
universalOptions.Addrs = config.SentinelAddresses
} else {
if len(config.ClusterNodes) > 0 {
logger.Info("To start as Cluster Mode")
var keyHashTag = "{SPLITIO}"
if config.ClusterKeyHashTag != "" {
keyHashTag = config.ClusterKeyHashTag
if len(keyHashTag) < 3 ||
string(keyHashTag[0]) != "{" ||
string(keyHashTag[len(keyHashTag)-1]) != "}" ||
strings.Count(keyHashTag, "{") != 1 ||
strings.Count(keyHashTag, "}") != 1 {
return nil, errors.New("keyHashTag is not valid")
}
}
prefix = keyHashTag + prefix
universalOptions.Addrs = config.ClusterNodes
} else {
logger.Info("To start as Single Mode")
universalOptions.Addrs = []string{fmt.Sprintf("%s:%d", config.Host, config.Port)}
}
}
rClient, err := redis.NewClient(universalOptions)
if err != nil {
logger.Error(err.Error())
}
helpers.EnsureConnected(rClient)
return redis.NewPrefixedRedisClient(rClient, prefix)
}

100
vendor/github.com/splitio/go-split-commons/v2/storage/redis/segments.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,100 @@
package redis
import (
"fmt"
"strconv"
"strings"
"sync"
"github.com/splitio/go-toolkit/v3/datastructures/set"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/redis"
)
// SegmentStorage is a redis implementation of a storage for segments
type SegmentStorage struct {
client redis.PrefixedRedisClient
logger logging.LoggerInterface
mutext *sync.RWMutex
}
// NewSegmentStorage creates a new RedisSegmentStorage and returns a reference to it
func NewSegmentStorage(redisClient *redis.PrefixedRedisClient, logger logging.LoggerInterface) *SegmentStorage {
return &SegmentStorage{
client: *redisClient,
logger: logger,
mutext: &sync.RWMutex{},
}
}
// ChangeNumber returns the changeNumber for a particular segment
func (r *SegmentStorage) ChangeNumber(segmentName string) (int64, error) {
segmentKey := strings.Replace(redisSegmentTill, "{segment}", segmentName, 1)
tillStr, err := r.client.Get(segmentKey)
if err != nil {
return -1, err
}
asInt, err := strconv.ParseInt(tillStr, 10, 64)
if err != nil {
r.logger.Error("Error retrieving till. Returning -1: ", err.Error())
return -1, err
}
return asInt, nil
}
// Keys returns segments keys for segment if it's present
func (r *SegmentStorage) Keys(segmentName string) *set.ThreadUnsafeSet {
keyToFetch := strings.Replace(redisSegment, "{segment}", segmentName, 1)
segmentKeys, err := r.client.SMembers(keyToFetch)
if len(segmentKeys) <= 0 {
r.logger.Debug(fmt.Sprintf("Nonexsitent segment requested: %s", segmentName))
return nil
}
if err != nil {
r.logger.Error(fmt.Sprintf("Error retrieving members from set %s", segmentName))
return nil
}
segment := set.NewSet()
for _, member := range segmentKeys {
segment.Add(member)
}
return segment
}
// SetChangeNumber sets the till value belong to segmentName
func (r *SegmentStorage) SetChangeNumber(segmentName string, changeNumber int64) error {
segmentKey := strings.Replace(redisSegmentTill, "{segment}", segmentName, 1)
return r.client.Set(segmentKey, changeNumber, 0)
}
// Update adds a new segment
func (r *SegmentStorage) Update(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, till int64) error {
r.mutext.Lock()
defer r.mutext.Unlock()
segmentKey := strings.Replace(redisSegment, "{segment}", name, 1)
if !toRemove.IsEmpty() {
_, err := r.client.SRem(segmentKey, toRemove.List()...)
if err != nil {
r.logger.Error(fmt.Sprintf("Error removing keys in redis: %s", err.Error()))
}
}
if !toAdd.IsEmpty() {
_, err := r.client.SAdd(segmentKey, toAdd.List()...)
if err != nil {
r.logger.Error(fmt.Sprintf("Error removing keys in redis: %s", err.Error()))
}
}
r.SetChangeNumber(name, till)
return nil
}
// SegmentContainsKey returns true if the segment contains a specific key
func (r *SegmentStorage) SegmentContainsKey(segmentName string, key string) (bool, error) {
segmentKey := strings.Replace(redisSegment, "{segment}", segmentName, 1)
exists := r.client.SIsMember(segmentKey, key)
return exists, nil
}
// CountRemovedKeys method
func (r *SegmentStorage) CountRemovedKeys(segmentName string) int64 { return 0 }

261
vendor/github.com/splitio/go-split-commons/v2/storage/redis/splits.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,261 @@
package redis
import (
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
"sync"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-toolkit/v3/datastructures/set"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/redis"
)
// SplitStorage is a redis-based implementation of split storage
type SplitStorage struct {
client *redis.PrefixedRedisClient
logger logging.LoggerInterface
mutext *sync.RWMutex
}
// NewSplitStorage creates a new RedisSplitStorage and returns a reference to it
func NewSplitStorage(redisClient *redis.PrefixedRedisClient, logger logging.LoggerInterface) *SplitStorage {
return &SplitStorage{
client: redisClient,
logger: logger,
mutext: &sync.RWMutex{},
}
}
// All returns a slice of splits dtos.
func (r *SplitStorage) All() []dtos.SplitDTO {
splits := make([]dtos.SplitDTO, 0)
keyPattern := strings.Replace(redisSplit, "{split}", "*", 1)
keys, err := r.client.Keys(keyPattern)
if err != nil {
r.logger.Error("Error fetching split keys. Returning empty split list")
return splits
}
rawSplits, err := r.client.MGet(keys)
if err != nil {
r.logger.Error("Could not get splits")
return splits
}
for idx, raw := range rawSplits {
var split dtos.SplitDTO
rawSplit, ok := rawSplits[idx].(string)
if ok {
err = json.Unmarshal([]byte(rawSplit), &split)
if err != nil {
r.logger.Error(fmt.Sprintf("Error parsing json for split %s", raw))
continue
}
}
splits = append(splits, split)
}
return splits
}
// ChangeNumber returns the latest split changeNumber
func (r *SplitStorage) ChangeNumber() (int64, error) {
val, err := r.client.Get(redisSplitTill)
if err != nil {
return -1, err
}
asInt, err := strconv.ParseInt(val, 10, 64)
if err != nil {
r.logger.Error("Could not parse Till value from redis")
return -1, err
}
return asInt, nil
}
// FetchMany retrieves features from redis storage
func (r *SplitStorage) FetchMany(features []string) map[string]*dtos.SplitDTO {
keysToFetch := make([]string, 0)
for _, feature := range features {
keysToFetch = append(keysToFetch, strings.Replace(redisSplit, "{split}", feature, 1))
}
rawSplits, err := r.client.MGet(keysToFetch)
if err != nil {
r.logger.Error(fmt.Sprintf("Could not fetch features from redis: %s", err.Error()))
return nil
}
splits := make(map[string]*dtos.SplitDTO)
for idx, feature := range features {
var split *dtos.SplitDTO
rawSplit, ok := rawSplits[idx].(string)
if ok {
err = json.Unmarshal([]byte(rawSplit), &split)
if err != nil {
r.logger.Error("Could not parse feature \"%s\" fetched from redis", feature)
return nil
}
}
splits[feature] = split
}
return splits
}
// KillLocally mock
func (r *SplitStorage) KillLocally(splitName string, defaultTreatment string, changeNumber int64) {
// @TODO Implement for Sync
}
// incr stores/increments trafficType in Redis
func (r *SplitStorage) incr(trafficType string) error {
key := strings.Replace(redisTrafficType, "{trafficType}", trafficType, 1)
_, err := r.client.Incr(key)
if err != nil {
r.logger.Error(fmt.Sprintf("Error storing trafficType %s in redis", trafficType))
r.logger.Error(err)
return errors.New("Error incrementing trafficType")
}
return nil
}
// decr decrements trafficType count in Redis
func (r *SplitStorage) decr(trafficType string) error {
key := strings.Replace(redisTrafficType, "{trafficType}", trafficType, 1)
val, _ := r.client.Decr(key)
if val <= 0 {
_, err := r.client.Del(key)
if err != nil {
r.logger.Verbose(fmt.Sprintf("Error removing trafficType %s in redis", trafficType))
}
}
return nil
}
// PutMany bulk stores splits in redis
func (r *SplitStorage) PutMany(splits []dtos.SplitDTO, changeNumber int64) {
r.mutext.Lock()
defer r.mutext.Unlock()
for _, split := range splits {
keyToStore := strings.Replace(redisSplit, "{split}", split.Name, 1)
raw, err := json.Marshal(split)
if err != nil {
r.logger.Error(fmt.Sprintf("Could not dump feature \"%s\" to json", split.Name))
continue
}
existing := r.Split(split.Name)
if existing != nil {
// If it's an update, we decrement the traffic type count of the existing split,
// and then add the updated one (as part of the normal flow), in case it's different.
r.decr(existing.TrafficTypeName)
}
r.incr(split.TrafficTypeName)
err = r.client.Set(keyToStore, raw, 0)
if err != nil {
r.logger.Error(fmt.Sprintf("Could not store split \"%s\" in redis: %s", split.Name, err.Error()))
}
}
err := r.client.Set(redisSplitTill, changeNumber, 0)
if err != nil {
r.logger.Error("Could not update split changenumber")
}
}
// Remove removes split item from redis
func (r *SplitStorage) Remove(splitName string) {
r.mutext.Lock()
defer r.mutext.Unlock()
keyToDelete := strings.Replace(redisSplit, "{split}", splitName, 1)
existing := r.Split(splitName)
if existing == nil {
r.logger.Warning("Tried to delete split " + splitName + " which doesn't exist. ignoring")
return
}
r.decr(existing.TrafficTypeName)
_, err := r.client.Del(keyToDelete)
if err != nil {
r.logger.Error(fmt.Sprintf("Error deleting split \"%s\".", splitName))
}
}
// SegmentNames returns a slice of strings with all the segment names
func (r *SplitStorage) SegmentNames() *set.ThreadUnsafeSet {
segmentNames := set.NewSet()
splits := r.All()
for _, split := range splits {
for _, condition := range split.Conditions {
for _, matcher := range condition.MatcherGroup.Matchers {
if matcher.UserDefinedSegment != nil {
segmentNames.Add(matcher.UserDefinedSegment.SegmentName)
}
}
}
}
return segmentNames
}
// SetChangeNumber sets the till value belong to segmentName
func (r *SplitStorage) SetChangeNumber(changeNumber int64) error {
return r.client.Set(redisSplitTill, changeNumber, 0)
}
// Split fetches a feature in redis and returns a pointer to a split dto
func (r *SplitStorage) Split(feature string) *dtos.SplitDTO {
keyToFetch := strings.Replace(redisSplit, "{split}", feature, 1)
val, err := r.client.Get(keyToFetch)
if err != nil {
r.logger.Error(fmt.Sprintf("Could not fetch feature %s from redis: %s", feature, err.Error()))
return nil
}
var split dtos.SplitDTO
err = json.Unmarshal([]byte(val), &split)
if err != nil {
r.logger.Error(fmt.Sprintf("Could not parse feature %s fetched from redis", feature))
return nil
}
return &split
}
// SplitNames returns a slice of strings with all the split names
func (r *SplitStorage) SplitNames() []string {
splitNames := make([]string, 0)
keyPattern := strings.Replace(redisSplit, "{split}", "*", 1)
keys, err := r.client.Keys(keyPattern)
if err == nil {
toRemove := strings.Replace(redisSplit, "{split}", "", 1) // Create a string with all the prefix to remove
for _, key := range keys {
splitNames = append(splitNames, strings.Replace(key, toRemove, "", 1)) // Extract split name from key
}
}
return splitNames
}
// TrafficTypeExists returns true or false depending on existence and counter
// of trafficType
func (r *SplitStorage) TrafficTypeExists(trafficType string) bool {
keyToFetch := strings.Replace(redisTrafficType, "{trafficType}", trafficType, 1)
res, err := r.client.Get(keyToFetch)
if err != nil {
r.logger.Error(fmt.Sprintf("Could not fetch trafficType \"%s\" from redis: %s", trafficType, err.Error()))
return false
}
val, err := strconv.ParseInt(res, 10, 64)
if err != nil {
r.logger.Error("TrafficType could not be converted")
return false
}
return val > 0
}

12
vendor/github.com/splitio/go-split-commons/v2/synchronizer/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,12 @@
package synchronizer
// Synchronizer interface for syncing data to and from splits servers
type Synchronizer interface {
SyncAll() error
SynchronizeSplits(till *int64) error
SynchronizeSegment(segmentName string, till *int64) error
StartPeriodicFetching()
StopPeriodicFetching()
StartPeriodicDataRecording()
StopPeriodicDataRecording()
}

80
vendor/github.com/splitio/go-split-commons/v2/synchronizer/local.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,80 @@
package synchronizer
import (
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-split-commons/v2/storage"
storageMock "github.com/splitio/go-split-commons/v2/storage/mocks"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/split"
"github.com/splitio/go-split-commons/v2/tasks"
"github.com/splitio/go-toolkit/v3/logging"
)
// Local implements Local Synchronizer
type Local struct {
splitTasks SplitTasks
workers Workers
logger logging.LoggerInterface
inMememoryFullQueue chan string
}
// NewLocal creates new Local
func NewLocal(
period int,
splitAPI *service.SplitAPI,
splitStorage storage.SplitStorage,
logger logging.LoggerInterface,
) Synchronizer {
metricStorageMock := storageMock.MockMetricStorage{
IncCounterCall: func(key string) {},
IncLatencyCall: func(metricName string, index int) {},
PopCountersCall: func() []dtos.CounterDTO { return make([]dtos.CounterDTO, 0, 0) },
PopGaugesCall: func() []dtos.GaugeDTO { return make([]dtos.GaugeDTO, 0, 0) },
PopLatenciesCall: func() []dtos.LatenciesDTO { return make([]dtos.LatenciesDTO, 0, 0) },
PutGaugeCall: func(key string, gauge float64) {},
}
metricsWrapper := storage.NewMetricWrapper(metricStorageMock, nil, logger)
workers := Workers{
SplitFetcher: split.NewSplitFetcher(splitStorage, splitAPI.SplitFetcher, metricsWrapper, logger),
}
return &Local{
splitTasks: SplitTasks{
SplitSyncTask: tasks.NewFetchSplitsTask(workers.SplitFetcher, period, logger),
},
workers: workers,
logger: logger,
}
}
// SyncAll syncs splits and segments
func (s *Local) SyncAll() error {
return s.workers.SplitFetcher.SynchronizeSplits(nil)
}
// StartPeriodicFetching starts periodic fetchers tasks
func (s *Local) StartPeriodicFetching() {
s.splitTasks.SplitSyncTask.Start()
}
// StopPeriodicFetching stops periodic fetchers tasks
func (s *Local) StopPeriodicFetching() {
s.splitTasks.SplitSyncTask.Stop(false)
}
// StartPeriodicDataRecording starts periodic recorders tasks
func (s *Local) StartPeriodicDataRecording() {
}
// StopPeriodicDataRecording stops periodic recorders tasks
func (s *Local) StopPeriodicDataRecording() {
}
// SynchronizeSplits syncs splits
func (s *Local) SynchronizeSplits(till *int64) error {
return s.workers.SplitFetcher.SynchronizeSplits(till)
}
// SynchronizeSegment syncs segment
func (s *Local) SynchronizeSegment(name string, till *int64) error {
return nil
}

184
vendor/github.com/splitio/go-split-commons/v2/synchronizer/manager.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,184 @@
package synchronizer
import (
"errors"
"sync/atomic"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/push"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-split-commons/v2/storage"
"github.com/splitio/go-toolkit/v3/logging"
)
const (
// Ready represents ready
Ready = iota
// StreamingReady ready
StreamingReady
// Error represents some error in SSE streaming
Error
)
const (
// Idle flags
Idle = iota
// Streaming flags
Streaming
// Polling flags
Polling
)
// Manager struct
type Manager struct {
synchronizer Synchronizer
logger logging.LoggerInterface
config conf.AdvancedConfig
pushManager push.Manager
managerStatus chan int
streamingStatus chan int
status atomic.Value
}
// NewSynchronizerManager creates new sync manager
func NewSynchronizerManager(
synchronizer Synchronizer,
logger logging.LoggerInterface,
config conf.AdvancedConfig,
authClient service.AuthClient,
splitStorage storage.SplitStorage,
managerStatus chan int,
) (*Manager, error) {
if managerStatus == nil || cap(managerStatus) < 1 {
return nil, errors.New("Status channel cannot be nil nor having capacity")
}
status := atomic.Value{}
status.Store(Idle)
manager := &Manager{
synchronizer: synchronizer,
logger: logger,
config: config,
managerStatus: managerStatus,
status: status,
}
if config.StreamingEnabled {
streamingStatus := make(chan int, 1000)
pushManager, err := push.NewPushManager(logger, synchronizer.SynchronizeSegment, synchronizer.SynchronizeSplits, splitStorage, &config, streamingStatus, authClient)
if err != nil {
return nil, err
}
manager.pushManager = pushManager
manager.streamingStatus = streamingStatus
}
return manager, nil
}
func (s *Manager) startPolling() {
s.status.Store(Polling)
s.pushManager.StopWorkers()
s.synchronizer.StartPeriodicFetching()
}
// IsRunning returns true if is in Streaming or Polling
func (s *Manager) IsRunning() bool {
return s.status.Load().(int) != Idle
}
// Start starts synchronization through Split
func (s *Manager) Start() {
if s.IsRunning() {
s.logger.Info("Manager is already running, skipping start")
return
}
select {
case <-s.managerStatus:
// Discarding previous status before starting
default:
}
err := s.synchronizer.SyncAll()
if err != nil {
s.managerStatus <- Error
return
}
s.logger.Debug("SyncAll Ready")
s.managerStatus <- Ready
s.synchronizer.StartPeriodicDataRecording()
if s.config.StreamingEnabled {
s.logger.Info("Start Streaming")
go s.pushManager.Start()
// Listens Streaming Status
for {
status := <-s.streamingStatus
switch status {
// Backoff is running -> start polling until auth is ok
case push.BackoffAuth:
fallthrough
// Backoff is running -> start polling until sse is connected
case push.BackoffSSE:
if s.status.Load().(int) != Polling {
s.logger.Info("Start periodic polling due backoff")
s.startPolling()
}
// SSE Streaming and workers are ready
case push.Ready:
// If Ready comes eventually when Backoff is done and polling is running
if s.status.Load().(int) == Polling {
s.synchronizer.StopPeriodicFetching()
}
s.logger.Info("SSE Streaming is ready")
s.status.Store(Streaming)
go s.synchronizer.SyncAll()
case push.StreamingDisabled:
fallthrough
// NonRetriableError occurs and it will switch to polling
case push.NonRetriableError:
s.pushManager.Stop()
s.logger.Info("Start periodic polling in Streaming")
s.startPolling()
return
// Publisher sends that there is no Notification Managers available
case push.PushIsDown:
// If streaming is already running, proceeding to stop workers
// and keeping SSE running
if s.status.Load().(int) == Streaming {
s.logger.Info("Start periodic polling in Streaming")
s.startPolling()
}
// Publisher sends that there are at least one Notification Manager available
case push.PushIsUp:
// If streaming is not already running, proceeding to start workers
if s.status.Load().(int) != Streaming {
s.logger.Info("Stop periodic polling")
s.pushManager.StartWorkers()
s.synchronizer.StopPeriodicFetching()
s.status.Store(Streaming)
go s.synchronizer.SyncAll()
}
// Reconnect received due error in streaming -> reconnecting
case push.Reconnect:
fallthrough
// Token expired -> reconnecting
case push.TokenExpiration:
s.pushManager.Stop()
go s.pushManager.Start()
}
}
} else {
s.logger.Info("Start periodic polling")
s.synchronizer.StartPeriodicFetching()
s.status.Store(Polling)
}
}
// Stop stop synchronizaation through Split
func (s *Manager) Stop() {
s.logger.Info("STOPPING MANAGER TASKS")
if s.pushManager != nil && s.pushManager.IsRunning() {
s.pushManager.Stop()
}
s.synchronizer.StopPeriodicFetching()
s.synchronizer.StopPeriodicDataRecording()
s.status.Store(Idle)
}

158
vendor/github.com/splitio/go-split-commons/v2/synchronizer/synchronizer.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,158 @@
package synchronizer
import (
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/event"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/impression"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/impressionscount"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/metric"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/segment"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/split"
"github.com/splitio/go-split-commons/v2/tasks"
"github.com/splitio/go-toolkit/v3/asynctask"
"github.com/splitio/go-toolkit/v3/logging"
)
// SplitTasks struct for tasks
type SplitTasks struct {
SplitSyncTask *asynctask.AsyncTask
SegmentSyncTask *asynctask.AsyncTask
TelemetrySyncTask *asynctask.AsyncTask
ImpressionSyncTask tasks.Task
EventSyncTask tasks.Task
ImpressionsCountSyncTask *asynctask.AsyncTask
}
// Workers struct for workers
type Workers struct {
SplitFetcher split.SplitFetcher
SegmentFetcher segment.SegmentFetcher
TelemetryRecorder metric.MetricRecorder
ImpressionRecorder impression.ImpressionRecorder
EventRecorder event.EventRecorder
ImpressionsCountRecorder impressionscount.ImpressionsCountRecorder
}
// SynchronizerImpl implements Synchronizer
type SynchronizerImpl struct {
splitTasks SplitTasks
workers Workers
logger logging.LoggerInterface
inMememoryFullQueue chan string
impressionBulkSize int64
eventBulkSize int64
}
// NewSynchronizer creates new SynchronizerImpl
func NewSynchronizer(
confAdvanced conf.AdvancedConfig,
splitTasks SplitTasks,
workers Workers,
logger logging.LoggerInterface,
inMememoryFullQueue chan string,
) Synchronizer {
return &SynchronizerImpl{
impressionBulkSize: confAdvanced.ImpressionsBulkSize,
eventBulkSize: confAdvanced.EventsBulkSize,
splitTasks: splitTasks,
workers: workers,
logger: logger,
inMememoryFullQueue: inMememoryFullQueue,
}
}
func (s *SynchronizerImpl) dataFlusher() {
for true {
msg := <-s.inMememoryFullQueue
switch msg {
case "EVENTS_FULL":
s.logger.Debug("FLUSHING storage queue")
err := s.workers.EventRecorder.SynchronizeEvents(s.eventBulkSize)
if err != nil {
s.logger.Error("Error flushing storage queue", err)
}
break
case "IMPRESSIONS_FULL":
s.logger.Debug("FLUSHING storage queue")
err := s.workers.ImpressionRecorder.SynchronizeImpressions(s.impressionBulkSize)
if err != nil {
s.logger.Error("Error flushing storage queue", err)
}
}
}
}
// SyncAll syncs splits and segments
func (s *SynchronizerImpl) SyncAll() error {
err := s.workers.SplitFetcher.SynchronizeSplits(nil)
if err != nil {
return err
}
return s.workers.SegmentFetcher.SynchronizeSegments()
}
// StartPeriodicFetching starts periodic fetchers tasks
func (s *SynchronizerImpl) StartPeriodicFetching() {
if s.splitTasks.SplitSyncTask != nil {
s.splitTasks.SplitSyncTask.Start()
}
if s.splitTasks.SegmentSyncTask != nil {
s.splitTasks.SegmentSyncTask.Start()
}
}
// StopPeriodicFetching stops periodic fetchers tasks
func (s *SynchronizerImpl) StopPeriodicFetching() {
if s.splitTasks.SplitSyncTask != nil {
s.splitTasks.SplitSyncTask.Stop(false)
}
if s.splitTasks.SegmentSyncTask != nil {
s.splitTasks.SegmentSyncTask.Stop(true)
}
}
// StartPeriodicDataRecording starts periodic recorders tasks
func (s *SynchronizerImpl) StartPeriodicDataRecording() {
if s.inMememoryFullQueue != nil {
go s.dataFlusher()
}
if s.splitTasks.ImpressionSyncTask != nil {
s.splitTasks.ImpressionSyncTask.Start()
}
if s.splitTasks.TelemetrySyncTask != nil {
s.splitTasks.TelemetrySyncTask.Start()
}
if s.splitTasks.EventSyncTask != nil {
s.splitTasks.EventSyncTask.Start()
}
if s.splitTasks.ImpressionsCountSyncTask != nil {
s.splitTasks.ImpressionsCountSyncTask.Start()
}
}
// StopPeriodicDataRecording stops periodic recorders tasks
func (s *SynchronizerImpl) StopPeriodicDataRecording() {
if s.splitTasks.ImpressionSyncTask != nil {
s.splitTasks.ImpressionSyncTask.Stop(true)
}
if s.splitTasks.TelemetrySyncTask != nil {
s.splitTasks.TelemetrySyncTask.Stop(false)
}
if s.splitTasks.EventSyncTask != nil {
s.splitTasks.EventSyncTask.Stop(true)
}
if s.splitTasks.ImpressionsCountSyncTask != nil {
s.splitTasks.ImpressionsCountSyncTask.Stop(true)
}
}
// SynchronizeSplits syncs splits
func (s *SynchronizerImpl) SynchronizeSplits(till *int64) error {
return s.workers.SplitFetcher.SynchronizeSplits(till)
}
// SynchronizeSegment syncs segment
func (s *SynchronizerImpl) SynchronizeSegment(name string, till *int64) error {
return s.workers.SegmentFetcher.SynchronizeSegment(name, till)
}

7
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/event/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,7 @@
package event
// EventRecorder interface
type EventRecorder interface {
SynchronizeEvents(bulkSize int64) error
FlushEvents(bulkSize int64) error
}

76
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/event/single.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,76 @@
package event
import (
"errors"
"time"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-split-commons/v2/storage"
"github.com/splitio/go-split-commons/v2/util"
"github.com/splitio/go-toolkit/v3/logging"
)
// RecorderSingle struct for event sync
type RecorderSingle struct {
eventStorage storage.EventStorageConsumer
eventRecorder service.EventsRecorder
metricsWrapper *storage.MetricWrapper
logger logging.LoggerInterface
metadata dtos.Metadata
}
// NewEventRecorderSingle creates new event synchronizer for posting events
func NewEventRecorderSingle(
eventStorage storage.EventStorageConsumer,
eventRecorder service.EventsRecorder,
metricsWrapper *storage.MetricWrapper,
logger logging.LoggerInterface,
metadata dtos.Metadata,
) EventRecorder {
return &RecorderSingle{
eventStorage: eventStorage,
eventRecorder: eventRecorder,
metricsWrapper: metricsWrapper,
logger: logger,
metadata: metadata,
}
}
// SynchronizeEvents syncs events
func (e *RecorderSingle) SynchronizeEvents(bulkSize int64) error {
queuedEvents, err := e.eventStorage.PopN(bulkSize)
if err != nil {
e.logger.Error("Error reading events queue", err)
return errors.New("Error reading events queue")
}
if len(queuedEvents) == 0 {
e.logger.Debug("No events fetched from queue. Nothing to send")
return nil
}
before := time.Now()
err = e.eventRecorder.Record(queuedEvents, e.metadata)
if err != nil {
if httpError, ok := err.(*dtos.HTTPError); ok {
e.metricsWrapper.StoreCounters(storage.PostEventsCounter, string(httpError.Code))
}
return err
}
bucket := util.Bucket(time.Now().Sub(before).Nanoseconds())
e.metricsWrapper.StoreLatencies(storage.PostEventsLatency, bucket)
e.metricsWrapper.StoreCounters(storage.PostEventsCounter, "ok")
return nil
}
// FlushEvents flushes events
func (e *RecorderSingle) FlushEvents(bulkSize int64) error {
for !e.eventStorage.Empty() {
err := e.SynchronizeEvents(bulkSize)
if err != nil {
return err
}
}
return nil
}

7
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/impression/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,7 @@
package impression
// ImpressionRecorder interface
type ImpressionRecorder interface {
SynchronizeImpressions(bulkSize int64) error
FlushImpressions(bulkSize int64) error
}

117
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/impression/single.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,117 @@
package impression
import (
"errors"
"time"
"github.com/splitio/go-split-commons/v2/conf"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-split-commons/v2/storage"
"github.com/splitio/go-split-commons/v2/util"
"github.com/splitio/go-toolkit/v3/logging"
)
const (
maxImpressionCacheSize = 500000
splitSDKImpressionsMode = "SplitSDKImpressionsMode"
)
// RecorderSingle struct for impression sync
type RecorderSingle struct {
impressionStorage storage.ImpressionStorageConsumer
impressionRecorder service.ImpressionsRecorder
metricsWrapper *storage.MetricWrapper
logger logging.LoggerInterface
metadata dtos.Metadata
mode string
}
// NewRecorderSingle creates new impression synchronizer for posting impressions
func NewRecorderSingle(
impressionStorage storage.ImpressionStorageConsumer,
impressionRecorder service.ImpressionsRecorder,
metricsWrapper *storage.MetricWrapper,
logger logging.LoggerInterface,
metadata dtos.Metadata,
managerConfig conf.ManagerConfig,
) ImpressionRecorder {
mode := conf.ImpressionsModeOptimized
if !util.ShouldBeOptimized(managerConfig) {
mode = conf.ImpressionsModeDebug
}
return &RecorderSingle{
impressionStorage: impressionStorage,
impressionRecorder: impressionRecorder,
metricsWrapper: metricsWrapper,
logger: logger,
metadata: metadata,
mode: mode,
}
}
// SynchronizeImpressions syncs impressions
func (i *RecorderSingle) SynchronizeImpressions(bulkSize int64) error {
queuedImpressions, err := i.impressionStorage.PopN(bulkSize)
if err != nil {
i.logger.Error("Error reading impressions queue", err)
return errors.New("Error reading impressions queue")
}
if len(queuedImpressions) == 0 {
i.logger.Debug("No impressions fetched from queue. Nothing to send")
return nil
}
impressionsToPost := make(map[string][]dtos.ImpressionDTO)
for _, impression := range queuedImpressions {
keyImpression := dtos.ImpressionDTO{
KeyName: impression.KeyName,
Treatment: impression.Treatment,
Time: impression.Time,
ChangeNumber: impression.ChangeNumber,
Label: impression.Label,
BucketingKey: impression.BucketingKey,
Pt: impression.Pt,
}
v, ok := impressionsToPost[impression.FeatureName]
if ok {
v = append(v, keyImpression)
} else {
v = []dtos.ImpressionDTO{keyImpression}
}
impressionsToPost[impression.FeatureName] = v
}
bulkImpressions := make([]dtos.ImpressionsDTO, 0)
for testName, testImpressions := range impressionsToPost {
bulkImpressions = append(bulkImpressions, dtos.ImpressionsDTO{
TestName: testName,
KeyImpressions: testImpressions,
})
}
before := time.Now()
err = i.impressionRecorder.Record(bulkImpressions, i.metadata, map[string]string{splitSDKImpressionsMode: i.mode})
if err != nil {
if httpError, ok := err.(*dtos.HTTPError); ok {
i.metricsWrapper.StoreCounters(storage.TestImpressionsCounter, string(httpError.Code))
}
return err
}
bucket := util.Bucket(time.Now().Sub(before).Nanoseconds())
i.metricsWrapper.StoreLatencies(storage.TestImpressionsLatency, bucket)
i.metricsWrapper.StoreCounters(storage.TestImpressionsCounter, "ok")
return nil
}
// FlushImpressions flushes impressions
func (i *RecorderSingle) FlushImpressions(bulkSize int64) error {
for !i.impressionStorage.Empty() {
err := i.SynchronizeImpressions(bulkSize)
if err != nil {
return err
}
}
return nil
}

6
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/impressionscount/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,6 @@
package impressionscount
// ImpressionsCountRecorder interface
type ImpressionsCountRecorder interface {
SynchronizeImpressionsCount() error
}

51
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/impressionscount/single.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,51 @@
package impressionscount
import (
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/provisional"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-toolkit/v3/logging"
)
// RecorderSingle struct for impressionsCount sync
type RecorderSingle struct {
impressionsCounter *provisional.ImpressionsCounter
impressionRecorder service.ImpressionsRecorder
metadata dtos.Metadata
logger logging.LoggerInterface
}
// NewRecorderSingle creates new impressionsCount synchronizer for posting impressionsCount
func NewRecorderSingle(
impressionsCounter *provisional.ImpressionsCounter,
impressionRecorder service.ImpressionsRecorder,
metadata dtos.Metadata,
logger logging.LoggerInterface,
) ImpressionsCountRecorder {
return &RecorderSingle{
impressionsCounter: impressionsCounter,
impressionRecorder: impressionRecorder,
metadata: metadata,
logger: logger,
}
}
// SynchronizeImpressionsCount syncs imp counts
func (m *RecorderSingle) SynchronizeImpressionsCount() error {
impressionsCount := m.impressionsCounter.PopAll()
impressionsInTimeFrame := make([]dtos.ImpressionsInTimeFrameDTO, 0)
for key, count := range impressionsCount {
impressionInTimeFrame := dtos.ImpressionsInTimeFrameDTO{
FeatureName: key.FeatureName,
RawCount: count,
TimeFrame: key.TimeFrame,
}
impressionsInTimeFrame = append(impressionsInTimeFrame, impressionInTimeFrame)
}
pf := dtos.ImpressionsCountDTO{
PerFeature: impressionsInTimeFrame,
}
return m.impressionRecorder.RecordImpressionsCount(pf, m.metadata)
}

6
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/metric/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,6 @@
package metric
// MetricRecorder interface
type MetricRecorder interface {
SynchronizeTelemetry() error
}

74
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/metric/single.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,74 @@
package metric
import (
"errors"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-split-commons/v2/storage"
)
// RecorderSingle struct for metric sync
type RecorderSingle struct {
metricStorage storage.MetricsStorageConsumer
metricRecorder service.MetricsRecorder
metadata dtos.Metadata
}
// NewRecorderSingle creates new metric synchronizer for posting metrics
func NewRecorderSingle(
metricStorage storage.MetricsStorageConsumer,
metricRecorder service.MetricsRecorder,
metadata dtos.Metadata,
) MetricRecorder {
return &RecorderSingle{
metricStorage: metricStorage,
metricRecorder: metricRecorder,
metadata: metadata,
}
}
func (m *RecorderSingle) synchronizeLatencies() error {
latencies := m.metricStorage.PopLatencies()
if len(latencies) > 0 {
err := m.metricRecorder.RecordLatencies(latencies, m.metadata)
return err
}
return nil
}
func (m *RecorderSingle) synchronizeGauges() error {
var errs []error
for _, gauge := range m.metricStorage.PopGauges() {
err := m.metricRecorder.RecordGauge(gauge, m.metadata)
if err != nil {
errs = append(errs, err)
}
}
if len(errs) > 0 {
return errors.New("Some gauges could not be posted")
}
return nil
}
func (m *RecorderSingle) synchronizeCounters() error {
counters := m.metricStorage.PopCounters()
if len(counters) > 0 {
err := m.metricRecorder.RecordCounters(counters, m.metadata)
return err
}
return nil
}
// SynchronizeTelemetry syncs telemetry
func (m *RecorderSingle) SynchronizeTelemetry() error {
err := m.synchronizeGauges()
if err != nil {
return err
}
err = m.synchronizeLatencies()
if err != nil {
return err
}
return m.synchronizeCounters()
}

8
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/segment/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,8 @@
package segment
// SegmentFetcher interface
type SegmentFetcher interface {
SynchronizeSegment(name string, till *int64) error
SynchronizeSegments() error
SegmentNames() []interface{}
}

139
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/segment/segment.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,139 @@
package segment
import (
"fmt"
"sync"
"time"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-split-commons/v2/storage"
"github.com/splitio/go-split-commons/v2/util"
"github.com/splitio/go-toolkit/v3/datastructures/set"
"github.com/splitio/go-toolkit/v3/logging"
)
// SegmentFetcherSimple struct for segment sync
type SegmentFetcherSimple struct {
splitStorage storage.SplitStorageConsumer
segmentStorage storage.SegmentStorage
segmentFetcher service.SegmentFetcher
metricsWrapper *storage.MetricWrapper
logger logging.LoggerInterface
}
// NewSegmentFetcher creates new segment synchronizer for processing segment updates
func NewSegmentFetcher(
splitStorage storage.SplitStorage,
segmentStorage storage.SegmentStorage,
segmentFetcher service.SegmentFetcher,
metricsWrapper *storage.MetricWrapper,
logger logging.LoggerInterface,
) SegmentFetcher {
return &SegmentFetcherSimple{
splitStorage: splitStorage,
segmentStorage: segmentStorage,
segmentFetcher: segmentFetcher,
metricsWrapper: metricsWrapper,
logger: logger,
}
}
func (s *SegmentFetcherSimple) processUpdate(segmentChanges *dtos.SegmentChangesDTO) {
name := segmentChanges.Name
oldSegment := s.segmentStorage.Keys(name)
if oldSegment == nil {
keys := set.NewSet()
for _, key := range segmentChanges.Added {
keys.Add(key)
}
s.logger.Debug(fmt.Sprintf("Segment [%s] doesn't exist now, it will add (%d) keys", name, keys.Size()))
s.segmentStorage.Update(name, keys, set.NewSet(), segmentChanges.Till)
} else {
toAdd := set.NewSet()
toRemove := set.NewSet()
// Segment exists, must add new members and remove old ones
for _, key := range segmentChanges.Added {
toAdd.Add(key)
}
for _, key := range segmentChanges.Removed {
toRemove.Add(key)
}
if toAdd.Size() > 0 || toRemove.Size() > 0 {
s.logger.Debug(fmt.Sprintf("Segment [%s] exists, it will be updated. %d keys added, %d keys removed", name, toAdd.Size(), toRemove.Size()))
s.segmentStorage.Update(name, toAdd, toRemove, segmentChanges.Till)
}
}
}
// SynchronizeSegment syncs segment
func (s *SegmentFetcherSimple) SynchronizeSegment(name string, till *int64) error {
for {
s.logger.Debug(fmt.Sprintf("Synchronizing segment %s", name))
changeNumber, _ := s.segmentStorage.ChangeNumber(name)
if changeNumber == 0 {
changeNumber = -1
}
if till != nil && *till < changeNumber {
return nil
}
before := time.Now()
segmentChanges, err := s.segmentFetcher.Fetch(name, changeNumber)
if err != nil {
if httpError, ok := err.(*dtos.HTTPError); ok {
s.metricsWrapper.StoreCounters(storage.SegmentChangesCounter, string(httpError.Code))
}
return err
}
s.processUpdate(segmentChanges)
bucket := util.Bucket(time.Now().Sub(before).Nanoseconds())
s.metricsWrapper.StoreLatencies(storage.SegmentChangesLatency, bucket)
s.metricsWrapper.StoreCounters(storage.SegmentChangesCounter, "ok")
if segmentChanges.Till == segmentChanges.Since || (till != nil && segmentChanges.Till >= *till) {
return nil
}
}
}
// SynchronizeSegments syncs segments at once
func (s *SegmentFetcherSimple) SynchronizeSegments() error {
// @TODO: add delays
segmentNames := s.splitStorage.SegmentNames().List()
s.logger.Debug("Segment Sync", segmentNames)
wg := sync.WaitGroup{}
wg.Add(len(segmentNames))
failedSegments := set.NewThreadSafeSet()
for _, name := range segmentNames {
conv, ok := name.(string)
if !ok {
s.logger.Warning("Skipping non-string segment present in storage at initialization-time!")
continue
}
go func(segmentName string) {
defer wg.Done() // Make sure the "finished" signal is always sent
ready := false
var err error
for !ready {
err = s.SynchronizeSegment(segmentName, nil)
if err != nil {
failedSegments.Add(segmentName)
}
return
}
}(conv)
}
wg.Wait()
if failedSegments.Size() > 0 {
return fmt.Errorf("The following segments failed to be fetched %v", failedSegments.List())
}
return nil
}
// SegmentNames returns all segments
func (s *SegmentFetcherSimple) SegmentNames() []interface{} {
return s.splitStorage.SegmentNames().List()
}

6
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/split/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,6 @@
package split
// SplitFetcher interface
type SplitFetcher interface {
SynchronizeSplits(till *int64) error
}

84
vendor/github.com/splitio/go-split-commons/v2/synchronizer/worker/split/split.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,84 @@
package split
import (
"time"
"github.com/splitio/go-split-commons/v2/dtos"
"github.com/splitio/go-split-commons/v2/service"
"github.com/splitio/go-split-commons/v2/storage"
"github.com/splitio/go-split-commons/v2/util"
"github.com/splitio/go-toolkit/v3/logging"
)
// SplitFetcherSimple struct for split sync
type SplitFetcherSimple struct {
splitStorage storage.SplitStorage
splitFetcher service.SplitFetcher
metricsWrapper *storage.MetricWrapper
logger logging.LoggerInterface
}
// NewSplitFetcher creates new split synchronizer for processing split updates
func NewSplitFetcher(
splitStorage storage.SplitStorage,
splitFetcher service.SplitFetcher,
metricsWrapper *storage.MetricWrapper,
logger logging.LoggerInterface,
) SplitFetcher {
return &SplitFetcherSimple{
splitStorage: splitStorage,
splitFetcher: splitFetcher,
metricsWrapper: metricsWrapper,
logger: logger,
}
}
func (s *SplitFetcherSimple) processUpdate(splits *dtos.SplitChangesDTO) {
inactiveSplits := make([]dtos.SplitDTO, 0)
activeSplits := make([]dtos.SplitDTO, 0)
for _, split := range splits.Splits {
if split.Status == "ACTIVE" {
activeSplits = append(activeSplits, split)
} else {
inactiveSplits = append(inactiveSplits, split)
}
}
// Add/Update active splits
s.splitStorage.PutMany(activeSplits, splits.Till)
// Remove inactive splits
for _, split := range inactiveSplits {
s.splitStorage.Remove(split.Name)
}
}
// SynchronizeSplits syncs splits
func (s *SplitFetcherSimple) SynchronizeSplits(till *int64) error {
// @TODO: add delays
for {
changeNumber, _ := s.splitStorage.ChangeNumber()
if changeNumber == 0 {
changeNumber = -1
}
if till != nil && *till < changeNumber {
return nil
}
before := time.Now()
splits, err := s.splitFetcher.Fetch(changeNumber)
if err != nil {
if httpError, ok := err.(*dtos.HTTPError); ok {
s.metricsWrapper.StoreCounters(storage.SplitChangesCounter, string(httpError.Code))
}
return err
}
s.processUpdate(splits)
bucket := util.Bucket(time.Now().Sub(before).Nanoseconds())
s.metricsWrapper.StoreCounters(storage.SplitChangesCounter, "ok")
s.metricsWrapper.StoreLatencies(storage.SplitChangesLatency, bucket)
if splits.Till == splits.Since || (till != nil && splits.Till >= *till) {
return nil
}
}
}

52
vendor/github.com/splitio/go-split-commons/v2/tasks/eventsync.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,52 @@
package tasks
import (
"fmt"
"sync"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/event"
"github.com/splitio/go-toolkit/v3/asynctask"
"github.com/splitio/go-toolkit/v3/logging"
)
// NewRecordEventsTask creates a new events recording task
func NewRecordEventsTask(
synchronizer event.EventRecorder,
bulkSize int64,
period int,
logger logging.LoggerInterface,
) Task {
record := func(logger logging.LoggerInterface) error {
return synchronizer.SynchronizeEvents(bulkSize)
}
onStop := func(logger logging.LoggerInterface) {
// All this function does is flush events which will clear the storage
synchronizer.FlushEvents(bulkSize)
}
return asynctask.NewAsyncTask("SubmitEvents", record, period, nil, onStop, logger)
}
// NewRecordEventsTasks creates a new splits fetching and storing task
func NewRecordEventsTasks(
recorder event.EventRecorder,
bulkSize int64,
period int,
logger logging.LoggerInterface,
totalTasks int) Task {
record := func(logger logging.LoggerInterface) error {
return recorder.SynchronizeEvents(bulkSize)
}
tasks := make([]Task, 0, totalTasks)
for i := 0; i < totalTasks; i++ {
logger.Info(fmt.Sprintf("Creating SubmitEvents_%d", i))
tasks = append(tasks, asynctask.NewAsyncTask(fmt.Sprintf("SubmitEvents_%d", i), record, period, nil, nil, logger))
}
return MultipleTask{
logger: logger,
tasks: tasks,
wg: &sync.WaitGroup{},
}
}

27
vendor/github.com/splitio/go-split-commons/v2/tasks/impressionscountsync.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,27 @@
package tasks
import (
"github.com/splitio/go-split-commons/v2/synchronizer/worker/impressionscount"
"github.com/splitio/go-toolkit/v3/asynctask"
"github.com/splitio/go-toolkit/v3/logging"
)
const (
period = 1800 // 30 min
)
// NewRecordImpressionsCountTask creates a new impressionsCount recording task
func NewRecordImpressionsCountTask(
recorder impressionscount.ImpressionsCountRecorder,
logger logging.LoggerInterface,
) *asynctask.AsyncTask {
record := func(logger logging.LoggerInterface) error {
return recorder.SynchronizeImpressionsCount()
}
onStop := func(logger logging.LoggerInterface) {
recorder.SynchronizeImpressionsCount()
}
return asynctask.NewAsyncTask("SubmitImpressionsCount", record, period, nil, onStop, logger)
}

52
vendor/github.com/splitio/go-split-commons/v2/tasks/impressionsync.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,52 @@
package tasks
import (
"fmt"
"sync"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/impression"
"github.com/splitio/go-toolkit/v3/asynctask"
"github.com/splitio/go-toolkit/v3/logging"
)
// NewRecordImpressionsTask creates a new splits fetching and storing task
func NewRecordImpressionsTask(
recorder impression.ImpressionRecorder,
period int,
logger logging.LoggerInterface,
bulkSize int64,
) Task {
record := func(logger logging.LoggerInterface) error {
return recorder.SynchronizeImpressions(bulkSize)
}
onStop := func(logger logging.LoggerInterface) {
// All this function does is flush impressions which will clear the storage
recorder.FlushImpressions(bulkSize)
}
return asynctask.NewAsyncTask("SubmitImpressions", record, period, nil, onStop, logger)
}
// NewRecordImpressionsTasks creates a new splits fetching and storing task
func NewRecordImpressionsTasks(
recorder impression.ImpressionRecorder,
period int,
logger logging.LoggerInterface,
bulkSize int64,
totalTasks int) Task {
record := func(logger logging.LoggerInterface) error {
return recorder.SynchronizeImpressions(bulkSize)
}
tasks := make([]Task, 0, totalTasks)
for i := 0; i < totalTasks; i++ {
logger.Info(fmt.Sprintf("Creating SubmitImpressions_%d", i))
tasks = append(tasks, asynctask.NewAsyncTask(fmt.Sprintf("SubmitImpressions_%d", i), record, period, nil, nil, logger))
}
return MultipleTask{
logger: logger,
tasks: tasks,
wg: &sync.WaitGroup{},
}
}

53
vendor/github.com/splitio/go-split-commons/v2/tasks/interface.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,53 @@
package tasks
import (
"sync"
"github.com/splitio/go-toolkit/v3/logging"
)
// Task interface
type Task interface {
Start()
Stop(blocking bool) error
IsRunning() bool
}
// MultipleTask struct
type MultipleTask struct {
tasks []Task
logger logging.LoggerInterface
wg *sync.WaitGroup
}
// IsRunning method
func (m MultipleTask) IsRunning() bool {
for _, t := range m.tasks {
if t.IsRunning() {
return true
}
}
return false
}
// Start method
func (m MultipleTask) Start() {
for _, t := range m.tasks {
m.wg.Add(1)
t.Start()
}
}
// Stop method
func (m MultipleTask) Stop(blocking bool) error {
for _, t := range m.tasks {
go func(t Task) {
t.Stop(blocking)
m.wg.Done()
}(t)
}
if blocking {
m.wg.Wait()
}
return nil
}

23
vendor/github.com/splitio/go-split-commons/v2/tasks/metricsync.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,23 @@
package tasks
import (
"github.com/splitio/go-split-commons/v2/synchronizer/worker/metric"
"github.com/splitio/go-toolkit/v3/asynctask"
"github.com/splitio/go-toolkit/v3/logging"
)
// NewRecordTelemetryTask creates a new telemtry recording task
func NewRecordTelemetryTask(
recorder metric.MetricRecorder,
period int,
logger logging.LoggerInterface,
) *asynctask.AsyncTask {
record := func(logger logging.LoggerInterface) error {
return recorder.SynchronizeTelemetry()
}
onStop := func(l logging.LoggerInterface) {
record(logger)
}
return asynctask.NewAsyncTask("SubmitTelemetry", record, period, nil, onStop, logger)
}

81
vendor/github.com/splitio/go-split-commons/v2/tasks/segmentsync.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,81 @@
package tasks
import (
"errors"
"fmt"
"sync/atomic"
"github.com/splitio/go-split-commons/v2/synchronizer/worker/segment"
"github.com/splitio/go-toolkit/v3/asynctask"
"github.com/splitio/go-toolkit/v3/logging"
"github.com/splitio/go-toolkit/v3/workerpool"
)
func updateSegments(
fetcher segment.SegmentFetcher,
admin *workerpool.WorkerAdmin,
logger logging.LoggerInterface,
) error {
segmentList := fetcher.SegmentNames()
for _, name := range segmentList {
ok := admin.QueueMessage(name)
if !ok {
logger.Error(
fmt.Sprintf("Segment %s could not be added because the job queue is full.\n", name),
fmt.Sprintf(
"You currently have %d segments and the queue size is %d.\n",
len(segmentList),
admin.QueueSize(),
),
"Please consider updating the segment queue size accordingly in the configuration options",
)
}
}
return nil
}
// NewFetchSegmentsTask creates a new segment fetching and storing task
func NewFetchSegmentsTask(
fetcher segment.SegmentFetcher,
period int,
workerCount int,
queueSize int,
logger logging.LoggerInterface,
) *asynctask.AsyncTask {
admin := atomic.Value{}
// After all segments are in sync, add workers to the pool that will keep them up to date
// periodically
onInit := func(logger logging.LoggerInterface) error {
admin.Store(workerpool.NewWorkerAdmin(queueSize, logger))
for i := 0; i < workerCount; i++ {
worker := NewSegmentWorker(
fmt.Sprintf("SegmentWorker_%d", i),
0,
fetcher.SynchronizeSegment,
)
admin.Load().(*workerpool.WorkerAdmin).AddWorker(worker)
}
return nil
}
update := func(logger logging.LoggerInterface) error {
wa, ok := admin.Load().(*workerpool.WorkerAdmin)
if !ok || wa == nil {
return errors.New("unable to type-assert worker manager")
}
return updateSegments(fetcher, wa, logger)
}
cleanup := func(logger logging.LoggerInterface) {
wa, ok := admin.Load().(*workerpool.WorkerAdmin)
if !ok || wa == nil {
logger.Error("unable to type-assert worker manager")
return
}
wa.StopAll(true)
}
return asynctask.NewAsyncTask("UpdateSegments", update, period, onInit, cleanup, logger)
}

20
vendor/github.com/splitio/go-split-commons/v2/tasks/splitsync.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,20 @@
package tasks
import (
"github.com/splitio/go-split-commons/v2/synchronizer/worker/split"
"github.com/splitio/go-toolkit/v3/asynctask"
"github.com/splitio/go-toolkit/v3/logging"
)
// NewFetchSplitsTask creates a new splits fetching and storing task
func NewFetchSplitsTask(
fetcher split.SplitFetcher,
period int,
logger logging.LoggerInterface,
) *asynctask.AsyncTask {
update := func(logger logging.LoggerInterface) error {
return fetcher.SynchronizeSplits(nil)
}
return asynctask.NewAsyncTask("UpdateSplits", update, period, nil, nil, logger)
}

47
vendor/github.com/splitio/go-split-commons/v2/tasks/worker.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,47 @@
package tasks
import (
"errors"
)
// SegmentWorker struct contains resources and functions for fetching segments and storing them
type SegmentWorker struct {
name string
failureTime int64
toExecute func(name string, till *int64) error
}
// NewSegmentWorker some
func NewSegmentWorker(name string, failureTime int64, toExecute func(name string, till *int64) error) *SegmentWorker {
return &SegmentWorker{
name: name,
failureTime: failureTime,
toExecute: toExecute,
}
}
// Name Returns the name of the worker
func (w *SegmentWorker) Name() string {
return w.name
}
// FailureTime Returns how much time should be waited after an error, before the worker resumes execution
func (w *SegmentWorker) FailureTime() int64 {
return w.failureTime
}
// DoWork performs the actual work and returns an error if something goes wrong
func (w *SegmentWorker) DoWork(msg interface{}) error {
segmentName, ok := msg.(string)
if !ok {
return errors.New("segment name popped from queue is not a string")
}
return w.toExecute(segmentName, nil)
}
// OnError callback does nothing
func (w *SegmentWorker) OnError(e error) {}
// Cleanup callback does nothing
func (w *SegmentWorker) Cleanup() error { return nil }

43
vendor/github.com/splitio/go-split-commons/v2/util/metrics.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,43 @@
package util
var latencyBuckets = [23]float64{
1.00,
1.50,
2.25,
3.38,
5.06,
7.59,
11.39,
17.09,
25.63,
38.44,
57.67,
86.50,
129.75,
194.62,
291.93,
437.89,
656.84,
985.26,
1477.89,
2216.84,
3325.26,
4987.89,
7481.83,
}
// Bucket returns the bucket where the received latency falls
func Bucket(latency int64) int {
floatLatency := float64(latency) / 1000 // Convert to millisencods
index := 0
for index < len(latencyBuckets) && floatLatency > latencyBuckets[index] {
index++
}
if index == len(latencyBuckets) {
return index - 1
}
return index
}

30
vendor/github.com/splitio/go-split-commons/v2/util/mode.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,30 @@
package util
import (
"strings"
"github.com/splitio/go-split-commons/v2/conf"
)
// ShouldAddPreviousTime returns if previous time should be set up or not depending on operationMode
func ShouldAddPreviousTime(managerConfig conf.ManagerConfig) bool {
switch strings.ToLower(managerConfig.OperationMode) {
case conf.ProducerSync:
fallthrough
case conf.Standalone:
return true
default:
return false
}
}
// ShouldBeOptimized returns if should dedupe impressions or not depending on configs
func ShouldBeOptimized(managerConfig conf.ManagerConfig) bool {
if !ShouldAddPreviousTime(managerConfig) {
return false
}
if strings.ToLower(managerConfig.ImpressionsMode) == conf.ImpressionsModeOptimized {
return true
}
return false
}

11
vendor/github.com/splitio/go-split-commons/v2/util/time.go сгенерированный поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,11 @@
package util
import "time"
const dedupWindowSizeMs = 3600 * 1000
// TruncateTimeFrame truncates de time frame received with the time window
func TruncateTimeFrame(timestampInNs int64) int64 {
timestampInMs := timestampInNs / int64(time.Millisecond)
return timestampInMs - (timestampInMs % dedupWindowSizeMs)
}