Doug Lauder
2023-03-22 17:22:27 -04:00
коммит произвёл GitHub
родитель b61c096497
Коммит c943ed6859
13276 изменённых файлов: 1695615 добавлений и 223189 удалений

51
server/platform/services/cache/cache.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,51 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package cache
import (
"errors"
"time"
"github.com/mattermost/mattermost-server/v6/model"
)
// ErrKeyNotFound is the error when the given key is not found
var ErrKeyNotFound = errors.New("key not found")
// Cache is a representation of a cache store that aims to replace cache.Cache
type Cache interface {
// Purge is used to completely clear the cache.
Purge() error
// Set adds the given key and value to the store without an expiry. If the key already exists,
// it will overwrite the previous value.
Set(key string, value any) error
// SetWithDefaultExpiry adds the given key and value to the store with the default expiry. If
// the key already exists, it will overwrite the previous value
SetWithDefaultExpiry(key string, value any) error
// SetWithExpiry adds the given key and value to the cache with the given expiry. If the key
// already exists, it will overwrite the previous value
SetWithExpiry(key string, value any, ttl time.Duration) error
// Get the content stored in the cache for the given key, and decode it into the value interface.
// Return ErrKeyNotFound if the key is missing from the cache
Get(key string, value any) error
// Remove deletes the value for a given key.
Remove(key string) error
// Keys returns a slice of the keys in the cache.
Keys() ([]string, error)
// Len returns the number of items in the cache.
Len() (int, error)
// GetInvalidateClusterEvent returns the cluster event configured when this cache was created.
GetInvalidateClusterEvent() model.ClusterEvent
// Name returns the name of the cache
Name() string
}

248
server/platform/services/cache/lru.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,248 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package cache
import (
"container/list"
"sync"
"time"
"github.com/tinylib/msgp/msgp"
"github.com/vmihailenco/msgpack/v5"
"github.com/mattermost/mattermost-server/v6/model"
)
// LRU is a thread-safe fixed size LRU cache.
type LRU struct {
lock sync.RWMutex
size int
len int
currentGeneration int64
evictList *list.List
items map[string]*list.Element
defaultExpiry time.Duration
name string
invalidateClusterEvent model.ClusterEvent
}
// LRUOptions contains options for initializing LRU cache
type LRUOptions struct {
Name string
Size int
DefaultExpiry time.Duration
InvalidateClusterEvent model.ClusterEvent
// StripedBuckets is used only by LRUStriped and shouldn't be greater than the number
// of CPUs available on the machine running this cache.
StripedBuckets int
}
// entry is used to hold a value in the evictList.
type entry struct {
key string
value []byte
expires time.Time
generation int64
}
// NewLRU creates an LRU of the given size.
func NewLRU(opts LRUOptions) Cache {
return &LRU{
name: opts.Name,
size: opts.Size,
evictList: list.New(),
items: make(map[string]*list.Element, opts.Size),
defaultExpiry: opts.DefaultExpiry,
invalidateClusterEvent: opts.InvalidateClusterEvent,
}
}
// Purge is used to completely clear the cache.
func (l *LRU) Purge() error {
l.lock.Lock()
defer l.lock.Unlock()
l.len = 0
l.currentGeneration++
return nil
}
// Set adds the given key and value to the store without an expiry. If the key already exists,
// it will overwrite the previous value.
func (l *LRU) Set(key string, value any) error {
return l.SetWithExpiry(key, value, 0)
}
// SetWithDefaultExpiry adds the given key and value to the store with the default expiry. If
// the key already exists, it will overwrite the previous value
func (l *LRU) SetWithDefaultExpiry(key string, value any) error {
return l.SetWithExpiry(key, value, l.defaultExpiry)
}
// SetWithExpiry adds the given key and value to the cache with the given expiry. If the key
// already exists, it will overwrite the previous value
func (l *LRU) SetWithExpiry(key string, value any, ttl time.Duration) error {
return l.set(key, value, ttl)
}
// Get the content stored in the cache for the given key, and decode it into the value interface.
// return ErrKeyNotFound if the key is missing from the cache
func (l *LRU) Get(key string, value any) error {
return l.get(key, value)
}
// Remove deletes the value for a key.
func (l *LRU) Remove(key string) error {
l.lock.Lock()
defer l.lock.Unlock()
if ent, ok := l.items[key]; ok {
l.removeElement(ent)
}
return nil
}
// Keys returns a slice of the keys in the cache.
func (l *LRU) Keys() ([]string, error) {
l.lock.RLock()
defer l.lock.RUnlock()
keys := make([]string, l.len)
i := 0
for ent := l.evictList.Back(); ent != nil; ent = ent.Prev() {
e := ent.Value.(*entry)
if e.generation == l.currentGeneration {
keys[i] = e.key
i++
}
}
return keys, nil
}
// Len returns the number of items in the cache.
func (l *LRU) Len() (int, error) {
l.lock.RLock()
defer l.lock.RUnlock()
return l.len, nil
}
// GetInvalidateClusterEvent returns the cluster event configured when this cache was created.
func (l *LRU) GetInvalidateClusterEvent() model.ClusterEvent {
return l.invalidateClusterEvent
}
// Name returns the name of the cache
func (l *LRU) Name() string {
return l.name
}
func (l *LRU) set(key string, value any, ttl time.Duration) error {
var expires time.Time
if ttl > 0 {
expires = time.Now().Add(ttl)
}
var buf []byte
var err error
// We use a fast path for hot structs.
if msgpVal, ok := value.(msgp.Marshaler); ok {
buf, err = msgpVal.MarshalMsg(nil)
} else {
// Slow path for other structs.
buf, err = msgpack.Marshal(value)
}
if err != nil {
return err
}
l.lock.Lock()
defer l.lock.Unlock()
// Check for existing item, ignoring expiry since we'd update anyway.
if ent, ok := l.items[key]; ok {
l.evictList.MoveToFront(ent)
e := ent.Value.(*entry)
e.value = buf
e.expires = expires
if e.generation != l.currentGeneration {
e.generation = l.currentGeneration
l.len++
}
return nil
}
// Add new item
ent := &entry{key, buf, expires, l.currentGeneration}
entry := l.evictList.PushFront(ent)
l.items[key] = entry
l.len++
if l.evictList.Len() > l.size {
l.removeElement(l.evictList.Back())
}
return nil
}
func (l *LRU) get(key string, value any) error {
val, err := l.getItem(key)
if err != nil {
return err
}
// We use a fast path for hot structs.
if msgpVal, ok := value.(msgp.Unmarshaler); ok {
_, err := msgpVal.UnmarshalMsg(val)
return err
}
// This is ugly and makes the cache package aware of the model package.
// But this is due to 2 things.
// 1. The msgp package works on methods on structs rather than functions.
// 2. Our cache interface passes pointers to empty pointers, and not pointers
// to values. This is mainly how all our model structs are passed around.
// It might be technically possible to use values _just_ for hot structs
// like these and then return a pointer while returning from the cache function,
// but it will make the codebase inconsistent, and has some edge-cases to take care of.
switch v := value.(type) {
case **model.User:
var u model.User
_, err := u.UnmarshalMsg(val)
*v = &u
return err
case *map[string]*model.User:
var u model.UserMap
_, err := u.UnmarshalMsg(val)
*v = u
return err
}
// Slow path for other structs.
return msgpack.Unmarshal(val, value)
}
func (l *LRU) getItem(key string) ([]byte, error) {
l.lock.Lock()
defer l.lock.Unlock()
ent, ok := l.items[key]
if !ok {
return nil, ErrKeyNotFound
}
e := ent.Value.(*entry)
if e.generation != l.currentGeneration || (!e.expires.IsZero() && time.Now().After(e.expires)) {
l.removeElement(ent)
return nil, ErrKeyNotFound
}
l.evictList.MoveToFront(ent)
return e.value, nil
}
func (l *LRU) removeElement(e *list.Element) {
l.evictList.Remove(e)
kv := e.Value.(*entry)
if kv.generation == l.currentGeneration {
l.len--
}
delete(l.items, kv.key)
}

151
server/platform/services/cache/lru_striped.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,151 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package cache
import (
"fmt"
"math"
"time"
"github.com/cespare/xxhash/v2"
"github.com/mattermost/mattermost-server/v6/model"
)
// LRUStriped keeps LRU caches in buckets in order to lower mutex contention.
// This is achieved by hashing the input key to map it to a dedicated bucket.
// Each bucket (an LRU cache) has its own lock that helps distributing the lock
// contention on multiple threads/cores, leading to less wait times.
//
// LRUStriped implements the Cache interface with the same behavior as LRU.
//
// Note that, because of it's distributed nature, the fixed size cannot be strictly respected
// and you may have a tiny bit more space for keys than you defined through LRUOptions.
// Bucket size is computed as follows: (size / nbuckets) + (size % nbuckets)
//
// Because of this size limit per bucket, and because of the nature of the data, you
// may have buckets filled unevenly, and because of this, keys will be evicted from the entire
// cache where a simple LRU wouldn't have. Example:
//
// Two buckets B1 and B2, of max size 2 each, meaning, theoretically, a max size of 4:
// - Say you have a set of 3 keys, they could fill an entire LRU cache.
// - But if all those keys are assigned to a single bucket B1, the first key will be evicted from B1
// - B2 will remain empty, even though there was enough memory allocated
//
// With 4 buckets and random UUIDs as keys, the amount of false evictions is around 5%.
//
// By default, the number of buckets equals the number of cpus returned from runtime.NumCPU.
//
// This struct is lock-free and intended to be used without lock.
type LRUStriped struct {
buckets []*LRU
name string
invalidateClusterEvent model.ClusterEvent
}
func (L LRUStriped) hashkeyMapHash(key string) uint64 {
return xxhash.Sum64String(key)
}
func (L LRUStriped) keyBucket(key string) *LRU {
return L.buckets[L.hashkeyMapHash(key)%uint64(len(L.buckets))]
}
// Purge loops through each LRU cache for purging. Since LRUStriped doesn't use any lock,
// each LRU bucket is purged after another one, which means that keys could still
// be present after a call to Purge.
func (L LRUStriped) Purge() error {
for _, lru := range L.buckets {
lru.Purge() // errors from purging LRU can be ignored as they always return nil
}
return nil
}
// Set does the same as LRU.Set
func (L LRUStriped) Set(key string, value any) error {
return L.keyBucket(key).Set(key, value)
}
// SetWithDefaultExpiry does the same as LRU.SetWithDefaultExpiry
func (L LRUStriped) SetWithDefaultExpiry(key string, value any) error {
return L.keyBucket(key).SetWithDefaultExpiry(key, value)
}
// SetWithExpiry does the same as LRU.SetWithExpiry
func (L LRUStriped) SetWithExpiry(key string, value any, ttl time.Duration) error {
return L.keyBucket(key).SetWithExpiry(key, value, ttl)
}
// Get does the same as LRU.Get
func (L LRUStriped) Get(key string, value any) error {
return L.keyBucket(key).Get(key, value)
}
// Remove does the same as LRU.Remove
func (L LRUStriped) Remove(key string) error {
return L.keyBucket(key).Remove(key)
}
// Keys does the same as LRU.Keys. However, because this is lock-free, keys might be
// inserted or removed from a previously scanned LRU cache.
// This is not as precise as using a single LRU instance.
func (L LRUStriped) Keys() ([]string, error) {
var keys []string
for _, lru := range L.buckets {
k, _ := lru.Keys() // Keys never returns any error
keys = append(keys, k...)
}
return keys, nil
}
// Len does the same as LRU.Len. As for LRUStriped.Keys, this call cannot be precise.
func (L LRUStriped) Len() (int, error) {
var size int
for _, lru := range L.buckets {
s, _ := lru.Len() // Len never returns any error
size += s
}
return size, nil
}
// GetInvalidateClusterEvent does the same as LRU.GetInvalidateClusterEvent
func (L LRUStriped) GetInvalidateClusterEvent() model.ClusterEvent {
return L.invalidateClusterEvent
}
// Name does the same as LRU.Name
func (L LRUStriped) Name() string {
return L.name
}
// NewLRUStriped creates a striped LRU cache using the special LRUOptions.StripedBuckets value.
// See LRUStriped and LRUOptions for more details.
//
// Not that in order to prevent false eviction, this LRU cache adds 10% (computation is rounded up) of the
// requested size to the total cache size.
func NewLRUStriped(opts LRUOptions) (Cache, error) {
if opts.StripedBuckets == 0 {
return nil, fmt.Errorf("number of buckets is mandatory")
}
if opts.Size < opts.StripedBuckets {
return nil, fmt.Errorf("cache size must at least be equal to the number of buckets")
}
// add 10% to the total size, before splitting
opts.Size += int(math.Ceil(float64(opts.Size) * 10.0 / 100.0))
// now this is the size for each bucket
opts.Size = (opts.Size / opts.StripedBuckets) + (opts.Size % opts.StripedBuckets)
buckets := make([]*LRU, opts.StripedBuckets)
for i := 0; i < opts.StripedBuckets; i++ {
buckets[i] = NewLRU(opts).(*LRU)
}
return LRUStriped{
buckets: buckets,
invalidateClusterEvent: opts.InvalidateClusterEvent,
name: opts.Name,
}, nil
}

91
server/platform/services/cache/lru_striped_bench_test.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,91 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package cache_test
import (
"fmt"
"runtime"
"sync"
"testing"
"github.com/cespare/xxhash/v2"
"github.com/mattermost/mattermost-server/v6/server/platform/services/cache"
)
const (
m = 500_000
)
func BenchmarkLRUStriped(b *testing.B) {
opts := cache.LRUOptions{
Name: "",
Size: 128,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
StripedBuckets: runtime.NumCPU() - 1,
}
cache, err := cache.NewLRUStriped(opts)
if err != nil {
panic(err)
}
// prepare keys and initial cache values and set routine
keys := make([]string, 0, m)
// bucketKeys is to demonstrate that splitted locks is working correctly
// by assigning one sequence of key for each bucket.
bucketKeys := make([][]string, opts.StripedBuckets)
for i := 0; i < m; i++ {
key := fmt.Sprintf("%d-key-%d", i, i)
keys = append(keys, key)
bucketKey := xxhash.Sum64String(key) % uint64(opts.StripedBuckets)
bucketKeys[bucketKey] = append(bucketKeys[bucketKey], key)
}
for i := 0; i < opts.Size; i++ {
cache.Set(keys[i], "preflight")
}
wgGet := &sync.WaitGroup{}
wgSet := &sync.WaitGroup{}
// need buffered chan because if the set routine finished before we write into the chan,
// we're left without any consumer, making any write to the chan waiting forever.
stopSet := make(chan bool, 1)
set := func() {
defer wgSet.Done()
for i := 0; i < m; i++ {
select {
case <-stopSet:
return
default:
_ = cache.Set(keys[i], "ignored")
}
}
}
get := func(bucket int) {
defer wgGet.Done()
var out string
for i := 0; i < m; i++ {
_ = cache.Get(bucketKeys[bucket][i%opts.Size], &out)
}
}
b.StopTimer()
b.ResetTimer()
for i := 0; i < b.N; i++ {
wgSet.Add(1)
go set()
for j := 0; j < opts.StripedBuckets; j++ {
wgGet.Add(1)
go get(j)
}
b.StartTimer()
wgGet.Wait()
b.StopTimer()
stopSet <- true
wgSet.Wait()
}
}

129
server/platform/services/cache/lru_striped_test.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,129 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package cache
import (
"fmt"
"hash/maphash"
"testing"
"github.com/cespare/xxhash/v2"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost-server/v6/model"
)
func makeLRUPredictableTestData(num int) [][2]string {
kv := make([][2]string, num)
for i := 0; i < len(kv); i++ {
kv[i] = [2]string{
fmt.Sprintf("%d-key-%d", i, i),
fmt.Sprintf("%d-val-%d", i, i),
}
}
return kv
}
func TestNewLRUStriped(t *testing.T) {
scache, err := NewLRUStriped(LRUOptions{StripedBuckets: 3, Size: 20})
require.NoError(t, err)
cache := scache.(LRUStriped)
require.Len(t, cache.buckets, 3)
assert.Equal(t, 8, cache.buckets[0].size)
assert.Equal(t, 8, cache.buckets[1].size)
assert.Equal(t, 8, cache.buckets[2].size)
}
func TestLRUStripedKeyDistribution(t *testing.T) {
dataset := makeLRUPredictableTestData(100)
scache, err := NewLRUStriped(LRUOptions{StripedBuckets: 4, Size: len(dataset)})
require.NoError(t, err)
cache := scache.(LRUStriped)
for _, kv := range dataset {
require.NoError(t, cache.Set(kv[0], kv[1]))
var out string
require.NoError(t, cache.Get(kv[0], &out))
require.Equal(t, kv[1], out)
}
require.Len(t, cache.buckets, 4)
acc := 0
for i := 0; i < 4; i++ {
clen, err := cache.buckets[i].Len()
acc += clen
assert.NoError(t, err)
assert.GreaterOrEqual(t, clen, len(dataset)/2/4, "at least 50%/nbuckets of all keys in each bucket")
}
// because of the limited size of each bucket and the nature of our data,
// we may have around 10% of our keys evicted in this scenario. removing 1% because we cannot predict
// accurately what is happening with random data.
assert.GreaterOrEqual(t, acc, len(dataset)-(len(dataset)*1.0/100.0))
}
func TestLRUStriped_Size(t *testing.T) {
scache, err := NewLRUStriped(LRUOptions{StripedBuckets: 2, Size: 128})
require.NoError(t, err)
cache := scache.(LRUStriped)
acc := 0
for _, bucket := range cache.buckets {
acc += bucket.size
}
assert.Equal(t, 128+13+1, acc) // +10% +modulo padding
}
func TestLRUStriped_HashKey(t *testing.T) {
scache, err := NewLRUStriped(LRUOptions{StripedBuckets: 2, Size: 128})
require.NoError(t, err)
cache := scache.(LRUStriped)
first := cache.hashkeyMapHash("key")
cache.hashkeyMapHash("other_key_to_ensure_that_result_its_not_dependent_on_previous_input")
second := cache.hashkeyMapHash("key")
require.Equal(t, first, second)
}
func TestLRUStriped_Get(t *testing.T) {
cache, err := NewLRUStriped(LRUOptions{StripedBuckets: 4, Size: 128})
require.NoError(t, err)
var out string
require.Equal(t, ErrKeyNotFound, cache.Get("key", &out))
require.Zero(t, out)
require.NoError(t, cache.Set("key", "value"))
require.NoError(t, cache.Get("key", &out))
require.Equal(t, "value", out)
}
var hashSink uint64
func BenchmarkSum64(b *testing.B) {
cases := []string{
"1",
"22",
"333",
model.NewId(),
model.NewId() + model.NewId(),
}
for _, case_ := range cases {
b.Run(fmt.Sprintf("maphash_string_len_%d", len(case_)), func(b *testing.B) {
seed := maphash.MakeSeed()
b.ResetTimer()
for i := 0; i < b.N; i++ {
var h maphash.Hash
h.SetSeed(seed)
h.WriteString(case_) // documentation and code says it never fails
hashSink = h.Sum64()
}
})
b.Run(fmt.Sprintf("xxhash_string_len_%d", len(case_)), func(b *testing.B) {
for i := 0; i < b.N; i++ {
hashSink = xxhash.Sum64String(case_)
}
})
}
}

646
server/platform/services/cache/lru_test.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,646 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package cache
import (
"fmt"
"sync"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost-server/v6/model"
)
func TestLRU(t *testing.T) {
l := NewLRU(LRUOptions{
Size: 128,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
for i := 0; i < 256; i++ {
err := l.Set(fmt.Sprintf("%d", i), i)
require.NoError(t, err)
}
size, err := l.Len()
require.NoError(t, err)
require.Equalf(t, size, 128, "bad len: %v", size)
keys, err := l.Keys()
require.NoError(t, err)
for i, k := range keys {
var v int
err = l.Get(k, &v)
require.NoError(t, err, "bad key: %v", k)
require.Equalf(t, fmt.Sprintf("%d", v), k, "bad key: %v", k)
require.Equalf(t, i+128, v, "bad value: %v", k)
}
for i := 0; i < 128; i++ {
var v int
err = l.Get(fmt.Sprintf("%d", i), &v)
require.Equal(t, ErrKeyNotFound, err, "should be evicted %v: %v", i, err)
}
for i := 128; i < 256; i++ {
var v int
err = l.Get(fmt.Sprintf("%d", i), &v)
require.NoError(t, err, "should not be evicted %v: %v", i, err)
}
for i := 128; i < 192; i++ {
l.Remove(fmt.Sprintf("%d", i))
var v int
err = l.Get(fmt.Sprintf("%d", i), &v)
require.Equal(t, ErrKeyNotFound, err, "should be deleted %v: %v", i, err)
}
var v int
err = l.Get("192", &v) // expect 192 to be last key in l.Keys()
require.NoError(t, err, "should exist")
require.Equalf(t, 192, v, "bad value: %v", v)
keys, err = l.Keys()
require.NoError(t, err)
for i, k := range keys {
require.Falsef(t, i < 63 && k != fmt.Sprintf("%d", i+193), "out of order key: %v", k)
require.Falsef(t, i == 63 && k != "192", "out of order key: %v", k)
}
l.Purge()
size, err = l.Len()
require.NoError(t, err)
require.Equalf(t, size, 0, "bad len: %v", size)
err = l.Get("200", &v)
require.Equal(t, err, ErrKeyNotFound, "should contain nothing")
err = l.Set("201", 301)
require.NoError(t, err)
err = l.Get("201", &v)
require.NoError(t, err)
require.Equal(t, 301, v)
}
func TestLRUExpire(t *testing.T) {
l := NewLRU(LRUOptions{
Size: 128,
DefaultExpiry: 1 * time.Second,
InvalidateClusterEvent: "",
})
l.SetWithDefaultExpiry("1", 1)
l.SetWithExpiry("3", 3, 0*time.Second)
time.Sleep(time.Second * 2)
var r1 int
err := l.Get("1", &r1)
require.Equal(t, err, ErrKeyNotFound, "should not exist")
var r2 int
err2 := l.Get("3", &r2)
require.NoError(t, err2, "should exist")
require.Equal(t, 3, r2)
}
func TestLRUMarshalUnMarshal(t *testing.T) {
l := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
value1 := map[string]any{
"key1": 1,
"key2": "value2",
}
err := l.Set("test", value1)
require.NoError(t, err)
var value2 map[string]any
err = l.Get("test", &value2)
require.NoError(t, err)
assert.EqualValues(t, 1, value2["key1"])
v2, ok := value2["key2"].(string)
require.True(t, ok, "unable to cast value")
assert.Equal(t, "value2", v2)
post := model.Post{
Id: "id",
CreateAt: 11111,
UpdateAt: 11111,
DeleteAt: 11111,
EditAt: 111111,
IsPinned: true,
UserId: "UserId",
ChannelId: "ChannelId",
RootId: "RootId",
OriginalId: "OriginalId",
Message: "OriginalId",
MessageSource: "MessageSource",
Type: "Type",
Props: map[string]any{
"key": "val",
},
Hashtags: "Hashtags",
Filenames: []string{"item1", "item2"},
FileIds: []string{"item1", "item2"},
PendingPostId: "PendingPostId",
HasReactions: true,
ReplyCount: 11111,
Metadata: &model.PostMetadata{
Embeds: []*model.PostEmbed{
{
Type: "Type",
URL: "URL",
Data: "some data",
},
{
Type: "Type 2",
URL: "URL 2",
Data: "some data 2",
},
},
Emojis: []*model.Emoji{
{
Id: "id",
Name: "name",
},
},
Files: nil,
Images: map[string]*model.PostImage{
"key": {
Width: 1,
Height: 1,
Format: "format",
FrameCount: 1,
},
"key2": {
Width: 999,
Height: 888,
Format: "format 2",
FrameCount: 1000,
},
},
Reactions: []*model.Reaction{
{
UserId: "user_id",
PostId: "post_id",
EmojiName: "emoji_name",
CreateAt: 111,
},
},
},
}
err = l.Set("post", post.Clone())
require.NoError(t, err)
var p model.Post
err = l.Get("post", &p)
require.NoError(t, err)
require.Equal(t, post.Clone(), p.Clone())
session := &model.Session{
Id: "ty7ia14yuty5bmpt8wmz6da1fw",
Token: "79c3iq6nzpycmkkawudanqhg5c",
CreateAt: 1595445296960,
ExpiresAt: 1598296496960,
LastActivityAt: 1595445296960,
UserId: "rpgh1q5ra38y9xjn9z8fjctezr",
Roles: "system_admin system_user",
IsOAuth: false,
ExpiredNotify: false,
Props: map[string]string{
"csrf": "33zb7h7rk3rfffztojn5pxbkxe",
"isMobile": "false",
"isSaml": "false",
"is_guest": "false",
"os": "",
"platform": "Windows",
},
}
err = l.Set("session", session)
require.NoError(t, err)
var s = &model.Session{}
err = l.Get("session", s)
require.NoError(t, err)
require.Equal(t, session, s)
user := &model.User{
Id: "id",
CreateAt: 11111,
UpdateAt: 11111,
DeleteAt: 11111,
Username: "username",
Password: "password",
AuthService: "AuthService",
AuthData: nil,
Email: "Email",
EmailVerified: true,
Nickname: "Nickname",
FirstName: "FirstName",
LastName: "LastName",
Position: "Position",
Roles: "Roles",
AllowMarketing: true,
Props: map[string]string{
"key0": "value0",
},
NotifyProps: map[string]string{
"key0": "value0",
},
LastPasswordUpdate: 111111,
LastPictureUpdate: 111111,
FailedAttempts: 111111,
Locale: "Locale",
MfaActive: true,
MfaSecret: "MfaSecret",
LastActivityAt: 111111,
IsBot: true,
TermsOfServiceId: "TermsOfServiceId",
TermsOfServiceCreateAt: 111111,
}
err = l.Set("user", user)
require.NoError(t, err)
var u *model.User
err = l.Get("user", &u)
require.NoError(t, err)
// msgp returns an empty map instead of a nil map.
// This does not make an actual difference in terms of functionality.
u.Timezone = nil
require.Equal(t, user, u)
tt := make(map[string]*model.User)
tt["1"] = u
err = l.Set("mm", model.UserMap(tt))
require.NoError(t, err)
var out map[string]*model.User
err = l.Get("mm", &out)
require.NoError(t, err)
out["1"].Timezone = nil
require.Equal(t, tt, out)
}
func BenchmarkLRU(b *testing.B) {
value1 := "simplestring"
b.Run("simple=new", func(b *testing.B) {
for i := 0; i < b.N; i++ {
l2 := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
err := l2.Set("test", value1)
require.NoError(b, err)
var val string
err = l2.Get("test", &val)
require.NoError(b, err)
}
})
type obj struct {
Field1 int
Field2 string
Field3 struct {
Field4 int
Field5 string
}
Field6 map[string]string
}
value2 := obj{
1,
"field2",
struct {
Field4 int
Field5 string
}{
6,
"field5 is a looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong string",
},
map[string]string{
"key0": "value0",
"key1": "value value1",
"key2": "value value value2",
"key3": "value value value value3",
"key4": "value value value value value4",
"key5": "value value value value value value5",
"key6": "value value value value value value value6",
"key7": "value value value value value value value value7",
"key8": "value value value value value value value value value8",
"key9": "value value value value value value value value value value9",
},
}
b.Run("complex=new", func(b *testing.B) {
for i := 0; i < b.N; i++ {
l2 := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
err := l2.Set("test", value2)
require.NoError(b, err)
var val obj
err = l2.Get("test", &val)
require.NoError(b, err)
}
})
user := &model.User{
Id: "id",
CreateAt: 11111,
UpdateAt: 11111,
DeleteAt: 11111,
Username: "username",
Password: "password",
AuthService: "AuthService",
AuthData: nil,
Email: "Email",
EmailVerified: true,
Nickname: "Nickname",
FirstName: "FirstName",
LastName: "LastName",
Position: "Position",
Roles: "Roles",
AllowMarketing: true,
Props: map[string]string{
"key0": "value0",
"key1": "value value1",
"key2": "value value value2",
"key3": "value value value value3",
"key4": "value value value value value4",
"key5": "value value value value value value5",
"key6": "value value value value value value value6",
"key7": "value value value value value value value value7",
"key8": "value value value value value value value value value8",
"key9": "value value value value value value value value value value9",
},
NotifyProps: map[string]string{
"key0": "value0",
"key1": "value value1",
"key2": "value value value2",
"key3": "value value value value3",
"key4": "value value value value value4",
"key5": "value value value value value value5",
"key6": "value value value value value value value6",
"key7": "value value value value value value value value7",
"key8": "value value value value value value value value value8",
"key9": "value value value value value value value value value value9",
},
LastPasswordUpdate: 111111,
LastPictureUpdate: 111111,
FailedAttempts: 111111,
Locale: "Locale",
Timezone: map[string]string{
"key0": "value0",
"key1": "value value1",
"key2": "value value value2",
"key3": "value value value value3",
"key4": "value value value value value4",
"key5": "value value value value value value5",
"key6": "value value value value value value value6",
"key7": "value value value value value value value value7",
"key8": "value value value value value value value value value8",
"key9": "value value value value value value value value value value9",
},
MfaActive: true,
MfaSecret: "MfaSecret",
LastActivityAt: 111111,
IsBot: true,
BotDescription: "field5 is a looooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooong string",
BotLastIconUpdate: 111111,
TermsOfServiceId: "TermsOfServiceId",
TermsOfServiceCreateAt: 111111,
}
b.Run("User=new", func(b *testing.B) {
for i := 0; i < b.N; i++ {
l2 := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
err := l2.Set("test", user)
require.NoError(b, err)
var val model.User
err = l2.Get("test", &val)
require.NoError(b, err)
}
})
uMap := map[string]*model.User{
"id1": {
Id: "id1",
CreateAt: 1111,
UpdateAt: 1112,
Username: "user1",
Password: "pass",
},
"id2": {
Id: "id2",
CreateAt: 1113,
UpdateAt: 1114,
Username: "user2",
Password: "pass2",
},
}
b.Run("UserMap=new", func(b *testing.B) {
for i := 0; i < b.N; i++ {
l2 := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
err := l2.Set("test", model.UserMap(uMap))
require.NoError(b, err)
var val map[string]*model.User
err = l2.Get("test", &val)
require.NoError(b, err)
}
})
post := &model.Post{
Id: "id",
CreateAt: 11111,
UpdateAt: 11111,
DeleteAt: 11111,
EditAt: 111111,
IsPinned: true,
UserId: "UserId",
ChannelId: "ChannelId",
RootId: "RootId",
OriginalId: "OriginalId",
Message: "OriginalId",
MessageSource: "MessageSource",
Type: "Type",
Props: map[string]any{
"key": "val",
},
Hashtags: "Hashtags",
Filenames: []string{"item1", "item2"},
FileIds: []string{"item1", "item2"},
PendingPostId: "PendingPostId",
HasReactions: true,
// Transient data populated before sending a post to the client
ReplyCount: 11111,
Metadata: &model.PostMetadata{
Embeds: []*model.PostEmbed{
{
Type: "Type",
URL: "URL",
Data: "some data",
},
{
Type: "Type 2",
URL: "URL 2",
Data: "some data 2",
},
},
Emojis: []*model.Emoji{
{
Id: "id",
Name: "name",
},
},
Files: nil,
Images: map[string]*model.PostImage{
"key": {
Width: 1,
Height: 1,
Format: "format",
FrameCount: 1,
},
"key2": {
Width: 999,
Height: 888,
Format: "format 2",
FrameCount: 1000,
},
},
Reactions: []*model.Reaction{},
},
}
b.Run("Post=new", func(b *testing.B) {
for i := 0; i < b.N; i++ {
l2 := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
err := l2.Set("test", post)
require.NoError(b, err)
var val model.Post
err = l2.Get("test", &val)
require.NoError(b, err)
}
})
status := model.Status{
UserId: "UserId",
Status: "Status",
Manual: true,
LastActivityAt: 111111,
ActiveChannel: "ActiveChannel",
}
b.Run("Status=new", func(b *testing.B) {
for i := 0; i < b.N; i++ {
l2 := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
err := l2.Set("test", status)
require.NoError(b, err)
var val *model.Status
err = l2.Get("test", &val)
require.NoError(b, err)
}
})
session := model.Session{
Id: "ty7ia14yuty5bmpt8wmz6da1fw",
Token: "79c3iq6nzpycmkkawudanqhg5c",
CreateAt: 1595445296960,
ExpiresAt: 1598296496960,
LastActivityAt: 1595445296960,
UserId: "rpgh1q5ra38y9xjn9z8fjctezr",
Roles: "system_admin system_user",
IsOAuth: false,
ExpiredNotify: false,
Props: map[string]string{
"csrf": "33zb7h7rk3rfffztojn5pxbkxe",
"isMobile": "false",
"isSaml": "false",
"is_guest": "false",
"os": "",
"platform": "Windows",
},
}
b.Run("Session=new", func(b *testing.B) {
for i := 0; i < b.N; i++ {
l2 := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
err := l2.Set("test", &session)
require.NoError(b, err)
var val *model.Session
err = l2.Get("test", &val)
require.NoError(b, err)
}
})
}
func TestLRURace(t *testing.T) {
l2 := NewLRU(LRUOptions{
Size: 1,
DefaultExpiry: 0,
InvalidateClusterEvent: "",
})
var wg sync.WaitGroup
l2.Set("test", "value1")
wg.Add(2)
go func() {
defer wg.Done()
value1 := "simplestring"
err := l2.Set("test", value1)
require.NoError(t, err)
}()
go func() {
defer wg.Done()
var val string
err := l2.Get("test", &val)
require.NoError(t, err)
}()
wg.Wait()
}

170
server/platform/services/cache/mocks/Cache.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,170 @@
// Code generated by mockery v1.0.0. DO NOT EDIT.
package mocks
import (
time "time"
mock "github.com/stretchr/testify/mock"
)
// Cache is an autogenerated mock type for the Cache type
type Cache struct {
mock.Mock
}
// Get provides a mock function with given fields: key, value
func (_m *Cache) Get(key string, value any) error {
ret := _m.Called(key, value)
var r0 error
if rf, ok := ret.Get(0).(func(string, any) error); ok {
r0 = rf(key, value)
} else {
r0 = ret.Error(0)
}
return r0
}
// GetInvalidateClusterEvent provides a mock function with given fields:
func (_m *Cache) GetInvalidateClusterEvent() string {
ret := _m.Called()
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Keys provides a mock function with given fields:
func (_m *Cache) Keys() ([]string, error) {
ret := _m.Called()
var r0 []string
if rf, ok := ret.Get(0).(func() []string); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]string)
}
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Len provides a mock function with given fields:
func (_m *Cache) Len() (int, error) {
ret := _m.Called()
var r0 int
if rf, ok := ret.Get(0).(func() int); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(int)
}
var r1 error
if rf, ok := ret.Get(1).(func() error); ok {
r1 = rf()
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// Name provides a mock function with given fields:
func (_m *Cache) Name() string {
ret := _m.Called()
var r0 string
if rf, ok := ret.Get(0).(func() string); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(string)
}
return r0
}
// Purge provides a mock function with given fields:
func (_m *Cache) Purge() error {
ret := _m.Called()
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// Remove provides a mock function with given fields: key
func (_m *Cache) Remove(key string) error {
ret := _m.Called(key)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(key)
} else {
r0 = ret.Error(0)
}
return r0
}
// Set provides a mock function with given fields: key, value
func (_m *Cache) Set(key string, value any) error {
ret := _m.Called(key, value)
var r0 error
if rf, ok := ret.Get(0).(func(string, any) error); ok {
r0 = rf(key, value)
} else {
r0 = ret.Error(0)
}
return r0
}
// SetWithDefaultExpiry provides a mock function with given fields: key, value
func (_m *Cache) SetWithDefaultExpiry(key string, value any) error {
ret := _m.Called(key, value)
var r0 error
if rf, ok := ret.Get(0).(func(string, any) error); ok {
r0 = rf(key, value)
} else {
r0 = ret.Error(0)
}
return r0
}
// SetWithExpiry provides a mock function with given fields: key, value, ttl
func (_m *Cache) SetWithExpiry(key string, value any, ttl time.Duration) error {
ret := _m.Called(key, value, ttl)
var r0 error
if rf, ok := ret.Get(0).(func(string, any, time.Duration) error); ok {
r0 = rf(key, value, ttl)
} else {
r0 = ret.Error(0)
}
return r0
}

65
server/platform/services/cache/mocks/Provider.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,65 @@
// Code generated by mockery v1.0.0. DO NOT EDIT.
package mocks
import (
mock "github.com/stretchr/testify/mock"
cache "github.com/mattermost/mattermost-server/v6/server/platform/services/cache"
)
// Provider is an autogenerated mock type for the Provider type
type Provider struct {
mock.Mock
}
// Close provides a mock function with given fields:
func (_m *Provider) Close() error {
ret := _m.Called()
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// Connect provides a mock function with given fields:
func (_m *Provider) Connect() error {
ret := _m.Called()
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
}
// NewCache provides a mock function with given fields: opts
func (_m *Provider) NewCache(opts *cache.CacheOptions) (cache.Cache, error) {
ret := _m.Called(opts)
var r0 cache.Cache
if rf, ok := ret.Get(0).(func(*cache.CacheOptions) cache.Cache); ok {
r0 = rf(opts)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(cache.Cache)
}
}
var r1 error
if rf, ok := ret.Get(1).(func(*cache.CacheOptions) error); ok {
r1 = rf(opts)
} else {
r1 = ret.Error(1)
}
return r0, r1
}

67
server/platform/services/cache/provider.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,67 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package cache
import (
"time"
"github.com/mattermost/mattermost-server/v6/model"
)
// CacheOptions contains options for initializing a cache
type CacheOptions struct {
Size int
DefaultExpiry time.Duration
Name string
InvalidateClusterEvent model.ClusterEvent
Striped bool
StripedBuckets int
}
// Provider is a provider for Cache
type Provider interface {
// NewCache creates a new cache with given options.
NewCache(opts *CacheOptions) (Cache, error)
// Connect opens a new connection to the cache using specific provider parameters.
Connect() error
// Close releases any resources used by the cache provider.
Close() error
}
type cacheProvider struct {
}
// NewProvider creates a new CacheProvider
func NewProvider() Provider {
return &cacheProvider{}
}
// NewCache creates a new cache with given opts
func (c *cacheProvider) NewCache(opts *CacheOptions) (Cache, error) {
if opts.Striped {
return NewLRUStriped(LRUOptions{
Name: opts.Name,
Size: opts.Size,
DefaultExpiry: opts.DefaultExpiry,
InvalidateClusterEvent: opts.InvalidateClusterEvent,
StripedBuckets: opts.StripedBuckets,
})
}
return NewLRU(LRUOptions{
Name: opts.Name,
Size: opts.Size,
DefaultExpiry: opts.DefaultExpiry,
InvalidateClusterEvent: opts.InvalidateClusterEvent,
}), nil
}
// Connect opens a new connection to the cache using specific provider parameters.
func (c *cacheProvider) Connect() error {
return nil
}
// Close releases any resources used by the cache provider.
func (c *cacheProvider) Close() error {
return nil
}

187
server/platform/services/cache/provider_test.go поставляемый Обычный файл
Просмотреть файл

@@ -0,0 +1,187 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package cache
import (
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/mattermost/mattermost-server/v6/model"
)
func TestNewCache(t *testing.T) {
t.Run("with only size option given", func(t *testing.T) {
p := NewProvider()
size := 1
c, err := p.NewCache(&CacheOptions{
Size: size,
})
require.NoError(t, err)
err = c.Set("key1", "val1")
require.NoError(t, err)
err = c.Set("key2", "val2")
require.NoError(t, err)
err = c.Set("key3", "val3")
require.NoError(t, err)
l, err := c.Len()
require.NoError(t, err)
require.Equal(t, size, l)
})
t.Run("with only size option given", func(t *testing.T) {
p := NewProvider()
size := 1
c, err := p.NewCache(&CacheOptions{
Size: size,
})
require.NoError(t, err)
err = c.Set("key1", "val1")
require.NoError(t, err)
err = c.Set("key2", "val2")
require.NoError(t, err)
err = c.Set("key3", "val3")
require.NoError(t, err)
l, err := c.Len()
require.NoError(t, err)
require.Equal(t, size, l)
})
t.Run("with all options specified", func(t *testing.T) {
p := NewProvider()
size := 1
expiry := 1 * time.Second
event := model.ClusterEvent("clusterEvent")
c, err := p.NewCache(&CacheOptions{
Size: size,
Name: "name",
DefaultExpiry: expiry,
InvalidateClusterEvent: event,
})
require.NoError(t, err)
require.Equal(t, event, c.GetInvalidateClusterEvent())
err = c.SetWithDefaultExpiry("key1", "val1")
require.NoError(t, err)
err = c.SetWithDefaultExpiry("key2", "val2")
require.NoError(t, err)
err = c.SetWithDefaultExpiry("key3", "val3")
require.NoError(t, err)
l, err := c.Len()
require.NoError(t, err)
require.Equal(t, size, l)
time.Sleep(expiry + 1*time.Second)
var v string
err = c.Get("key1", &v)
require.Equal(t, ErrKeyNotFound, err)
err = c.Get("key2", &v)
require.Equal(t, ErrKeyNotFound, err)
err = c.Get("key3", &v)
require.Equal(t, ErrKeyNotFound, err)
})
}
func TestNewCache_Striped(t *testing.T) {
t.Run("with only size option given", func(t *testing.T) {
p := NewProvider()
size := 1
c, err := p.NewCache(&CacheOptions{
Size: size,
Striped: true,
StripedBuckets: 1,
})
require.NoError(t, err)
err = c.Set("key1", "val1")
require.NoError(t, err)
err = c.Set("key2", "val2")
require.NoError(t, err)
err = c.Set("key3", "val3")
require.NoError(t, err)
l, err := c.Len()
require.NoError(t, err)
require.Equal(t, size+1, l) // +10% from striping
})
t.Run("with only size option given", func(t *testing.T) {
p := NewProvider()
size := 1
c, err := p.NewCache(&CacheOptions{
Size: size,
Striped: true,
StripedBuckets: 1,
})
require.NoError(t, err)
err = c.Set("key1", "val1")
require.NoError(t, err)
err = c.Set("key2", "val2")
require.NoError(t, err)
err = c.Set("key3", "val3")
require.NoError(t, err)
l, err := c.Len()
require.NoError(t, err)
require.Equal(t, size+1, l) // +10% rounded up from striped lru
})
t.Run("with all options specified", func(t *testing.T) {
p := NewProvider()
size := 1
expiry := 1 * time.Second
event := model.ClusterEvent("clusterEvent")
c, err := p.NewCache(&CacheOptions{
Size: size,
Name: "name",
DefaultExpiry: expiry,
InvalidateClusterEvent: event,
Striped: true,
StripedBuckets: 1,
})
require.NoError(t, err)
require.Equal(t, event, c.GetInvalidateClusterEvent())
err = c.SetWithDefaultExpiry("key1", "val1")
require.NoError(t, err)
err = c.SetWithDefaultExpiry("key2", "val2")
require.NoError(t, err)
err = c.SetWithDefaultExpiry("key3", "val3")
require.NoError(t, err)
l, err := c.Len()
require.NoError(t, err)
require.Equal(t, size+1, l) // +10% from striping
time.Sleep(expiry + 1*time.Second)
var v string
err = c.Get("key1", &v)
require.Equal(t, ErrKeyNotFound, err)
err = c.Get("key2", &v)
require.Equal(t, ErrKeyNotFound, err)
err = c.Get("key3", &v)
require.Equal(t, ErrKeyNotFound, err)
})
}
func TestConnectClose(t *testing.T) {
p := NewProvider()
err := p.Connect()
require.NoError(t, err)
err = p.Close()
require.NoError(t, err)
}