mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[GH-13070] Migrate emojiIdCacheByName cache from store/sqlstore/emoji_store.go to the new store/localcachelayer (#13125)

Просмотр исходного кода 
Automatic Merge
Этот коммит содержится в:
larkox
2019-11-19 11:26:52 +01:00
коммит произвёл mattermod
родитель 1aca43d401
Коммит c40f0a4aea
7 изменённых файлов: 264 добавлений и 129 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
model/cluster_message.go
Просмотреть файл

@@ -24,6 +24,8 @@ const (
CLUSTER_EVENT_CLEAR_SESSION_CACHE_FOR_USER = "clear_session_user"
CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES = "inv_roles"
CLUSTER_EVENT_INVALIDATE_CACHE_FOR_SCHEMES = "inv_schemes"
CLUSTER_EVENT_INVALIDATE_CACHE_FOR_EMOJIS_BY_ID = "inv_emojis_by_id"
CLUSTER_EVENT_INVALIDATE_CACHE_FOR_EMOJIS_ID_BY_NAME = "inv_emojis_id_by_name"
CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBER_COUNTS = "inv_channel_member_counts"
CLUSTER_EVENT_CLEAR_SESSION_CACHE_FOR_ALL_USERS = "inv_all_user_sessions"
CLUSTER_EVENT_INSTALL_PLUGIN = "install_plugin"

100
store/localcachelayer/emoji_layer.go Обычный файл
Просмотреть файл

@@ -0,0 +1,100 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package localcachelayer
import (
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store"
)
type LocalCacheEmojiStore struct {
store.EmojiStore
rootStore *LocalCacheStore
}
func (es *LocalCacheEmojiStore) handleClusterInvalidateEmojiById(msg *model.ClusterMessage) {
if msg.Data == CLEAR_CACHE_MESSAGE_DATA {
es.rootStore.emojiCacheById.Purge()
} else {
es.rootStore.emojiCacheById.Remove(msg.Data)
}
}
func (es *LocalCacheEmojiStore) handleClusterInvalidateEmojiIdByName(msg *model.ClusterMessage) {
if msg.Data == CLEAR_CACHE_MESSAGE_DATA {
es.rootStore.emojiIdCacheByName.Purge()
} else {
es.rootStore.emojiIdCacheByName.Remove(msg.Data)
}
}
func (es LocalCacheEmojiStore) Get(id string, allowFromCache bool) (*model.Emoji, *model.AppError) {
if allowFromCache {
if emoji, ok := es.getFromCacheById(id); ok {
return emoji, nil
}
}
emoji, err := es.EmojiStore.Get(id, allowFromCache)
if allowFromCache && err == nil {
es.addToCache(emoji)
}
return emoji, err
}
func (es LocalCacheEmojiStore) GetByName(name string, allowFromCache bool) (*model.Emoji, *model.AppError) {
if id, ok := model.GetSystemEmojiId(name); ok {
return es.Get(id, allowFromCache)
}
if allowFromCache {
if emoji, ok := es.getFromCacheByName(name); ok {
return emoji, nil
}
}
emoji, err := es.EmojiStore.GetByName(name, allowFromCache)
if allowFromCache && err == nil {
es.addToCache(emoji)
}
return emoji, err
}
func (es LocalCacheEmojiStore) Delete(emoji *model.Emoji, time int64) *model.AppError {
err := es.EmojiStore.Delete(emoji, time)
if err == nil {
es.removeFromCache(emoji)
}
return err
}
func (es LocalCacheEmojiStore) addToCache(emoji *model.Emoji) {
es.rootStore.doStandardAddToCache(es.rootStore.emojiCacheById, emoji.Id, emoji)
es.rootStore.doStandardAddToCache(es.rootStore.emojiIdCacheByName, emoji.Name, emoji.Id)
}
func (es LocalCacheEmojiStore) getFromCacheById(id string) (*model.Emoji, bool) {
if emoji := es.rootStore.doStandardReadCache(es.rootStore.emojiCacheById, id); emoji != nil {
return emoji.(*model.Emoji), true
}
return nil, false
}
func (es LocalCacheEmojiStore) getFromCacheByName(name string) (*model.Emoji, bool) {
if emojiId := es.rootStore.doStandardReadCache(es.rootStore.emojiIdCacheByName, name); emojiId != nil {
return es.getFromCacheById(emojiId.(string))
}
return nil, false
}
func (es LocalCacheEmojiStore) removeFromCache(emoji *model.Emoji) {
es.rootStore.doInvalidateCacheCluster(es.rootStore.emojiCacheById, emoji.Id)
es.rootStore.doInvalidateCacheCluster(es.rootStore.emojiIdCacheByName, emoji.Name)
}

136
store/localcachelayer/emoji_layer_test.go Обычный файл
Просмотреть файл

@@ -0,0 +1,136 @@
// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved.
// See License.txt for license information.
package localcachelayer
import (
"testing"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store/storetest"
"github.com/mattermost/mattermost-server/store/storetest/mocks"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestEmojiStore(t *testing.T) {
StoreTest(t, storetest.TestEmojiStore)
}
func TestEmojiStoreCache(t *testing.T) {
fakeEmoji := model.Emoji{Id: "123", Name: "name123"}
t.Run("first call by id not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
emoji, err := cachedStore.Emoji().Get("123", true)
require.Nil(t, err)
assert.Equal(t, emoji, &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
emoji, err = cachedStore.Emoji().Get("123", true)
require.Nil(t, err)
assert.Equal(t, emoji, &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
})
t.Run("first call by name not cached, second cached and returning same data", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
emoji, err := cachedStore.Emoji().GetByName("name123", true)
require.Nil(t, err)
assert.Equal(t, emoji, &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
emoji, err = cachedStore.Emoji().GetByName("name123", true)
require.Nil(t, err)
assert.Equal(t, emoji, &fakeEmoji)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
})
t.Run("first call by id not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Emoji().Get("123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().Get("123", false)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call by name not cached, second force no cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Emoji().GetByName("name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().GetByName("name123", false)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
})
t.Run("first call by id force no cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Emoji().Get("123", false)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().Get("123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
cachedStore.Emoji().Get("123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call by id force no cached, second not cached, third cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Emoji().GetByName("name123", false)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().GetByName("name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
cachedStore.Emoji().GetByName("name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
})
t.Run("first call by id, second call by name cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Emoji().Get("123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().GetByName("name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 0)
})
t.Run("first call by name, second call by id cached", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Emoji().GetByName("name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().Get("123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 0)
})
t.Run("first call by id not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Emoji().Get("123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 1)
cachedStore.Emoji().Delete(&fakeEmoji, 0)
cachedStore.Emoji().Get("123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "Get", 2)
})
t.Run("first call by name not cached, invalidate, and then not cached again", func(t *testing.T) {
mockStore := getMockStore()
cachedStore := NewLocalCacheLayer(mockStore, nil, nil)
cachedStore.Emoji().GetByName("name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 1)
cachedStore.Emoji().Delete(&fakeEmoji, 0)
cachedStore.Emoji().GetByName("name123", true)
mockStore.Emoji().(*mocks.EmojiStore).AssertNumberOfCalls(t, "GetByName", 2)
})
}

17
store/localcachelayer/layer.go
Просмотреть файл

@@ -20,6 +20,9 @@ const (
SCHEME_CACHE_SIZE = 20000
SCHEME_CACHE_SEC = 30 * 60
EMOJI_CACHE_SIZE = 5000
EMOJI_CACHE_SEC = 30 * 60
CHANNEL_MEMBERS_COUNTS_CACHE_SIZE = model.CHANNEL_CACHE_SIZE
CHANNEL_MEMBERS_COUNTS_CACHE_SEC = 30 * 60
@@ -36,6 +39,9 @@ type LocalCacheStore struct {
roleCache *utils.Cache
scheme LocalCacheSchemeStore
schemeCache *utils.Cache
emoji LocalCacheEmojiStore
emojiCacheById *utils.Cache
emojiIdCacheByName *utils.Cache
channel LocalCacheChannelStore
channelMemberCountsCache *utils.Cache
}
@@ -52,6 +58,9 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf
localCacheStore.role = LocalCacheRoleStore{RoleStore: baseStore.Role(), rootStore: &localCacheStore}
localCacheStore.schemeCache = utils.NewLruWithParams(SCHEME_CACHE_SIZE, "Scheme", SCHEME_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_SCHEMES)
localCacheStore.scheme = LocalCacheSchemeStore{SchemeStore: baseStore.Scheme(), rootStore: &localCacheStore}
localCacheStore.emojiCacheById = utils.NewLruWithParams(EMOJI_CACHE_SIZE, "EmojiById", EMOJI_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_EMOJIS_BY_ID)
localCacheStore.emojiIdCacheByName = utils.NewLruWithParams(EMOJI_CACHE_SIZE, "EmojiByName", EMOJI_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_EMOJIS_ID_BY_NAME)
localCacheStore.emoji = LocalCacheEmojiStore{EmojiStore: baseStore.Emoji(), rootStore: &localCacheStore}
localCacheStore.channelMemberCountsCache = utils.NewLruWithParams(CHANNEL_MEMBERS_COUNTS_CACHE_SIZE, "ChannelMemberCounts", CHANNEL_MEMBERS_COUNTS_CACHE_SEC, model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBER_COUNTS)
localCacheStore.channel = LocalCacheChannelStore{ChannelStore: baseStore.Channel(), rootStore: &localCacheStore}
@@ -59,6 +68,8 @@ func NewLocalCacheLayer(baseStore store.Store, metrics einterfaces.MetricsInterf
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_REACTIONS, localCacheStore.reaction.handleClusterInvalidateReaction)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_ROLES, localCacheStore.role.handleClusterInvalidateRole)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_SCHEMES, localCacheStore.scheme.handleClusterInvalidateScheme)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_EMOJIS_BY_ID, localCacheStore.emoji.handleClusterInvalidateEmojiById)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_EMOJIS_ID_BY_NAME, localCacheStore.emoji.handleClusterInvalidateEmojiIdByName)
cluster.RegisterClusterMessageHandler(model.CLUSTER_EVENT_INVALIDATE_CACHE_FOR_CHANNEL_MEMBER_COUNTS, localCacheStore.channel.handleClusterInvalidateChannelMemberCounts)
}
return localCacheStore
@@ -76,6 +87,10 @@ func (s LocalCacheStore) Scheme() store.SchemeStore {
return s.scheme
}
func (s LocalCacheStore) Emoji() store.EmojiStore {
return s.emoji
}
func (s LocalCacheStore) Channel() store.ChannelStore {
return s.channel
}
@@ -130,5 +145,7 @@ func (s *LocalCacheStore) doClearCacheCluster(cache *utils.Cache) {
func (s *LocalCacheStore) Invalidate() {
s.doClearCacheCluster(s.reactionCache)
s.doClearCacheCluster(s.emojiCacheById)
s.doClearCacheCluster(s.emojiIdCacheByName)
s.doClearCacheCluster(s.channelMemberCountsCache)
}

9
store/localcachelayer/main_test.go
Просмотреть файл

@@ -41,6 +41,15 @@ func getMockStore() *mocks.Store {
mockSchemesStore.On("PermanentDeleteAll").Return(nil)
mockStore.On("Scheme").Return(&mockSchemesStore)
fakeEmoji := model.Emoji{Id: "123", Name: "name123"}
mockEmojiStore := mocks.EmojiStore{}
mockEmojiStore.On("Get", "123", true).Return(&fakeEmoji, nil)
mockEmojiStore.On("Get", "123", false).Return(&fakeEmoji, nil)
mockEmojiStore.On("GetByName", "name123", true).Return(&fakeEmoji, nil)
mockEmojiStore.On("GetByName", "name123", false).Return(&fakeEmoji, nil)
mockEmojiStore.On("Delete", &fakeEmoji, int64(0)).Return(nil)
mockStore.On("Emoji").Return(&mockEmojiStore)
mockCount := int64(10)
mockChannelStore := mocks.ChannelStore{}
mockChannelStore.On("ClearCaches").Return()

73
store/sqlstore/emoji_store.go
Просмотреть файл

@@ -11,17 +11,8 @@ import (
"github.com/mattermost/mattermost-server/einterfaces"
"github.com/mattermost/mattermost-server/model"
"github.com/mattermost/mattermost-server/store"
"github.com/mattermost/mattermost-server/utils"
)
const (
EMOJI_CACHE_SIZE = 5000
EMOJI_CACHE_SEC = 1800 // 30 mins
)
var emojiCacheById = utils.NewLru(EMOJI_CACHE_SIZE)
var emojiIdCacheByName = utils.NewLru(EMOJI_CACHE_SIZE)
type SqlEmojiStore struct {
SqlStore
metrics einterfaces.MetricsInterface
@@ -66,26 +57,10 @@ func (es SqlEmojiStore) Save(emoji *model.Emoji) (*model.Emoji, *model.AppError)
}
func (es SqlEmojiStore) Get(id string, allowFromCache bool) (*model.Emoji, *model.AppError) {
if allowFromCache {
if emoji, ok := es.getFromCacheById(id); ok {
return emoji, nil
}
}
return es.getBy("Id", id, allowFromCache)
}
func (es SqlEmojiStore) GetByName(name string, allowFromCache bool) (*model.Emoji, *model.AppError) {
if id, ok := model.GetSystemEmojiId(name); ok {
return es.Get(id, allowFromCache)
}
if allowFromCache {
if emoji, ok := es.getFromCacheByName(name); ok {
return emoji, nil
}
}
return es.getBy("Name", name, allowFromCache)
}
@@ -139,8 +114,6 @@ func (es SqlEmojiStore) Delete(emoji *model.Emoji, time int64) *model.AppError {
return model.NewAppError("SqlEmojiStore.Delete", "store.sql_emoji.delete.no_results", nil, "id="+emoji.Id, http.StatusBadRequest)
}
es.removeFromCache(emoji)
return nil
}
@@ -193,51 +166,5 @@ func (es SqlEmojiStore) getBy(what string, key interface{}, addToCache bool) (*m
return nil, model.NewAppError("SqlEmojiStore.GetByName", "store.sql_emoji.get.app_error", nil, "key="+fmt.Sprintf("%v", key)+", "+err.Error(), status)
}
if addToCache {
es.addToCache(emoji)
}
return emoji, nil
}
func (es SqlEmojiStore) addToCache(emoji *model.Emoji) {
emojiCacheById.AddWithExpiresInSecs(emoji.Id, emoji, EMOJI_CACHE_SEC)
emojiIdCacheByName.AddWithExpiresInSecs(emoji.Name, emoji.Id, EMOJI_CACHE_SEC)
}
func (es SqlEmojiStore) getFromCacheById(id string) (*model.Emoji, bool) {
if cacheItem, ok := emojiCacheById.Get(id); ok {
es.incrementMemCacheHitCounter("Emoji")
return cacheItem.(*model.Emoji), true
}
es.incrementMemCacheMissCounter("Emoji")
return nil, false
}
func (es SqlEmojiStore) getFromCacheByName(name string) (*model.Emoji, bool) {
if id, ok := emojiIdCacheByName.Get(name); ok {
return es.getFromCacheById(id.(string))
}
es.incrementMemCacheMissCounter("Emoji")
return nil, false
}
func (es SqlEmojiStore) incrementMemCacheHitCounter(cache string) {
if es.metrics == nil {
return
}
es.metrics.IncrementMemCacheHitCounter(cache)
}
func (es SqlEmojiStore) incrementMemCacheMissCounter(cache string) {
if es.metrics == nil {
return
}
es.metrics.IncrementMemCacheMissCounter(cache)
}
func (es SqlEmojiStore) removeFromCache(emoji *model.Emoji) {
emojiCacheById.Remove(emoji.Id)
emojiIdCacheByName.Remove(emoji.Name)
}

56
store/storetest/emoji_store.go
Просмотреть файл

@@ -21,7 +21,6 @@ func TestEmojiStore(t *testing.T, ss store.Store) {
t.Run("EmojiGetMultipleByName", func(t *testing.T) { testEmojiGetMultipleByName(t, ss) })
t.Run("EmojiGetList", func(t *testing.T) { testEmojiGetList(t, ss) })
t.Run("EmojiSearch", func(t *testing.T) { testEmojiSearch(t, ss) })
t.Run("EmojiCaching", func(t *testing.T) { testEmojiCaching(t, ss) })
}
func testEmojiSaveDelete(t *testing.T, ss store.Store) {
@@ -91,61 +90,6 @@ func testEmojiGet(t *testing.T, ss store.Store) {
}
}
func testEmojiCaching(t *testing.T, ss store.Store) {
emojis := make([]*model.Emoji, 3)
for i := range emojis {
emojis[i] = &model.Emoji{
CreatorId: model.NewId(),
Name: model.NewId(),
}
}
for _, emoji := range emojis {
_, err := ss.Emoji().Save(emoji)
require.Nil(t, err)
}
defer func() {
for _, emoji := range emojis {
err := ss.Emoji().Delete(emoji, time.Now().Unix())
require.Nil(t, err)
}
}()
var retrievedEmoji *model.Emoji
var cachedEmoji *model.Emoji
var err *model.AppError
for _, emoji := range emojis {
cachedEmoji, err = ss.Emoji().Get(emoji.Id, true)
assert.Nilf(t, err, "should be able to retrieve emoji with id %v", emoji.Id)
retrievedEmoji, err = ss.Emoji().Get(emoji.Id, false)
if assert.Nilf(t, err, "should be able to retrieve emoji with id %v", emoji.Id) {
assert.Falsef(t, retrievedEmoji == cachedEmoji, "should not be the same as cached with id %v", emoji.Id)
}
retrievedEmoji, err = ss.Emoji().Get(emoji.Id, true)
if assert.Nilf(t, err, "should be able to retrieve emoji with id %v", emoji.Id) {
assert.Truef(t, retrievedEmoji == cachedEmoji, "should be the cached emoji with id %v", emoji.Id)
}
retrievedEmoji, err = ss.Emoji().GetByName(emoji.Name, false)
if assert.Nilf(t, err, "should be able to retrieve emoji with name %v", emoji.Name) {
assert.Falsef(t, retrievedEmoji == cachedEmoji, "should not be the same as cached with name %v", emoji.Name)
}
retrievedEmoji, _ = ss.Emoji().GetByName(emoji.Name, true)
if assert.Nilf(t, err, "should be able to retrieve emoji with name %v", emoji.Name) {
assert.Truef(t, retrievedEmoji == cachedEmoji, "should be the cached emoji with name %v", emoji.Name)
}
}
_, err = ss.Emoji().Get(model.NewId(), false)
assert.NotNilf(t, err, "should not retrieve emoji with unsaved ID")
_, err = ss.Emoji().GetByName(model.NewId(), false)
assert.NotNilf(t, err, "should not retrieve emoji with unsaved name")
}
func testEmojiGetByName(t *testing.T, ss store.Store) {
emojis := []model.Emoji{
{