Merge 3.6.2 into master (#5211)

* Add webhook cache * Add channel by name cache * Fxing profiles in channels cache * Fix merge
2017-01-27 14:07:34 -05:00
--- a/store/sql_channel_store.go
+++ b/store/sql_channel_store.go
@@ -38,11 +38,13 @@ type SqlChannelStore struct {
 var channelMemberCountsCache = utils.NewLru(CHANNEL_MEMBERS_COUNTS_CACHE_SIZE)
 var allChannelMembersForUserCache = utils.NewLru(ALL_CHANNEL_MEMBERS_FOR_USER_CACHE_SIZE)
 var channelCache = utils.NewLru(CHANNEL_CACHE_SIZE)
+var channelByNameCache = utils.NewLru(CHANNEL_CACHE_SIZE)

 func ClearChannelCaches() {
 	channelMemberCountsCache.Purge()
 	allChannelMembersForUserCache.Purge()
 	channelCache.Purge()
+	channelByNameCache.Purge()
 }

 func NewSqlChannelStore(sqlStore *SqlStore) ChannelStore {
@@ -306,6 +308,10 @@ func (us SqlChannelStore) InvalidateChannel(id string) {
 	channelCache.Remove(id)
 }

+func (us SqlChannelStore) InvalidateChannelByName(teamId, name string) {
+	channelCache.Remove(teamId + name)
+}
+
 func (s SqlChannelStore) Get(id string, allowFromCache bool) StoreChannel {
 	return s.get(id, false, allowFromCache)
 }
@@ -539,15 +545,15 @@ func (s SqlChannelStore) GetTeamChannels(teamId string) StoreChannel {
 	return storeChannel
 }

-func (s SqlChannelStore) GetByName(teamId string, name string) StoreChannel {
-	return s.getByName(teamId, name, false)
+func (s SqlChannelStore) GetByName(teamId string, name string, allowFromCache bool) StoreChannel {
+	return s.getByName(teamId, name, false, allowFromCache)
 }

-func (s SqlChannelStore) GetByNameIncludeDeleted(teamId string, name string) StoreChannel {
-	return s.getByName(teamId, name, true)
+func (s SqlChannelStore) GetByNameIncludeDeleted(teamId string, name string, allowFromCache bool) StoreChannel {
+	return s.getByName(teamId, name, true, allowFromCache)
 }

-func (s SqlChannelStore) getByName(teamId string, name string, includeDeleted bool) StoreChannel {
+func (s SqlChannelStore) getByName(teamId string, name string, includeDeleted bool, allowFromCache bool) StoreChannel {
 	storeChannel := make(StoreChannel, 1)

 	var query string
@@ -562,6 +568,23 @@ func (s SqlChannelStore) getByName(teamId string, name string, includeDeleted bo

 		channel := model.Channel{}

+		if allowFromCache {
+			metrics := einterfaces.GetMetricsInterface()
+			if cacheItem, ok := channelByNameCache.Get(teamId + name); ok {
+				if metrics != nil {
+					metrics.IncrementMemCacheHitCounter("Channel By Name")
+				}
+				result.Data = cacheItem.(*model.Channel)
+				storeChannel <- result
+				close(storeChannel)
+				return
+			} else {
+				if metrics != nil {
+					metrics.IncrementMemCacheMissCounter("Channel By Name")
+				}
+			}
+		}
+
 		if err := s.GetReplica().SelectOne(&channel, query, map[string]interface{}{"TeamId": teamId, "Name": name}); err != nil {
 			if err == sql.ErrNoRows {
 				result.Err = model.NewLocAppError("SqlChannelStore.GetByName", MISSING_CHANNEL_ERROR, nil, "teamId="+teamId+", "+"name="+name+", "+err.Error())
@@ -570,6 +593,7 @@ func (s SqlChannelStore) getByName(teamId string, name string, includeDeleted bo
 			}
 		} else {
 			result.Data = &channel
+			channelByNameCache.AddWithExpiresInSecs(teamId+name, &channel, CHANNEL_CACHE_SEC)
 		}

 		storeChannel <- result
--- a/store/sql_channel_store_test.go
+++ b/store/sql_channel_store_test.go
@@ -352,7 +352,7 @@ func TestChannelStoreGetByName(t *testing.T) {
 	o1.Type = model.CHANNEL_OPEN
 	Must(store.Channel().Save(&o1))

-	r1 := <-store.Channel().GetByName(o1.TeamId, o1.Name)
+	r1 := <-store.Channel().GetByName(o1.TeamId, o1.Name, true)
 	if r1.Err != nil {
 		t.Fatal(r1.Err)
 	} else {
@@ -361,13 +361,25 @@ func TestChannelStoreGetByName(t *testing.T) {
 		}
 	}

-	if err := (<-store.Channel().GetByName(o1.TeamId, "")).Err; err == nil {
+	if err := (<-store.Channel().GetByName(o1.TeamId, "", true)).Err; err == nil {
+		t.Fatal("Missing id should have failed")
+	}
+
+	if r1 := <-store.Channel().GetByName(o1.TeamId, o1.Name, false); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		if r1.Data.(*model.Channel).ToJson() != o1.ToJson() {
+			t.Fatal("invalid returned channel")
+		}
+	}
+
+	if err := (<-store.Channel().GetByName(o1.TeamId, "", false)).Err; err == nil {
 		t.Fatal("Missing id should have failed")
 	}

 	Must(store.Channel().Delete(r1.Data.(*model.Channel).Id, model.GetMillis()))

-	if err := (<-store.Channel().GetByName(o1.TeamId, "")).Err; err == nil {
+	if err := (<-store.Channel().GetByName(o1.TeamId, "", false)).Err; err == nil {
 		t.Fatal("Deleted channel should not be returned by GetByName()")
 	}
 }
--- a/store/sql_webhook_store.go
+++ b/store/sql_webhook_store.go
@@ -4,13 +4,26 @@
 package store

 import (
+	"github.com/mattermost/platform/einterfaces"
 	"github.com/mattermost/platform/model"
+	"github.com/mattermost/platform/utils"
 )

 type SqlWebhookStore struct {
 	*SqlStore
 }

+const (
+	WEBHOOK_CACHE_SIZE = 25000
+	WEBHOOK_CACHE_SEC  = 900 // 15 minutes
+)
+
+var webhookCache = utils.NewLru(WEBHOOK_CACHE_SIZE)
+
+func ClearWebhookCaches() {
+	webhookCache.Purge()
+}
+
 func NewSqlWebhookStore(sqlStore *SqlStore) WebhookStore {
 	s := &SqlWebhookStore{sqlStore}

@@ -54,6 +67,10 @@ func (s SqlWebhookStore) CreateIndexesIfNotExists() {
 	s.CreateIndexIfNotExists("idx_outgoing_webhook_delete_at", "OutgoingWebhooks", "DeleteAt")
 }

+func (s SqlWebhookStore) InvalidateWebhookCache(webhookId string) {
+	webhookCache.Remove(webhookId)
+}
+
 func (s SqlWebhookStore) SaveIncoming(webhook *model.IncomingWebhook) StoreChannel {
 	storeChannel := make(StoreChannel, 1)

@@ -88,18 +105,39 @@ func (s SqlWebhookStore) SaveIncoming(webhook *model.IncomingWebhook) StoreChann
 	return storeChannel
 }

-func (s SqlWebhookStore) GetIncoming(id string) StoreChannel {
+func (s SqlWebhookStore) GetIncoming(id string, allowFromCache bool) StoreChannel {
 	storeChannel := make(StoreChannel, 1)

 	go func() {
 		result := StoreResult{}

+		if allowFromCache {
+			metrics := einterfaces.GetMetricsInterface()
+			if cacheItem, ok := webhookCache.Get(id); ok {
+				if metrics != nil {
+					metrics.IncrementMemCacheHitCounter("Webhook")
+				}
+				result.Data = cacheItem.(*model.IncomingWebhook)
+				storeChannel <- result
+				close(storeChannel)
+				return
+			} else {
+				if metrics != nil {
+					metrics.IncrementMemCacheMissCounter("Webhook")
+				}
+			}
+		}
+
 		var webhook model.IncomingWebhook

 		if err := s.GetReplica().SelectOne(&webhook, "SELECT * FROM IncomingWebhooks WHERE Id = :Id AND DeleteAt = 0", map[string]interface{}{"Id": id}); err != nil {
 			result.Err = model.NewLocAppError("SqlWebhookStore.GetIncoming", "store.sql_webhooks.get_incoming.app_error", nil, "id="+id+", err="+err.Error())
 		}

+		if result.Err == nil {
+			webhookCache.AddWithExpiresInSecs(id, &webhook, WEBHOOK_CACHE_SEC)
+		}
+
 		result.Data = &webhook

 		storeChannel <- result
--- a/store/sql_webhook_store_test.go
+++ b/store/sql_webhook_store_test.go
@@ -35,7 +35,7 @@ func TestWebhookStoreGetIncoming(t *testing.T) {

 	o1 = (<-store.Webhook().SaveIncoming(o1)).Data.(*model.IncomingWebhook)

-	if r1 := <-store.Webhook().GetIncoming(o1.Id); r1.Err != nil {
+	if r1 := <-store.Webhook().GetIncoming(o1.Id, false); r1.Err != nil {
 		t.Fatal(r1.Err)
 	} else {
 		if r1.Data.(*model.IncomingWebhook).CreateAt != o1.CreateAt {
@@ -43,7 +43,19 @@ func TestWebhookStoreGetIncoming(t *testing.T) {
 		}
 	}

-	if err := (<-store.Webhook().GetIncoming("123")).Err; err == nil {
+	if r1 := <-store.Webhook().GetIncoming(o1.Id, true); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		if r1.Data.(*model.IncomingWebhook).CreateAt != o1.CreateAt {
+			t.Fatal("invalid returned webhook")
+		}
+	}
+
+	if err := (<-store.Webhook().GetIncoming("123", false)).Err; err == nil {
+		t.Fatal("Missing id should have failed")
+	}
+
+	if err := (<-store.Webhook().GetIncoming("123", true)).Err; err == nil {
 		t.Fatal("Missing id should have failed")
 	}
 }
@@ -85,7 +97,7 @@ func TestWebhookStoreDeleteIncoming(t *testing.T) {

 	o1 = (<-store.Webhook().SaveIncoming(o1)).Data.(*model.IncomingWebhook)

-	if r1 := <-store.Webhook().GetIncoming(o1.Id); r1.Err != nil {
+	if r1 := <-store.Webhook().GetIncoming(o1.Id, true); r1.Err != nil {
 		t.Fatal(r1.Err)
 	} else {
 		if r1.Data.(*model.IncomingWebhook).CreateAt != o1.CreateAt {
@@ -97,7 +109,9 @@ func TestWebhookStoreDeleteIncoming(t *testing.T) {
 		t.Fatal(r2.Err)
 	}

-	if r3 := (<-store.Webhook().GetIncoming(o1.Id)); r3.Err == nil {
+	ClearWebhookCaches()
+
+	if r3 := (<-store.Webhook().GetIncoming(o1.Id, true)); r3.Err == nil {
 		t.Log(r3.Data)
 		t.Fatal("Missing id should have failed")
 	}
@@ -113,7 +127,7 @@ func TestWebhookStoreDeleteIncomingByUser(t *testing.T) {

 	o1 = (<-store.Webhook().SaveIncoming(o1)).Data.(*model.IncomingWebhook)

-	if r1 := <-store.Webhook().GetIncoming(o1.Id); r1.Err != nil {
+	if r1 := <-store.Webhook().GetIncoming(o1.Id, true); r1.Err != nil {
 		t.Fatal(r1.Err)
 	} else {
 		if r1.Data.(*model.IncomingWebhook).CreateAt != o1.CreateAt {
@@ -125,7 +139,9 @@ func TestWebhookStoreDeleteIncomingByUser(t *testing.T) {
 		t.Fatal(r2.Err)
 	}

-	if r3 := (<-store.Webhook().GetIncoming(o1.Id)); r3.Err == nil {
+	ClearWebhookCaches()
+
+	if r3 := (<-store.Webhook().GetIncoming(o1.Id, true)); r3.Err == nil {
 		t.Log(r3.Data)
 		t.Fatal("Missing id should have failed")
 	}
--- a/store/store.go
+++ b/store/store.go
@@ -87,12 +87,13 @@ type ChannelStore interface {
 	Update(channel *model.Channel) StoreChannel
 	Get(id string, allowFromCache bool) StoreChannel
 	InvalidateChannel(id string)
+	InvalidateChannelByName(teamId, name string)
 	GetFromMaster(id string) StoreChannel
 	Delete(channelId string, time int64) StoreChannel
 	SetDeleteAt(channelId string, deleteAt int64, updateAt int64) StoreChannel
 	PermanentDeleteByTeam(teamId string) StoreChannel
-	GetByName(team_id string, name string) StoreChannel
-	GetByNameIncludeDeleted(team_id string, name string) StoreChannel
+	GetByName(team_id string, name string, allowFromCache bool) StoreChannel
+	GetByNameIncludeDeleted(team_id string, name string, allowFromCache bool) StoreChannel
 	GetDeletedByName(team_id string, name string) StoreChannel
 	GetChannels(teamId string, userId string) StoreChannel
 	GetMoreChannels(teamId string, userId string, offset int, limit int) StoreChannel
@@ -245,7 +246,7 @@ type SystemStore interface {

 type WebhookStore interface {
 	SaveIncoming(webhook *model.IncomingWebhook) StoreChannel
-	GetIncoming(id string) StoreChannel
+	GetIncoming(id string, allowFromCache bool) StoreChannel
 	GetIncomingByTeam(teamId string) StoreChannel
 	GetIncomingByChannel(channelId string) StoreChannel
 	DeleteIncoming(webhookId string, time int64) StoreChannel
@@ -259,6 +260,7 @@ type WebhookStore interface {
 	UpdateOutgoing(hook *model.OutgoingWebhook) StoreChannel
 	AnalyticsIncomingCount(teamId string) StoreChannel
 	AnalyticsOutgoingCount(teamId string) StoreChannel
+	InvalidateWebhookCache(webhook string)
 }

 type CommandStore interface {