Add store unit tests and add make target for testing store with postgres (#5925)

* Add store unit tests and add make target for testing store with postgres

* Remove postgres target form test-server target

* Fix audit test

Makefile

@@ -206,6 +206,22 @@ test-te: start-docker prepare-enterprise
 		fi; \
 	done
 
+test-postgres: start-docker prepare-enterprise
+	@echo Testing Postgres
+
+	@sed -i'' -e 's|"DriverName": "mysql"|"DriverName": "postgres"|g' config/config.json
+	@sed -i'' -e 's|"DataSource": "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8"|"DataSource": "postgres://mmuser:mostest@dockerhost:5432?sslmode=disable"|g' config/config.json
+
+	$(GO) test $(GOFLAGS) -run=$(TESTS) -test.v -test.timeout=2000s -covermode=count -coverprofile=cprofile.out -coverpkg=$(ALL_PACKAGES_COMMA) github.com/mattermost/platform/store || exit 1; \
+	if [ -f cprofile.out ]; then \
+		tail -n +2 cprofile.out >> cover.out; \
+		rm cprofile.out; \
+	fi; \
+
+	@sed -i'' -e 's|"DataSource": "postgres://mmuser:mostest@dockerhost:5432?sslmode=disable"|"DataSource": "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8"|g' config/config.json
+	@sed -i'' -e 's|"DriverName": "postgres"|"DriverName": "mysql"|g' config/config.json
+	@rm config/config.json-e
+
 test-ee: start-docker prepare-enterprise
 	@echo Testing EE
 

store/sql_audit_store_test.go

@@ -50,7 +50,7 @@ func TestSqlAuditStore(t *testing.T) {
 	result = <-c
 	audits = result.Data.(model.Audits)
 
-	if len(audits) <= 4 {
+	if len(audits) < 4 {
 		t.Fatal("Failed to save and retrieve 4 audit logs")
 	}
 

store/sql_channel_store.go

@@ -1257,58 +1257,6 @@ func (s SqlChannelStore) PermanentDeleteMembersByUser(userId string) StoreChanne
 	return storeChannel
 }
 
-func (s SqlChannelStore) SetLastViewedAt(channelId string, userId string, newLastViewedAt int64) StoreChannel {
-	storeChannel := make(StoreChannel, 1)
-
-	go func() {
-		result := StoreResult{}
-
-		var query string
-
-		if utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_POSTGRES {
-			query = `UPDATE
-				ChannelMembers
-			SET
-			    MentionCount = 0,
-			    MsgCount = Channels.TotalMsgCount - (SELECT COUNT(*)
-			    					 FROM Posts
-			    					 WHERE ChannelId = :ChannelId
-			    					 AND CreateAt > :NewLastViewedAt),
-			    LastViewedAt = :NewLastViewedAt
-			FROM
-				Channels
-			WHERE
-			    Channels.Id = ChannelMembers.ChannelId
-			        AND UserId = :UserId
-			        AND ChannelId = :ChannelId`
-		} else if utils.Cfg.SqlSettings.DriverName == model.DATABASE_DRIVER_MYSQL {
-			query = `UPDATE
-				ChannelMembers, Channels
-			SET
-			    ChannelMembers.MentionCount = 0,
-			    ChannelMembers.MsgCount = Channels.TotalMsgCount - (SELECT COUNT(*)
-										FROM Posts
-										WHERE ChannelId = :ChannelId
-										AND CreateAt > :NewLastViewedAt),
-			    ChannelMembers.LastViewedAt = :NewLastViewedAt
-			WHERE
-			    Channels.Id = ChannelMembers.ChannelId
-			        AND UserId = :UserId
-			        AND ChannelId = :ChannelId`
-		}
-
-		_, err := s.GetMaster().Exec(query, map[string]interface{}{"ChannelId": channelId, "UserId": userId, "NewLastViewedAt": newLastViewedAt})
-		if err != nil {
-			result.Err = model.NewLocAppError("SqlChannelStore.SetLastViewedAt", "store.sql_channel.set_last_viewed_at.app_error", nil, "channel_id="+channelId+", user_id="+userId+", "+err.Error())
-		}
-
-		storeChannel <- result
-		close(storeChannel)
-	}()
-
-	return storeChannel
-}
-
 func (s SqlChannelStore) UpdateLastViewedAt(channelIds []string, userId string) StoreChannel {
 	storeChannel := make(StoreChannel, 1)
 

store/sql_channel_store_test.go

@@ -149,10 +149,14 @@ func TestChannelStoreUpdate(t *testing.T) {
 	o1.DisplayName = "Name"
 	o1.Name = "a" + model.NewId() + "b"
 	o1.Type = model.CHANNEL_OPEN
+	Must(store.Channel().Save(&o1))
 
-	if err := (<-store.Channel().Save(&o1)).Err; err != nil {
+	o2 := model.Channel{}
-		t.Fatal(err)
+	o2.TeamId = o1.TeamId
-	}
+	o2.DisplayName = "Name"
+	o2.Name = "a" + model.NewId() + "b"
+	o2.Type = model.CHANNEL_OPEN
+	Must(store.Channel().Save(&o2))
 
 	time.Sleep(100 * time.Millisecond)
 
@@ -169,6 +173,11 @@ func TestChannelStoreUpdate(t *testing.T) {
 	if err := (<-store.Channel().Update(&o1)).Err; err == nil {
 		t.Fatal("Update should have faile because id change")
 	}
+
+	o2.Name = o1.Name
+	if err := (<-store.Channel().Update(&o2)).Err; err == nil {
+		t.Fatal("Update should have failed because of existing name")
+	}
 }
 
 func TestGetChannelUnread(t *testing.T) {
@@ -321,6 +330,15 @@ func TestChannelStoreGet(t *testing.T) {
 			t.Fatal("too little")
 		}
 	}
+
+	if r3 := <-store.Channel().GetTeamChannels(o1.TeamId); r3.Err != nil {
+		t.Fatal(r3.Err)
+	} else {
+		channels := r3.Data.(*model.ChannelList)
+		if len(*channels) == 0 {
+			t.Fatal("too little")
+		}
+	}
 }
 
 func TestChannelStoreGetForPost(t *testing.T) {
@@ -422,6 +440,10 @@ func TestChannelStoreDelete(t *testing.T) {
 	if cresult.Err.Id != "store.sql_channel.get_channels.not_found.app_error" {
 		t.Fatal("no channels should be found")
 	}
+
+	if r := <-store.Channel().PermanentDeleteByTeam(o1.TeamId); r.Err != nil {
+		t.Fatal(r.Err)
+	}
 }
 
 func TestChannelStoreGetByName(t *testing.T) {
@@ -1173,6 +1195,37 @@ func TestChannelStoreIncrementMentionCount(t *testing.T) {
 	}
 }
 
+func TestUpdateChannelMember(t *testing.T) {
+	Setup()
+
+	userId := model.NewId()
+
+	c1 := &model.Channel{
+		TeamId:      model.NewId(),
+		DisplayName: model.NewId(),
+		Name:        model.NewId(),
+		Type:        model.CHANNEL_OPEN,
+	}
+	Must(store.Channel().Save(c1))
+
+	m1 := &model.ChannelMember{
+		ChannelId:   c1.Id,
+		UserId:      userId,
+		NotifyProps: model.GetDefaultChannelNotifyProps(),
+	}
+	Must(store.Channel().SaveMember(m1))
+
+	m1.NotifyProps["test"] = "sometext"
+	if result := <-store.Channel().UpdateMember(m1); result.Err != nil {
+		t.Fatal(result.Err)
+	}
+
+	m1.UserId = ""
+	if result := <-store.Channel().UpdateMember(m1); result.Err == nil {
+		t.Fatal("bad user id - should fail")
+	}
+}
+
 func TestGetMember(t *testing.T) {
 	Setup()
 
@@ -1231,6 +1284,26 @@ func TestGetMember(t *testing.T) {
 	} else if member.UserId != userId {
 		t.Fatal("should've gotten member for user")
 	}
+
+	if result := <-store.Channel().GetAllChannelMembersNotifyPropsForChannel(c2.Id, false); result.Err != nil {
+		t.Fatal(result.Err)
+	} else {
+		props := result.Data.(map[string]model.StringMap)
+		if len(props) == 0 {
+			t.Fatal("should not be empty")
+		}
+	}
+
+	if result := <-store.Channel().GetAllChannelMembersNotifyPropsForChannel(c2.Id, true); result.Err != nil {
+		t.Fatal(result.Err)
+	} else {
+		props := result.Data.(map[string]model.StringMap)
+		if len(props) == 0 {
+			t.Fatal("should not be empty")
+		}
+	}
+
+	store.Channel().InvalidateCacheForChannelMembersNotifyProps(c2.Id)
 }
 
 func TestChannelStoreGetMemberForPost(t *testing.T) {

store/sql_store_test.go

@@ -25,12 +25,20 @@ func Setup() {
 }
 
 func TestSqlStore1(t *testing.T) {
+	utils.TranslationsPreInit()
 	utils.LoadConfig("config.json")
 	utils.Cfg.SqlSettings.Trace = true
 
 	store := NewSqlStore()
 	store.Close()
 
+	utils.Cfg.SqlSettings.DataSourceReplicas = []string{utils.Cfg.SqlSettings.DataSource}
+
+	store = NewSqlStore()
+	store.TotalMasterDbConnections()
+	store.TotalReadDbConnections()
+	store.Close()
+
 	utils.LoadConfig("config.json")
 }
 

store/sql_system_store.go

@@ -34,7 +34,7 @@ func (s SqlSystemStore) Save(system *model.System) StoreChannel {
 		result := StoreResult{}
 
 		if err := s.GetMaster().Insert(system); err != nil {
-			result.Err = model.NewLocAppError("SqlSystemStore.Save", "store.sql_system.save.app_error", nil, "")
+			result.Err = model.NewLocAppError("SqlSystemStore.Save", "store.sql_system.save.app_error", nil, err.Error())
 		}
 
 		storeChannel <- result

store/sql_upgrade.go

@@ -47,7 +47,7 @@ func UpgradeDatabase(sqlStore *SqlStore) {
 	// If the SchemaVersion is empty this this is the first time it has ran
 	// so lets set it to the current version.
 	if sqlStore.SchemaVersion == "" {
-		if result := <-sqlStore.system.Save(&model.System{Name: "Version", Value: model.CurrentVersion}); result.Err != nil {
+		if result := <-sqlStore.system.SaveOrUpdate(&model.System{Name: "Version", Value: model.CurrentVersion}); result.Err != nil {
 			l4g.Critical(result.Err.Error())
 			time.Sleep(time.Second)
 			os.Exit(EXIT_VERSION_SAVE_MISSING)
@@ -66,7 +66,7 @@ func UpgradeDatabase(sqlStore *SqlStore) {
 }
 
 func saveSchemaVersion(sqlStore *SqlStore, version string) {
-	if result := <-sqlStore.system.Update(&model.System{Name: "Version", Value: model.CurrentVersion}); result.Err != nil {
+	if result := <-sqlStore.system.Update(&model.System{Name: "Version", Value: version}); result.Err != nil {
 		l4g.Critical(result.Err.Error())
 		time.Sleep(time.Second)
 		os.Exit(EXIT_VERSION_SAVE)

store/sql_upgrade_test.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2017 Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package store
+
+import (
+	"testing"
+
+	"github.com/mattermost/platform/model"
+)
+
+func TestStoreUpgrade(t *testing.T) {
+	Setup()
+
+	saveSchemaVersion(store.(*SqlStore), VERSION_3_0_0)
+	UpgradeDatabase(store.(*SqlStore))
+
+	store.(*SqlStore).SchemaVersion = ""
+	UpgradeDatabase(store.(*SqlStore))
+}
+
+func TestSaveSchemaVersion(t *testing.T) {
+	Setup()
+
+	saveSchemaVersion(store.(*SqlStore), VERSION_3_0_0)
+	if result := <-store.System().Get(); result.Err != nil {
+		t.Fatal(result.Err)
+	} else {
+		props := result.Data.(model.StringMap)
+		if props["Version"] != VERSION_3_0_0 {
+			t.Fatal("version not updated")
+		}
+	}
+
+	if store.(*SqlStore).SchemaVersion != VERSION_3_0_0 {
+		t.Fatal("version not updated")
+	}
+
+	saveSchemaVersion(store.(*SqlStore), model.CurrentVersion)
+}

store/sql_user_store.go

@@ -4,7 +4,6 @@
 package store
 
 import (
-	"crypto/md5"
 	"database/sql"
 	"fmt"
 	"net/http"
@@ -426,58 +425,6 @@ func (us SqlUserStore) GetAll() StoreChannel {
 	return storeChannel
 }
 
-func (s SqlUserStore) GetEtagForDirectProfiles(userId string) StoreChannel {
-	storeChannel := make(StoreChannel, 1)
-
-	go func() {
-		result := StoreResult{}
-
-		var ids []string
-		_, err := s.GetReplica().Select(ids, `
-			SELECT
-			    Id
-			FROM
-			    Users
-			WHERE
-			    Id IN (SELECT DISTINCT
-			            UserId
-			        FROM
-			            ChannelMembers
-			        WHERE
-			            ChannelMembers.UserId != :UserId
-			                AND ChannelMembers.ChannelId IN (SELECT 
-			                    Channels.Id
-			                FROM
-			                    Channels,
-			                    ChannelMembers
-			                WHERE
-			                    Channels.Type = 'D'
-			                        AND Channels.Id = ChannelMembers.ChannelId
-			                        AND ChannelMembers.UserId = :UserId))
-			        OR Id IN (SELECT
-			            Name
-			        FROM
-			            Preferences
-			        WHERE
-			            UserId = :UserId
-			                AND Category = 'direct_channel_show')
-			ORDER BY UpdateAt DESC
-        `, map[string]interface{}{"UserId": userId})
-
-		if err != nil || len(ids) == 0 {
-			result.Data = fmt.Sprintf("%v.%v.0.%v.%v", model.CurrentVersion, model.GetMillis(), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
-		} else {
-			allIds := strings.Join(ids, "")
-			result.Data = fmt.Sprintf("%v.%x.%v.%v.%v", model.CurrentVersion, md5.Sum([]byte(allIds)), len(ids), utils.Cfg.PrivacySettings.ShowFullName, utils.Cfg.PrivacySettings.ShowEmailAddress)
-		}
-
-		storeChannel <- result
-		close(storeChannel)
-	}()
-
-	return storeChannel
-}
-
 func (s SqlUserStore) GetEtagForAllProfiles() StoreChannel {
 	storeChannel := make(StoreChannel, 1)
 

store/sql_user_store_test.go

@@ -130,6 +130,10 @@ func TestUserStoreUpdate(t *testing.T) {
 			t.Fatal("Email should have been updated as the update is trusted")
 		}
 	}
+
+	if result := <-store.User().UpdateLastPictureUpdate(u1.Id); result.Err != nil {
+		t.Fatal("Update should not have failed")
+	}
 }
 
 func TestUserStoreUpdateUpdateAt(t *testing.T) {
@@ -217,20 +221,39 @@ func TestUserCount(t *testing.T) {
 	}
 }
 
-func TestUserStoreGetAllProfiles(t *testing.T) {
+func TestGetAllUsingAuthService(t *testing.T) {
 	Setup()
 
-	teamId := model.NewId()
+	u1 := &model.User{}
+	u1.Email = model.NewId()
+	u1.AuthService = "someservice"
+	Must(store.User().Save(u1))
+
+	u2 := &model.User{}
+	u2.Email = model.NewId()
+	u2.AuthService = "someservice"
+	Must(store.User().Save(u2))
+
+	if r1 := <-store.User().GetAllUsingAuthService(u1.AuthService); r1.Err != nil {
+		t.Fatal(r1.Err)
+	} else {
+		users := r1.Data.([]*model.User)
+		if len(users) < 2 {
+			t.Fatal("invalid returned users")
+		}
+	}
+}
+
+func TestUserStoreGetAllProfiles(t *testing.T) {
+	Setup()
 
 	u1 := &model.User{}
 	u1.Email = model.NewId()
 	Must(store.User().Save(u1))
-	Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}))
 
 	u2 := &model.User{}
 	u2.Email = model.NewId()
 	Must(store.User().Save(u2))
-	Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}))
 
 	if r1 := <-store.User().GetAllProfiles(0, 100); r1.Err != nil {
 		t.Fatal(r1.Err)
@@ -249,6 +272,34 @@ func TestUserStoreGetAllProfiles(t *testing.T) {
 			t.Fatal("invalid returned users, limit did not work")
 		}
 	}
+
+	if r2 := <-store.User().GetAll(); r2.Err != nil {
+		t.Fatal(r2.Err)
+	} else {
+		users := r2.Data.([]*model.User)
+		if len(users) < 2 {
+			t.Fatal("invalid returned users")
+		}
+	}
+
+	etag := ""
+	if r2 := <-store.User().GetEtagForAllProfiles(); r2.Err != nil {
+		t.Fatal(r2.Err)
+	} else {
+		etag = r2.Data.(string)
+	}
+
+	u3 := &model.User{}
+	u3.Email = model.NewId()
+	Must(store.User().Save(u3))
+
+	if r2 := <-store.User().GetEtagForAllProfiles(); r2.Err != nil {
+		t.Fatal(r2.Err)
+	} else {
+		if etag == r2.Data.(string) {
+			t.Fatal("etags should not match")
+		}
+	}
 }
 
 func TestUserStoreGetProfiles(t *testing.T) {
@@ -293,6 +344,26 @@ func TestUserStoreGetProfiles(t *testing.T) {
 			t.Fatal("should have returned empty map")
 		}
 	}
+
+	etag := ""
+	if r2 := <-store.User().GetEtagForProfiles(teamId); r2.Err != nil {
+		t.Fatal(r2.Err)
+	} else {
+		etag = r2.Data.(string)
+	}
+
+	u3 := &model.User{}
+	u3.Email = model.NewId()
+	Must(store.User().Save(u3))
+	Must(store.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u3.Id}))
+
+	if r2 := <-store.User().GetEtagForProfiles(teamId); r2.Err != nil {
+		t.Fatal(r2.Err)
+	} else {
+		if etag == r2.Data.(string) {
+			t.Fatal("etags should not match")
+		}
+	}
 }
 
 func TestUserStoreGetProfilesInChannel(t *testing.T) {
@@ -502,6 +573,7 @@ func TestUserStoreGetAllProfilesInChannel(t *testing.T) {
 		}
 	}
 
+	store.User().InvalidateProfilesInChannelCacheByUser(u1.Id)
 	store.User().InvalidateProfilesInChannelCache(c2.Id)
 }
 

store/store.go

@@ -126,7 +126,6 @@ type ChannelStore interface {
 	PermanentDeleteMembersByUser(userId string) StoreChannel
 	PermanentDeleteMembersByChannel(channelId string) StoreChannel
 	UpdateLastViewedAt(channelIds []string, userId string) StoreChannel
-	SetLastViewedAt(channelId string, userId string, newLastViewedAt int64) StoreChannel
 	IncrementMentionCount(channelId string, userId string) StoreChannel
 	AnalyticsTypeCount(teamId string, channelType string) StoreChannel
 	ExtraUpdateByUser(userId string, time int64) StoreChannel