From be0d13578dad9f374e035e1230b934cf5685b1e1 Mon Sep 17 00:00:00 2001 From: sowmiyamuthuraman <32141844+sowmiyamuthuraman@users.noreply.github.com> Date: Wed, 18 Sep 2019 23:32:22 +0530 Subject: [PATCH 01/10] Refactor "app/authorization.go" to use structured logging (#12233) --- app/authorization.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/app/authorization.go b/app/authorization.go index 995cc9e3bb..de1fc96e74 100644 --- a/app/authorization.go +++ b/app/authorization.go @@ -4,7 +4,6 @@ package app import ( - "fmt" "net/http" "strings" @@ -192,8 +191,7 @@ func (a *App) RolesGrantPermission(roleNames []string, permissionId string) bool if err != nil { // This should only happen if something is very broken. We can't realistically // recover the situation, so deny permission and log an error. - mlog.Error("Failed to get roles from database with role names: " + strings.Join(roleNames, ",")) - mlog.Error(fmt.Sprint(err)) + mlog.Error("Failed to get roles from database with role names: "+strings.Join(roleNames, ",")+" ", mlog.Err(err)) return false } From ecbc5ef5e19b1a51a181e71b14d5a9ecf6bc9b88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Wed, 18 Sep 2019 20:27:32 +0200 Subject: [PATCH 02/10] Fixing autocomplete endpoint for in/out of channels (#12147) --- api4/user.go | 21 +-- api4/user_test.go | 329 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 236 insertions(+), 114 deletions(-) diff --git a/api4/user.go b/api4/user.go index bfdd7baad8..315c36e970 100644 --- a/api4/user.go +++ b/api4/user.go @@ -806,6 +806,13 @@ func autocompleteUsers(c *Context, w http.ResponseWriter, r *http.Request) { var autocomplete model.UserAutocomplete + var err *model.AppError + options, err = c.App.RestrictUsersSearchByPermissions(c.App.Session.UserId, options) + if err != nil { + c.Err = err + return + } + if len(channelId) > 0 { // Applying the provided teamId here is useful for DMs and GMs which don't belong // to a team. Applying it when the channel does belong to a team makes less sense, @@ -819,13 +826,6 @@ func autocompleteUsers(c *Context, w http.ResponseWriter, r *http.Request) { autocomplete.Users = result.InChannel autocomplete.OutOfChannel = result.OutOfChannel } else if len(teamId) > 0 { - var err *model.AppError - options, err = c.App.RestrictUsersSearchByPermissions(c.App.Session.UserId, options) - if err != nil { - c.Err = err - return - } - result, err := c.App.AutocompleteUsersInTeam(teamId, name, options) if err != nil { c.Err = err @@ -834,13 +834,6 @@ func autocompleteUsers(c *Context, w http.ResponseWriter, r *http.Request) { autocomplete.Users = result.InTeam } else { - var err *model.AppError - options, err = c.App.RestrictUsersSearchByPermissions(c.App.Session.UserId, options) - if err != nil { - c.Err = err - return - } - result, err := c.App.SearchUsersInTeam("", name, options) if err != nil { c.Err = err diff --git a/api4/user_test.go b/api4/user_test.go index 505c654b1e..4628be1fa3 100644 --- a/api4/user_test.go +++ b/api4/user_test.go @@ -997,133 +997,262 @@ func findUserInList(id string, users []*model.User) bool { return false } -func TestAutocompleteUsers(t *testing.T) { +func TestAutocompleteUsersInChannel(t *testing.T) { th := Setup().InitBasic() defer th.TearDown() teamId := th.BasicTeam.Id channelId := th.BasicChannel.Id username := th.BasicUser.Username + newUser := th.CreateUser() - rusers, resp := th.Client.AutocompleteUsersInChannel(teamId, channelId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - - if len(rusers.Users) != 1 { - t.Fatal("should have returned 1 user") + tt := []struct { + Name string + TeamId string + ChannelId string + Username string + ExpectedResults int + MoreThan bool + }{ + { + "Autocomplete in channel for specific username", + teamId, + channelId, + username, + 1, + false, + }, + { + "Search for not valid username", + teamId, + channelId, + "amazonses", + 0, + false, + }, + { + "Search for all users", + teamId, + channelId, + "", + 2, + true, + }, + { + "Search all in specific channel", + "", + channelId, + "", + 2, + true, + }, } - rusers, resp = th.Client.AutocompleteUsersInChannel(teamId, channelId, "amazonses", model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - if len(rusers.Users) != 0 { - t.Fatal("should have returned 0 users") + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + th.LoginBasic() + rusers, resp := th.Client.AutocompleteUsersInChannel(tc.TeamId, tc.ChannelId, tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + if tc.MoreThan { + assert.True(t, len(rusers.Users) >= tc.ExpectedResults) + } else { + assert.Len(t, rusers.Users, tc.ExpectedResults) + } + th.Client.Logout() + _, resp = th.Client.AutocompleteUsersInChannel(tc.TeamId, tc.ChannelId, tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckUnauthorizedStatus(t, resp) + + th.Client.Login(newUser.Email, newUser.Password) + _, resp = th.Client.AutocompleteUsersInChannel(tc.TeamId, tc.ChannelId, tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckForbiddenStatus(t, resp) + }) } - rusers, resp = th.Client.AutocompleteUsersInChannel(teamId, channelId, "", model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - if len(rusers.Users) < 2 { - t.Fatal("should have many users") - } + t.Run("Check against privacy config settings", func(t *testing.T) { + th.App.UpdateConfig(func(cfg *model.Config) { *cfg.PrivacySettings.ShowFullName = false }) - rusers, resp = th.Client.AutocompleteUsersInChannel("", channelId, "", model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - if len(rusers.Users) < 2 { - t.Fatal("should have many users") - } + th.LoginBasic() + rusers, resp := th.Client.AutocompleteUsersInChannel(teamId, channelId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) - rusers, resp = th.Client.AutocompleteUsersInTeam(teamId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) + assert.Equal(t, rusers.Users[0].FirstName, "", "should not show first/last name") + assert.Equal(t, rusers.Users[0].LastName, "", "should not show first/last name") + }) - if len(rusers.Users) != 1 { - t.Fatal("should have returned 1 user") - } + t.Run("Check OutOfChannel results with/without VIEW_MEMBERS permissions", func(t *testing.T) { + permissionsUser := th.CreateUser() + th.SystemAdminClient.DemoteUserToGuest(permissionsUser.Id) + permissionsUser.Roles = "system_guest" + th.LinkUserToTeam(permissionsUser, th.BasicTeam) + th.AddUserToChannel(permissionsUser, th.BasicChannel) - rusers, resp = th.Client.AutocompleteUsers(username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) + otherUser := th.CreateUser() + th.LinkUserToTeam(otherUser, th.BasicTeam) - if len(rusers.Users) != 1 { - t.Fatal("should have returned 1 users") - } + th.Client.Login(permissionsUser.Email, permissionsUser.Password) - rusers, resp = th.Client.AutocompleteUsers("", model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) + rusers, resp := th.Client.AutocompleteUsersInChannel(teamId, channelId, "", model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + assert.Len(t, rusers.OutOfChannel, 1) - if len(rusers.Users) < 2 { - t.Fatal("should have returned many users") - } + defaultRolePermissions := th.SaveDefaultRolePermissions() + defer func() { + th.RestoreDefaultRolePermissions(defaultRolePermissions) + }() - rusers, resp = th.Client.AutocompleteUsersInTeam(teamId, "amazonses", model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - if len(rusers.Users) != 0 { - t.Fatal("should have returned 0 users") - } + th.RemovePermissionFromRole(model.PERMISSION_VIEW_MEMBERS.Id, model.SYSTEM_USER_ROLE_ID) + th.RemovePermissionFromRole(model.PERMISSION_VIEW_MEMBERS.Id, model.TEAM_USER_ROLE_ID) - rusers, resp = th.Client.AutocompleteUsersInTeam(teamId, "", model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - if len(rusers.Users) < 2 { - t.Fatal("should have many users") - } + rusers, resp = th.Client.AutocompleteUsersInChannel(teamId, channelId, "", model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + assert.Len(t, rusers.OutOfChannel, 0) - th.Client.Logout() - _, resp = th.Client.AutocompleteUsersInChannel(teamId, channelId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckUnauthorizedStatus(t, resp) + th.App.GetOrCreateDirectChannel(permissionsUser.Id, otherUser.Id) - _, resp = th.Client.AutocompleteUsersInTeam(teamId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckUnauthorizedStatus(t, resp) - - _, resp = th.Client.AutocompleteUsers(username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckUnauthorizedStatus(t, resp) - - user := th.CreateUser() - th.Client.Login(user.Email, user.Password) - _, resp = th.Client.AutocompleteUsersInChannel(teamId, channelId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckForbiddenStatus(t, resp) - - _, resp = th.Client.AutocompleteUsersInTeam(teamId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckForbiddenStatus(t, resp) - - _, resp = th.Client.AutocompleteUsers(username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - - _, resp = th.SystemAdminClient.AutocompleteUsersInChannel(teamId, channelId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - - _, resp = th.SystemAdminClient.AutocompleteUsersInTeam(teamId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - - _, resp = th.SystemAdminClient.AutocompleteUsers(username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - - // Check against privacy config settings - th.App.UpdateConfig(func(cfg *model.Config) { *cfg.PrivacySettings.ShowFullName = false }) - - th.LoginBasic() - - rusers, resp = th.Client.AutocompleteUsers(username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - - if rusers.Users[0].FirstName != "" || rusers.Users[0].LastName != "" { - t.Fatal("should not show first/last name") - } - - rusers, resp = th.Client.AutocompleteUsersInChannel(teamId, channelId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - - if rusers.Users[0].FirstName != "" || rusers.Users[0].LastName != "" { - t.Fatal("should not show first/last name") - } - - rusers, resp = th.Client.AutocompleteUsersInTeam(teamId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") - CheckNoError(t, resp) - - if rusers.Users[0].FirstName != "" || rusers.Users[0].LastName != "" { - t.Fatal("should not show first/last name") - } + rusers, resp = th.Client.AutocompleteUsersInChannel(teamId, channelId, "", model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + assert.Len(t, rusers.OutOfChannel, 1) + }) t.Run("user must have access to team id, especially when it does not match channel's team id", func(t *testing.T) { - rusers, resp = th.Client.AutocompleteUsersInChannel("otherTeamId", channelId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") + _, resp := th.Client.AutocompleteUsersInChannel("otherTeamId", channelId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") CheckErrorMessage(t, resp, "api.context.permissions.app_error") }) } +func TestAutocompleteUsersInTeam(t *testing.T) { + th := Setup().InitBasic() + defer th.TearDown() + teamId := th.BasicTeam.Id + username := th.BasicUser.Username + newUser := th.CreateUser() + + tt := []struct { + Name string + TeamId string + Username string + ExpectedResults int + MoreThan bool + }{ + { + "specific username", + teamId, + username, + 1, + false, + }, + { + "not valid username", + teamId, + "amazonses", + 0, + false, + }, + { + "all users in team", + teamId, + "", + 2, + true, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + th.LoginBasic() + rusers, resp := th.Client.AutocompleteUsersInTeam(tc.TeamId, tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + if tc.MoreThan { + assert.True(t, len(rusers.Users) >= tc.ExpectedResults) + } else { + assert.Len(t, rusers.Users, tc.ExpectedResults) + } + th.Client.Logout() + _, resp = th.Client.AutocompleteUsersInTeam(tc.TeamId, tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckUnauthorizedStatus(t, resp) + + th.Client.Login(newUser.Email, newUser.Password) + _, resp = th.Client.AutocompleteUsersInTeam(tc.TeamId, tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckForbiddenStatus(t, resp) + }) + } + + t.Run("Check against privacy config settings", func(t *testing.T) { + th.App.UpdateConfig(func(cfg *model.Config) { *cfg.PrivacySettings.ShowFullName = false }) + + th.LoginBasic() + rusers, resp := th.Client.AutocompleteUsersInTeam(teamId, username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + + assert.Equal(t, rusers.Users[0].FirstName, "", "should not show first/last name") + assert.Equal(t, rusers.Users[0].LastName, "", "should not show first/last name") + }) +} + +func TestAutocompleteUsers(t *testing.T) { + th := Setup().InitBasic() + defer th.TearDown() + username := th.BasicUser.Username + newUser := th.CreateUser() + + tt := []struct { + Name string + Username string + ExpectedResults int + MoreThan bool + }{ + { + "specific username", + username, + 1, + false, + }, + { + "not valid username", + "amazonses", + 0, + false, + }, + { + "all users in team", + "", + 2, + true, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + th.LoginBasic() + rusers, resp := th.Client.AutocompleteUsers(tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + if tc.MoreThan { + assert.True(t, len(rusers.Users) >= tc.ExpectedResults) + } else { + assert.Len(t, rusers.Users, tc.ExpectedResults) + } + + th.Client.Logout() + _, resp = th.Client.AutocompleteUsers(tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckUnauthorizedStatus(t, resp) + + th.Client.Login(newUser.Email, newUser.Password) + _, resp = th.Client.AutocompleteUsers(tc.Username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + }) + } + + t.Run("Check against privacy config settings", func(t *testing.T) { + th.App.UpdateConfig(func(cfg *model.Config) { *cfg.PrivacySettings.ShowFullName = false }) + + th.LoginBasic() + rusers, resp := th.Client.AutocompleteUsers(username, model.USER_SEARCH_DEFAULT_LIMIT, "") + CheckNoError(t, resp) + + assert.Equal(t, rusers.Users[0].FirstName, "", "should not show first/last name") + assert.Equal(t, rusers.Users[0].LastName, "", "should not show first/last name") + }) +} + func TestGetProfileImage(t *testing.T) { th := Setup().InitBasic() defer th.TearDown() From d1017a089a84c4b07377b6654023c01c051dbeec Mon Sep 17 00:00:00 2001 From: Ankit R Gadiya Date: Thu, 19 Sep 2019 14:07:02 +0530 Subject: [PATCH 03/10] Moved Push to S3, Clean Checkout and Build Docker before Test (#12027) --- build/Jenkinsfile.pr | 744 +++++++++++++++++++++---------------------- 1 file changed, 372 insertions(+), 372 deletions(-) diff --git a/build/Jenkinsfile.pr b/build/Jenkinsfile.pr index 2fc80b6789..c380471cd4 100644 --- a/build/Jenkinsfile.pr +++ b/build/Jenkinsfile.pr @@ -7,413 +7,413 @@ def utils = new org.mattermost.Utils() def rnd = UUID.randomUUID().toString() pipeline { - agent { - label 'default-mm-builder' - } + agent { + label 'default-mm-builder' + } - options { - buildDiscarder(logRotator(numToKeepStr: '3')) - timeout(time: 1, unit: 'HOURS') - } + options { + buildDiscarder(logRotator(numToKeepStr: '3')) + timeout(time: 1, unit: 'HOURS') + } - environment { - COMPOSE_PROJECT_NAME="${rnd}-${env.BUILD_NUMBER}" - } + environment { + COMPOSE_PROJECT_NAME="${rnd}-${env.BUILD_NUMBER}" + } - stages { - stage('Setup') { - steps { - script { - utils.stopOldBuilds() - } - cleanWs notFailBuild: true - sh """ - mkdir -p src/github.com/mattermost/mattermost-server - mkdir -p src/github.com/mattermost/mattermost-webapp - mkdir -p src/github.com/mattermost/enterprise - """ - dir('src/github.com/mattermost/mattermost-server') { - checkout scm - } - dir('src/github.com/mattermost/mattermost-webapp') { - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/mattermost-webapp.git']]] - } - dir('src/github.com/mattermost/enterprise') { - checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/enterprise.git']]] - } - dir('src/github.com/mattermost/enterprise') { - ansiColor('xterm') { - sh """ - #!/bin/bash -ex - git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout $env.CHANGE_TARGET || git checkout master || echo 1 - export EE_GIT_COMMIT=\$(git rev-parse HEAD) + stages { + stage('Setup') { + steps { + script { + utils.stopOldBuilds() + } + cleanWs notFailBuild: true + sh """ + mkdir -p src/github.com/mattermost/mattermost-server + mkdir -p src/github.com/mattermost/mattermost-webapp + mkdir -p src/github.com/mattermost/enterprise + """ + dir('src/github.com/mattermost/mattermost-server') { + checkout scm + } + dir('src/github.com/mattermost/mattermost-webapp') { + checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/mattermost-webapp.git']]] + } + dir('src/github.com/mattermost/enterprise') { + checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/enterprise.git']]] + } + dir('src/github.com/mattermost/enterprise') { + ansiColor('xterm') { + sh """ + #!/bin/bash -ex + git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout $env.CHANGE_TARGET || git checkout master || echo 1 + export EE_GIT_COMMIT=\$(git rev-parse HEAD) - echo EE Commit: \${EE_GIT_COMMIT} - """ - } - } - dir('src/github.com/mattermost/mattermost-webapp') { - withDockerContainer(args: '', image: 'mattermost/mattermost-build-webapp:oct-2-2018') { - ansiColor('xterm') { - sh """ - #!/bin/bash -ex - git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout $env.CHANGE_TARGET || git checkout master - rm -rf ./dist - export WEBAPP_GIT_COMMIT=\$(git rev-parse HEAD) + echo EE Commit: \${EE_GIT_COMMIT} + """ + } + } + dir('src/github.com/mattermost/mattermost-webapp') { + withDockerContainer(args: '', image: 'mattermost/mattermost-build-webapp:oct-2-2018') { + ansiColor('xterm') { + sh """ + #!/bin/bash -ex + git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout $env.CHANGE_TARGET || git checkout master + rm -rf ./dist + export WEBAPP_GIT_COMMIT=\$(git rev-parse HEAD) - echo Webapp Commit: \${WEBAPP_GIT_COMMIT} + echo Webapp Commit: \${WEBAPP_GIT_COMMIT} - curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build - """ - } - } - } - } - } + curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build + """ + } + } + } + } + } - stage('Check i18n') { - environment { - GOPATH = "/go" - } + stage('Check i18n') { + environment { + GOPATH = "/go" + } - steps { - withDockerContainer(args: '-u root --privileged -v ${WORKSPACE}/src:/go/src/', image: 'mattermost/mattermost-build-server:feb-28-2019') { - ansiColor('xterm') { - sh """ - cd /go/src/github.com/mattermost/mattermost-server - cp i18n/en.json /tmp/en.json - make i18n-extract - diff /tmp/en.json i18n/en.json - """ - } - } - } - } + steps { + withDockerContainer(args: '-u root --privileged -v ${WORKSPACE}/src:/go/src/', image: 'mattermost/mattermost-build-server:feb-28-2019') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server + cp i18n/en.json /tmp/en.json + make i18n-extract + diff /tmp/en.json i18n/en.json + """ + } + } + } + } - stage('Build') { - environment { - GOPATH = "/go" - } + stage('Build') { + environment { + GOPATH = "/go" + } - steps { - withDockerContainer(args: '-u root --privileged -v ${WORKSPACE}/src:/go/src/', image: 'mattermost/mattermost-build-server:feb-28-2019') { - ansiColor('xterm') { - sh """ - cd /go/src/github.com/mattermost/mattermost-server - make config-reset - make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' - make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' - make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' - """ - } - } - } - } + steps { + withDockerContainer(args: '-u root --privileged -v ${WORKSPACE}/src:/go/src/', image: 'mattermost/mattermost-build-server:feb-28-2019') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server + make config-reset + make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' + make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' + make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' + """ + } + } + } + } - stage('Test') { - environment { - GOPATH = "/go" - TEST_DATABASE_MYSQL_DSN = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" - TEST_DATABASE_POSTGRESQL_DSN = "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" - TEST_DATABASE_MYSQL_ROOT_PASSWD = "mostest" - CI_INBUCKET_HOST = "inbucket" - CI_MINIO_HOST = "minio" - CI_INBUCKET_PORT = "10080" - CI_MINIO_PORT = "9000" - CI_INBUCKET_SMTP_PORT = "10025" - CI_LDAP_HOST = "openldap" - IS_CI = true - MM_SQLSETTINGS_DATASOURCE = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8" - MM_EMAILSETTINGS_SMTPSERVER = "inbucket" - MM_EMAILSETTINGS_SMTPPORT = "10025" - MM_ELASTICSEARCHSETTINGS_CONNECTIONURL = "http://elasticsearch:9200" - LDAP_DATA = "test" - } + stage('Push to S3') { + stages { + stage('Pull request') { + when { + allOf { + expression { env.CHANGE_ID != null } + expression { env.CHANGE_TARGET != null } + } + } + steps { + dir('src/github.com/mattermost/mattermost-server/dist') { + step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[ + bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_ID}", + excludedFile: '', + flatten: true, + gzipFiles: false, + keepForever: false, + managedArtifacts: false, + noUploadOnFailure: true, + selectedRegion: 'us-east-1', + showDirectlyInBrowser: false, + sourceFile: '*.tar.gz', + storageClass: 'STANDARD', + uploadFromSlave: false, + useServerSideEncryption: false, + userMetadata: [[key: 'Cache-Control', value: 'no-cache']] + ], [ + bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_BRANCH}", + excludedFile: '', + flatten: true, + gzipFiles: false, + keepForever: false, + managedArtifacts: false, + noUploadOnFailure: true, + selectedRegion: 'us-east-1', + showDirectlyInBrowser: false, + sourceFile: '*.tar.gz', + storageClass: 'STANDARD', + uploadFromSlave: false, + useServerSideEncryption: false, + userMetadata: [[key: 'Cache-Control', value: 'no-cache']] + ]], profileName: 'Releases', userMetadAta: []]) + } + } + } + stage('Branch') { + when { + expression { env.CHANGE_ID == null } + } + steps { + dir('src/github.com/mattermost/mattermost-server/dist') { + step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[ + bucket: "releases.mattermost.com/mattermost-platform-pr/${BRANCH_NAME}", + excludedFile: '', + flatten: true, + gzipFiles: false, + keepForever: false, + managedArtifacts: false, + noUploadOnFailure: true, + selectedRegion: 'us-east-1', + showDirectlyInBrowser: false, + sourceFile: '*.tar.gz', + storageClass: 'STANDARD', + uploadFromSlave: false, + useServerSideEncryption: false, + userMetadata: [[key: 'Cache-Control', value: 'no-cache']] + ]], profileName: 'Releases', userMetadAta: []]) + } + } + } + } + } - steps { - dir('src/github.com/mattermost/mattermost-server/build') { - ansiColor('xterm') { - sh """ - /usr/local/bin/docker-compose --no-ansi run --rm start_dependencies - /usr/local/bin/docker-compose --no-ansi ps - cat ../tests/${LDAP_DATA}-data.ldif | /usr/local/bin/docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - """ - } - } + stage('Clean checkout') { + when { + expression { env.CHANGE_ID != null } + } + // We need to perform a clean checkout here to ge the original git commit hash from the PR + // Jenkins now merges master in top of the PR and this generate a new git hash + // We need to do that to build the docker image based on the original git commit and then this will be used by + // mattermod to update the test server. + steps { + sh """ + mkdir -p /tmp/mattermost-server + """ + dir('/tmp/mattermost-server') { + checkout([$class: 'GitSCM', branches: [[name: 'FETCH_HEAD']], + doGenerateSubmoduleConfigurations: false, extensions: [], + submoduleCfg: [], userRemoteConfigs: [ + [refspec: "+refs/pull/${CHANGE_ID}/head:refs/remotes/origin/PR-${CHANGE_ID}", + credentialsId: "310159d3-f7c5-4f5d-bfa1-151e3ef2db57",url: "https://github.com/mattermost/mattermost-server.git"]]]) + sh 'git rev-parse --short HEAD' + } + } + } - dir('src/github.com/mattermost/mattermost-server') { - ansiColor('xterm') { - sh """ - echo "Creating databases" - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test' - echo "Importing postgres dump from version 5.0" - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres psql -U mmuser -d migrated < \$(pwd)/scripts/mattermost-postgresql-5.0.sql - """ - } - } + stage('Build Docker Image') { + environment { + GIT_COMMIT_SHORT = sh( + script: "cd /tmp/mattermost-server && printf \$(git rev-parse --short HEAD)", + returnStdout: true + ) + } + when { + expression { env.CHANGE_ID != null } + } + steps { + dir('src/github.com/mattermost/mattermost-server') { + withCredentials([usernamePassword(credentialsId: 'matterbuild-docker-hub', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASS')]) { + sh 'docker login --username ${DOCKER_USER} --password ${DOCKER_PASS}' + sh """ + docker build --no-cache --build-arg MM_PACKAGE=https://releases.mattermost.com/mattermost-platform-pr/${CHANGE_ID}/mattermost-enterprise-linux-amd64.tar.gz -t mattermost/mattermost-enterprise-edition:${GIT_COMMIT_SHORT} build + docker push mattermost/mattermost-enterprise-edition:${GIT_COMMIT_SHORT} + docker logout + """ + } + } + } + } - withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { - ansiColor('xterm') { - sh """ - cd /go/src/github.com/mattermost/mattermost-server - echo "Setting up config for db migration" - export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10\" - export MM_SQLSETTINGS_DRIVERNAME=\"postgres\" - make ARGS="config get SqlSettings.DataSource" run-cli - echo "Running the migration" - make ARGS="version" run-cli + stage('Test') { + environment { + GOPATH = "/go" + TEST_DATABASE_MYSQL_DSN = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" + TEST_DATABASE_POSTGRESQL_DSN = "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" + TEST_DATABASE_MYSQL_ROOT_PASSWD = "mostest" + CI_INBUCKET_HOST = "inbucket" + CI_MINIO_HOST = "minio" + CI_INBUCKET_PORT = "10080" + CI_MINIO_PORT = "9000" + CI_INBUCKET_SMTP_PORT = "10025" + CI_LDAP_HOST = "openldap" + IS_CI = true + MM_SQLSETTINGS_DATASOURCE = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8" + MM_EMAILSETTINGS_SMTPSERVER = "inbucket" + MM_EMAILSETTINGS_SMTPPORT = "10025" + MM_ELASTICSEARCHSETTINGS_CONNECTIONURL = "http://elasticsearch:9200" + LDAP_DATA = "test" + } - echo "Setting up config for fresh db setup" - export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10\" - make ARGS="config get SqlSettings.DataSource" run-cli + steps { + dir('src/github.com/mattermost/mattermost-server/build') { + ansiColor('xterm') { + sh """ + /usr/local/bin/docker-compose --no-ansi run --rm start_dependencies + /usr/local/bin/docker-compose --no-ansi ps + cat ../tests/${LDAP_DATA}-data.ldif | /usr/local/bin/docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + """ + } + } - echo "Setting up fresh db" - make ARGS="version" run-cli - """ - } - } + dir('src/github.com/mattermost/mattermost-server') { + ansiColor('xterm') { + sh """ + echo "Creating databases" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test' + echo "Importing postgres dump from version 5.0" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres psql -U mmuser -d migrated < \$(pwd)/scripts/mattermost-postgresql-5.0.sql + """ + } + } + + withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server + echo "Setting up config for db migration" + export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10\" + export MM_SQLSETTINGS_DRIVERNAME=\"postgres\" + make ARGS="config get SqlSettings.DataSource" run-cli + echo "Running the migration" + make ARGS="version" run-cli + + echo "Setting up config for fresh db setup" + export MM_SQLSETTINGS_DATASOURCE=\"postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10\" + make ARGS="config get SqlSettings.DataSource" run-cli + + echo "Setting up fresh db" + make ARGS="version" run-cli + """ + } + } - dir('src/github.com/mattermost/mattermost-server') { - ansiColor('xterm') { - sh """ - echo "Generating dump" - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql + dir('src/github.com/mattermost/mattermost-server') { + ansiColor('xterm') { + sh """ + echo "Generating dump" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql - echo "Removing databases created for db comparison" - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres sh -c \'exec echo \"DROP DATABASE migrated; DROP DATABASE latest;\" | exec psql -U mmuser mattermost_test\' + echo "Removing databases created for db comparison" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T postgres sh -c \'exec echo \"DROP DATABASE migrated; DROP DATABASE latest;\" | exec psql -U mmuser mattermost_test\' - echo "Generating diff" - diff migrated.sql latest.sql > diff.txt - export diffErrorCode=\$? + echo "Generating diff" + diff migrated.sql latest.sql > diff.txt + export diffErrorCode=\$? - if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi + if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi - exit \$diffErrorCode - """ - } - } + exit \$diffErrorCode + """ + } + } - dir('src/github.com/mattermost/mattermost-server') { - ansiColor('xterm') { - sh """ - echo "Creating databases" - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -uroot -pmostest -e \"CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser\" - echo "Importing mysql dump from version 5.0" - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D migrated -uroot -pmostest < \$(pwd)/scripts/mattermost-mysql-5.0.sql - """ - } - } + dir('src/github.com/mattermost/mattermost-server') { + ansiColor('xterm') { + sh """ + echo "Creating databases" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -uroot -pmostest -e \"CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser\" + echo "Importing mysql dump from version 5.0" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -D migrated -uroot -pmostest < \$(pwd)/scripts/mattermost-mysql-5.0.sql + """ + } + } - withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { - ansiColor('xterm') { - sh """ - cd /go/src/github.com/mattermost/mattermost-server - echo "Setting up config for db migration" - export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\" - export MM_SQLSETTINGS_DRIVERNAME=\"mysql\" - make ARGS="config get SqlSettings.DataSource" run-cli - echo "Running the migration" - make ARGS="version" run-cli + withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server + echo "Setting up config for db migration" + export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\" + export MM_SQLSETTINGS_DRIVERNAME=\"mysql\" + make ARGS="config get SqlSettings.DataSource" run-cli + echo "Running the migration" + make ARGS="version" run-cli - echo "Setting up config for fresh db setup" - export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\" - make ARGS="config get SqlSettings.DataSource" run-cli + echo "Setting up config for fresh db setup" + export MM_SQLSETTINGS_DATASOURCE=\"mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s\" + make ARGS="config get SqlSettings.DataSource" run-cli - echo "Setting up fresh db" - make ARGS="version" run-cli - """ - } - } + echo "Setting up fresh db" + make ARGS="version" run-cli + """ + } + } - dir('src/github.com/mattermost/mattermost-server') { - ansiColor('xterm') { - sh """ - echo "Generating dump" - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql + dir('src/github.com/mattermost/mattermost-server') { + ansiColor('xterm') { + sh """ + echo "Generating dump" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql - echo "Removing databases created for db comparison" - /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -uroot -pmostest -e \"DROP DATABASE migrated; DROP DATABASE latest\" + echo "Removing databases created for db comparison" + /usr/local/bin/docker-compose --no-ansi -f build/docker-compose.yml exec -T mysql mysql -uroot -pmostest -e \"DROP DATABASE migrated; DROP DATABASE latest\" - echo "Generating diff" - diff migrated.sql latest.sql > diff.txt - export diffErrorCode=\$? + echo "Generating diff" + diff migrated.sql latest.sql > diff.txt + export diffErrorCode=\$? - if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi + if [ \$diffErrorCode -eq 0 ]; then echo \"Both schemas are same\";else cat diff.txt; fi - exit \$diffErrorCode - """ - } - } + exit \$diffErrorCode + """ + } + } - withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { - ansiColor('xterm') { - sh """ - cd /go/src/github.com/mattermost/mattermost-server + withDockerContainer(args: "-u root --privileged --net ${COMPOSE_PROJECT_NAME}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:feb-28-2019') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server - mkdir -p client/plugins - cat config/config.json + mkdir -p client/plugins + cat config/config.json - make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE= - """ - } - withCredentials([string(credentialsId: 'CODECOV_TOKEN', variable: 'CODECOV')]) { - sh ''' - cd /go/src/github.com/mattermost/mattermost-server - curl -s https://codecov.io/bash | bash -s - -t $CODECOV || echo 'Codecov failed to upload' - ''' - } - } - } - } + make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE= + """ + } + withCredentials([string(credentialsId: 'CODECOV_TOKEN', variable: 'CODECOV')]) { + sh ''' + cd /go/src/github.com/mattermost/mattermost-server + curl -s https://codecov.io/bash | bash -s - -t $CODECOV || echo 'Codecov failed to upload' + ''' + } + } + } + } + } - stage('Push to S3') { - stages { - stage('Pull request') { - when { - allOf { - expression { env.CHANGE_ID != null } - expression { env.CHANGE_TARGET != null } - } - } - steps { - dir('src/github.com/mattermost/mattermost-server/dist') { - step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[ - bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_ID}", - excludedFile: '', - flatten: true, - gzipFiles: false, - keepForever: false, - managedArtifacts: false, - noUploadOnFailure: true, - selectedRegion: 'us-east-1', - showDirectlyInBrowser: false, - sourceFile: '*.tar.gz', - storageClass: 'STANDARD', - uploadFromSlave: false, - useServerSideEncryption: false, - userMetadata: [[key: 'Cache-Control', value: 'no-cache']] - ], [ - bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_BRANCH}", - excludedFile: '', - flatten: true, - gzipFiles: false, - keepForever: false, - managedArtifacts: false, - noUploadOnFailure: true, - selectedRegion: 'us-east-1', - showDirectlyInBrowser: false, - sourceFile: '*.tar.gz', - storageClass: 'STANDARD', - uploadFromSlave: false, - useServerSideEncryption: false, - userMetadata: [[key: 'Cache-Control', value: 'no-cache']] - ]], profileName: 'Releases', userMetadAta: []]) - } - } - } - stage('Branch') { - when { - expression { env.CHANGE_ID == null } - } - steps { - dir('src/github.com/mattermost/mattermost-server/dist') { - step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[ - bucket: "releases.mattermost.com/mattermost-platform-pr/${BRANCH_NAME}", - excludedFile: '', - flatten: true, - gzipFiles: false, - keepForever: false, - managedArtifacts: false, - noUploadOnFailure: true, - selectedRegion: 'us-east-1', - showDirectlyInBrowser: false, - sourceFile: '*.tar.gz', - storageClass: 'STANDARD', - uploadFromSlave: false, - useServerSideEncryption: false, - userMetadata: [[key: 'Cache-Control', value: 'no-cache']] - ]], profileName: 'Releases', userMetadAta: []]) - } - } - } - } - } + post { + always { + dir('src/github.com/mattermost/mattermost-server') { + junit allowEmptyResults: true, healthScaleFactor: 0.0, testResults: 'report*.xml' + archiveArtifacts 'report*.xml' + } + dir('src/github.com/mattermost/mattermost-server/build') { + ansiColor('xterm') { + sh """ + # Capture docker logs + docker-compose logs --tail="all" -t --no-color > docker-compose_logs + docker ps -a --no-trunc > docker_ps + docker stats -a --no-stream > docker_stats + tar -czvf docker_logs.tar.gz docker-compose_logs docker_ps docker_stats - stage('Clean checkout') { - when { - expression { env.CHANGE_ID != null } - } - // We need to perform a clean checkout here to ge the original git commit hash from the PR - // Jenkins now merges master in top of the PR and this generate a new git hash - // We need to do that to build the docker image based on the original git commit and then this will be used by - // mattermod to update the test server. - steps { - sh """ - mkdir -p /tmp/mattermost-server - """ - dir('/tmp/mattermost-server') { - checkout([$class: 'GitSCM', branches: [[name: 'FETCH_HEAD']], - doGenerateSubmoduleConfigurations: false, extensions: [], - submoduleCfg: [], userRemoteConfigs: [ - [refspec: "+refs/pull/${CHANGE_ID}/head:refs/remotes/origin/PR-${CHANGE_ID}", - credentialsId: "310159d3-f7c5-4f5d-bfa1-151e3ef2db57",url: "https://github.com/mattermost/mattermost-server.git"]]]) - sh 'git rev-parse --short HEAD' - } - } - } - - stage('Build Docker Image') { - environment { - GIT_COMMIT_SHORT = sh( - script: "cd /tmp/mattermost-server && printf \$(git rev-parse --short HEAD)", - returnStdout: true - ) - } - when { - expression { env.CHANGE_ID != null } - } - steps { - dir('src/github.com/mattermost/mattermost-server') { - withCredentials([usernamePassword(credentialsId: 'matterbuild-docker-hub', usernameVariable: 'DOCKER_USER', passwordVariable: 'DOCKER_PASS')]) { - sh 'docker login --username ${DOCKER_USER} --password ${DOCKER_PASS}' - sh """ - docker build --no-cache --build-arg MM_PACKAGE=https://releases.mattermost.com/mattermost-platform-pr/${CHANGE_ID}/mattermost-enterprise-linux-amd64.tar.gz -t mattermost/mattermost-enterprise-edition:${GIT_COMMIT_SHORT} build - docker push mattermost/mattermost-enterprise-edition:${GIT_COMMIT_SHORT} - docker logout - """ - } - } - } - } - } - - post { - always { - dir('src/github.com/mattermost/mattermost-server') { - junit allowEmptyResults: true, healthScaleFactor: 0.0, testResults: 'report*.xml' - archiveArtifacts 'report*.xml' - } - dir('src/github.com/mattermost/mattermost-server/build') { - ansiColor('xterm') { - sh """ - # Capture docker logs - docker-compose logs --tail="all" -t --no-color > docker-compose_logs - docker ps -a --no-trunc > docker_ps - docker stats -a --no-stream > docker_stats - tar -czvf docker_logs.tar.gz docker-compose_logs docker_ps docker_stats - - docker-compose --no-ansi down -v - """ - } - archiveArtifacts 'docker_logs.tar.gz' - } - } - cleanup { - cleanWs notFailBuild: true - } - } + docker-compose --no-ansi down -v + """ + } + archiveArtifacts 'docker_logs.tar.gz' + } + } + cleanup { + cleanWs notFailBuild: true + } + } } From e42c40b4b57bdec69f9bfa908eff77695044aac2 Mon Sep 17 00:00:00 2001 From: tejashreecd Date: Thu, 19 Sep 2019 16:13:21 +0530 Subject: [PATCH 04/10] [MM-11854] Migrates some CI/CD steps to circleci (#11978) * [MM-11854] Migrates some CI/CD steps to circleci * [MM-11854] Empty commit to trigger CI/CD build * update circleci config * update build image --- .circleci/config.yml | 220 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 218 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b86d194ae8..a1e2b2f67f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,8 +1,224 @@ version: 2.1 +orbs: + aws-s3: circleci/aws-s3@1.0.11 + +executors: + ubuntu: + working_directory: ~/go/src/github.com/mattermost/ + machine: + image: "ubuntu-1604:201903-01" + environment: + COMPOSE_PROJECT_NAME: "circleci" + jobs: + setup: + working_directory: /go/src/github.com/mattermost/mattermost-server + docker: + - image: mattermost/mattermost-build-webapp:oct-2-2018 + steps: + - checkout + - run: | + cd ../ + mkdir -p ~/.ssh/ + echo -e "Host github.com\n\tStrictHostKeyChecking no\n" > ~/.ssh/config + git clone git@github.com:mattermost/mattermost-webapp.git + cd mattermost-webapp + git checkout $CIRCLE_BRANCH || git checkout master + export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD) + echo "$WEBAPP_GIT_COMMIT" + curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build + - persist_to_workspace: + root: /go/src/github.com/mattermost + paths: + - mattermost-webapp + - mattermost-server + + # TODO: enable this step when the i18n-extract works with mattermost-server only + # and not depend on both mm-server/enterprise + # check-i18n: + # docker: + # - image: circleci/golang:1.12 + # working_directory: /go/src/github.com/mattermost/ + # steps: + # - attach_workspace: + # at: /go/src/github.com/mattermost/ + # - run: + # command: | + # cd mattermost-server + # cp i18n/en.json /tmp/en.json + # make i18n-extract + # diff /tmp/en.json i18n/en.json + build: docker: - - image: circleci/golang:1.12 + - image: mattermost/mattermost-build-server:feb-28-2019 + working_directory: /go/src/github.com/mattermost steps: - - run: echo "skipping build. PR \#11978 in progress." + - attach_workspace: + at: /go/src/github.com/mattermost/ + - run: + command: | + cd mattermost-server + make config-reset + make check-style BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' + GOFLAGS=-p=8 make build BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' + GOFLAGS=-p=8 make package BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' + - store_artifacts: + path: /go/src/github.com/mattermost/mattermost-server/dist/mattermost-team-linux-amd64.tar.gz + - store_artifacts: + path: /go/src/github.com/mattermost/mattermost-server/dist/mattermost-team-osx-amd64.tar.gz + - store_artifacts: + path: /go/src/github.com/mattermost/mattermost-server/dist/mattermost-team-windows-amd64.zip + - persist_to_workspace: + root: /go/src/github.com/mattermost + paths: + - mattermost-server + - mattermost-webapp + + test: + executor: + name: ubuntu + steps: + - attach_workspace: + at: ~/go/src/github.com/mattermost + - run: + name: Run Docker compose + command: | + cd mattermost-server/build + docker-compose --no-ansi run --rm start_dependencies + cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker-compose --no-ansi ps + background: true + - run: + name: Wait for docker compose + command: | + sleep 5 + docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" + docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" + - run: + name: Run Tests + command: | + ulimit -n 8096 + mkdir -p mattermost-server/client/plugins + docker run -it --net circleci_mm-test \ + --env TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \ + --env TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \ + --env TEST_DATABASE_MYSQL_ROOT_PASSWD=mostest \ + --env CI_INBUCKET_HOST=inbucket \ + --env CI_MINIO_HOST=minio \ + --env CI_INBUCKET_PORT=10080 \ + --env CI_MINIO_PORT=9000 \ + --env CI_INBUCKET_SMTP_PORT=10025 \ + --env CI_LDAP_HOST=openldap \ + --env IS_CI="true" \ + --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8" \ + --env MM_EMAILSETTINGS_SMTPSERVER=inbucket \ + --env MM_EMAILSETTINGS_SMTPPORT=10025 \ + --env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \ + -v ~/go/src:/go/src \ + -w /go/src/github.com/mattermost/mattermost-server \ + mattermost/mattermost-build-server:feb-28-2019 \ + bash -c 'ulimit -n 8096; make test-server BUILD_NUMBER="$CIRCLE_BRANCH-$CIRCLE_PREVIOUS_BUILD_NUM" TESTFLAGS= TESTFLAGSEE=' + no_output_timeout: 1h + - run: + name: Capture docker logs + when: always + command: | + cd mattermost-server/build + # Capture docker logs + mkdir -p logs + docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs + docker ps -a --no-trunc > logs/docker_ps + docker stats -a --no-stream > logs/docker_stats + tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats + - store_artifacts: + path: ~/go/src/github.com/mattermost/mattermost-server/build/logs + - run: + when: always + command: | + cd mattermost-server + mkdir -p test-results + cp report.xml test-results + - store_test_results: + path: ~/go/src/github.com/mattermost/mattermost-server/test-results/ + - store_artifacts: + path: ~/go/src/github.com/mattermost/mattermost-server/test-results/ + + upload-s3-sha: + docker: + - image: 'circleci/python:2.7' + working_directory: ~/go/src/github.com/mattermost/enterprise + steps: + - attach_workspace: + at: ~/go/src/github.com/mattermost/ + - run: + command: | + cd ~/go/src/github.com/mattermost/mattermost-server/dist/ + rm -rf mattermost + - aws-s3/copy: + from: ~/go/src/github.com/mattermost/mattermost-server/dist/ + to: "s3://releases.mattermost.com/mattermost-platform-pr/commit/${CIRCLE_SHA1}/" + arguments: --acl public-read --cache-control "no-cache" --recursive + + upload-s3: + docker: + - image: 'circleci/python:2.7' + working_directory: ~/go/src/github.com/mattermost/enterprise + steps: + - attach_workspace: + at: ~/go/src/github.com/mattermost/ + - run: + command: | + cd ~/go/src/github.com/mattermost/mattermost-server/dist/ + rm -rf mattermost + - aws-s3/copy: + from: ~/go/src/github.com/mattermost/mattermost-server/dist/ + to: s3://releases.mattermost.com/mattermost-platform-pr/$(echo "${CIRCLE_BRANCH}" | sed 's/pull\//PR-/g')/ + arguments: --acl public-read --cache-control "no-cache" --recursive + + build-docker: + working_directory: ~/ + docker: + - image: circleci/buildpack-deps:stretch + steps: + - attach_workspace: + at: . + - setup_remote_docker + - run: + command: | + export TAG="${CIRCLE_SHA1:0:7}" + cd mattermost-server + export MM_PACKAGE=https://releases.mattermost.com/mattermost-platform-pr/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz + docker build --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mattermost-team-edition:${TAG} build + echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin + docker push mattermost/mattermost-team-edition:${TAG} + + +workflows: + version: 2 + untagged-build: + jobs: + - setup + # - check-i18n: + # requires: + # - setup + - test: + requires: + - setup + - build: + requires: + - test + - upload-s3-sha: + context: mattermost-ci-s3 + requires: + - build + - upload-s3: + context: mattermost-ci-s3 + requires: + - build + - build-docker: + context: matterbuild-docker + requires: + - upload-s3-sha From 9b5b2831eec220582e39d626a3ff16c3efc02744 Mon Sep 17 00:00:00 2001 From: Akil Darjean Date: Thu, 19 Sep 2019 12:01:30 -0500 Subject: [PATCH 05/10] Migrate tests from 'app/command_mute_test.go' to use testify (#12240) --- app/command_mute_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/app/command_mute_test.go b/app/command_mute_test.go index cef422468c..5c7d95d47d 100644 --- a/app/command_mute_test.go +++ b/app/command_mute_test.go @@ -23,13 +23,13 @@ func TestMuteCommandNoChannel(t *testing.T) { channel1 := th.BasicChannel channel1M, channel1MError := th.App.GetChannelMember(channel1.Id, th.BasicUser.Id) - if channel1MError != nil { - t.Fatal("User is not a member of channel 1") - } - - if channel1M.NotifyProps[model.MARK_UNREAD_NOTIFY_PROP] == model.CHANNEL_NOTIFY_MENTION { - t.Fatal("channel shouldn't be muted on initial setup") - } + assert.Nil(t, channel1MError, "User is not a member of channel 1") + assert.NotEqual( + t, + channel1M.NotifyProps[model.MARK_UNREAD_NOTIFY_PROP], + model.CHANNEL_NOTIFY_MENTION, + "Channel shouldn't be muted on initial setup", + ) cmd := &MuteProvider{} resp := cmd.DoCommand(th.App, &model.CommandArgs{ From 22ec2d169cdd346dbf44cf358f21e3376d1f3329 Mon Sep 17 00:00:00 2001 From: Ogundele Olumide Date: Thu, 19 Sep 2019 23:33:24 +0100 Subject: [PATCH 06/10] chore: migrate t.fatal to testify (#12231) - convert t.fatal to require or assert --- app/plugin_api_test.go | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/app/plugin_api_test.go b/app/plugin_api_test.go index 8277a2897b..d913c03142 100644 --- a/app/plugin_api_test.go +++ b/app/plugin_api_test.go @@ -340,13 +340,11 @@ func TestPluginAPISavePluginConfig(t *testing.T) { pluginConfigJsonString := `{"mystringsetting": "str", "MyIntSetting": 32, "myboolsetting": true}` var pluginConfig map[string]interface{} - if err := json.Unmarshal([]byte(pluginConfigJsonString), &pluginConfig); err != nil { - t.Fatal(err) - } + err := json.Unmarshal([]byte(pluginConfigJsonString), &pluginConfig) + require.NoError(t, err) - if err := api.SavePluginConfig(pluginConfig); err != nil { - t.Fatal(err) - } + appErr := api.SavePluginConfig(pluginConfig) + require.Nil(t, appErr) type Configuration struct { MyStringSetting string @@ -355,14 +353,12 @@ func TestPluginAPISavePluginConfig(t *testing.T) { } savedConfiguration := new(Configuration) - if err := api.LoadPluginConfiguration(savedConfiguration); err != nil { - t.Fatal(err) - } + err = api.LoadPluginConfiguration(savedConfiguration) + require.NoError(t, err) expectedConfiguration := new(Configuration) - if err := json.Unmarshal([]byte(pluginConfigJsonString), &expectedConfiguration); err != nil { - t.Fatal(err) - } + err = json.Unmarshal([]byte(pluginConfigJsonString), &expectedConfiguration) + require.NoError(t, err) assert.Equal(t, expectedConfiguration, savedConfiguration) } @@ -387,9 +383,9 @@ func TestPluginAPIGetPluginConfig(t *testing.T) { pluginConfigJsonString := `{"mystringsetting": "str", "myintsetting": 32, "myboolsetting": true}` var pluginConfig map[string]interface{} - if err := json.Unmarshal([]byte(pluginConfigJsonString), &pluginConfig); err != nil { - t.Fatal(err) - } + err := json.Unmarshal([]byte(pluginConfigJsonString), &pluginConfig) + require.NoError(t, err) + th.App.UpdateConfig(func(cfg *model.Config) { cfg.PluginSettings.Plugins["pluginid"] = pluginConfig }) @@ -403,9 +399,9 @@ func TestPluginAPILoadPluginConfiguration(t *testing.T) { defer th.TearDown() var pluginJson map[string]interface{} - if err := json.Unmarshal([]byte(`{"mystringsetting": "str", "MyIntSetting": 32, "myboolsetting": true}`), &pluginJson); err != nil { - t.Fatal(err) - } + err := json.Unmarshal([]byte(`{"mystringsetting": "str", "MyIntSetting": 32, "myboolsetting": true}`), &pluginJson) + require.NoError(t, err) + th.App.UpdateConfig(func(cfg *model.Config) { cfg.PluginSettings.Plugins["testloadpluginconfig"] = pluginJson }) @@ -474,9 +470,9 @@ func TestPluginAPILoadPluginConfigurationDefaults(t *testing.T) { defer th.TearDown() var pluginJson map[string]interface{} - if err := json.Unmarshal([]byte(`{"mystringsetting": "override"}`), &pluginJson); err != nil { - t.Fatal(err) - } + err := json.Unmarshal([]byte(`{"mystringsetting": "override"}`), &pluginJson) + require.NoError(t, err) + th.App.UpdateConfig(func(cfg *model.Config) { cfg.PluginSettings.Plugins["testloadpluginconfig"] = pluginJson }) From 33cf37bbc0dce62ee6ff48c3b66d987c49fea8f4 Mon Sep 17 00:00:00 2001 From: Jesse Hallam Date: Thu, 19 Sep 2019 22:02:19 -0300 Subject: [PATCH 07/10] MM: 16479: ensure replacement file finishes io.Copy (#12249) * simplify FileWillBeUploaded * MM-16479: ensure replacement file finishes io.Copy --- plugin/client_rpc.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/plugin/client_rpc.go b/plugin/client_rpc.go index a43107c93c..1bf1f4e435 100644 --- a/plugin/client_rpc.go +++ b/plugin/client_rpc.go @@ -395,26 +395,31 @@ func (g *hooksRPCClient) FileWillBeUploaded(c *Context, info *model.FileInfo, fi serveIOReader(file, uploadedFileConnection) }() + replacementDone := make(chan bool) replacementFileStreamId := g.muxBroker.NextId() go func() { + defer close(replacementDone) + replacementFileConnection, err := g.muxBroker.Accept(replacementFileStreamId) if err != nil { g.log.Error("Plugin failed to serve replacement file stream. MuxBroker could not Accept connection", mlog.Err(err)) return } defer replacementFileConnection.Close() - if _, err := io.Copy(output, replacementFileConnection); err != nil && err != io.EOF { + if _, err := io.Copy(output, replacementFileConnection); err != nil { g.log.Error("Error reading replacement file.", mlog.Err(err)) } }() _args := &Z_FileWillBeUploadedArgs{c, info, uploadedFileStreamId, replacementFileStreamId} _returns := &Z_FileWillBeUploadedReturns{A: _args.B} - if g.implemented[FileWillBeUploadedId] { - if err := g.client.Call("Plugin.FileWillBeUploaded", _args, _returns); err != nil { - g.log.Error("RPC call FileWillBeUploaded to plugin failed.", mlog.Err(err)) - } + if err := g.client.Call("Plugin.FileWillBeUploaded", _args, _returns); err != nil { + g.log.Error("RPC call FileWillBeUploaded to plugin failed.", mlog.Err(err)) } + + // Ensure the io.Copy from the replacementFileConnection above completes. + <-replacementDone + return _returns.A, _returns.B } From 05fae599b8191ae9cafdacdaab46e2cb9a0d7e4c Mon Sep 17 00:00:00 2001 From: Ben Schumacher Date: Fri, 20 Sep 2019 06:22:40 +0200 Subject: [PATCH 08/10] [MM-18625] Fix database source parsing on go 1.12.8 (#12250) --- config/database.go | 22 ++++++++++++++-------- config/database_test.go | 19 ++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/config/database.go b/config/database.go index 15a8b64134..4c61375007 100644 --- a/config/database.go +++ b/config/database.go @@ -8,6 +8,7 @@ import ( "database/sql" "io/ioutil" "net/url" + "regexp" "strings" "github.com/jmoiron/sqlx" @@ -22,6 +23,8 @@ import ( _ "github.com/lib/pq" ) +var tcpStripper = regexp.MustCompile(`@tcp\((.*)\)`) + // DatabaseStore is a config store backed by a database. type DatabaseStore struct { commonStore @@ -101,23 +104,22 @@ func initializeConfigurationsTable(db *sqlx.DB) error { // By contrast, a Postgres DSN is returned unmodified. func parseDSN(dsn string) (string, string, error) { // Treat the DSN as the URL that it is. - u, err := url.Parse(dsn) - if err != nil { - return "", "", errors.Wrap(err, "failed to parse DSN as URL") + s := strings.SplitN(dsn, "://", 2) + if len(s) != 2 { + errors.New("failed to parse DSN as URL") } - scheme := u.Scheme + scheme := s[0] switch scheme { case "mysql": // Strip off the mysql:// for the dsn with which to connect. - u.Scheme = "" - dsn = strings.TrimPrefix(u.String(), "//") + dsn = s[1] case "postgres": // No changes required default: - return "", "", errors.Wrapf(err, "unsupported scheme %s", scheme) + return "", "", errors.Errorf("unsupported scheme %s", scheme) } return scheme, dsn, nil @@ -293,7 +295,11 @@ func (ds *DatabaseStore) RemoveFile(name string) error { // String returns the path to the database backing the config, masking the password. func (ds *DatabaseStore) String() string { - u, _ := url.Parse(ds.originalDsn) + // Remove @tcp and the parentheses from the host and parse the rest as a URL + u, err := url.Parse(tcpStripper.ReplaceAllString(ds.originalDsn, `@$1`)) + if err != nil { + return "(omitted due to error parsing the DSN)" + } // Strip out the password to avoid leaking in logs. u.User = url.User(u.User.Username()) diff --git a/config/database_test.go b/config/database_test.go index 106152e889..6119fc2305 100644 --- a/config/database_test.go +++ b/config/database_test.go @@ -6,7 +6,6 @@ package config_test import ( "bytes" "fmt" - "net/url" "os" "strings" "testing" @@ -137,6 +136,11 @@ func TestDatabaseStoreNew(t *testing.T) { _, err := config.NewDatabaseStore("invalid") require.Error(t, err) }) + + t.Run("unsupported scheme with valid data source", func(t *testing.T) { + _, err := config.NewDatabaseStore(fmt.Sprintf("invalid://%s", *sqlSettings.DataSource)) + require.Error(t, err) + }) } func TestDatabaseStoreGet(t *testing.T) { @@ -930,14 +934,11 @@ func TestDatabaseStoreString(t *testing.T) { sqlSettings := mainHelper.GetSqlSettings() ds, err := config.NewDatabaseStore(fmt.Sprintf("%s://%s", *sqlSettings.DriverName, *sqlSettings.DataSource)) require.NoError(t, err) + require.NotNil(t, ds) defer ds.Close() - actualStringURL, err := url.Parse(ds.String()) - require.NoError(t, err) - - assert.Equal(t, *sqlSettings.DriverName, actualStringURL.Scheme) - actualUsername := actualStringURL.User.Username() - actualPassword, _ := actualStringURL.User.Password() - assert.NotEmpty(t, actualUsername) - assert.Empty(t, actualPassword, "should mask password") + maskedDSN := ds.String() + assert.True(t, strings.HasPrefix(maskedDSN, "mysql://")) + assert.True(t, strings.Contains(maskedDSN, "mmuser")) + assert.False(t, strings.Contains(maskedDSN, "mostest")) } From e236eb74fa0d18374b77a6dbee578b3322113d12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Espino?= Date: Fri, 20 Sep 2019 15:09:58 +0200 Subject: [PATCH 09/10] Add prometheous metrics for each api handler (#12254) --- api4/handlers.go | 5 +++++ einterfaces/metrics.go | 1 + web/handlers.go | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/api4/handlers.go b/api4/handlers.go index acdf1d03d5..41bf74f758 100644 --- a/api4/handlers.go +++ b/api4/handlers.go @@ -18,6 +18,7 @@ func (api *API) ApiHandler(h func(*Context, http.ResponseWriter, *http.Request)) handler := &web.Handler{ GetGlobalAppOptions: api.GetGlobalAppOptions, HandleFunc: h, + HandlerName: web.GetHandlerName(h), RequireSession: false, TrustRequester: false, RequireMfa: false, @@ -35,6 +36,7 @@ func (api *API) ApiSessionRequired(h func(*Context, http.ResponseWriter, *http.R handler := &web.Handler{ GetGlobalAppOptions: api.GetGlobalAppOptions, HandleFunc: h, + HandlerName: web.GetHandlerName(h), RequireSession: true, TrustRequester: false, RequireMfa: true, @@ -54,6 +56,7 @@ func (api *API) ApiSessionRequiredMfa(h func(*Context, http.ResponseWriter, *htt handler := &web.Handler{ GetGlobalAppOptions: api.GetGlobalAppOptions, HandleFunc: h, + HandlerName: web.GetHandlerName(h), RequireSession: true, TrustRequester: false, RequireMfa: false, @@ -73,6 +76,7 @@ func (api *API) ApiHandlerTrustRequester(h func(*Context, http.ResponseWriter, * handler := &web.Handler{ GetGlobalAppOptions: api.GetGlobalAppOptions, HandleFunc: h, + HandlerName: web.GetHandlerName(h), RequireSession: false, TrustRequester: true, RequireMfa: false, @@ -91,6 +95,7 @@ func (api *API) ApiSessionRequiredTrustRequester(h func(*Context, http.ResponseW handler := &web.Handler{ GetGlobalAppOptions: api.GetGlobalAppOptions, HandleFunc: h, + HandlerName: web.GetHandlerName(h), RequireSession: true, TrustRequester: true, RequireMfa: true, diff --git a/einterfaces/metrics.go b/einterfaces/metrics.go index b604b895be..b6319225bc 100644 --- a/einterfaces/metrics.go +++ b/einterfaces/metrics.go @@ -44,4 +44,5 @@ type MetricsInterface interface { IncrementPostsSearchCounter() ObservePostsSearchDuration(elapsed float64) ObserveStoreMethodDuration(method string, success string, elapsed float64) + ObserveApiEndpointDuration(endpoint string, elapsed float64) } diff --git a/web/handlers.go b/web/handlers.go index 9269a04e93..a666501f6d 100644 --- a/web/handlers.go +++ b/web/handlers.go @@ -6,6 +6,9 @@ package web import ( "fmt" "net/http" + "reflect" + "runtime" + "strings" "time" "github.com/NYTimes/gziphandler" @@ -16,10 +19,20 @@ import ( "github.com/mattermost/mattermost-server/utils" ) +func GetHandlerName(h func(*Context, http.ResponseWriter, *http.Request)) string { + handlerName := runtime.FuncForPC(reflect.ValueOf(h).Pointer()).Name() + pos := strings.LastIndex(handlerName, ".") + if pos != -1 && len(handlerName) > pos { + handlerName = handlerName[pos+1:] + } + return handlerName +} + func (w *Web) NewHandler(h func(*Context, http.ResponseWriter, *http.Request)) http.Handler { return &Handler{ GetGlobalAppOptions: w.GetGlobalAppOptions, HandleFunc: h, + HandlerName: GetHandlerName(h), RequireSession: false, TrustRequester: false, RequireMfa: false, @@ -35,6 +48,7 @@ func (w *Web) NewStaticHandler(h func(*Context, http.ResponseWriter, *http.Reque return &Handler{ GetGlobalAppOptions: w.GetGlobalAppOptions, HandleFunc: h, + HandlerName: GetHandlerName(h), RequireSession: false, TrustRequester: false, RequireMfa: false, @@ -47,6 +61,7 @@ func (w *Web) NewStaticHandler(h func(*Context, http.ResponseWriter, *http.Reque type Handler struct { GetGlobalAppOptions app.AppOptionCreator HandleFunc func(*Context, http.ResponseWriter, *http.Request) + HandlerName string RequireSession bool TrustRequester bool RequireMfa bool @@ -192,6 +207,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if r.URL.Path != model.API_URL_SUFFIX+"/websocket" { elapsed := float64(time.Since(now)) / float64(time.Second) c.App.Metrics.ObserveHttpRequestDuration(elapsed) + c.App.Metrics.ObserveApiEndpointDuration(h.HandlerName, elapsed) } } } @@ -249,6 +265,7 @@ func (w *Web) ApiHandler(h func(*Context, http.ResponseWriter, *http.Request)) h handler := &Handler{ GetGlobalAppOptions: w.GetGlobalAppOptions, HandleFunc: h, + HandlerName: GetHandlerName(h), RequireSession: false, TrustRequester: false, RequireMfa: false, @@ -267,6 +284,7 @@ func (w *Web) ApiHandlerTrustRequester(h func(*Context, http.ResponseWriter, *ht handler := &Handler{ GetGlobalAppOptions: w.GetGlobalAppOptions, HandleFunc: h, + HandlerName: GetHandlerName(h), RequireSession: false, TrustRequester: true, RequireMfa: false, @@ -284,6 +302,7 @@ func (w *Web) ApiSessionRequired(h func(*Context, http.ResponseWriter, *http.Req handler := &Handler{ GetGlobalAppOptions: w.GetGlobalAppOptions, HandleFunc: h, + HandlerName: GetHandlerName(h), RequireSession: true, TrustRequester: false, RequireMfa: true, @@ -302,6 +321,7 @@ func (w *Web) apiHandlerTrustRequester(h func(*Context, http.ResponseWriter, *ht handler := &Handler{ GetGlobalAppOptions: w.GetGlobalAppOptions, HandleFunc: h, + HandlerName: GetHandlerName(h), RequireSession: false, TrustRequester: true, RequireMfa: false, From 32bc5854464ffe45da92559018230059dc35cd0e Mon Sep 17 00:00:00 2001 From: Shota Gvinepadze Date: Fri, 20 Sep 2019 17:48:19 +0400 Subject: [PATCH 10/10] [MM-18121] Fix config set panic (#12083) * Fix config set panic * Change error message * Add a test case * Update error string --- cmd/mattermost/commands/config.go | 17 +++++++++++------ cmd/mattermost/commands/config_test.go | 6 ++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cmd/mattermost/commands/config.go b/cmd/mattermost/commands/config.go index 494e45a433..81ed6d7236 100644 --- a/cmd/mattermost/commands/config.go +++ b/cmd/mattermost/commands/config.go @@ -21,6 +21,8 @@ import ( "github.com/mattermost/viper" ) +const noSettingsNamed = "unable to find a setting named: %s" + var ConfigCmd = &cobra.Command{ Use: "config", Short: "Configuration", @@ -296,7 +298,7 @@ func updateConfigValue(configSetting string, newVal []string, oldConfig, newConf func UpdateMap(configMap map[string]interface{}, configSettings []string, newVal []string) error { res, ok := configMap[configSettings[0]] if !ok { - return fmt.Errorf("unable to find a setting with that name %s", configSettings[0]) + return fmt.Errorf(noSettingsNamed, configSettings[0]) } value := reflect.ValueOf(res) @@ -308,6 +310,9 @@ func UpdateMap(configMap map[string]interface{}, configSettings []string, newVal if len(configSettings) == 1 { return errors.New("unable to set multiple settings at once") } + if value.Len() == 0 { + return fmt.Errorf(noSettingsNamed, configSettings[1]) + } return UpdateMap(res.(map[string]interface{}), configSettings[1:], newVal) case reflect.Int: @@ -319,7 +324,7 @@ func UpdateMap(configMap map[string]interface{}, configSettings []string, newVal configMap[configSettings[0]] = val return nil } - return fmt.Errorf("unable to find a setting with that name %s", configSettings[0]) + return fmt.Errorf(noSettingsNamed, configSettings[0]) case reflect.Int64: if len(configSettings) == 1 { @@ -330,7 +335,7 @@ func UpdateMap(configMap map[string]interface{}, configSettings []string, newVal configMap[configSettings[0]] = int64(val) return nil } - return fmt.Errorf("unable to find a setting with that name %s", configSettings[0]) + return fmt.Errorf(noSettingsNamed, configSettings[0]) case reflect.Bool: if len(configSettings) == 1 { @@ -341,21 +346,21 @@ func UpdateMap(configMap map[string]interface{}, configSettings []string, newVal configMap[configSettings[0]] = val return nil } - return fmt.Errorf("unable to find a setting with that name %s", configSettings[0]) + return fmt.Errorf(noSettingsNamed, configSettings[0]) case reflect.String: if len(configSettings) == 1 { configMap[configSettings[0]] = newVal[0] return nil } - return fmt.Errorf("unable to find a setting with that name %s", configSettings[0]) + return fmt.Errorf(noSettingsNamed, configSettings[0]) case reflect.Slice: if len(configSettings) == 1 { configMap[configSettings[0]] = newVal return nil } - return fmt.Errorf("unable to find a setting with that name %s", configSettings[0]) + return fmt.Errorf(noSettingsNamed, configSettings[0]) default: return errors.New("type not supported yet") diff --git a/cmd/mattermost/commands/config_test.go b/cmd/mattermost/commands/config_test.go index d5cb45ccc5..607f4f4a70 100644 --- a/cmd/mattermost/commands/config_test.go +++ b/cmd/mattermost/commands/config_test.go @@ -141,6 +141,12 @@ func TestConfigSet(t *testing.T) { assert.NotContains(t, string(output), "invalid-key") }) + t.Run("Error when the parameter of an unknown plugin is set", func(t *testing.T) { + output, err := th.RunCommandWithOutput(t, "config", "set", "PluginSettings.Plugins.someplugin", "true") + assert.Error(t, err) + assert.NotContains(t, string(output), "panic") + }) + t.Run("Error when the wrong locale is set", func(t *testing.T) { th.CheckCommand(t, "config", "set", "LocalizationSettings.DefaultServerLocale", "es") assert.Error(t, th.RunCommand(t, "config", "set", "LocalizationSettings.DefaultServerLocale", "invalid-key"))