From 8374b6221e5e9f8fd07cc44f3b0d1f0452bd8db8 Mon Sep 17 00:00:00 2001 From: Jesse Hallam Date: Mon, 10 Dec 2018 11:58:09 -0500 Subject: [PATCH] unittest using externally managed database (#9400) * MM-12083: unittest using externally managed database * cherry-pick Makefile changes from @cpanato * Jenkins changes (#9915) * add docker compose * udpate * when using minio dont need to set the region * update * add wait for it script * using old minio * add new jenkins file * update makefile * add dockerfile * rename the docker-compose proj to avoid colision (#9917) * rename the docker-compose proj to avoid colision * enable debug * enable debug to double checkt the branchs and fix docker-compose name (#9919) * add ee hash to check (#9920) * fix name (#9921) * update jenkins file to push from branch and prs * if a new push comes in stop the running build * split mysql and postgres variables * add script to run jenkins-like env in local dev env * update docker-compose project name to use uuid to make it more randon * fix DCNAME definition * update elasticsearch docker image * revert test * tidy up stages, and wait for mysql differently * update docker image and add check for postgres * checking if is ready * update docker compose to have a wait for deps * add readme and rename dockerfile * fix -unittest setup * using mm docker image * restore parallel unit tests at the package level Spin up a dedicated database for each package under test to avoid races in accessing the same tables. Simplify the interface for configuring the test database to just a DSN instead of multiple exports for each field. * try to work around root mysql access in CI * update local-test-env.sh too * MYSQL_ROOT_HOST: % * fix missing quotes * setting some memory limits for mysql * revert memory docker compose does not support * fix env name for postgres * expose errors in app/export_test.go * fix test label, better error checking on teardown * increase query timeout for tests * fix export_test * update local dev script * add configurable mysql root passwd --- Makefile | 89 ++++++--- app/export_test.go | 11 +- build/Dockerfile.buildenv | 3 + build/{Jenkinsfile => Jenkinsfile.k8s} | 0 build/Jenkinsfile.pr | 244 +++++++++++++++++++++++++ build/README.md | 8 + build/docker-compose.yml | 82 +++++++++ build/local-test-env.sh | 89 +++++++++ scripts/test.sh | 2 +- store/sqlstore/store_test.go | 45 ++--- store/storetest/docker.go | 143 --------------- store/storetest/settings.go | 233 +++++++++++++++++++++++ store/storetest/user_store.go | 164 ++++++++--------- testlib/helper.go | 27 ++- testlib/store.go | 3 +- web/handlers_test.go | 3 + 16 files changed, 846 insertions(+), 300 deletions(-) create mode 100644 build/Dockerfile.buildenv rename build/{Jenkinsfile => Jenkinsfile.k8s} (100%) create mode 100644 build/Jenkinsfile.pr create mode 100644 build/README.md create mode 100644 build/docker-compose.yml create mode 100755 build/local-test-env.sh delete mode 100644 store/storetest/docker.go create mode 100644 store/storetest/settings.go diff --git a/Makefile b/Makefile index 0bc56e3ad3..343ed785dd 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ ROOT := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) +IS_CI ?= false # Build Flags BUILD_NUMBER ?= $(BUILD_NUMBER:) BUILD_DATE = $(shell date -u) @@ -100,47 +101,55 @@ all: run ## Alias for 'run'. include build/*.mk start-docker: ## Starts the docker containers for local development. +ifeq ($(IS_CI),false) @echo Starting docker containers - @if [ $(shell docker ps -a | grep -ci mattermost-mysql) -eq 0 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql$$ | wc -l) -eq 0 ]; then \ echo starting mattermost-mysql; \ - docker run --name mattermost-mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=mostest \ - -e MYSQL_USER=mmuser -e MYSQL_PASSWORD=mostest -e MYSQL_DATABASE=mattermost_test -d mysql:5.7 > /dev/null; \ - elif [ $(shell docker ps | grep -ci mattermost-mysql) -eq 0 ]; then \ + docker run --name mattermost-mysql -p 3306:3306 \ + -e MYSQL_ROOT_PASSWORD=mostest \ + -e MYSQL_USER=mmuser \ + -e MYSQL_PASSWORD=mostest \ + -e MYSQL_DATABASE=mattermost_test \ + -d mysql:5.7 > /dev/null; \ + elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-mysql$$ | wc -l) -eq 0 ]; then \ echo restarting mattermost-mysql; \ docker start mattermost-mysql > /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-postgres) -eq 0 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres$$ | wc -l) -eq 0 ]; then \ echo starting mattermost-postgres; \ - docker run --name mattermost-postgres -p 5432:5432 -e POSTGRES_USER=mmuser -e POSTGRES_PASSWORD=mostest -e POSTGRES_DB=mattermost_test \ - -d postgres:9.4 > /dev/null; \ - elif [ $(shell docker ps | grep -ci mattermost-postgres) -eq 0 ]; then \ + docker run --name mattermost-postgres -p 5432:5432 \ + -e POSTGRES_USER=mmuser \ + -e POSTGRES_PASSWORD=mostest \ + -e POSTGRES_DB=mattermost_test \ + -d postgres:9.4 > /dev/null; \ + elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-postgres$$ | wc -l) -eq 0 ]; then \ echo restarting mattermost-postgres; \ docker start mattermost-postgres > /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-inbucket) -eq 0 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-inbucket$$ | wc -l) -eq 0 ]; then \ echo starting mattermost-inbucket; \ docker run --name mattermost-inbucket -p 9000:10080 -p 2500:10025 -d jhillyerd/inbucket:release-1.2.0 > /dev/null; \ - elif [ $(shell docker ps | grep -ci mattermost-inbucket) -eq 0 ]; then \ + elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-inbucket$$ | wc -l) -eq 0 ]; then \ echo restarting mattermost-inbucket; \ docker start mattermost-inbucket > /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-minio) -eq 0 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-minio$$ | wc -l) -eq 0 ]; then \ echo starting mattermost-minio; \ docker run --name mattermost-minio -p 9001:9000 -e "MINIO_ACCESS_KEY=minioaccesskey" \ -e "MINIO_SECRET_KEY=miniosecretkey" -d minio/minio:RELEASE.2018-05-25T19-49-13Z server /data > /dev/null; \ docker exec -it mattermost-minio /bin/sh -c "mkdir -p /data/mattermost-test" > /dev/null; \ - elif [ $(shell docker ps | grep -ci mattermost-minio) -eq 0 ]; then \ + elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-minio$$ | wc -l) -eq 0 ]; then \ echo restarting mattermost-minio; \ docker start mattermost-minio > /dev/null; \ fi ifeq ($(BUILD_ENTERPRISE_READY),true) @echo Ldap test user test.one - @if [ $(shell docker ps -a | grep -ci mattermost-openldap) -eq 0 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-openldap$$ | wc -l) -eq 0 ]; then \ echo starting mattermost-openldap; \ docker run --name mattermost-openldap -p 389:389 -p 636:636 \ -e LDAP_TLS_VERIFY_CLIENT="never" \ @@ -155,48 +164,61 @@ ifeq ($(BUILD_ENTERPRISE_READY),true) docker exec -ti mattermost-openldap bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';\ docker exec -ti mattermost-openldap bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest';\ docker exec -ti mattermost-openldap bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';\ - elif [ $(shell docker ps | grep -ci mattermost-openldap) -eq 0 ]; then \ + elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-openldap$$ | wc -l) -eq 0 ]; then \ echo restarting mattermost-openldap; \ docker start mattermost-openldap > /dev/null; \ sleep 10; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-elasticsearch) -eq 0 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-elasticsearch$$ | wc -l) -eq 0 ]; then \ echo starting mattermost-elasticsearch; \ docker run --name mattermost-elasticsearch -p 9200:9200 -e "http.host=0.0.0.0" -e "transport.host=127.0.0.1" -e "ES_JAVA_OPTS=-Xms250m -Xmx250m" -d grundleborg/elasticsearch:latest > /dev/null; \ - elif [ $(shell docker ps | grep -ci mattermost-elasticsearch) -eq 0 ]; then \ + elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-elasticsearch$$ | wc -l) -eq 0 ]; then \ echo restarting mattermost-elasticsearch; \ docker start mattermost-elasticsearch> /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-redis) -eq 0 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-redis$$ | wc -l) -eq 0 ]; then \ echo starting mattermost-redis; \ docker run --name mattermost-redis -p 6379:6379 -d redis > /dev/null; \ - elif [ $(shell docker ps | grep -ci mattermost-redis) -eq 0 ]; then \ + elif [ $(shell docker ps --no-trunc --quiet --filter name=^/mattermost-redis$$ | wc -l) -eq 0 ]; then \ echo restarting mattermost-redis; \ docker start mattermost-redis > /dev/null; \ fi endif +else + @echo CI Build: skipping docker start +endif stop-docker: ## Stops the docker containers for local development. @echo Stopping docker containers - @if [ $(shell docker ps -a | grep -ci mattermost-mysql) -eq 1 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql$$ | wc -l) -eq 1 ]; then \ echo stopping mattermost-mysql; \ docker stop mattermost-mysql > /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-postgres) -eq 1 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql-unittest$$ | wc -l) -eq 1 ]; then \ + echo stopping mattermost-mysql-unittest; \ + docker stop mattermost-mysql-unittest > /dev/null; \ + fi + + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres$$ | wc -l) -eq 1 ]; then \ echo stopping mattermost-postgres; \ docker stop mattermost-postgres > /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-openldap) -eq 1 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres-unittest$$ | wc -l) -eq 1 ]; then \ + echo stopping mattermost-postgres-unittest; \ + docker stop mattermost-postgres-unittest > /dev/null; \ + fi + + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-openldap$$ | wc -l) -eq 1 ]; then \ echo stopping mattermost-openldap; \ docker stop mattermost-openldap > /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-inbucket) -eq 1 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-inbucket$$ | wc -l) -eq 1 ]; then \ echo stopping mattermost-inbucket; \ docker stop mattermost-inbucket > /dev/null; \ fi @@ -206,26 +228,43 @@ stop-docker: ## Stops the docker containers for local development. docker stop mattermost-minio > /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-elasticsearch) -eq 1 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-elasticsearch$$ | wc -l) -eq 1 ]; then \ echo stopping mattermost-elasticsearch; \ docker stop mattermost-elasticsearch > /dev/null; \ fi + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-redis$$ | wc -l) -eq 1 ]; then \ + echo stopping mattermost-redis; \ + docker stop mattermost-redis > /dev/null; \ + fi + clean-docker: ## Deletes the docker containers for local development. @echo Removing docker containers - @if [ $(shell docker ps -a | grep -ci mattermost-mysql) -eq 1 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql$$ | wc -l) -eq 1 ]; then \ echo removing mattermost-mysql; \ docker stop mattermost-mysql > /dev/null; \ docker rm -v mattermost-mysql > /dev/null; \ fi - @if [ $(shell docker ps -a | grep -ci mattermost-postgres) -eq 1 ]; then \ + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-mysql-unittest$$ | wc -l) -eq 1 ]; then \ + echo removing mattermost-mysql-unittest; \ + docker stop mattermost-mysql-unittest > /dev/null; \ + docker rm -v mattermost-mysql-unittest > /dev/null; \ + fi + + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres$$ | wc -l) -eq 1 ]; then \ echo removing mattermost-postgres; \ docker stop mattermost-postgres > /dev/null; \ docker rm -v mattermost-postgres > /dev/null; \ fi + @if [ $(shell docker ps -a --no-trunc --quiet --filter name=^/mattermost-postgres-unittest$$ | wc -l) -eq 1 ]; then \ + echo removing mattermost-postgres-unittest; \ + docker stop mattermost-postgres-unittest > /dev/null; \ + docker rm -v mattermost-postgres-unittest > /dev/null; \ + fi + @if [ $(shell docker ps -a | grep -ci mattermost-openldap) -eq 1 ]; then \ echo removing mattermost-openldap; \ docker stop mattermost-openldap > /dev/null; \ diff --git a/app/export_test.go b/app/export_test.go index 1be0d79836..3d269115bf 100644 --- a/app/export_test.go +++ b/app/export_test.go @@ -88,7 +88,8 @@ func TestExportUserChannels(t *testing.T) { th.App.Srv.Store.Channel().SaveMember(&channelMember) th.App.Srv.Store.Preference().Save(&preferences) th.App.UpdateChannelMemberNotifyProps(notifyProps, channel.Id, user.Id) - exportData, _ := th.App.buildUserChannelMemberships(user.Id, team.Id) + exportData, err := th.App.buildUserChannelMemberships(user.Id, team.Id) + require.Nil(t, err) assert.Equal(t, len(*exportData), 3) for _, data := range *exportData { if *data.Name == channelName { @@ -158,15 +159,15 @@ func TestExportCustomEmoji(t *testing.T) { filePath := "../demo.json" - fileWriter, _ := os.Create(filePath) + fileWriter, err := os.Create(filePath) + require.Nil(t, err) defer os.Remove(filePath) pathToEmojiDir := "../data/emoji/" dirNameToExportEmoji := "exported_emoji_test" - - err := th.App.ExportCustomEmoji(fileWriter, filePath, pathToEmojiDir, dirNameToExportEmoji) defer os.RemoveAll("../" + dirNameToExportEmoji) - if err != nil { + + if err := th.App.ExportCustomEmoji(fileWriter, filePath, pathToEmojiDir, dirNameToExportEmoji); err != nil { t.Fatal(err) } } diff --git a/build/Dockerfile.buildenv b/build/Dockerfile.buildenv new file mode 100644 index 0000000000..e6ddfc0574 --- /dev/null +++ b/build/Dockerfile.buildenv @@ -0,0 +1,3 @@ +FROM golang:1.11 + +RUN apt-get update && apt-get install -y make git apt-transport-https ca-certificates curl software-properties-common build-essential zip wget xmlsec1 diff --git a/build/Jenkinsfile b/build/Jenkinsfile.k8s similarity index 100% rename from build/Jenkinsfile rename to build/Jenkinsfile.k8s diff --git a/build/Jenkinsfile.pr b/build/Jenkinsfile.pr new file mode 100644 index 0000000000..3a27197c8e --- /dev/null +++ b/build/Jenkinsfile.pr @@ -0,0 +1,244 @@ +#!/usr/bin/env groovy + +@Library('shared-pipelines') _ +import org.mattermost.Utils +import java.util.*; +def utils = new org.mattermost.Utils() +def rnd = UUID.randomUUID().toString() + +pipeline { + agent { + label 'default-mm-builder' + } + + options { buildDiscarder(logRotator(numToKeepStr: '3')) } + + environment { + DCNAME = "${rnd}" + } + + stages { + stage('Setup') { + steps { + script { + utils.stopOldBuilds() + } + cleanWs notFailBuild: true + sh """ + mkdir -p src/github.com/mattermost/mattermost-server + mkdir -p src/github.com/mattermost/mattermost-webapp + mkdir -p src/github.com/mattermost/enterprise + """ + dir('src/github.com/mattermost/mattermost-server') { + checkout scm + } + dir('src/github.com/mattermost/mattermost-webapp') { + checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/mattermost-webapp.git']]] + } + dir('src/github.com/mattermost/enterprise') { + checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/enterprise.git']]] + } + dir('src/github.com/mattermost/enterprise') { + ansiColor('xterm') { + sh """ + #!/bin/bash -ex + git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout master || echo 1 + export EE_GIT_COMMIT=\$(git rev-parse HEAD) + + echo EE Commit: \${EE_GIT_COMMIT} + """ + } + } + dir('src/github.com/mattermost/mattermost-webapp') { + withDockerContainer(args: '', image: 'mattermost/mattermost-build-webapp:oct-2-2018') { + ansiColor('xterm') { + sh """ + #!/bin/bash -ex + git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout master || echo 1 + rm -rf ./dist + export WEBAPP_GIT_COMMIT=\$(git rev-parse HEAD) + + echo Webapp Commit: \${WEBAPP_GIT_COMMIT} + + curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build || exit 1 + """ + } + } + } + } + } + + stage('Build') { + environment { + GOPATH = "/go" + } + + steps { + withDockerContainer(args: '-u root --privileged -v ${WORKSPACE}/src:/go/src/', image: 'mattermost/mattermost-build-server:dec-7-2018') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server + mv config/default.json config/config.json || echo "" + make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1 + make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1 + make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1 + """ + } + } + } + } + + stage('Test') { + environment { + GOPATH = "/go" + TEST_DATABASE_MYSQL_DSN = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" + TEST_DATABASE_POSTGRESQL_DSN = "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" + TEST_DATABASE_MYSQL_ROOT_PASSWD = "mostest" + CI_INBUCKET_HOST = "inbucket" + CI_MINIO_HOST = "minio" + CI_INBUCKET_PORT = "10080" + CI_MINIO_PORT = "9000" + CI_LDAP_HOST = "openldap" + IS_CI = true + } + + steps { + dir('src/github.com/mattermost/mattermost-server/build') { + ansiColor('xterm') { + sh """ + COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose run --rm start_dependencies + COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose up -d mysql postgres minio inbucket openldap elasticsearch redis + + docker ps + + docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'; + docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'; + docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + """ + } + } + + withDockerContainer(args: "-u root --privileged --net ${DCNAME}-${BUILD_NUMBER}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:dec-7-2018') { + ansiColor('xterm') { + sh """ + cd /go/src/github.com/mattermost/mattermost-server + + sed -i'.bak' 's|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|g' config/config.json; + sed -i'.bak' 's|"SMTPServer": "dockerhost",|"SMTPServer": "inbucket",|g' config/config.json; + sed -i'.bak' 's|"SMTPPort": "2500",|"SMTPPort": "10025",|g' config/config.json; + sed -i'.bak' 's|"ConnectionUrl": "http://dockerhost:9200",|"ConnectionUrl": "http://elasticsearch:9200",|g' config/config.json; + mkdir -p client/plugins + + cat config/config.json + + make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE= || exit 1 + """ + } + } + } + } + + stage('Push to S3') { + stages { + stage('Pull request') { + when { + allOf { + expression { env.CHANGE_ID != null } + expression { env.CHANGE_TARGET != null } + } + } + steps { + dir('src/github.com/mattermost/mattermost-server/dist') { + step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[ + bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_ID}", + excludedFile: '', + flatten: true, + gzipFiles: false, + keepForever: false, + managedArtifacts: false, + noUploadOnFailure: true, + selectedRegion: 'us-east-1', + showDirectlyInBrowser: false, + sourceFile: '*.tar.gz', + storageClass: 'STANDARD', + uploadFromSlave: false, + useServerSideEncryption: false, + userMetadata: [[key: 'Cache-Control', value: 'no-cache']] + ], [ + bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_BRANCH}", + excludedFile: '', + flatten: true, + gzipFiles: false, + keepForever: false, + managedArtifacts: false, + noUploadOnFailure: true, + selectedRegion: 'us-east-1', + showDirectlyInBrowser: false, + sourceFile: '*.tar.gz', + storageClass: 'STANDARD', + uploadFromSlave: false, + useServerSideEncryption: false, + userMetadata: [[key: 'Cache-Control', value: 'no-cache']] + ]], profileName: 'Releases', userMetadAta: []]) + } + } + } + stage('Branch') { + when { + expression { env.CHANGE_ID == null } + } + steps { + dir('src/github.com/mattermost/mattermost-server/dist') { + step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[ + bucket: "releases.mattermost.com/mattermost-platform-pr/${BRANCH_NAME}", + excludedFile: '', + flatten: true, + gzipFiles: false, + keepForever: false, + managedArtifacts: false, + noUploadOnFailure: true, + selectedRegion: 'us-east-1', + showDirectlyInBrowser: false, + sourceFile: '*.tar.gz', + storageClass: 'STANDARD', + uploadFromSlave: false, + useServerSideEncryption: false, + userMetadata: [[key: 'Cache-Control', value: 'no-cache']] + ]], profileName: 'Releases', userMetadAta: []]) + } + } + } + } + } + + stage('Trigger docker image') { + when { + expression { env.CHANGE_ID != null } + } + steps { + build job: '../../mk/mattermost-enterprise-edition-release', parameters: [string(name: 'RELEASE', value: "${CHANGE_ID}"), booleanParam(name: 'FROM_PR', value: true)], propagate: false, wait: false + } + } + } + + post { + always { + dir('src/github.com/mattermost/mattermost-server') { + junit allowEmptyResults: true, healthScaleFactor: 0.0, testResults: 'report*.xml' + } + dir('src/github.com/mattermost/mattermost-server/build') { + ansiColor('xterm') { + sh """ + COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose down -v + """ + } + } + } + cleanup { + cleanWs notFailBuild: true + } + } +} diff --git a/build/README.md b/build/README.md new file mode 100644 index 0000000000..fe0fd83c50 --- /dev/null +++ b/build/README.md @@ -0,0 +1,8 @@ +## About this folder + +This folder contains some files that we use to build the `mattermost-server` using `Jenkins` and other files like privacy policy and licenses. + +PRs opened against the `mattermost-server` repository will use the file called `Jenkinsfile.pr` + +The `Dockerfile` in this folder (`Dockerfile.buildenv`) is the build environment for our current builds you can find the docker image to downalod [here](https://hub.docker.com/r/mattermost/mattermost-build-server/tags/) or build your own. + diff --git a/build/docker-compose.yml b/build/docker-compose.yml new file mode 100644 index 0000000000..75483d5cc5 --- /dev/null +++ b/build/docker-compose.yml @@ -0,0 +1,82 @@ +version: '3.4' +services: + mysql: + image: "mysql:5.7" + restart: always + container_name: mysql-$DCNAME-$BUILD_NUMBER + networks: + - mm-test + environment: + MYSQL_ROOT_HOST: "%" + MYSQL_ROOT_PASSWORD: mostest + MYSQL_PASSWORD: mostest + MYSQL_USER: mmuser + MYSQL_DATABASE: mattermost_test + postgres: + image: "postgres:9.4" + restart: always + container_name: postgres-$DCNAME-$BUILD_NUMBER + networks: + - mm-test + environment: + POSTGRES_USER: mmuser + POSTGRES_PASSWORD: mostest + POSTGRES_DB: mattermost_test + minio: + image: "minio/minio:RELEASE.2018-05-25T19-49-13Z" + command: "server /data" + container_name: minio-$DCNAME-$BUILD_NUMBER + networks: + - mm-test + environment: + MINIO_ACCESS_KEY: minioaccesskey + MINIO_SECRET_KEY: miniosecretkey + inbucket: + image: "jhillyerd/inbucket:release-1.2.0" + restart: always + container_name: inbucket-$DCNAME-$BUILD_NUMBER + networks: + - mm-test + openldap: + image: "osixia/openldap:1.1.6" + restart: always + container_name: openldap-$DCNAME-$BUILD_NUMBER + networks: + - mm-test + environment: + LDAP_TLS_VERIFY_CLIENT: "never" + LDAP_ORGANISATION: "Mattermost Test" + LDAP_DOMAIN: "mm.test.com" + LDAP_ADMIN_PASSWORD: "mostest" + elasticsearch: + image: "mattermost/mattermost-elasticsearch-docker:6.5.1" + container_name: elasticsearch-$DCNAME-$BUILD_NUMBER + networks: + - mm-test + environment: + http.host: "0.0.0.0" + transport.host: "127.0.0.1" + ES_JAVA_OPTS: "-Xms250m -Xmx250m" + redis: + image: redis + networks: + - mm-test + container_name: redis-$DCNAME-$BUILD_NUMBER + + start_dependencies: + image: mattermost/mattermost-wait-for-dep:latest + networks: + - mm-test + depends_on: + - mysql + - postgres + - minio + - inbucket + - openldap + - elasticsearch + - redis + command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 redis:6379 + +networks: + mm-test: + driver: bridge diff --git a/build/local-test-env.sh b/build/local-test-env.sh new file mode 100755 index 0000000000..be5b10b262 --- /dev/null +++ b/build/local-test-env.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +export DCNAME=localdev +export BUILD_NUMBER=null +local_cmdname=${0##*/} + +usage() +{ + cat << USAGE >&2 +Usage: + $local_cmdname up/down +USAGE + exit 1 +} + +up() +{ + COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose run --rm start_dependencies + COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose up -d mysql postgres minio inbucket openldap elasticsearch redis + + docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' + docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest' + docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest' + + echo "Changing the config.json" + sed -i'.bak' 's|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json + sed -i'.bak' 's|"SMTPServer": "dockerhost",|"SMTPServer": "inbucket",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json + sed -i'.bak' 's|"SMTPPort": "2500",|"SMTPPort": "10025",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json + sed -i'.bak' 's|"ConnectionUrl": "http://dockerhost:9200",|"ConnectionUrl": "http://elasticsearch:9200",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json + + docker run -it -u root \ + --privileged \ + -v $GOPATH:/go \ + -w /go/src/github.com/mattermost/mattermost-server/ \ + --net $DCNAME-$BUILD_NUMBER\_mm-test \ + -e GOPATH="/go" \ + -e TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \ + -e TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \ + -e TEST_DATABASE_MYSQL_ROOT_PASSWD="mostest" + -e CI_INBUCKET_HOST="inbucket" \ + -e CI_MINIO_HOST="minio" \ + -e CI_INBUCKET_PORT="10080" \ + -e CI_MINIO_PORT="9000" \ + -e CI_LDAP_HOST="openldap" \ + -e IS_CI=true \ + mattermost/mattermost-build-server:dec-7-2018 /bin/bash +} + +down() +{ + COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose down + + echo "Reverting the changes in the config.json" + sed -i'.bak' 's|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json + sed -i'.bak' 's|"SMTPServer": "inbucket",|"SMTPServer": "dockerhost",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json + sed -i'.bak' 's|"SMTPPort": "10025",|"SMTPPort": "2500",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json + sed -i'.bak' 's|"ConnectionUrl": "http://elasticsearch:9200",|"ConnectionUrl": "http://dockerhost:9200",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + up) + echo "Starting Containers" + up + break + ;; + down) + echo "Stopping Containers" + down + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$1" == "" ]]; then + usage +fi diff --git a/scripts/test.sh b/scripts/test.sh index 7d4d620d11..c38f131e17 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -15,7 +15,7 @@ echo "Packages to test: $PACKAGES" find . -name 'cprofile*.out' -exec sh -c 'rm "{}"' \; find . -type d -name data -not -path './vendor/*' | xargs rm -rf -$GO test $GOFLAGS -run=$TESTS $TESTFLAGS -p 1 -v -timeout=2000s -covermode=count -coverpkg=$PACKAGES_COMMA -exec $DIR/test-xprog.sh $PACKAGES 2>&1 | tee output +$GO test $GOFLAGS -run=$TESTS $TESTFLAGS -v -timeout=2000s -covermode=count -coverpkg=$PACKAGES_COMMA -exec $DIR/test-xprog.sh $PACKAGES 2>&1 | tee output EXIT_STATUS=$? cat output | $GOPATH/bin/go-junit-report > report.xml diff --git a/store/sqlstore/store_test.go b/store/sqlstore/store_test.go index 55002aee26..9ac04bf2e7 100644 --- a/store/sqlstore/store_test.go +++ b/store/sqlstore/store_test.go @@ -15,23 +15,15 @@ import ( "github.com/mattermost/mattermost-server/utils" ) -var storeTypes = []*struct { +type storeType struct { Name string - Func func() (*storetest.RunningContainer, *model.SqlSettings, error) - Container *storetest.RunningContainer + SqlSettings *model.SqlSettings SqlSupplier *SqlSupplier Store store.Store -}{ - { - Name: "MySQL", - Func: storetest.NewMySQLContainer, - }, - { - Name: "PostgreSQL", - Func: storetest.NewPostgreSQLContainer, - }, } +var storeTypes []*storeType + func StoreTest(t *testing.T, f func(*testing.T, store.Store)) { defer func() { if err := recover(); err != nil { @@ -59,6 +51,15 @@ func StoreTestWithSqlSupplier(t *testing.T, f func(*testing.T, store.Store, stor } func initStores() { + storeTypes = append(storeTypes, &storeType{ + Name: "MySQL", + SqlSettings: storetest.MakeSqlSettings(model.DATABASE_DRIVER_MYSQL), + }) + storeTypes = append(storeTypes, &storeType{ + Name: "PostgreSQL", + SqlSettings: storetest.MakeSqlSettings(model.DATABASE_DRIVER_POSTGRES), + }) + defer func() { if err := recover(); err != nil { tearDownStores() @@ -66,29 +67,18 @@ func initStores() { } }() var wg sync.WaitGroup - errCh := make(chan error, len(storeTypes)) - wg.Add(len(storeTypes)) for _, st := range storeTypes { st := st + wg.Add(1) go func() { defer wg.Done() - container, settings, err := st.Func() - if err != nil { - errCh <- err - return - } - st.Container = container - st.SqlSupplier = NewSqlSupplier(*settings, nil) + st.SqlSupplier = NewSqlSupplier(*st.SqlSettings, nil) st.Store = store.NewLayeredStore(st.SqlSupplier, nil, nil) + st.Store.DropAllTables() st.Store.MarkSystemRanUnitTests() }() } wg.Wait() - select { - case err := <-errCh: - panic(err) - default: - } } var tearDownStoresOnce sync.Once @@ -103,9 +93,6 @@ func tearDownStores() { if st.Store != nil { st.Store.Close() } - if st.Container != nil { - st.Container.Stop() - } wg.Done() }() } diff --git a/store/storetest/docker.go b/store/storetest/docker.go deleted file mode 100644 index f5b8807d0e..0000000000 --- a/store/storetest/docker.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package storetest - -import ( - "encoding/json" - "fmt" - "io" - "net" - "os/exec" - "strings" - "time" - - "github.com/mattermost/mattermost-server/mlog" - "github.com/mattermost/mattermost-server/model" -) - -type Container struct { - Id string - NetworkSettings struct { - Ports map[string][]struct { - HostPort string - } - } -} - -type RunningContainer struct { - Container -} - -func (c *RunningContainer) Stop() error { - mlog.Info(fmt.Sprintf("Removing container: %v", c.Id)) - return exec.Command("docker", "rm", "-f", c.Id).Run() -} - -func NewMySQLContainer() (*RunningContainer, *model.SqlSettings, error) { - container, err := runContainer([]string{ - "-e", "MYSQL_ROOT_PASSWORD=mostest", - "-e", "MYSQL_USER=mmuser", - "-e", "MYSQL_PASSWORD=mostest", - "-e", "MYSQL_DATABASE=mattermost_test", - "--tmpfs", "/var/lib/mysql", - "mysql:5.7", - }) - if err != nil { - return nil, nil, err - } - mlog.Info("Waiting for mysql connectivity") - port := container.NetworkSettings.Ports["3306/tcp"][0].HostPort - if err := waitForPort(port); err != nil { - container.Stop() - return nil, nil, err - } - return container, databaseSettings("mysql", "mmuser:mostest@tcp(127.0.0.1:"+port+")/mattermost_test?charset=utf8mb4,utf8"), nil -} - -func NewPostgreSQLContainer() (*RunningContainer, *model.SqlSettings, error) { - container, err := runContainer([]string{ - "-e", "POSTGRES_USER=mmuser", - "-e", "POSTGRES_PASSWORD=mostest", - "--tmpfs", "/var/lib/postgresql/data", - "postgres:9.4", - }) - if err != nil { - return nil, nil, err - } - mlog.Info("Waiting for postgres connectivity") - port := container.NetworkSettings.Ports["5432/tcp"][0].HostPort - if err := waitForPort(port); err != nil { - container.Stop() - return nil, nil, err - } - return container, databaseSettings("postgres", "postgres://mmuser:mostest@127.0.0.1:"+port+"?sslmode=disable"), nil -} - -func databaseSettings(driver, dataSource string) *model.SqlSettings { - settings := &model.SqlSettings{ - DriverName: &driver, - DataSource: &dataSource, - DataSourceReplicas: []string{}, - DataSourceSearchReplicas: []string{}, - MaxIdleConns: new(int), - ConnMaxLifetimeMilliseconds: new(int), - MaxOpenConns: new(int), - Trace: false, - AtRestEncryptKey: model.NewRandomString(32), - QueryTimeout: new(int), - } - *settings.MaxIdleConns = 10 - *settings.ConnMaxLifetimeMilliseconds = 3600000 - *settings.MaxOpenConns = 100 - *settings.QueryTimeout = 10 - return settings -} - -func runContainer(args []string) (*RunningContainer, error) { - name := "mattermost-storetest-" + model.NewId() - dockerArgs := append([]string{"run", "-d", "-P", "--name", name}, args...) - out, err := exec.Command("docker", dockerArgs...).Output() - if err != nil { - return nil, err - } - id := strings.TrimSpace(string(out)) - out, err = exec.Command("docker", "inspect", id).Output() - if err != nil { - exec.Command("docker", "rm", "-f", id).Run() - return nil, err - } - var containers []Container - if err := json.Unmarshal(out, &containers); err != nil { - exec.Command("docker", "rm", "-f", id).Run() - return nil, err - } - mlog.Info(fmt.Sprintf("Running container: %v", id)) - return &RunningContainer{containers[0]}, nil -} - -func waitForPort(port string) error { - deadline := time.Now().Add(time.Minute * 10) - for time.Now().Before(deadline) { - conn, err := net.DialTimeout("tcp", "127.0.0.1:"+port, time.Minute) - if err != nil { - return err - } - if err = conn.SetReadDeadline(time.Now().Add(time.Millisecond * 500)); err != nil { - return err - } - _, err = conn.Read(make([]byte, 1)) - conn.Close() - if err == nil { - return nil - } - if e, ok := err.(net.Error); ok && e.Timeout() { - return nil - } - if err != io.EOF { - return err - } - time.Sleep(time.Millisecond * 200) - } - return fmt.Errorf("timeout waiting for port %v", port) -} diff --git a/store/storetest/settings.go b/store/storetest/settings.go new file mode 100644 index 0000000000..faef15e1c5 --- /dev/null +++ b/store/storetest/settings.go @@ -0,0 +1,233 @@ +// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package storetest + +import ( + "flag" + "fmt" + "net/url" + "os" + "path" + + "database/sql" + + "github.com/go-sql-driver/mysql" + _ "github.com/go-sql-driver/mysql" + _ "github.com/lib/pq" + "github.com/pkg/errors" + + "github.com/mattermost/mattermost-server/model" +) + +const ( + defaultMysqlDSN = "mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" + defaultPostgresqlDSN = "postgres://mmuser:mostest@dockerhost:5432/mattermost_test?sslmode=disable&connect_timeout=10" + defaultMysqlRootPWD = "passwd" +) + +func getEnv(name, defaultValue string) string { + if value := os.Getenv(name); value != "" { + return value + } else { + return defaultValue + } +} + +func log(message string) { + verbose := false + if verboseFlag := flag.Lookup("test.v"); verboseFlag != nil { + verbose = verboseFlag.Value.String() == "true" + } + if verboseFlag := flag.Lookup("v"); verboseFlag != nil { + verbose = verboseFlag.Value.String() == "true" + } + + if verbose { + fmt.Println(message) + } +} + +// MySQLSettings returns the database settings to connect to the MySQL unittesting database. +// The database name is generated randomly and must be created before use. +func MySQLSettings() *model.SqlSettings { + dsn := getEnv("TEST_DATABASE_MYSQL_DSN", defaultMysqlDSN) + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + panic("failed to parse dsn " + dsn + ": " + err.Error()) + } + + cfg.DBName = "db" + model.NewId() + + return databaseSettings("mysql", cfg.FormatDSN()) +} + +// PostgresSQLSettings returns the database settings to connect to the PostgreSQL unittesting database. +// The database name is generated randomly and must be created before use. +func PostgreSQLSettings() *model.SqlSettings { + dsn := getEnv("TEST_DATABASE_POSTGRESQL_DSN", defaultPostgresqlDSN) + dsnUrl, err := url.Parse(dsn) + if err != nil { + panic("failed to parse dsn " + dsn + ": " + err.Error()) + } + + // Generate a random database name + dsnUrl.Path = "db" + model.NewId() + + return databaseSettings("postgres", dsnUrl.String()) +} + +func mySQLRootDSN(dsn string) string { + rootPwd := getEnv("TEST_DATABASE_MYSQL_ROOT_PASSWD", defaultMysqlRootPWD) + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + panic("failed to parse dsn " + dsn + ": " + err.Error()) + } + + cfg.User = "root" + cfg.Passwd = rootPwd + cfg.DBName = "mysql" + + return cfg.FormatDSN() +} + +func postgreSQLRootDSN(dsn string) string { + dsnUrl, err := url.Parse(dsn) + if err != nil { + panic("failed to parse dsn " + dsn + ": " + err.Error()) + } + + // // Assume the unittesting database has the same password. + // password := "" + // if dsnUrl.User != nil { + // password, _ = dsnUrl.User.Password() + // } + + // dsnUrl.User = url.UserPassword("", password) + dsnUrl.Path = "postgres" + + return dsnUrl.String() +} + +func mySQLDSNDatabase(dsn string) string { + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + panic("failed to parse dsn " + dsn + ": " + err.Error()) + } + + return cfg.DBName +} + +func postgreSQLDSNDatabase(dsn string) string { + dsnUrl, err := url.Parse(dsn) + if err != nil { + panic("failed to parse dsn " + dsn + ": " + err.Error()) + } + + return path.Base(dsnUrl.Path) +} + +func databaseSettings(driver, dataSource string) *model.SqlSettings { + settings := &model.SqlSettings{ + DriverName: &driver, + DataSource: &dataSource, + DataSourceReplicas: []string{}, + DataSourceSearchReplicas: []string{}, + MaxIdleConns: new(int), + ConnMaxLifetimeMilliseconds: new(int), + MaxOpenConns: new(int), + Trace: false, + AtRestEncryptKey: model.NewRandomString(32), + QueryTimeout: new(int), + } + *settings.MaxIdleConns = 10 + *settings.ConnMaxLifetimeMilliseconds = 3600000 + *settings.MaxOpenConns = 100 + *settings.QueryTimeout = 60 + + return settings +} + +// execAsRoot executes the given sql as root against the testing database +func execAsRoot(settings *model.SqlSettings, sqlCommand string) error { + var dsn string + var driver = *settings.DriverName + + switch driver { + case model.DATABASE_DRIVER_MYSQL: + dsn = mySQLRootDSN(*settings.DataSource) + case model.DATABASE_DRIVER_POSTGRES: + dsn = postgreSQLRootDSN(*settings.DataSource) + default: + return fmt.Errorf("unsupported driver %s", driver) + } + + db, err := sql.Open(driver, dsn) + if err != nil { + return errors.Wrapf(err, "failed to connect to %s database as root", driver) + } + defer db.Close() + if _, err = db.Exec(sqlCommand); err != nil { + return errors.Wrapf(err, "failed to execute `%s` against %s database as root", sqlCommand, driver) + } + + return nil +} + +// MakeSqlSettings creates a randomly named database and returns the corresponding sql settings +func MakeSqlSettings(driver string) *model.SqlSettings { + var settings *model.SqlSettings + var dbName string + + switch driver { + case model.DATABASE_DRIVER_MYSQL: + settings = MySQLSettings() + dbName = mySQLDSNDatabase(*settings.DataSource) + case model.DATABASE_DRIVER_POSTGRES: + settings = PostgreSQLSettings() + dbName = postgreSQLDSNDatabase(*settings.DataSource) + default: + panic("unsupported driver " + driver) + } + + if err := execAsRoot(settings, "CREATE DATABASE "+dbName); err != nil { + panic("failed to create temporary database " + dbName + ": " + err.Error()) + } + + switch driver { + case model.DATABASE_DRIVER_MYSQL: + if err := execAsRoot(settings, "GRANT ALL PRIVILEGES ON "+dbName+".* TO 'mmuser'"); err != nil { + panic("failed to grant mmuser permission to " + dbName + ":" + err.Error()) + } + case model.DATABASE_DRIVER_POSTGRES: + if err := execAsRoot(settings, "GRANT ALL PRIVILEGES ON DATABASE \""+dbName+"\" TO mmuser"); err != nil { + panic("failed to grant mmuser permission to " + dbName + ":" + err.Error()) + } + default: + panic("unsupported driver " + driver) + } + + log("Created temporary database " + dbName) + + return settings +} + +func CleanupSqlSettings(settings *model.SqlSettings) { + var driver = *settings.DriverName + var dbName string + + switch driver { + case model.DATABASE_DRIVER_MYSQL: + dbName = mySQLDSNDatabase(*settings.DataSource) + case model.DATABASE_DRIVER_POSTGRES: + dbName = postgreSQLDSNDatabase(*settings.DataSource) + default: + panic("unsupported driver " + driver) + } + + if err := execAsRoot(settings, "DROP DATABASE "+dbName); err != nil { + panic("failed to drop temporary database " + dbName + ": " + err.Error()) + } + + log("Dropped temporary database " + dbName) +} diff --git a/store/storetest/user_store.go b/store/storetest/user_store.go index 390adf3bca..a7abf37e03 100644 --- a/store/storetest/user_store.go +++ b/store/storetest/user_store.go @@ -77,7 +77,7 @@ func testUserStoreSave(t *testing.T, ss store.Store) { if err := (<-ss.User().Save(&u1)).Err; err != nil { t.Fatal("couldn't save user", err) } - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, maxUsersPerTeam)) @@ -108,7 +108,7 @@ func testUserStoreSave(t *testing.T, ss store.Store) { if err := (<-ss.User().Save(&u1)).Err; err != nil { t.Fatal("couldn't save item", err) } - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, maxUsersPerTeam)) } @@ -119,7 +119,7 @@ func testUserStoreSave(t *testing.T, ss store.Store) { if err := (<-ss.User().Save(&u1)).Err; err != nil { t.Fatal("couldn't save item", err) } - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() if err := (<-ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, maxUsersPerTeam)).Err; err == nil { t.Fatal("should be the limit") @@ -130,14 +130,14 @@ func testUserStoreUpdate(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() u2.AuthService = "ldap" store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u2.Id}, -1)) time.Sleep(100 * time.Millisecond) @@ -166,7 +166,7 @@ func testUserStoreUpdate(t *testing.T, ss store.Store) { oldEmail := u3.Email u3.AuthService = "gitlab" store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u3.Id}, -1)) u3.Email = MakeEmail() @@ -198,7 +198,7 @@ func testUserStoreUpdateUpdateAt(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1)) time.Sleep(10 * time.Millisecond) @@ -221,7 +221,7 @@ func testUserStoreUpdateFailedPasswordAttempts(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1)) if err := (<-ss.User().UpdateFailedPasswordAttempts(u1.Id, 3)).Err; err != nil { @@ -242,7 +242,7 @@ func testUserStoreGet(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1)) if r1 := <-ss.User().Get(u1.Id); r1.Err != nil { @@ -262,7 +262,7 @@ func testUserCount(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1)) if result := <-ss.User().GetTotalUsersCount(); result.Err != nil { @@ -278,13 +278,13 @@ func testGetAllUsingAuthService(t *testing.T, ss store.Store) { u1.Email = MakeEmail() u1.AuthService = "someservice" store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() u2 := &model.User{} u2.Email = MakeEmail() u2.AuthService = "someservice" store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() if r1 := <-ss.User().GetAllUsingAuthService(u1.AuthService); r1.Err != nil { t.Fatal(r1.Err) @@ -300,12 +300,12 @@ func testUserStoreGetAllProfiles(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() if r1 := <-ss.User().GetAllProfiles(0, 100); r1.Err != nil { t.Fatal(r1.Err) @@ -344,7 +344,7 @@ func testUserStoreGetAllProfiles(t *testing.T, ss store.Store) { u3 := &model.User{} u3.Email = MakeEmail() store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() if r2 := <-ss.User().GetEtagForAllProfiles(); r2.Err != nil { t.Fatal(r2.Err) @@ -361,13 +361,13 @@ func testUserStoreGetProfiles(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) if r1 := <-ss.User().GetProfiles(teamId, 0, 100); r1.Err != nil { @@ -408,7 +408,7 @@ func testUserStoreGetProfiles(t *testing.T, ss store.Store) { u3 := &model.User{} u3.Email = MakeEmail() store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u3.Id}, -1)) if r2 := <-ss.User().GetEtagForProfiles(teamId); r2.Err != nil { @@ -426,13 +426,13 @@ func testUserStoreGetProfilesInChannel(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) c1 := model.Channel{} @@ -504,13 +504,13 @@ func testUserStoreGetProfilesInChannelByStatus(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) c1 := model.Channel{} @@ -586,13 +586,13 @@ func testUserStoreGetProfilesWithoutTeam(t *testing.T, ss store.Store) { u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() u2 := &model.User{} u2.Username = "a000000001" + model.NewId() u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() if r1 := <-ss.User().GetProfilesWithoutTeam(0, 100); r1.Err != nil { t.Fatal(r1.Err) @@ -623,13 +623,13 @@ func testUserStoreGetAllProfilesInChannel(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) c1 := model.Channel{} @@ -713,13 +713,13 @@ func testUserStoreGetProfilesNotInChannel(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) c1 := model.Channel{} @@ -808,13 +808,13 @@ func testUserStoreGetProfilesByIds(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) if r1 := <-ss.User().GetProfileByIds([]string{u1.Id}, false); r1.Err != nil { @@ -953,14 +953,14 @@ func testUserStoreGetProfilesByUsernames(t *testing.T, ss store.Store) { u1.Email = MakeEmail() u1.Username = "username1" + model.NewId() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() u2.Username = "username2" + model.NewId() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) if r1 := <-ss.User().GetProfilesByUsernames([]string{u1.Username, u2.Username}, teamId); r1.Err != nil { @@ -999,7 +999,7 @@ func testUserStoreGetProfilesByUsernames(t *testing.T, ss store.Store) { u3.Email = MakeEmail() u3.Username = "username3" + model.NewId() store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: team2Id, UserId: u3.Id}, -1)) if r1 := <-ss.User().GetProfilesByUsernames([]string{u1.Username, u3.Username}, ""); r1.Err != nil { @@ -1040,13 +1040,13 @@ func testUserStoreGetSystemAdminProfiles(t *testing.T, ss store.Store) { u1.Email = MakeEmail() u1.Roles = model.SYSTEM_USER_ROLE_ID + " " + model.SYSTEM_ADMIN_ROLE_ID store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) if r1 := <-ss.User().GetSystemAdminProfiles(); r1.Err != nil { @@ -1065,7 +1065,7 @@ func testUserStoreGetByEmail(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamid, UserId: u1.Id}, -1)) if err := (<-ss.User().GetByEmail(u1.Email)).Err; err != nil { @@ -1087,7 +1087,7 @@ func testUserStoreGetByAuthData(t *testing.T, ss store.Store) { u1.AuthData = &auth u1.AuthService = "service" store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) if err := (<-ss.User().GetByAuth(u1.AuthData, u1.AuthService)).Err; err != nil { @@ -1107,7 +1107,7 @@ func testUserStoreGetByUsername(t *testing.T, ss store.Store) { u1.Email = MakeEmail() u1.Username = model.NewId() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) if err := (<-ss.User().GetByUsername(u1.Username)).Err; err != nil { @@ -1129,7 +1129,7 @@ func testUserStoreGetForLogin(t *testing.T, ss store.Store) { AuthData: &auth, } store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() auth2 := model.NewId() @@ -1140,7 +1140,7 @@ func testUserStoreGetForLogin(t *testing.T, ss store.Store) { AuthData: &auth2, } store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() if result := <-ss.User().GetForLogin(u1.Username, true, true); result.Err != nil { t.Fatal("Should have gotten user by username", result.Err) @@ -1170,7 +1170,7 @@ func testUserStoreUpdatePassword(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) hashedPassword := model.HashPassword("newpwd") @@ -1193,7 +1193,7 @@ func testUserStoreDelete(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: model.NewId(), UserId: u1.Id}, -1)) if err := (<-ss.User().PermanentDelete(u1.Id)).Err; err != nil { @@ -1207,7 +1207,7 @@ func testUserStoreUpdateAuthData(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) service := "someservice" @@ -1252,14 +1252,14 @@ func testUserUnreadCount(t *testing.T, ss store.Store) { u1.Username = "user1" + model.NewId() u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) u2 := &model.User{} u2.Email = MakeEmail() u2.Username = "user2" + model.NewId() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u2.Id}, -1)) if err := (<-ss.Channel().Save(&c1, -1)).Err; err != nil { @@ -1330,7 +1330,7 @@ func testUserStoreUpdateMfaSecret(t *testing.T, ss store.Store) { u1 := model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(&u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() time.Sleep(100 * time.Millisecond) @@ -1348,7 +1348,7 @@ func testUserStoreUpdateMfaActive(t *testing.T, ss store.Store) { u1 := model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(&u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() time.Sleep(100 * time.Millisecond) @@ -1370,7 +1370,7 @@ func testUserStoreGetRecentlyActiveUsersForTeam(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Status().SaveOrUpdate(&model.Status{UserId: u1.Id, Status: model.STATUS_ONLINE, Manual: false, LastActivityAt: model.GetMillis(), ActiveChannel: ""})) tid := model.NewId() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1)) @@ -1384,7 +1384,7 @@ func testUserStoreGetNewUsersForTeam(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Status().SaveOrUpdate(&model.Status{UserId: u1.Id, Status: model.STATUS_ONLINE, Manual: false, LastActivityAt: model.GetMillis(), ActiveChannel: ""})) tid := model.NewId() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1)) @@ -1419,14 +1419,14 @@ func testUserStoreSearch(t *testing.T, ss store.Store) { Email: "harold" + model.NewId() + "@simulator.amazonses.com", } store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() u2 := &model.User{ Username: "jim-bobby" + model.NewId(), Email: MakeEmail(), } store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() u3 := &model.User{ Username: "jimbo3" + model.NewId(), @@ -1434,7 +1434,7 @@ func testUserStoreSearch(t *testing.T, ss store.Store) { DeleteAt: 1, } store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() u5 := &model.User{ Username: "yu" + model.NewId(), @@ -1444,7 +1444,7 @@ func testUserStoreSearch(t *testing.T, ss store.Store) { Email: MakeEmail(), } store.Must(ss.User().Save(u5)) - defer ss.User().PermanentDelete(u5.Id) + defer func() { store.Must(ss.User().PermanentDelete(u5.Id)) }() u6 := &model.User{ Username: "underscore" + model.NewId(), @@ -1454,7 +1454,7 @@ func testUserStoreSearch(t *testing.T, ss store.Store) { Email: MakeEmail(), } store.Must(ss.User().Save(u6)) - defer ss.User().PermanentDelete(u6.Id) + defer func() { store.Must(ss.User().PermanentDelete(u6.Id)) }() tid := model.NewId() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1)) @@ -1716,14 +1716,14 @@ func testUserStoreSearchNotInChannel(t *testing.T, ss store.Store) { Email: "harold" + model.NewId() + "@simulator.amazonses.com", } store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() u2 := &model.User{ Username: "jim2-bobby" + model.NewId(), Email: MakeEmail(), } store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() u3 := &model.User{ Username: "jimbo3" + model.NewId(), @@ -1731,7 +1731,7 @@ func testUserStoreSearchNotInChannel(t *testing.T, ss store.Store) { DeleteAt: 1, } store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() tid := model.NewId() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1)) @@ -1923,14 +1923,14 @@ func testUserStoreSearchInChannel(t *testing.T, ss store.Store) { Email: "harold" + model.NewId() + "@simulator.amazonses.com", } store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() u2 := &model.User{ Username: "jim-bobby" + model.NewId(), Email: MakeEmail(), } store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() u3 := &model.User{ Username: "jimbo3" + model.NewId(), @@ -1938,7 +1938,7 @@ func testUserStoreSearchInChannel(t *testing.T, ss store.Store) { DeleteAt: 1, } store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() tid := model.NewId() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u1.Id}, -1)) @@ -2069,14 +2069,14 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) { Email: "harold" + model.NewId() + "@simulator.amazonses.com", } store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() u2 := &model.User{ Username: "jim-bobby" + model.NewId(), Email: MakeEmail(), } store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() u3 := &model.User{ Username: "jimbo3" + model.NewId(), @@ -2084,7 +2084,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) { DeleteAt: 1, } store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() u4 := &model.User{ Username: "simon" + model.NewId(), @@ -2092,7 +2092,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) { DeleteAt: 0, } store.Must(ss.User().Save(u4)) - defer ss.User().PermanentDelete(u4.Id) + defer func() { store.Must(ss.User().PermanentDelete(u4.Id)) }() u5 := &model.User{ Username: "yu" + model.NewId(), @@ -2102,7 +2102,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) { Email: MakeEmail(), } store.Must(ss.User().Save(u5)) - defer ss.User().PermanentDelete(u5.Id) + defer func() { store.Must(ss.User().PermanentDelete(u5.Id)) }() u6 := &model.User{ Username: "underscore" + model.NewId(), @@ -2112,7 +2112,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) { Email: MakeEmail(), } store.Must(ss.User().Save(u6)) - defer ss.User().PermanentDelete(u6.Id) + defer func() { store.Must(ss.User().PermanentDelete(u6.Id)) }() teamId1 := model.NewId() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId1, UserId: u1.Id}, -1)) @@ -2186,7 +2186,7 @@ func testUserStoreSearchNotInTeam(t *testing.T, ss store.Store) { []*model.User{}, }, { - "search jimb, team1", + "search jimb, team2", teamId2, "jimb", &model.UserSearchOptions{ @@ -2241,14 +2241,14 @@ func testUserStoreSearchWithoutTeam(t *testing.T, ss store.Store) { Email: "harold" + model.NewId() + "@simulator.amazonses.com", } store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() u2 := &model.User{ Username: "jim2-bobby" + model.NewId(), Email: MakeEmail(), } store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() u3 := &model.User{ Username: "jimbo3" + model.NewId(), @@ -2256,7 +2256,7 @@ func testUserStoreSearchWithoutTeam(t *testing.T, ss store.Store) { DeleteAt: 1, } store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() tid := model.NewId() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: tid, UserId: u3.Id}, -1)) @@ -2329,7 +2329,7 @@ func testUserStoreAnalyticsGetInactiveUsersCount(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() var count int64 @@ -2343,7 +2343,7 @@ func testUserStoreAnalyticsGetInactiveUsersCount(t *testing.T, ss store.Store) { u2.Email = MakeEmail() u2.DeleteAt = model.GetMillis() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() if result := <-ss.User().AnalyticsGetInactiveUsersCount(); result.Err != nil { t.Fatal(result.Err) @@ -2375,12 +2375,12 @@ func testUserStoreAnalyticsGetSystemAdminCount(t *testing.T, ss store.Store) { if err := (<-ss.User().Save(&u1)).Err; err != nil { t.Fatal("couldn't save user", err) } - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() if err := (<-ss.User().Save(&u2)).Err; err != nil { t.Fatal("couldn't save user", err) } - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() if result := <-ss.User().AnalyticsGetSystemAdminCount(); result.Err != nil { t.Fatal(result.Err) @@ -2398,14 +2398,14 @@ func testUserStoreGetProfilesNotInTeam(t *testing.T, ss store.Store) { u1 := &model.User{} u1.Email = MakeEmail() store.Must(ss.User().Save(u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u1.Id}, -1)) store.Must(ss.User().UpdateUpdateAt(u1.Id)) u2 := &model.User{} u2.Email = MakeEmail() store.Must(ss.User().Save(u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.User().UpdateUpdateAt(u2.Id)) var initialUsersNotInTeam int @@ -2512,7 +2512,7 @@ func testUserStoreGetProfilesNotInTeam(t *testing.T, ss store.Store) { u3 := &model.User{} u3.Email = MakeEmail() store.Must(ss.User().Save(u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() store.Must(ss.Team().SaveMember(&model.TeamMember{TeamId: teamId, UserId: u3.Id}, -1)) store.Must(ss.User().UpdateUpdateAt(u3.Id)) @@ -2550,13 +2550,13 @@ func testUserStoreClearAllCustomRoleAssignments(t *testing.T, ss store.Store) { } store.Must(ss.User().Save(&u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() store.Must(ss.User().Save(&u2)) - defer ss.User().PermanentDelete(u2.Id) + defer func() { store.Must(ss.User().PermanentDelete(u2.Id)) }() store.Must(ss.User().Save(&u3)) - defer ss.User().PermanentDelete(u3.Id) + defer func() { store.Must(ss.User().PermanentDelete(u3.Id)) }() store.Must(ss.User().Save(&u4)) - defer ss.User().PermanentDelete(u4.Id) + defer func() { store.Must(ss.User().PermanentDelete(u4.Id)) }() require.Nil(t, (<-ss.User().ClearAllCustomRoleAssignments()).Err) @@ -2584,7 +2584,7 @@ func testUserStoreGetAllAfter(t *testing.T, ss store.Store) { Roles: "system_user system_admin system_post_all", } store.Must(ss.User().Save(&u1)) - defer ss.User().PermanentDelete(u1.Id) + defer func() { store.Must(ss.User().PermanentDelete(u1.Id)) }() r1 := <-ss.User().GetAllAfter(10000, strings.Repeat("0", 26)) require.Nil(t, r1.Err) diff --git a/testlib/helper.go b/testlib/helper.go index bcc19f60a7..adb7177ef8 100644 --- a/testlib/helper.go +++ b/testlib/helper.go @@ -4,6 +4,7 @@ package testlib import ( + "flag" "os" "testing" @@ -21,11 +22,12 @@ type MainHelper struct { SqlSupplier *sqlstore.SqlSupplier ClusterInterface *FakeClusterInterface - container *storetest.RunningContainer - status int + status int } func NewMainHelper() *MainHelper { + flag.Parse() + // Setup a global logger to catch tests logging outside of app context // The global logger will be stomped by apps initalizing but that's fine for testing. // Ideally this won't happen. @@ -38,21 +40,19 @@ func NewMainHelper() *MainHelper { utils.TranslationsPreInit() - container, settings, err := storetest.NewMySQLContainer() - if err != nil { - panic("failed to start mysql container: " + err.Error()) - } + settings := storetest.MakeSqlSettings(model.DATABASE_DRIVER_MYSQL) - testClusterInterface := &FakeClusterInterface{} - testStoreSqlSupplier := sqlstore.NewSqlSupplier(*settings, nil) - testStore := &TestStore{store.NewLayeredStore(testStoreSqlSupplier, nil, testClusterInterface)} + clusterInterface := &FakeClusterInterface{} + sqlSupplier := sqlstore.NewSqlSupplier(*settings, nil) + testStore := &TestStore{ + store.NewLayeredStore(sqlSupplier, nil, clusterInterface), + } return &MainHelper{ Settings: settings, Store: testStore, - SqlSupplier: testStoreSqlSupplier, - ClusterInterface: testClusterInterface, - container: container, + SqlSupplier: sqlSupplier, + ClusterInterface: clusterInterface, } } @@ -61,8 +61,7 @@ func (h *MainHelper) Main(m *testing.M) { } func (h *MainHelper) Close() error { - h.container.Stop() - h.container = nil + storetest.CleanupSqlSettings(h.Settings) os.Exit(h.status) diff --git a/testlib/store.go b/testlib/store.go index 5d20d3754d..cfc9430bdc 100644 --- a/testlib/store.go +++ b/testlib/store.go @@ -11,5 +11,6 @@ type TestStore struct { store.Store } -func (*TestStore) Close() { +func (s *TestStore) Close() { + // Don't propagate to the underlying store, since this instance is persistent. } diff --git a/web/handlers_test.go b/web/handlers_test.go index 2cd3b19595..9741241818 100644 --- a/web/handlers_test.go +++ b/web/handlers_test.go @@ -11,6 +11,7 @@ import ( "github.com/mattermost/mattermost-server/app" "github.com/mattermost/mattermost-server/model" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func handlerForHTTPErrors(c *Context, w http.ResponseWriter, r *http.Request) { @@ -19,6 +20,7 @@ func handlerForHTTPErrors(c *Context, w http.ResponseWriter, r *http.Request) { func TestHandlerServeHTTPErrors(t *testing.T) { s, err := app.NewServer(app.StoreOverride(mainHelper.Store), app.DisableConfigWatch) + require.Nil(t, err) defer s.Shutdown() web := New(s, s.AppOptions, s.Router) @@ -62,6 +64,7 @@ func handlerForHTTPSecureTransport(c *Context, w http.ResponseWriter, r *http.Re func TestHandlerServeHTTPSecureTransport(t *testing.T) { s, err := app.NewServer(app.StoreOverride(mainHelper.Store), app.DisableConfigWatch) + require.Nil(t, err) defer s.Shutdown() a := s.FakeApp()