unittest using externally managed database (#9400)

* MM-12083: unittest using externally managed database * cherry-pick Makefile changes from @cpanato * Jenkins changes (#9915) * add docker compose * udpate * when using minio dont need to set the region * update * add wait for it script * using old minio * add new jenkins file * update makefile * add dockerfile * rename the docker-compose proj to avoid colision (#9917) * rename the docker-compose proj to avoid colision * enable debug * enable debug to double checkt the branchs and fix docker-compose name (#9919) * add ee hash to check (#9920) * fix name (#9921) * update jenkins file to push from branch and prs * if a new push comes in stop the running build * split mysql and postgres variables * add script to run jenkins-like env in local dev env * update docker-compose project name to use uuid to make it more randon * fix DCNAME definition * update elasticsearch docker image * revert test * tidy up stages, and wait for mysql differently * update docker image and add check for postgres * checking if is ready * update docker compose to have a wait for deps * add readme and rename dockerfile * fix -unittest setup * using mm docker image * restore parallel unit tests at the package level Spin up a dedicated database for each package under test to avoid races in accessing the same tables. Simplify the interface for configuring the test database to just a DSN instead of multiple exports for each field. * try to work around root mysql access in CI * update local-test-env.sh too * MYSQL_ROOT_HOST: % * fix missing quotes * setting some memory limits for mysql * revert memory docker compose does not support * fix env name for postgres * expose errors in app/export_test.go * fix test label, better error checking on teardown * increase query timeout for tests * fix export_test * update local dev script * add configurable mysql root passwd
2018-12-10 11:58:09 -05:00
--- a/build/Dockerfile.buildenv
+++ b/build/Dockerfile.buildenv
@@ -0,0 +1,3 @@
+FROM golang:1.11
+
+RUN apt-get update && apt-get install -y make git apt-transport-https ca-certificates curl software-properties-common build-essential zip wget xmlsec1
--- a/build/Jenkinsfile.k8s
+++ b/build/Jenkinsfile.k8s
--- a/build/Jenkinsfile.pr
+++ b/build/Jenkinsfile.pr
@@ -0,0 +1,244 @@
+#!/usr/bin/env groovy
+
+@Library('shared-pipelines') _
+import org.mattermost.Utils
+import java.util.*;
+def utils = new org.mattermost.Utils()
+def rnd = UUID.randomUUID().toString()
+
+pipeline {
+    agent {
+        label 'default-mm-builder'
+    }
+
+    options { buildDiscarder(logRotator(numToKeepStr: '3')) }
+
+    environment {
+        DCNAME = "${rnd}"
+    }
+
+    stages {
+        stage('Setup') {
+            steps {
+                script {
+                    utils.stopOldBuilds()
+                }
+                cleanWs notFailBuild: true
+                sh """
+                    mkdir -p src/github.com/mattermost/mattermost-server
+                    mkdir -p src/github.com/mattermost/mattermost-webapp
+                    mkdir -p src/github.com/mattermost/enterprise
+                """
+                dir('src/github.com/mattermost/mattermost-server') {
+                    checkout scm
+                }
+                dir('src/github.com/mattermost/mattermost-webapp') {
+                    checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/mattermost-webapp.git']]]
+                }
+                dir('src/github.com/mattermost/enterprise') {
+                    checkout changelog: false, poll: false, scm: [$class: 'GitSCM', branches: [[name: '*/master']], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: '310159d3-f7c5-4f5d-bfa1-151e3ef2db57', url: 'https://github.com/mattermost/enterprise.git']]]
+                }
+                dir('src/github.com/mattermost/enterprise') {
+                    ansiColor('xterm') {
+                        sh """
+                            #!/bin/bash -ex
+                            git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout master || echo 1
+                            export EE_GIT_COMMIT=\$(git rev-parse HEAD)
+
+                            echo EE Commit: \${EE_GIT_COMMIT}
+                        """
+                    }
+                }
+                dir('src/github.com/mattermost/mattermost-webapp') {
+                    withDockerContainer(args: '', image: 'mattermost/mattermost-build-webapp:oct-2-2018') {
+                        ansiColor('xterm') {
+                            sh """
+                                #!/bin/bash -ex
+                                git checkout $env.BRANCH_NAME || git checkout $env.CHANGE_BRANCH || git checkout $env.GIT_BRANCH || git checkout master || echo 1
+                                rm -rf ./dist
+                                export WEBAPP_GIT_COMMIT=\$(git rev-parse HEAD)
+
+                                echo Webapp Commit: \${WEBAPP_GIT_COMMIT}
+
+                                curl -f -o ./dist.tar.gz https://releases.mattermost.com/mattermost-webapp/commit/\${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || make node_modules test build || exit 1
+                            """
+                        }
+                    }
+                }
+            }
+        }
+
+        stage('Build') {
+            environment {
+                GOPATH = "/go"
+            }
+
+            steps {
+                withDockerContainer(args: '-u root --privileged -v ${WORKSPACE}/src:/go/src/', image: 'mattermost/mattermost-build-server:dec-7-2018') {
+                    ansiColor('xterm') {
+                        sh """
+                            cd /go/src/github.com/mattermost/mattermost-server
+                            mv config/default.json config/config.json || echo ""
+                            make check-style BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1
+                            make build BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1
+                            make package BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' || exit 1
+                        """
+                    }
+                }
+            }
+        }
+
+        stage('Test') {
+            environment {
+                GOPATH = "/go"
+                TEST_DATABASE_MYSQL_DSN = "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s"
+                TEST_DATABASE_POSTGRESQL_DSN = "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10"
+                TEST_DATABASE_MYSQL_ROOT_PASSWD = "mostest"
+                CI_INBUCKET_HOST = "inbucket"
+                CI_MINIO_HOST = "minio"
+                CI_INBUCKET_PORT = "10080"
+                CI_MINIO_PORT = "9000"
+                CI_LDAP_HOST = "openldap"
+                IS_CI = true
+            }
+
+            steps {
+                dir('src/github.com/mattermost/mattermost-server/build') {
+                    ansiColor('xterm') {
+                        sh """
+                            COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose run --rm start_dependencies
+                            COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose up -d mysql postgres minio inbucket openldap elasticsearch redis
+
+                            docker ps
+
+                            docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+                            docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+                            docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest';
+                            docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+                            docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest';
+                            docker exec -t openldap-${DCNAME}-${env.BUILD_NUMBER} bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+                        """
+                    }
+                }
+
+                withDockerContainer(args: "-u root --privileged --net ${DCNAME}-${BUILD_NUMBER}_mm-test -v ${WORKSPACE}/src:/go/src/", image: 'mattermost/mattermost-build-server:dec-7-2018') {
+                    ansiColor('xterm') {
+                    sh """
+                            cd /go/src/github.com/mattermost/mattermost-server
+
+                            sed -i'.bak' 's|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|g' config/config.json;
+                            sed -i'.bak' 's|"SMTPServer": "dockerhost",|"SMTPServer": "inbucket",|g' config/config.json;
+                            sed -i'.bak' 's|"SMTPPort": "2500",|"SMTPPort": "10025",|g' config/config.json;
+                            sed -i'.bak' 's|"ConnectionUrl": "http://dockerhost:9200",|"ConnectionUrl": "http://elasticsearch:9200",|g' config/config.json;
+                            mkdir -p client/plugins
+
+                            cat config/config.json
+
+                            make test-server BUILD_NUMBER='${BRANCH_NAME}-${BUILD_NUMBER}' TESTFLAGS= TESTFLAGSEE= || exit 1
+                        """
+                    }
+                }
+            }
+        }
+
+        stage('Push to S3') {
+            stages {
+                stage('Pull request') {
+                    when {
+                        allOf {
+                            expression { env.CHANGE_ID != null }
+                            expression { env.CHANGE_TARGET != null }
+                        }
+                    }
+                    steps {
+                        dir('src/github.com/mattermost/mattermost-server/dist') {
+                            step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[
+                                bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_ID}",
+                                excludedFile: '',
+                                flatten: true,
+                                gzipFiles: false,
+                                keepForever: false,
+                                managedArtifacts: false,
+                                noUploadOnFailure: true,
+                                selectedRegion: 'us-east-1',
+                                showDirectlyInBrowser: false,
+                                sourceFile: '*.tar.gz',
+                                storageClass: 'STANDARD',
+                                uploadFromSlave: false,
+                                useServerSideEncryption: false,
+                                userMetadata: [[key: 'Cache-Control', value: 'no-cache']]
+                            ], [
+                                bucket: "releases.mattermost.com/mattermost-platform-pr/${CHANGE_BRANCH}",
+                                excludedFile: '',
+                                flatten: true,
+                                gzipFiles: false,
+                                keepForever: false,
+                                managedArtifacts: false,
+                                noUploadOnFailure: true,
+                                selectedRegion: 'us-east-1',
+                                showDirectlyInBrowser: false,
+                                sourceFile: '*.tar.gz',
+                                storageClass: 'STANDARD',
+                                uploadFromSlave: false,
+                                useServerSideEncryption: false,
+                                userMetadata: [[key: 'Cache-Control', value: 'no-cache']]
+                            ]], profileName: 'Releases', userMetadAta: []])
+                        }
+                    }
+                }
+                stage('Branch') {
+                    when {
+                        expression { env.CHANGE_ID == null }
+                    }
+                    steps {
+                        dir('src/github.com/mattermost/mattermost-server/dist') {
+                            step([$class: 'S3BucketPublisher', dontWaitForConcurrentBuildCompletion: false, entries: [[
+                                bucket: "releases.mattermost.com/mattermost-platform-pr/${BRANCH_NAME}",
+                                excludedFile: '',
+                                flatten: true,
+                                gzipFiles: false,
+                                keepForever: false,
+                                managedArtifacts: false,
+                                noUploadOnFailure: true,
+                                selectedRegion: 'us-east-1',
+                                showDirectlyInBrowser: false,
+                                sourceFile: '*.tar.gz',
+                                storageClass: 'STANDARD',
+                                uploadFromSlave: false,
+                                useServerSideEncryption: false,
+                                userMetadata: [[key: 'Cache-Control', value: 'no-cache']]
+                            ]], profileName: 'Releases', userMetadAta: []])
+                        }
+                    }
+                }
+            }
+        }
+
+        stage('Trigger docker image') {
+            when {
+                expression { env.CHANGE_ID != null }
+            }
+            steps {
+                build job: '../../mk/mattermost-enterprise-edition-release', parameters: [string(name: 'RELEASE', value: "${CHANGE_ID}"), booleanParam(name: 'FROM_PR', value: true)], propagate: false, wait: false
+            }
+        }
+    }
+
+    post {
+        always {
+            dir('src/github.com/mattermost/mattermost-server') {
+                junit allowEmptyResults: true, healthScaleFactor: 0.0, testResults: 'report*.xml'
+            }
+            dir('src/github.com/mattermost/mattermost-server/build') {
+                ansiColor('xterm') {
+                    sh """
+                        COMPOSE_PROJECT_NAME=${DCNAME}-${env.BUILD_NUMBER} docker-compose down -v
+                    """
+                }
+            }
+        }
+        cleanup {
+            cleanWs notFailBuild: true
+        }
+    }
+}
--- a/build/README.md
+++ b/build/README.md
@@ -0,0 +1,8 @@
+## About this folder
+
+This folder contains some files that we use to build the `mattermost-server` using `Jenkins` and other files like privacy policy and licenses.
+
+PRs opened against the `mattermost-server` repository will use the file called `Jenkinsfile.pr`
+
+The `Dockerfile` in this folder (`Dockerfile.buildenv`) is the build environment for our current builds you can find the docker image to downalod [here](https://hub.docker.com/r/mattermost/mattermost-build-server/tags/) or build your own.
+
--- a/build/docker-compose.yml
+++ b/build/docker-compose.yml
@@ -0,0 +1,82 @@
+version: '3.4'
+services:
+  mysql:
+    image: "mysql:5.7"
+    restart: always
+    container_name: mysql-$DCNAME-$BUILD_NUMBER
+    networks:
+      - mm-test
+    environment:
+      MYSQL_ROOT_HOST: "%"
+      MYSQL_ROOT_PASSWORD: mostest
+      MYSQL_PASSWORD: mostest
+      MYSQL_USER: mmuser
+      MYSQL_DATABASE: mattermost_test
+  postgres:
+    image: "postgres:9.4"
+    restart: always
+    container_name: postgres-$DCNAME-$BUILD_NUMBER
+    networks:
+      - mm-test
+    environment:
+      POSTGRES_USER: mmuser
+      POSTGRES_PASSWORD: mostest
+      POSTGRES_DB: mattermost_test
+  minio:
+    image: "minio/minio:RELEASE.2018-05-25T19-49-13Z"
+    command: "server /data"
+    container_name: minio-$DCNAME-$BUILD_NUMBER
+    networks:
+      - mm-test
+    environment:
+      MINIO_ACCESS_KEY: minioaccesskey
+      MINIO_SECRET_KEY: miniosecretkey
+  inbucket:
+    image: "jhillyerd/inbucket:release-1.2.0"
+    restart: always
+    container_name: inbucket-$DCNAME-$BUILD_NUMBER
+    networks:
+      - mm-test
+  openldap:
+    image: "osixia/openldap:1.1.6"
+    restart: always
+    container_name: openldap-$DCNAME-$BUILD_NUMBER
+    networks:
+      - mm-test
+    environment:
+      LDAP_TLS_VERIFY_CLIENT: "never"
+      LDAP_ORGANISATION: "Mattermost Test"
+      LDAP_DOMAIN: "mm.test.com"
+      LDAP_ADMIN_PASSWORD: "mostest"
+  elasticsearch:
+    image: "mattermost/mattermost-elasticsearch-docker:6.5.1"
+    container_name: elasticsearch-$DCNAME-$BUILD_NUMBER
+    networks:
+      - mm-test
+    environment:
+      http.host: "0.0.0.0"
+      transport.host: "127.0.0.1"
+      ES_JAVA_OPTS: "-Xms250m -Xmx250m"
+  redis:
+    image: redis
+    networks:
+      - mm-test
+    container_name: redis-$DCNAME-$BUILD_NUMBER
+
+  start_dependencies:
+    image: mattermost/mattermost-wait-for-dep:latest
+    networks:
+      - mm-test
+    depends_on:
+      - mysql
+      - postgres
+      - minio
+      - inbucket
+      - openldap
+      - elasticsearch
+      - redis
+    command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 redis:6379
+
+networks:
+  mm-test:
+    driver: bridge
--- a/build/local-test-env.sh
+++ b/build/local-test-env.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+
+export DCNAME=localdev
+export BUILD_NUMBER=null
+local_cmdname=${0##*/}
+
+usage()
+{
+    cat << USAGE >&2
+Usage:
+    $local_cmdname up/down
+USAGE
+    exit 1
+}
+
+up()
+{
+  COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose run --rm start_dependencies
+  COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose up -d mysql postgres minio inbucket openldap elasticsearch redis
+
+  docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: organizationalunit" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
+  docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test1\nmail: success+testone@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
+  docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'
+  docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: uid=test.two,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: iNetOrgPerson\nsn: User\ncn: Test2\nmail: success+testtwo@simulator.amazonses.com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
+  docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'ldappasswd -s Password1 -D "cn=admin,dc=mm,dc=test,dc=com" -x "uid=test.two,ou=testusers,dc=mm,dc=test,dc=com" -w mostest'
+  docker exec -t openldap-$DCNAME-$BUILD_NUMBER bash -c 'echo -e "dn: cn=tgroup,ou=testusers,dc=mm,dc=test,dc=com\nobjectclass: groupOfUniqueNames\nuniqueMember: uid=test.one,ou=testusers,dc=mm,dc=test,dc=com" | ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'
+
+  echo "Changing the config.json"
+  sed -i'.bak' 's|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
+  sed -i'.bak' 's|"SMTPServer": "dockerhost",|"SMTPServer": "inbucket",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
+  sed -i'.bak' 's|"SMTPPort": "2500",|"SMTPPort": "10025",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
+  sed -i'.bak' 's|"ConnectionUrl": "http://dockerhost:9200",|"ConnectionUrl": "http://elasticsearch:9200",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
+
+  docker run -it -u root \
+   --privileged \
+   -v $GOPATH:/go \
+   -w /go/src/github.com/mattermost/mattermost-server/ \
+   --net $DCNAME-$BUILD_NUMBER\_mm-test \
+   -e GOPATH="/go" \
+   -e TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \
+   -e TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \
+   -e TEST_DATABASE_MYSQL_ROOT_PASSWD="mostest"
+   -e CI_INBUCKET_HOST="inbucket" \
+   -e CI_MINIO_HOST="minio" \
+   -e CI_INBUCKET_PORT="10080" \
+   -e CI_MINIO_PORT="9000" \
+   -e CI_LDAP_HOST="openldap" \
+   -e IS_CI=true \
+   mattermost/mattermost-build-server:dec-7-2018 /bin/bash
+}
+
+down()
+{
+  COMPOSE_PROJECT_NAME=$DCNAME-$BUILD_NUMBER docker-compose down
+
+  echo "Reverting the changes in the config.json"
+  sed -i'.bak' 's|mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8|mmuser:mostest@tcp(dockerhost:3306)/mattermost_test?charset=utf8mb4,utf8|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
+  sed -i'.bak' 's|"SMTPServer": "inbucket",|"SMTPServer": "dockerhost",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
+  sed -i'.bak' 's|"SMTPPort": "10025",|"SMTPPort": "2500",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
+  sed -i'.bak' 's|"ConnectionUrl": "http://elasticsearch:9200",|"ConnectionUrl": "http://dockerhost:9200",|g' $GOPATH/src/github.com/mattermost/mattermost-server/config/config.json
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+    case "$1" in
+        up)
+        echo "Starting Containers"
+        up
+        break
+        ;;
+        down)
+        echo "Stopping Containers"
+        down
+        break
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+if [[ "$1" == "" ]]; then
+    usage
+fi