447 строки
19 KiB
YAML
447 строки
19 KiB
YAML
version: 2.1
|
|
|
|
orbs:
|
|
aws-s3: circleci/aws-s3@1.0.11
|
|
owasp: entur/owasp@0.0.10
|
|
|
|
executors:
|
|
ubuntu:
|
|
working_directory: ~/mattermost/
|
|
machine:
|
|
image: "ubuntu-1604:201903-01"
|
|
environment:
|
|
COMPOSE_PROJECT_NAME: "circleci"
|
|
resource_class: "large"
|
|
|
|
jobs:
|
|
setup:
|
|
working_directory: ~/mattermost/mattermost-server
|
|
docker:
|
|
- image: mattermost/mattermost-build-webapp:oct-2-2018
|
|
resource_class: xlarge
|
|
steps:
|
|
- checkout
|
|
- run: |
|
|
cd ../
|
|
git clone https://github.com/mattermost/mattermost-webapp.git
|
|
cd mattermost-webapp
|
|
git checkout $CIRCLE_BRANCH || git checkout master
|
|
export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
|
|
echo "$WEBAPP_GIT_COMMIT"
|
|
export CURL_FAILED=0
|
|
curl -f -o ./dist.tar.gz https://pr-builds.mattermost.com/mattermost-webapp/commit/${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || export CURL_FAILED=1
|
|
|
|
if [ $CURL_FAILED -eq 1 ]
|
|
then
|
|
npm ci && cd node_modules/mattermost-redux && npm install && npm run build && cd ../.. && make build
|
|
fi
|
|
- persist_to_workspace:
|
|
root: ~/mattermost
|
|
paths:
|
|
- mattermost-webapp
|
|
- mattermost-server
|
|
|
|
check-deps:
|
|
parameters:
|
|
cve_data_directory:
|
|
type: string
|
|
default: "~/.owasp/dependency-check-data"
|
|
working_directory: ~/mattermost/mattermost-server
|
|
executor: owasp/default
|
|
environment:
|
|
version_url: "https://jeremylong.github.io/DependencyCheck/current.txt"
|
|
executable_url: "https://dl.bintray.com/jeremy-long/owasp/dependency-check-VERSION-release.zip"
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/mattermost/
|
|
- run:
|
|
name: Install Go
|
|
command: sudo apt-get update && sudo apt-get install golang
|
|
- owasp/with_commandline:
|
|
steps:
|
|
# Taken from https://github.com/entur/owasp-orb/blob/master/src/%40orb.yml#L349-L361
|
|
- owasp/generate_cache_keys:
|
|
cache_key: commmandline-default-cache-key-v6
|
|
- owasp/restore_owasp_cache
|
|
- run:
|
|
name: Update OWASP Dependency-Check Database
|
|
command: ~/.owasp/dependency-check/bin/dependency-check.sh --data << parameters.cve_data_directory >> --updateonly
|
|
- owasp/store_owasp_cache:
|
|
cve_data_directory: <<parameters.cve_data_directory>>
|
|
- run:
|
|
name: Run OWASP Dependency-Check Analyzer
|
|
command: |
|
|
~/.owasp/dependency-check/bin/dependency-check.sh --data << parameters.cve_data_directory >> \
|
|
--format ALL --noupdate --enableExperimental --suppression ./dependency-suppression.xml \
|
|
--scan './**/*' || true
|
|
- owasp/collect_reports:
|
|
persist_to_workspace: false
|
|
- run:
|
|
name: Post results to Mattermost
|
|
command: ./scripts/depcheck-alert.sh
|
|
|
|
# TODO: enable this step when the i18n-extract works with mattermost-server only
|
|
# and not depend on both mm-server/enterprise
|
|
# check-i18n:
|
|
# docker:
|
|
# - image: circleci/golang:1.12
|
|
# working_directory: ~/mattermost/
|
|
# steps:
|
|
# - attach_workspace:
|
|
# at: ~/mattermost/
|
|
# - run:
|
|
# command: |
|
|
# cd mattermost-server
|
|
# cp i18n/en.json /tmp/en.json
|
|
# make i18n-extract
|
|
# diff /tmp/en.json i18n/en.json
|
|
|
|
check-app-layers:
|
|
docker:
|
|
- image: mattermost/mattermost-build-server:20200322_golang-1.14.1
|
|
working_directory: ~/mattermost
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/mattermost/
|
|
- run:
|
|
command: |
|
|
cd mattermost-server
|
|
make app-layers
|
|
if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
|
|
|
|
build:
|
|
docker:
|
|
- image: mattermost/mattermost-build-server:20200322_golang-1.14.1
|
|
resource_class: xlarge
|
|
working_directory: ~/mattermost
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/mattermost/
|
|
- run:
|
|
command: |
|
|
echo "Installing golangci-lint"
|
|
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b /usr/local/bin v1.23.6 # Keep this in sync with Jenkinsfile.pr
|
|
echo "Installing mattermost-govet"
|
|
export GOBIN=${PWD}/mattermost-server/bin
|
|
GO111MODULE=off go get -u github.com/mattermost/mattermost-govet
|
|
export PATH=$PATH:$GOBIN
|
|
cd mattermost-server
|
|
make config-reset
|
|
make check-style BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true
|
|
make build BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}'
|
|
make package BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}'
|
|
- store_artifacts:
|
|
path: ~/mattermost/mattermost-server/dist/mattermost-team-linux-amd64.tar.gz
|
|
- store_artifacts:
|
|
path: ~/mattermost/mattermost-server/dist/mattermost-team-osx-amd64.tar.gz
|
|
- store_artifacts:
|
|
path: ~/mattermost/mattermost-server/dist/mattermost-team-windows-amd64.zip
|
|
- persist_to_workspace:
|
|
root: ~/mattermost
|
|
paths:
|
|
- mattermost-server
|
|
- mattermost-webapp
|
|
|
|
test:
|
|
executor:
|
|
name: ubuntu
|
|
parameters:
|
|
dbdriver:
|
|
type: string
|
|
dbsource:
|
|
type: string
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/mattermost
|
|
- run:
|
|
name: Run Docker compose
|
|
command: |
|
|
cd mattermost-server/build
|
|
docker-compose --no-ansi run --rm start_dependencies
|
|
cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
|
|
docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
|
|
docker-compose --no-ansi ps
|
|
background: true
|
|
- run:
|
|
name: Wait for docker compose
|
|
command: |
|
|
sleep 5
|
|
docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
|
|
docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
|
|
- run:
|
|
name: Run Tests
|
|
command: |
|
|
ulimit -n 8096
|
|
mkdir -p mattermost-server/client/plugins
|
|
docker run -it --net circleci_mm-test \
|
|
--env TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \
|
|
--env TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \
|
|
--env TEST_DATABASE_MYSQL_ROOT_PASSWD=mostest \
|
|
--env CI_INBUCKET_HOST=inbucket \
|
|
--env CI_MINIO_HOST=minio \
|
|
--env CI_INBUCKET_PORT=10080 \
|
|
--env CI_MINIO_PORT=9000 \
|
|
--env CI_INBUCKET_SMTP_PORT=10025 \
|
|
--env CI_LDAP_HOST=openldap \
|
|
--env IS_CI="true" \
|
|
--env MM_EMAILSETTINGS_SMTPSERVER=inbucket \
|
|
--env MM_EMAILSETTINGS_SMTPPORT=10025 \
|
|
--env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \
|
|
--env MM_SQLSETTINGS_DATASOURCE="<<parameters.dbsource>>" \
|
|
--env MM_SQLSETTINGS_DRIVERNAME=<<parameters.dbdriver>> \
|
|
--env GOBIN=/mattermost/mattermost-server/bin \
|
|
-v ~/mattermost:/mattermost \
|
|
-w /mattermost/mattermost-server \
|
|
mattermost/mattermost-build-server:20200322_golang-1.14.1 \
|
|
bash -c 'ulimit -n 8096; make test-server BUILD_NUMBER="$CIRCLE_BRANCH-$CIRCLE_PREVIOUS_BUILD_NUM" TESTFLAGS= TESTFLAGSEE='
|
|
no_output_timeout: 1h
|
|
- run:
|
|
name: Capture docker logs
|
|
when: always
|
|
command: |
|
|
cd mattermost-server/build
|
|
# Capture docker logs
|
|
mkdir -p logs
|
|
docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs
|
|
docker ps -a --no-trunc > logs/docker_ps
|
|
docker stats -a --no-stream > logs/docker_stats
|
|
tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats
|
|
- store_artifacts:
|
|
path: ~/mattermost/mattermost-server/build/logs
|
|
- run:
|
|
when: always
|
|
command: |
|
|
cd mattermost-server
|
|
mkdir -p test-results
|
|
cp report.xml test-results
|
|
- store_test_results:
|
|
path: ~/mattermost/mattermost-server/test-results/
|
|
- store_artifacts:
|
|
path: ~/mattermost/mattermost-server/test-results/
|
|
|
|
test-schema:
|
|
executor:
|
|
name: ubuntu
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/mattermost
|
|
- run:
|
|
name: Run Docker compose
|
|
command: |
|
|
cd mattermost-server/build
|
|
docker-compose --no-ansi run --rm start_dependencies
|
|
cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
|
|
docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
|
|
docker-compose --no-ansi ps
|
|
background: true
|
|
- run:
|
|
name: Wait for docker compose
|
|
command: |
|
|
sleep 5
|
|
docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
|
|
docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
|
|
- run:
|
|
name: Postgres schema migration validation
|
|
command: |
|
|
ulimit -n 8096
|
|
mkdir -p mattermost-server/client/plugins
|
|
cd mattermost-server/build
|
|
echo "Creating databases"
|
|
docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
|
|
echo "Importing postgres dump from version 5.0"
|
|
docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-5.0.sql
|
|
docker run -it --net circleci_mm-test \
|
|
--env CI_INBUCKET_HOST=inbucket \
|
|
--env CI_MINIO_HOST=minio \
|
|
--env CI_INBUCKET_PORT=10080 \
|
|
--env CI_MINIO_PORT=9000 \
|
|
--env CI_INBUCKET_SMTP_PORT=10025 \
|
|
--env CI_LDAP_HOST=openldap \
|
|
--env IS_CI="true" \
|
|
--env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
|
|
--env MM_EMAILSETTINGS_SMTPSERVER=inbucket \
|
|
--env MM_SQLSETTINGS_DRIVERNAME=postgres \
|
|
--env MM_EMAILSETTINGS_SMTPPORT=10025 \
|
|
--env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \
|
|
-v ~/mattermost:/mattermost \
|
|
-w /mattermost/mattermost-server \
|
|
mattermost/mattermost-build-server:20200322_golang-1.14.1 \
|
|
bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10" ARGS="version" run-cli'
|
|
echo "Generating dump"
|
|
docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
|
|
docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
|
|
echo "Removing databases created for db comparison"
|
|
docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
|
|
|
|
echo "Generating diff"
|
|
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
|
|
no_output_timeout: 1h
|
|
- run:
|
|
name: MySQL schema migration validation
|
|
command: |
|
|
ulimit -n 8096
|
|
mkdir -p mattermost-server/client/plugins
|
|
cd mattermost-server/build
|
|
echo "Creating databases"
|
|
docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
|
|
echo "Importing mysql dump from version 5.0"
|
|
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-5.0.sql
|
|
docker run -it --net circleci_mm-test \
|
|
--env CI_INBUCKET_HOST=inbucket \
|
|
--env CI_MINIO_HOST=minio \
|
|
--env CI_INBUCKET_PORT=10080 \
|
|
--env CI_MINIO_PORT=9000 \
|
|
--env CI_INBUCKET_SMTP_PORT=10025 \
|
|
--env CI_LDAP_HOST=openldap \
|
|
--env IS_CI="true" \
|
|
--env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
|
|
--env MM_EMAILSETTINGS_SMTPSERVER=inbucket \
|
|
--env MM_SQLSETTINGS_DRIVERNAME=mysql \
|
|
--env MM_EMAILSETTINGS_SMTPPORT=10025 \
|
|
--env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \
|
|
-v ~/mattermost:/mattermost \
|
|
-w /mattermost/mattermost-server \
|
|
mattermost/mattermost-build-server:20200322_golang-1.14.1 \
|
|
bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" ARGS="version" run-cli'
|
|
|
|
echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest"
|
|
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
|
|
docker-compose --no-ansi exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
|
|
|
|
echo "Generating dump"
|
|
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
|
|
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
|
|
|
|
echo "Removing databases created for db comparison"
|
|
docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
|
|
|
|
echo "Generating diff"
|
|
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
|
|
no_output_timeout: 1h
|
|
|
|
upload-s3-sha:
|
|
docker:
|
|
- image: 'circleci/python:2.7'
|
|
working_directory: ~/mattermost/enterprise
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/mattermost/
|
|
- run:
|
|
command: |
|
|
cd ~/mattermost/mattermost-server/dist/
|
|
rm -rf mattermost
|
|
- aws-s3/copy:
|
|
from: ~/mattermost/mattermost-server/dist/
|
|
to: "s3://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/"
|
|
arguments: --acl public-read --cache-control "no-cache" --recursive
|
|
|
|
upload-s3:
|
|
docker:
|
|
- image: 'circleci/python:2.7'
|
|
working_directory: ~/mattermost/enterprise
|
|
steps:
|
|
- attach_workspace:
|
|
at: ~/mattermost/
|
|
- run:
|
|
command: |
|
|
cd ~/mattermost/mattermost-server/dist/
|
|
rm -rf mattermost
|
|
- aws-s3/copy:
|
|
from: ~/mattermost/mattermost-server/dist/
|
|
to: s3://pr-builds.mattermost.com/mattermost-server/$(echo "${CIRCLE_BRANCH}" | sed 's/pull\//PR-/g')/
|
|
arguments: --acl public-read --cache-control "no-cache" --recursive
|
|
|
|
build-docker:
|
|
working_directory: ~/
|
|
docker:
|
|
- image: circleci/buildpack-deps:stretch
|
|
resource_class: xlarge
|
|
steps:
|
|
- attach_workspace:
|
|
at: .
|
|
- setup_remote_docker
|
|
- run:
|
|
command: |
|
|
export TAG="${CIRCLE_SHA1:0:7}"
|
|
cd mattermost-server
|
|
export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz
|
|
docker build --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mattermost-team-edition:${TAG} build
|
|
echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin
|
|
docker push mattermost/mattermost-team-edition:${TAG}
|
|
|
|
trigger-ee-tests:
|
|
machine: true
|
|
steps:
|
|
- run:
|
|
name: Trigger enterprise
|
|
command: |
|
|
curl \
|
|
--request POST \
|
|
--url https://circleci.com/api/v2/project/gh/mattermost/enterprise/pipeline \
|
|
--user ${CIRCLECI_TOKEN} \
|
|
--data parameters[external_branch]=$CIRCLE_BRANCH \
|
|
--data parameters[external_sha]=$CIRCLE_SHA1
|
|
|
|
workflows:
|
|
version: 2
|
|
untagged-build:
|
|
jobs:
|
|
- setup:
|
|
filters:
|
|
branches:
|
|
ignore:
|
|
- master
|
|
- /^release-.*/
|
|
- check-deps:
|
|
context: sast-webhook
|
|
requires:
|
|
- setup
|
|
# - check-i18n:
|
|
# requires:
|
|
# - setup
|
|
- check-app-layers:
|
|
requires:
|
|
- setup
|
|
- build:
|
|
requires:
|
|
- check-app-layers
|
|
- upload-s3-sha:
|
|
context: mattermost-ci-pr-builds-s3
|
|
requires:
|
|
- build
|
|
- upload-s3:
|
|
context: mattermost-ci-pr-builds-s3
|
|
requires:
|
|
- build
|
|
- build-docker:
|
|
context: matterbuild-docker
|
|
requires:
|
|
- upload-s3-sha
|
|
- test:
|
|
name: test-mysql
|
|
dbdriver: mysql
|
|
dbsource: "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8"
|
|
requires:
|
|
- check-app-layers
|
|
- test:
|
|
name: test-postgres
|
|
dbdriver: postgres
|
|
dbsource: "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10"
|
|
requires:
|
|
- check-app-layers
|
|
- test-schema:
|
|
requires:
|
|
- check-app-layers
|
|
- trigger-ee-tests:
|
|
context: matterbuild-circleci-token
|
|
requires:
|
|
- test-mysql
|
|
- test-postgres
|
|
- test-schema
|
|
- upload-s3
|
|
- upload-s3-sha
|
|
- build-docker
|
|
filters:
|
|
branches:
|
|
only: /^pull\/.*$/
|