[feat] CI essential multi-product architecture changes (#20783)
* [feat] essential multi-product architecture changes This takes care of aligning mattermost-server CircleCI pipeline to the mutlti-product architecture. Specifically the following are performed: - seperation of mattermost-server and mattermost-webapp clone/setup stages - introduction of clone/setup stage for focalboard - change to using cimg/base image instaed of mattermost/mattermost-build-webapp when cloning/setup mattermost-server - introduction of `configure-multirepo` stage, in which multi-module workspaces (golang) configuration is being performed for mattermost-server and focalboard - introduction of parallel build of focalboard along with mattermost-server - reorganisation of step dependencies, to accomodate `configure-multirepo` step Ticket: https://mattermost.atlassian.net/browse/CLD-3801 Signed-off-by: Akis Maziotis <akis.maziotis@mattermost.com>
Этот коммит содержится в:
Akis Maziotis
коммит произвёл
GitHub
GitHub
родитель
55b3961b98
Коммит
1ef24221e3
@@ -14,14 +14,24 @@ executors:
|
||||
COMPOSE_PROJECT_NAME: "circleci"
|
||||
|
||||
jobs:
|
||||
setup:
|
||||
setup-mattermost-server:
|
||||
working_directory: ~/mattermost/mattermost-server
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d
|
||||
- image: cimg/base:2022.08@sha256:35e5e29930ab565475a4f2aa9b4124998ed67dbc7b0e2dd5f420a4189d08d0d2
|
||||
resource_class: xlarge
|
||||
# Use `--retry-all-errors` instead of `until` after curl version >= 7.71.0; `retry` will not work, since it only retries on transient errors, 403 is not one of them.
|
||||
steps:
|
||||
- checkout
|
||||
- persist_to_workspace:
|
||||
root: ~/mattermost
|
||||
paths:
|
||||
- mattermost-server
|
||||
setup-mattermost-webapp:
|
||||
working_directory: ~/mattermost/mattermost-webapp
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d
|
||||
resource_class: xlarge
|
||||
steps:
|
||||
- run: |
|
||||
cd ../
|
||||
git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git
|
||||
@@ -29,10 +39,8 @@ jobs:
|
||||
git checkout $CIRCLE_BRANCH || git checkout master
|
||||
export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
|
||||
echo "$WEBAPP_GIT_COMMIT"
|
||||
|
||||
FILE_DIST=dist.tar.gz
|
||||
curl --version
|
||||
|
||||
runtime="2 minute"
|
||||
endtime=$(date -ud "$runtime" +%s)
|
||||
while [[ $(date -u +%s) -le $endtime ]]; do
|
||||
@@ -53,7 +61,22 @@ jobs:
|
||||
root: ~/mattermost
|
||||
paths:
|
||||
- mattermost-webapp
|
||||
- mattermost-server
|
||||
setup-focalboard:
|
||||
working_directory: ~/mattermost/focalboard
|
||||
docker:
|
||||
- image: cimg/base:2022.08@sha256:35e5e29930ab565475a4f2aa9b4124998ed67dbc7b0e2dd5f420a4189d08d0d2
|
||||
resource_class: xlarge
|
||||
steps:
|
||||
- run: |
|
||||
cd ../
|
||||
git clone --depth=10 --no-single-branch https://github.com/mattermost/focalboard.git
|
||||
cd focalboard
|
||||
git checkout $CIRCLE_BRANCH || git checkout main
|
||||
echo $(git rev-parse HEAD)
|
||||
- persist_to_workspace:
|
||||
root: ~/mattermost
|
||||
paths:
|
||||
- focalboard
|
||||
|
||||
check-deps:
|
||||
parameters:
|
||||
@@ -133,7 +156,6 @@ jobs:
|
||||
cd mattermost-server
|
||||
make app-layers
|
||||
if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
|
||||
|
||||
check-go-mod-tidy:
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
|
||||
@@ -146,7 +168,6 @@ jobs:
|
||||
cd mattermost-server
|
||||
make modules-tidy
|
||||
if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; exit 1; fi
|
||||
|
||||
check-store-layers:
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
|
||||
@@ -159,7 +180,6 @@ jobs:
|
||||
cd mattermost-server
|
||||
make store-layers
|
||||
if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
|
||||
|
||||
check-mocks:
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
|
||||
@@ -172,7 +192,6 @@ jobs:
|
||||
cd mattermost-server
|
||||
make mocks
|
||||
if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi
|
||||
|
||||
check-email-templates:
|
||||
docker:
|
||||
- image: cimg/go:1.17-node
|
||||
@@ -186,7 +205,6 @@ jobs:
|
||||
sudo npm install -g mjml@4.9.0
|
||||
make build-templates
|
||||
if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi
|
||||
|
||||
check-gen-serialized:
|
||||
docker:
|
||||
- image: cimg/go:1.17
|
||||
@@ -199,7 +217,6 @@ jobs:
|
||||
cd mattermost-server
|
||||
make gen-serialized
|
||||
if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi
|
||||
|
||||
check-migrations:
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
|
||||
@@ -212,7 +229,6 @@ jobs:
|
||||
cd mattermost-server
|
||||
make migrations-extract
|
||||
if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi
|
||||
|
||||
# Dedicated job for golangci-lint it does not need anything only the server code for that
|
||||
# and to make more clear when the job fails
|
||||
check-golangci-lint:
|
||||
@@ -240,7 +256,6 @@ jobs:
|
||||
make config-reset
|
||||
make plugin-checker
|
||||
make vet BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
|
||||
|
||||
# Dedicate job for sentry it does not need anything only the server code for that
|
||||
# and to make more clear when the job fails
|
||||
sentry:
|
||||
@@ -254,7 +269,6 @@ jobs:
|
||||
command: |
|
||||
sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD`
|
||||
sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD`
|
||||
|
||||
build-api-spec:
|
||||
docker:
|
||||
- image: circleci/node:lts
|
||||
@@ -274,7 +288,7 @@ jobs:
|
||||
paths:
|
||||
- mattermost-api-reference
|
||||
|
||||
build:
|
||||
build-mattermost-server:
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
|
||||
resource_class: xlarge
|
||||
@@ -304,6 +318,38 @@ jobs:
|
||||
- mattermost-server
|
||||
- mattermost-webapp
|
||||
|
||||
build-focalboard:
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
|
||||
resource_class: xlarge
|
||||
working_directory: ~/mattermost
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: ~/mattermost/
|
||||
- run:
|
||||
command: |
|
||||
cd focalboard
|
||||
make server-linux
|
||||
configure-multirepo:
|
||||
docker:
|
||||
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
|
||||
working_directory: ~/mattermost
|
||||
steps:
|
||||
- attach_workspace:
|
||||
at: ~/mattermost/
|
||||
- run:
|
||||
command: |
|
||||
cd mattermost-server
|
||||
make setup-go-work
|
||||
cd ../focalboard
|
||||
EXCLUDE_ENTERPRISE=TRUE make templates-archive
|
||||
- persist_to_workspace:
|
||||
root: ~/mattermost
|
||||
paths:
|
||||
- mattermost-server
|
||||
- mattermost-webapp
|
||||
- focalboard
|
||||
|
||||
test:
|
||||
executor:
|
||||
name: ubuntu
|
||||
@@ -410,13 +456,11 @@ jobs:
|
||||
-w /mattermost/mattermost-server \
|
||||
mattermost/mattermost-build-server:20220415_golang-1.18.1 \
|
||||
bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='db migrate' run-cli"
|
||||
|
||||
echo "Generating dump"
|
||||
docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
|
||||
docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
|
||||
echo "Removing databases created for db comparison"
|
||||
docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
|
||||
|
||||
echo "Generating diff"
|
||||
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
|
||||
no_output_timeout: 1h
|
||||
@@ -430,7 +474,6 @@ jobs:
|
||||
echo "Importing mysql dump from version 6.0.0"
|
||||
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql
|
||||
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
|
||||
|
||||
docker run -it --net circleci_mm-test \
|
||||
--env-file="dotenv/test-schema-validation.env" \
|
||||
--env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
|
||||
@@ -439,14 +482,11 @@ jobs:
|
||||
-w /mattermost/mattermost-server \
|
||||
mattermost/mattermost-build-server:20220415_golang-1.18.1 \
|
||||
bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='db migrate' run-cli"
|
||||
|
||||
echo "Generating dump"
|
||||
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
|
||||
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
|
||||
|
||||
echo "Removing databases created for db comparison"
|
||||
docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
|
||||
|
||||
echo "Generating diff"
|
||||
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
|
||||
no_output_timeout: 1h
|
||||
@@ -495,19 +535,28 @@ jobs:
|
||||
- run:
|
||||
command: |
|
||||
export TAG="${CIRCLE_SHA1:0:7}"
|
||||
|
||||
cd mattermost-server
|
||||
|
||||
export DOCKER_CLI_EXPERIMENTAL=enabled
|
||||
echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin
|
||||
export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz
|
||||
docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} build
|
||||
|
||||
workflows:
|
||||
version: 2
|
||||
untagged-build:
|
||||
jobs:
|
||||
- setup:
|
||||
- setup-mattermost-server:
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- master
|
||||
- /^release-.*/
|
||||
- setup-mattermost-webapp:
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
- master
|
||||
- /^release-.*/
|
||||
- setup-focalboard:
|
||||
filters:
|
||||
branches:
|
||||
ignore:
|
||||
@@ -524,7 +573,8 @@ workflows:
|
||||
# - setup
|
||||
- check-go-mod-tidy:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- check-golangci-lint:
|
||||
filters:
|
||||
branches:
|
||||
@@ -533,29 +583,46 @@ workflows:
|
||||
- /^release-.*/
|
||||
- check-app-layers:
|
||||
requires:
|
||||
- setup
|
||||
- configure-multirepo
|
||||
- check-store-layers:
|
||||
requires:
|
||||
- setup
|
||||
- configure-multirepo
|
||||
- check-mocks:
|
||||
requires:
|
||||
- setup
|
||||
- configure-multirepo
|
||||
- check-email-templates:
|
||||
requires:
|
||||
- setup
|
||||
- configure-multirepo
|
||||
- check-gen-serialized:
|
||||
requires:
|
||||
- setup
|
||||
- configure-multirepo
|
||||
- check-mattermost-vet:
|
||||
requires:
|
||||
- setup
|
||||
- configure-multirepo
|
||||
- check-migrations:
|
||||
requires:
|
||||
- setup
|
||||
- configure-multirepo
|
||||
- build-api-spec:
|
||||
requires:
|
||||
- setup
|
||||
- build:
|
||||
- configure-multirepo
|
||||
- configure-multirepo:
|
||||
requires:
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- setup-focalboard
|
||||
- build-mattermost-server:
|
||||
requires:
|
||||
- check-go-mod-tidy
|
||||
- check-app-layers
|
||||
- check-store-layers
|
||||
- check-mocks
|
||||
- check-email-templates
|
||||
- check-gen-serialized
|
||||
- check-mattermost-vet
|
||||
- check-golangci-lint
|
||||
- check-migrations
|
||||
- build-api-spec
|
||||
- build-focalboard:
|
||||
requires:
|
||||
- check-go-mod-tidy
|
||||
- check-app-layers
|
||||
@@ -570,11 +637,13 @@ workflows:
|
||||
- upload-s3-sha:
|
||||
context: mattermost-ci-pr-builds-s3
|
||||
requires:
|
||||
- build
|
||||
- build-mattermost-server
|
||||
- build-focalboard
|
||||
- upload-s3:
|
||||
context: mattermost-ci-pr-builds-s3
|
||||
requires:
|
||||
- build
|
||||
- build-mattermost-server
|
||||
- build-focalboard
|
||||
- build-docker:
|
||||
context: matterbuild-docker
|
||||
requires:
|
||||
@@ -621,13 +690,21 @@ workflows:
|
||||
- check-gen-serialized
|
||||
release-build:
|
||||
jobs:
|
||||
- setup:
|
||||
- setup-mattermost-server:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /^release-.*/
|
||||
- cloud
|
||||
- setup-mattermost-webapp:
|
||||
filters:
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /^release-.*/
|
||||
- cloud
|
||||
|
||||
# Disabling check-deps since the new version is not on bintray and all builds will fail
|
||||
# we've are on top of this, and it's going to be temporary.
|
||||
#- check-deps:
|
||||
@@ -646,28 +723,36 @@ workflows:
|
||||
- cloud
|
||||
- check-go-mod-tidy:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- check-app-layers:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- check-store-layers:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- build-api-spec:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- check-mocks:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- check-email-templates:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- check-gen-serialized:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- check-mattermost-vet:
|
||||
requires:
|
||||
- setup
|
||||
- setup-mattermost-server
|
||||
- setup-mattermost-webapp
|
||||
- sentry:
|
||||
context: matterbuild-sentry
|
||||
requires:
|
||||
@@ -680,7 +765,7 @@ workflows:
|
||||
- check-mattermost-vet
|
||||
- check-golangci-lint
|
||||
- build-api-spec
|
||||
- build:
|
||||
- build-mattermost-server:
|
||||
requires:
|
||||
- check-go-mod-tidy
|
||||
- check-app-layers
|
||||
|
||||
Ссылка в новой задаче
Block a user