mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

[feat] CI essential multi-product architecture changes (#20783)

Просмотр исходного кода 
* [feat] essential multi-product architecture changes

This takes care of aligning mattermost-server CircleCI pipeline to the mutlti-product architecture.
Specifically the following are performed:

- seperation of mattermost-server and mattermost-webapp clone/setup stages
- introduction of clone/setup stage for focalboard
- change to using cimg/base image instaed of  mattermost/mattermost-build-webapp when cloning/setup mattermost-server
- introduction of `configure-multirepo` stage, in which multi-module workspaces (golang) configuration is being performed for mattermost-server and focalboard
- introduction of parallel build of focalboard along with mattermost-server
- reorganisation of step dependencies, to accomodate `configure-multirepo` step

Ticket: https://mattermost.atlassian.net/browse/CLD-3801
Signed-off-by: Akis Maziotis <akis.maziotis@mattermost.com>
Этот коммит содержится в:
Akis Maziotis
2022-08-11 18:16:59 +03:00
коммит произвёл GitHub
родитель 55b3961b98
Коммит 1ef24221e3
1 изменённых файлов: 132 добавлений и 47 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

179
.circleci/config.yml
Просмотреть файл

@@ -14,14 +14,24 @@ executors:
COMPOSE_PROJECT_NAME: "circleci" COMPOSE_PROJECT_NAME: "circleci"
jobs: jobs:
setup: setup-mattermost-server:
working_directory: ~/mattermost/mattermost-server working_directory: ~/mattermost/mattermost-server
docker: docker:
- image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d - image: cimg/base:2022.08@sha256:35e5e29930ab565475a4f2aa9b4124998ed67dbc7b0e2dd5f420a4189d08d0d2
resource_class: xlarge resource_class: xlarge
# Use `--retry-all-errors` instead of `until` after curl version >= 7.71.0; `retry` will not work, since it only retries on transient errors, 403 is not one of them. # Use `--retry-all-errors` instead of `until` after curl version >= 7.71.0; `retry` will not work, since it only retries on transient errors, 403 is not one of them.
steps: steps:
- checkout - checkout
- persist_to_workspace:
root: ~/mattermost
paths:
- mattermost-server
setup-mattermost-webapp:
working_directory: ~/mattermost/mattermost-webapp
docker:
- image: mattermost/mattermost-build-webapp:20220802_node-16.10.0@sha256:3272aa759f10c2ef1719ed08cc82ddb07224bec5be86f09800c72f5e2a623c3d
resource_class: xlarge
steps:
- run: | - run: |
cd ../ cd ../
git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git
@@ -29,10 +39,8 @@ jobs:
git checkout $CIRCLE_BRANCH || git checkout master git checkout $CIRCLE_BRANCH || git checkout master
export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD) export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
echo "$WEBAPP_GIT_COMMIT" echo "$WEBAPP_GIT_COMMIT"
FILE_DIST=dist.tar.gz FILE_DIST=dist.tar.gz
curl --version curl --version
runtime="2 minute" runtime="2 minute"
endtime=$(date -ud "$runtime" +%s) endtime=$(date -ud "$runtime" +%s)
while [[ $(date -u +%s) -le $endtime ]]; do while [[ $(date -u +%s) -le $endtime ]]; do
@@ -53,7 +61,22 @@ jobs:
root: ~/mattermost root: ~/mattermost
paths: paths:
- mattermost-webapp - mattermost-webapp
- mattermost-server setup-focalboard:
working_directory: ~/mattermost/focalboard
docker:
- image: cimg/base:2022.08@sha256:35e5e29930ab565475a4f2aa9b4124998ed67dbc7b0e2dd5f420a4189d08d0d2
resource_class: xlarge
steps:
- run: |
cd ../
git clone --depth=10 --no-single-branch https://github.com/mattermost/focalboard.git
cd focalboard
git checkout $CIRCLE_BRANCH || git checkout main
echo $(git rev-parse HEAD)
- persist_to_workspace:
root: ~/mattermost
paths:
- focalboard
check-deps: check-deps:
parameters: parameters:
@@ -133,7 +156,6 @@ jobs:
cd mattermost-server cd mattermost-server
make app-layers make app-layers
if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
check-go-mod-tidy: check-go-mod-tidy:
docker: docker:
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1 - image: mattermost/mattermost-build-server:20220415_golang-1.18.1
@@ -146,7 +168,6 @@ jobs:
cd mattermost-server cd mattermost-server
make modules-tidy make modules-tidy
if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; exit 1; fi if [[ -n $(git status --porcelain) ]]; then echo "Please tidy up the Go modules using make modules-tidy"; exit 1; fi
check-store-layers: check-store-layers:
docker: docker:
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1 - image: mattermost/mattermost-build-server:20220415_golang-1.18.1
@@ -159,7 +180,6 @@ jobs:
cd mattermost-server cd mattermost-server
make store-layers make store-layers
if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
check-mocks: check-mocks:
docker: docker:
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1 - image: mattermost/mattermost-build-server:20220415_golang-1.18.1
@@ -172,7 +192,6 @@ jobs:
cd mattermost-server cd mattermost-server
make mocks make mocks
if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi if [[ -n $(git status --porcelain) ]]; then echo "Please update the mocks using `make mocks`"; exit 1; fi
check-email-templates: check-email-templates:
docker: docker:
- image: cimg/go:1.17-node - image: cimg/go:1.17-node
@@ -186,7 +205,6 @@ jobs:
sudo npm install -g mjml@4.9.0 sudo npm install -g mjml@4.9.0
make build-templates make build-templates
if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi if [[ -n $(git status --porcelain) ]]; then echo "Please update the email templates using `make build-templates`"; exit 1; fi
check-gen-serialized: check-gen-serialized:
docker: docker:
- image: cimg/go:1.17 - image: cimg/go:1.17
@@ -199,7 +217,6 @@ jobs:
cd mattermost-server cd mattermost-server
make gen-serialized make gen-serialized
if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi if [[ -n $(git status --porcelain) ]]; then echo "Please update the serialized files using 'make gen-serialized'"; exit 1; fi
check-migrations: check-migrations:
docker: docker:
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1 - image: mattermost/mattermost-build-server:20220415_golang-1.18.1
@@ -212,7 +229,6 @@ jobs:
cd mattermost-server cd mattermost-server
make migrations-extract make migrations-extract
if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi if [[ -n $(git status --porcelain) ]]; then echo "Please update the migrations using make migrations-extract"; exit 1; fi
# Dedicated job for golangci-lint it does not need anything only the server code for that # Dedicated job for golangci-lint it does not need anything only the server code for that
# and to make more clear when the job fails # and to make more clear when the job fails
check-golangci-lint: check-golangci-lint:
@@ -240,7 +256,6 @@ jobs:
make config-reset make config-reset
make plugin-checker make plugin-checker
make vet BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml' make vet BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
# Dedicate job for sentry it does not need anything only the server code for that # Dedicate job for sentry it does not need anything only the server code for that
# and to make more clear when the job fails # and to make more clear when the job fails
sentry: sentry:
@@ -254,7 +269,6 @@ jobs:
command: | command: |
sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD` sentry-cli --log-level=debug releases new --finalize -p mattermost-server `git rev-parse HEAD`
sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD` sentry-cli --log-level=debug releases set-commits --auto `git rev-parse HEAD`
build-api-spec: build-api-spec:
docker: docker:
- image: circleci/node:lts - image: circleci/node:lts
@@ -274,7 +288,7 @@ jobs:
paths: paths:
- mattermost-api-reference - mattermost-api-reference
build: build-mattermost-server:
docker: docker:
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1 - image: mattermost/mattermost-build-server:20220415_golang-1.18.1
resource_class: xlarge resource_class: xlarge
@@ -304,6 +318,38 @@ jobs:
- mattermost-server - mattermost-server
- mattermost-webapp - mattermost-webapp
build-focalboard:
docker:
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
resource_class: xlarge
working_directory: ~/mattermost
steps:
- attach_workspace:
at: ~/mattermost/
- run:
command: |
cd focalboard
make server-linux
configure-multirepo:
docker:
- image: mattermost/mattermost-build-server:20220415_golang-1.18.1
working_directory: ~/mattermost
steps:
- attach_workspace:
at: ~/mattermost/
- run:
command: |
cd mattermost-server
make setup-go-work
cd ../focalboard
EXCLUDE_ENTERPRISE=TRUE make templates-archive
- persist_to_workspace:
root: ~/mattermost
paths:
- mattermost-server
- mattermost-webapp
- focalboard
test: test:
executor: executor:
name: ubuntu name: ubuntu
@@ -410,13 +456,11 @@ jobs:
-w /mattermost/mattermost-server \ -w /mattermost/mattermost-server \
mattermost/mattermost-build-server:20220415_golang-1.18.1 \ mattermost/mattermost-build-server:20220415_golang-1.18.1 \
bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='db migrate' run-cli" bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10' ARGS='db migrate' run-cli"
echo "Generating dump" echo "Generating dump"
docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
echo "Removing databases created for db comparison" echo "Removing databases created for db comparison"
docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test' docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
echo "Generating diff" echo "Generating diff"
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
no_output_timeout: 1h no_output_timeout: 1h
@@ -430,7 +474,6 @@ jobs:
echo "Importing mysql dump from version 6.0.0" echo "Importing mysql dump from version 6.0.0"
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-6.0.0.sql
docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')" docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "INSERT INTO Systems (Name, Value) VALUES ('Version', '6.0.0')"
docker run -it --net circleci_mm-test \ docker run -it --net circleci_mm-test \
--env-file="dotenv/test-schema-validation.env" \ --env-file="dotenv/test-schema-validation.env" \
--env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \ --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
@@ -439,14 +482,11 @@ jobs:
-w /mattermost/mattermost-server \ -w /mattermost/mattermost-server \
mattermost/mattermost-build-server:20220415_golang-1.18.1 \ mattermost/mattermost-build-server:20220415_golang-1.18.1 \
bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='db migrate' run-cli" bash -c "ulimit -n 8096; make ARGS='db migrate' run-cli && make MM_SQLSETTINGS_DATASOURCE='mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s' ARGS='db migrate' run-cli"
echo "Generating dump" echo "Generating dump"
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
echo "Removing databases created for db comparison" echo "Removing databases created for db comparison"
docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
echo "Generating diff" echo "Generating diff"
diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
no_output_timeout: 1h no_output_timeout: 1h
@@ -495,19 +535,28 @@ jobs:
- run: - run:
command: | command: |
export TAG="${CIRCLE_SHA1:0:7}" export TAG="${CIRCLE_SHA1:0:7}"
cd mattermost-server cd mattermost-server
export DOCKER_CLI_EXPERIMENTAL=enabled export DOCKER_CLI_EXPERIMENTAL=enabled
echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin
export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz
docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} build docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} build
workflows: workflows:
version: 2 version: 2
untagged-build: untagged-build:
jobs: jobs:
- setup: - setup-mattermost-server:
filters:
branches:
ignore:
- master
- /^release-.*/
- setup-mattermost-webapp:
filters:
branches:
ignore:
- master
- /^release-.*/
- setup-focalboard:
filters: filters:
branches: branches:
ignore: ignore:
@@ -524,7 +573,8 @@ workflows:
# - setup # - setup
- check-go-mod-tidy: - check-go-mod-tidy:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- check-golangci-lint: - check-golangci-lint:
filters: filters:
branches: branches:
@@ -533,29 +583,46 @@ workflows:
- /^release-.*/ - /^release-.*/
- check-app-layers: - check-app-layers:
requires: requires:
- setup - configure-multirepo
- check-store-layers: - check-store-layers:
requires: requires:
- setup - configure-multirepo
- check-mocks: - check-mocks:
requires: requires:
- setup - configure-multirepo
- check-email-templates: - check-email-templates:
requires: requires:
- setup - configure-multirepo
- check-gen-serialized: - check-gen-serialized:
requires: requires:
- setup - configure-multirepo
- check-mattermost-vet: - check-mattermost-vet:
requires: requires:
- setup - configure-multirepo
- check-migrations: - check-migrations:
requires: requires:
- setup - configure-multirepo
- build-api-spec: - build-api-spec:
requires: requires:
- setup - configure-multirepo
- build: - configure-multirepo:
requires:
- setup-mattermost-server
- setup-mattermost-webapp
- setup-focalboard
- build-mattermost-server:
requires:
- check-go-mod-tidy
- check-app-layers
- check-store-layers
- check-mocks
- check-email-templates
- check-gen-serialized
- check-mattermost-vet
- check-golangci-lint
- check-migrations
- build-api-spec
- build-focalboard:
requires: requires:
- check-go-mod-tidy - check-go-mod-tidy
- check-app-layers - check-app-layers
@@ -570,11 +637,13 @@ workflows:
- upload-s3-sha: - upload-s3-sha:
context: mattermost-ci-pr-builds-s3 context: mattermost-ci-pr-builds-s3
requires: requires:
- build - build-mattermost-server
- build-focalboard
- upload-s3: - upload-s3:
context: mattermost-ci-pr-builds-s3 context: mattermost-ci-pr-builds-s3
requires: requires:
- build - build-mattermost-server
- build-focalboard
- build-docker: - build-docker:
context: matterbuild-docker context: matterbuild-docker
requires: requires:
@@ -621,13 +690,21 @@ workflows:
- check-gen-serialized - check-gen-serialized
release-build: release-build:
jobs: jobs:
- setup: - setup-mattermost-server:
filters: filters:
branches: branches:
only: only:
- master - master
- /^release-.*/ - /^release-.*/
- cloud - cloud
- setup-mattermost-webapp:
filters:
branches:
only:
- master
- /^release-.*/
- cloud
# Disabling check-deps since the new version is not on bintray and all builds will fail # Disabling check-deps since the new version is not on bintray and all builds will fail
# we've are on top of this, and it's going to be temporary. # we've are on top of this, and it's going to be temporary.
#- check-deps: #- check-deps:
@@ -646,28 +723,36 @@ workflows:
- cloud - cloud
- check-go-mod-tidy: - check-go-mod-tidy:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- check-app-layers: - check-app-layers:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- check-store-layers: - check-store-layers:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- build-api-spec: - build-api-spec:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- check-mocks: - check-mocks:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- check-email-templates: - check-email-templates:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- check-gen-serialized: - check-gen-serialized:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- check-mattermost-vet: - check-mattermost-vet:
requires: requires:
- setup - setup-mattermost-server
- setup-mattermost-webapp
- sentry: - sentry:
context: matterbuild-sentry context: matterbuild-sentry
requires: requires:
@@ -680,7 +765,7 @@ workflows:
- check-mattermost-vet - check-mattermost-vet
- check-golangci-lint - check-golangci-lint
- build-api-spec - build-api-spec
- build: - build-mattermost-server:
requires: requires:
- check-go-mod-tidy - check-go-mod-tidy
- check-app-layers - check-app-layers