mirrors/mostlymatter
1
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

CLD-5783 Implement rolling upgrade tests (#27842)

Просмотр исходного кода 
* Make server.prepare idempotent, don't cleanup containers on server.start
* Implement rolling upgrade E2E tests
* Apply Akis' suggestions from code review
* Apply Saturn's suggestions from review
* Communicate RollingRelease on GHA and test automation results channel
* Implement E2E test case versioning for rolling release tests

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: Akis Maziotis <akis.maziotis@mattermost.com>
Этот коммит содержится в:
Mario Vitale
2024-08-07 14:08:50 +02:00
коммит произвёл GitHub
родитель 90884172ae
Коммит 39ca104688
11 изменённых файлов: 100 добавлений и 18 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

27
.github/workflows/e2e-fulltests-ci.yml поставляемый
Просмотреть файл

@@ -13,6 +13,10 @@ on:
type: string
description: If testing a PR, this must be specified as well.
required: false
ROLLING_RELEASE_FROM_TAG:
type: string
description: If specified, the server will be initialized with the given release tag, perform a rolling upgrade, then run the full E2E test.
required: false
MM_ENV:
type: string
required: false
@@ -56,11 +60,14 @@ jobs:
BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}"
REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}"
ROLLING_RELEASE_commit_sha: "${{ steps.generate.outputs.ROLLING_RELEASE_commit_sha }}"
ROLLING_RELEASE_SERVER_IMAGE: "${{ steps.generate.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
env:
GH_TOKEN: "${{ github.token }}"
REF: "${{ inputs.ref || github.sha }}"
PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
ROLLING_RELEASE_FROM_TAG: "${{ inputs.ROLLING_RELEASE_FROM_TAG }}"
# We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
TEST_FILTER_PR: >-
--stage="@prod"
@@ -103,7 +110,6 @@ jobs:
SERVER_IMAGE_TAG="${COMMIT_SHA::7}"
SERVER_IMAGE_ORG=mattermostdevelopment
BUILD_ID_SUFFIX="${REPORT_TYPE@L}-${SERVER}-ent"
STATUS_CHECK_CONTEXT="E2E Tests/test${MM_ENV:+/$MM_ENV_HASH}"
WORKERS_NUMBER=20
TEST_FILTER="$TEST_FILTER_PR"
COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
@@ -121,7 +127,7 @@ jobs:
SERVER_IMAGE_TAG=master
SERVER_IMAGE_ORG=mattermostdevelopment
BUILD_ID_SUFFIX="${_IS_TEST_UNSTABLE:+unstable-}daily-${SERVER}-ent"
STATUS_CHECK_CONTEXT="E2E Tests/test-${BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}"
BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
WORKERS_NUMBER=10 # Daily tests are not time critical, and it's more efficient to run on fewer workers
TEST_FILTER="${!_TEST_FILTER_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}"
TM4J_ENABLE=true
@@ -136,7 +142,7 @@ jobs:
SERVER_IMAGE_TAG="$(cut -c2- <<<$REF)" # Remove the leading 'v' from the given tag name, to generate the docker image tag
SERVER_IMAGE_ORG=mattermost
BUILD_ID_SUFFIX="release-${SERVER}-ent"
STATUS_CHECK_CONTEXT="E2E Tests/test-${BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}"
BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
WORKERS_NUMBER=20
TEST_FILTER="${!_TEST_FILTER_VARIABLE}"
TM4J_ENABLE=true
@@ -149,17 +155,28 @@ jobs:
echo "Fatal: unimplemented test type. Aborting."
exit 1
esac
if [ -n "$ROLLING_RELEASE_FROM_TAG" ]; then
ROLLING_RELEASE_COMMIT_SHA=$(git rev-parse --verify "$ROLLING_RELEASE_FROM_TAG")
ROLLING_RELEASE_SERVER_IMAGE_TAG=$(echo "$ROLLING_RELEASE_FROM_TAG" | sed 's/^v//') # Remove the leading 'v' from the given tag name, to generate the docker image tag
ROLLING_RELEASE_SERVER_IMAGE="mattermost/mattermost-enterprise-edition:${ROLLING_RELEASE_SERVER_IMAGE_TAG}"
BUILD_ID_SUFFIX="rolling${ROLLING_RELEASE_FROM_TAG/-/_}-$BUILD_ID_SUFFIX"
BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
### Run sanity assertions after variable generations
git show-ref --verify "refs/tags/${ROLLING_RELEASE_FROM_TAG}" # 'inputs.ROLLING_RELEASE_FROM_TAG' must be a tag, for release report types
fi
echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
echo "BRANCH=${BRANCH}" >> $GITHUB_OUTPUT
echo "SERVER_IMAGE=${SERVER_IMAGE_ORG}/mattermost-enterprise-edition:${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
echo "SERVER=${SERVER}" >> $GITHUB_OUTPUT
echo "server_uppercase=${SERVER@U}" >> $GITHUB_OUTPUT
echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT
echo "status_check_context=${STATUS_CHECK_CONTEXT}" >> $GITHUB_OUTPUT
echo "status_check_context=E2E Tests/test${BUILD_ID_SUFFIX_IN_STATUS_CHECK:+-$BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT
echo "TEST_FILTER=${TEST_FILTER}" >> $GITHUB_OUTPUT
echo "TM4J_ENABLE=${TM4J_ENABLE:-}" >> $GITHUB_OUTPUT
echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT
echo "ROLLING_RELEASE_commit_sha=${ROLLING_RELEASE_COMMIT_SHA}" >> $GITHUB_OUTPUT
echo "ROLLING_RELEASE_SERVER_IMAGE=${ROLLING_RELEASE_SERVER_IMAGE}" >> $GITHUB_OUTPUT
# BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
# Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
echo "BUILD_ID=${{ github.run_id }}_${{ github.run_attempt }}-${SERVER_IMAGE_TAG}-${BUILD_ID_SUFFIX}" >> $GITHUB_OUTPUT
@@ -200,6 +217,8 @@ jobs:
BRANCH: "${{ needs.generate-test-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
REPORT_TYPE: "${{ needs.generate-test-variables.outputs.REPORT_TYPE }}"
ROLLING_RELEASE_commit_sha: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_commit_sha }}"
ROLLING_RELEASE_SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
secrets:
MM_LICENSE: "${{ secrets[format('MM_E2E_TEST_LICENSE_{0}_ENT', needs.generate-test-variables.outputs.server_uppercase)] }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"

34
.github/workflows/e2e-tests-ci-template.yml поставляемый
Просмотреть файл

@@ -55,6 +55,12 @@ on:
REPORT_TYPE:
type: string
required: false
ROLLING_RELEASE_commit_sha:
type: string
required: false
ROLLING_RELEASE_SERVER_IMAGE:
type: string
required: false
secrets:
MM_LICENSE:
required: false
@@ -238,7 +244,7 @@ jobs:
os: [ubuntu-latest-8-cores]
worker_index: ${{ fromJSON(needs.generate-build-variables.outputs.workers) }} # https://docs.github.com/en/actions/learn-github-actions/expressions#example-returning-a-json-object
runs-on: "${{ matrix.os }}"
timeout-minutes: 60
timeout-minutes: 90
needs:
- cypress-check
- playwright-check
@@ -263,6 +269,8 @@ jobs:
CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
CWS_URL: "${{ secrets.CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}"
ROLLING_RELEASE_COMMIT_SHA: "${{ inputs.ROLLING_RELEASE_commit_sha }}"
ROLLING_RELEASE_SERVER_IMAGE: "${{ inputs.ROLLING_RELEASE_SERVER_IMAGE }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -287,7 +295,30 @@ jobs:
cache-dependency-path: "e2e-tests/cypress/package-lock.json"
- name: ci/e2e-test
run: |
make cloud-init
if [ -n "$ROLLING_RELEASE_SERVER_IMAGE" ]; then
echo "RollingRelease: checking out E2E test cases from revision ${ROLLING_RELEASE_COMMIT_SHA}, for initial smoketest"
git checkout "${ROLLING_RELEASE_COMMIT_SHA}" -- cypress/ && git status
(
echo "RollingRelease: running initial smoketest against image $ROLLING_RELEASE_SERVER_IMAGE"
export SERVER_IMAGE="$ROLLING_RELEASE_SERVER_IMAGE"
export TEST_FILTER=""
export AUTOMATION_DASHBOARD_URL=""
make
)
echo "RollingRelease: asserting smoketest result has zero failures."
FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
if [ "$FAILURES" -ne "0" ]; then
echo "RollingRelease: initial smoketest for rolling release E2E run has nonzero ($FAILURES) failures. Aborting test run." >&2
exit 1
fi
rm -rfv cypress/results
echo "RollingRelease: reset the E2E test cases to the revision to test"
git reset --hard HEAD && git status
echo "RollingRelease: smoketest completed. Starting full E2E tests."
fi
make
make cloud-teardown
- name: ci/e2e-test-store-results
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
@@ -343,6 +374,7 @@ jobs:
TYPE: "${{ inputs.REPORT_TYPE }}"
SERVER: "${{ inputs.SERVER }}"
SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}"
BRANCH: "${{ inputs.BRANCH }}"
BUILD_ID: "${{ inputs.BUILD_ID }}"