From 2765d9299185eb95037cd4689962d062fb744baa Mon Sep 17 00:00:00 2001 From: Jesse Hallam Date: Thu, 15 Jun 2023 15:27:52 -0300 Subject: [PATCH] Gotestsum (#22826) * disable coverage This reduces runtime of the server test suite from ~30m to ~10m, and as far as I can see: we discarded the coverage output anyway. * allow morph 60s to migrate when running tests * scripts/test.sh: drop COVERMODE Stop generating coverage data when running unit tests. It's likely we'll want this data back at some point, but for now it's unused and removing simplifies invoking tests for developers. * scripts/test.sh: remove cleanup steps * scripts/test.sh: drop TESTS parameter * scripts/test.sh: drop TESTFLAGS parameter * switch to gotestsum --- .github/workflows/mmctl-test-template.yml | 56 +++++++++----- .github/workflows/server-ci-template.yml | 8 ++ .github/workflows/server-test-template.yml | 52 +++++++++---- server/.gitignore | 1 + server/Makefile | 86 ++++++++++++++++------ server/scripts/test-xprog.sh | 8 -- server/scripts/test.sh | 35 --------- 7 files changed, 146 insertions(+), 100 deletions(-) delete mode 100755 server/scripts/test-xprog.sh delete mode 100755 server/scripts/test.sh diff --git a/.github/workflows/mmctl-test-template.yml b/.github/workflows/mmctl-test-template.yml index 5afcc2fd8b..7e7a779f37 100644 --- a/.github/workflows/mmctl-test-template.yml +++ b/.github/workflows/mmctl-test-template.yml @@ -2,22 +2,29 @@ name: mmctl CI on: workflow_call: inputs: + name: + required: true + type: string datasource: required: true type: string drivername: required: true type: string + logsartifact: + required: true + type: string env: go-version: "1.19.5" jobs: - run-mmctl-tests: + test: + name: ${{ inputs.name }} runs-on: ubuntu-latest-8-cores env: COMPOSE_PROJECT_NAME: ghactions BUILD_IMAGE: mattermost/mattermost-build-server:20230118_golang-1.19.5 steps: - - name: Checkout mattermost-server + - name: Checkout mattermost project uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - name: Setup Go uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0 @@ -26,25 +33,38 @@ jobs: - name: Run docker compose run: | cd server/build - docker-compose --no-ansi run --rm start_dependencies - cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --no-ansi ps - cd .. - - name: Wait for docker compose - run: | - until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done; - docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" - docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" + docker-compose --ansi never run --rm start_dependencies + cat ../tests/test-data.ldif | docker-compose --ansi never exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker-compose --ansi never ps - name: Run mmctl Tests run: | - mkdir -p client/plugins - cd server/build docker run --net ghactions_mm-test \ --ulimit nofile=8096:8096 \ - --env-file=dotenv/test.env \ + --env-file=server/build/dotenv/test.env \ --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \ - -v ~/work/mattermost:/mattermost \ - -w /mattermost/mattermost/server \ + -v $(go env GOCACHE):/go/cache \ + -e GOCACHE=/go/cache \ + -v $PWD:/mattermost \ + -w /mattermost/server \ $BUILD_IMAGE \ - make test-mmctl-coverage BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID MM_SERVER_PATH=/mattermost/mattermost/server + make test-mmctl-coverage BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID + - name: Stop docker compose + run: | + cd server/build + docker-compose --ansi never stop + - name: Archive logs + if: ${{ always() }} + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ inputs.logsartifact }} + path: | + server/gotestsum.json + server/report.xml + - name: Publish Test Report + uses: mikepenz/action-junit-report@c0e4b81aaa0067314a2d0d06e19b512c9d8af4f5 # v3.7.7 + if: success() || failure() # always run even if the previous step fails + with: + report_paths: server/report.xml + check_name: ${{ inputs.name }} (Results) + job_name: ${{ inputs.name }} diff --git a/.github/workflows/server-ci-template.yml b/.github/workflows/server-ci-template.yml index 008ce8dd00..e3b6ec5d15 100644 --- a/.github/workflows/server-ci-template.yml +++ b/.github/workflows/server-ci-template.yml @@ -230,29 +230,37 @@ jobs: needs: check-mattermost-vet uses: ./.github/workflows/server-test-template.yml with: + name: Postgres with binary parameters datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes drivername: postgres + logsartifact: postgres-binary-server-test-logs test-postgres-normal: name: Postgres needs: check-mattermost-vet uses: ./.github/workflows/server-test-template.yml with: + name: Postgres datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10 drivername: postgres + logsartifact: postgres-server-test-logs test-mysql: name: MySQL needs: check-mattermost-vet uses: ./.github/workflows/server-test-template.yml with: + name: MySQL datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true&maxAllowedPacket=4194304 drivername: mysql + logsartifact: mysql-server-test-logs test-mmctl: name: Run mmctl tests needs: check-mattermost-vet uses: ./.github/workflows/mmctl-test-template.yml with: + name: mmctl datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10 drivername: postgres + logsartifact: mmctl-test-logs build-mattermost-server: name: Build mattermost server app runs-on: ubuntu-latest-8-cores diff --git a/.github/workflows/server-test-template.yml b/.github/workflows/server-test-template.yml index 1382f1b1f6..80b2c70e46 100644 --- a/.github/workflows/server-test-template.yml +++ b/.github/workflows/server-test-template.yml @@ -2,16 +2,23 @@ name: Server Test Template on: workflow_call: inputs: + name: + required: true + type: string datasource: required: true type: string drivername: required: true type: string + logsartifact: + required: true + type: string env: go-version: "1.19.5" jobs: test: + name: ${{ inputs.name }} runs-on: ubuntu-latest-8-cores env: COMPOSE_PROJECT_NAME: ghactions @@ -27,31 +34,44 @@ jobs: - name: Run docker compose run: | cd server/build - docker-compose --no-ansi run --rm start_dependencies - cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; - docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; - docker-compose --no-ansi ps - cd .. - - name: Wait for docker compose - run: | - until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done; - docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" - docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" + docker-compose --ansi never run --rm start_dependencies + cat ../tests/test-data.ldif | docker-compose --ansi never exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; + docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test'; + docker-compose --ansi never ps - name: Run Tests run: | if [[ ${{ github.ref_name }} == 'master' ]]; then export RACE_MODE="-race" fi - mkdir -p client/plugins - cd server/build docker run --net ghactions_mm-test \ --ulimit nofile=8096:8096 \ - --env-file=dotenv/test.env \ + --env-file=server/build/dotenv/test.env \ --env MM_SQLSETTINGS_DRIVERNAME="${{ inputs.drivername }}" \ --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \ --env TEST_DATABASE_MYSQL_DSN="${{ inputs.datasource }}" \ --env TEST_DATABASE_POSTGRESQL_DSN="${{ inputs.datasource }}" \ - -v ~/work/mattermost:/mattermost \ - -w /mattermost/mattermost/server \ + -v $(go env GOCACHE):/go/cache \ + -e GOCACHE=/go/cache \ + -v $PWD:/mattermost \ + -w /mattermost/server \ $BUILD_IMAGE \ - make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID TESTFLAGS= TESTFLAGSEE= + make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID + - name: Stop docker compose + run: | + cd server/build + docker-compose --ansi never stop + - name: Archive logs + if: ${{ always() }} + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ inputs.logsartifact }} + path: | + server/gotestsum.json + server/report.xml + - name: Publish Test Report + uses: mikepenz/action-junit-report@c0e4b81aaa0067314a2d0d06e19b512c9d8af4f5 # v3.7.7 + if: success() || failure() # always run even if the previous step fails + with: + report_paths: server/report.xml + check_name: ${{ inputs.name }} (Results) + job_name: ${{ inputs.name }} diff --git a/server/.gitignore b/server/.gitignore index 396dd86d20..f965b70690 100644 --- a/server/.gitignore +++ b/server/.gitignore @@ -113,6 +113,7 @@ cprofile.out *.test webapp/coverage /report.xml +/gotestsum.json .agignore .ctags diff --git a/server/Makefile b/server/Makefile index 229a39895a..cc1efc344d 100644 --- a/server/Makefile +++ b/server/Makefile @@ -54,10 +54,17 @@ ifeq ($(BUILD_NUMBER),dev) export MM_FEATUREFLAGS_GRAPHQL = true endif +# Ensure developer invocation and tests are anchored. +MM_SERVER_PATH ?= $(ROOT) + +# Go test sum configuration +GOTESTSUM_FORMAT ?= testname +GOTESTSUM_JUNITFILE ?= report.xml +GOTESTSUM_JSONFILE ?= gotestsum.json + # mmctl -MM_SERVER_PATH ?= $(PWD) MMCTL_BUILD_TAGS = -MMCTL_TESTFLAGS = -timeout 30m -race -v +MMCTL_TESTFLAGS = -timeout 30m -race MMCTL_PKG = github.com/mattermost/mattermost/server/v8/cmd/mmctl/commands LDFLAGS += -X "$(MMCTL_PKG).gitCommit=$(BUILD_HASH)" LDFLAGS += -X "$(MMCTL_PKG).gitTreeState=$(GIT_TREESTATE)" @@ -143,9 +150,6 @@ DIST_PATH_OSX_AMD64=$(DIST_ROOT)/osx_amd64/mattermost DIST_PATH_OSX_ARM64=$(DIST_ROOT)/osx_arm64/mattermost DIST_PATH_WIN=$(DIST_ROOT)/windows/mattermost -# Tests -TESTS=. - # Packages lists TE_PACKAGES=$(shell $(GO) list ./... | grep -vE 'server/v8/cmd/mmctl') SUITE_PACKAGES=$(shell $(GO) list ./...| grep -vE 'server/v8/cmd/mmctl') @@ -431,10 +435,10 @@ endif check-style: plugin-checker vet golangci-lint ## Runs style/lint checks -go-junit-report: - $(GO) install github.com/jstemmer/go-junit-report@v1.0.0 +gotestsum: + $(GO) install gotest.tools/gotestsum@v1.7.0 -test-compile: ## Compile tests. +test-compile: gotestsum ## Compile tests. @echo COMPILE TESTS for package in $(SUITE_PACKAGES) $(EE_PACKAGES); do \ @@ -451,15 +455,23 @@ modules-tidy: mv channels/imports/imports.go.orig channels/imports/imports.go; \ fi; -test-server-pre: check-prereqs-enterprise start-docker go-junit-report ## Runs tests. +test-server-pre: check-prereqs-enterprise start-docker gotestsum ## Runs tests. ifeq ($(BUILD_ENTERPRISE_READY),true) @echo Running all tests else @echo Running only TE tests endif +test-server-race: export MM_SERVER_PATH := $(MM_SERVER_PATH) +test-server-race: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) +test-server-race: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) +test-server-race: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE) test-server-race: test-server-pre - ./scripts/test.sh "$(GO)" "-race $(GOFLAGS)" "$(TE_PACKAGES) $(EE_PACKAGES)" "$(TESTS)" "$(TESTFLAGS)" "$(GOBIN)" "90m" +ifeq ($(IS_CI),true) + GOMAXPROCS=4 $(GOBIN)/gotestsum --packages="$(TE_PACKAGES) $(EE_PACKAGES)" -- -race $(GOFLAGS) -timeout=90m +else + $(GOBIN)/gotestsum --packages="$(TE_PACKAGES) $(EE_PACKAGES)" -- -race $(GOFLAGS) -timeout=90m +endif ifneq ($(IS_CI),true) ifneq ($(MM_NO_DOCKER),true) ifneq ($(TEMP_DOCKER_SERVICES),) @@ -469,8 +481,12 @@ ifneq ($(IS_CI),true) endif endif +test-server: export MM_SERVER_PATH := $(MM_SERVER_PATH) +test-server: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) +test-server: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) +test-server: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE) test-server: test-server-pre - ./scripts/test.sh "$(GO)" "$(GOFLAGS)" "$(SUITE_PACKAGES) $(EE_PACKAGES)" "$(TESTS)" "$(TESTFLAGS)" "$(GOBIN)" "90m" + $(GOBIN)/gotestsum --packages="$(SUITE_PACKAGES) $(EE_PACKAGES)" -- $(GOFLAGS) -timeout=90m ifneq ($(IS_CI),true) ifneq ($(MM_NO_DOCKER),true) ifneq ($(TEMP_DOCKER_SERVICES),) @@ -480,17 +496,25 @@ ifneq ($(IS_CI),true) endif endif -test-server-ee: check-prereqs-enterprise start-docker go-junit-report ## Runs EE tests. +test-server-ee: export MM_SERVER_PATH := $(MM_SERVER_PATH) +test-server-ee: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) +test-server-ee: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) +test-server-ee: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE) +test-server-ee: check-prereqs-enterprise start-docker gotestsum ## Runs EE tests. @echo Running only EE tests - ./scripts/test.sh "$(GO)" "$(GOFLAGS)" "$(EE_PACKAGES)" "$(TESTS)" "$(TESTFLAGS)" "$(GOBIN)" "20m" + $(GOBIN)/gotestsum --packages="$(EE_PACKAGES)" -- $(GOFLAGS) -timeout=20m +test-server-quick: export MM_SERVER_PATH := $(MM_SERVER_PATH) +test-server-quick: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) +test-server-quick: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) +test-server-quick: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE) test-server-quick: check-prereqs-enterprise ## Runs only quick tests. ifeq ($(BUILD_ENTERPRISE_READY),true) @echo Running all tests - $(GO) test $(GOFLAGS) -short $(SUITE_PACKAGES) $(EE_PACKAGES) + $(GOBIN)/gotestsum --packages="$(SUITE_PACKAGES) $(EE_PACKAGES)" -- $(GOFLAGS) -short else @echo Running only TE tests - $(GO) test $(GOFLAGS) -short $(SUITE_PACKAGES) + $(GOBIN)/gotestsum --packages="$(SUITE_PACKAGES)" -- $(GOFLAGS) -short endif internal-test-web-client: ## Runs web client tests. @@ -530,21 +554,37 @@ inject-test-data: # add test data to the local instance. @echo Login with a regular account username=user-1 password=SampleUs@r-1 @echo ======================================================================== -test-mmctl-unit: +test-mmctl-unit: export MM_SERVER_PATH := $(MM_SERVER_PATH) +test-mmctl-unit: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) +test-mmctl-unit: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) +test-mmctl-unit: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE) +test-mmctl-unit: gotestsum @echo Running mmctl unit tests - $(GO) test $(MMCTL_TESTFLAGS) -tags 'unit $(MMCTL_BUILD_TAGS)' $(MMCTL_PACKAGES) + $(GOBIN)/gotestsum --packages="$(MMCTL_PACKAGES)" -- -tags 'unit $(MMCTL_BUILD_TAGS)' $(MMCTL_TESTFLAGS) -test-mmctl-e2e: start-docker +test-mmctl-e2e: export MM_SERVER_PATH := $(MM_SERVER_PATH) +test-mmctl-e2e: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) +test-mmctl-e2e: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) +test-mmctl-e2e: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE) +test-mmctl-e2e: gotestsum start-docker @echo Running mmctl e2e tests - MM_SERVER_PATH=$(MM_SERVER_PATH) $(GO) test $(MMCTL_TESTFLAGS) -tags 'e2e $(MMCTL_BUILD_TAGS)' $(MMCTL_PACKAGES) + $(GOBIN)/gotestsum --packages="$(MMCTL_PACKAGES)" -- -tags 'e2e $(MMCTL_BUILD_TAGS)' $(MMCTL_TESTFLAGS) -test-mmctl: start-docker +test-mmctl: export MM_SERVER_PATH := $(MM_SERVER_PATH) +test-mmctl-e2e: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) +test-mmctl-e2e: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) +test-mmctl-e2e: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE) +test-mmctl: gotestsum start-docker @echo Running all mmctl tests - MM_SERVER_PATH=$(MM_SERVER_PATH) $(GO) test $(MMCTL_TESTFLAGS) -tags 'unit e2e $(MMCTL_BUILD_TAGS)' $(MMCTL_PACKAGES) + $(GOBIN)/gotestsum --packages="$(MMCTL_PACKAGES)" -- -tags 'unit e2e $(MMCTL_BUILD_TAGS)' $(MMCTL_TESTFLAGS) -test-mmctl-coverage: start-docker +test-mmctl-coverage: export MM_SERVER_PATH := $(MM_SERVER_PATH) +test-mmctl-e2e: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT) +test-mmctl-e2e: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE) +test-mmctl-e2e: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE) +test-mmctl-coverage: gotestsum start-docker @echo Running all mmctl tests with coverage - MM_SERVER_PATH=$(MM_SERVER_PATH) $(GO) test $(MMCTL_TESTFLAGS) -tags 'unit e2e $(MMCTL_BUILD_TAGS)' -coverprofile=mmctlcover.out $(MMCTL_PACKAGES) + $(GOBIN)/gotestsum --packages="$(MMCTL_PACKAGES)" -- -tags 'unit e2e $(MMCTL_BUILD_TAGS)' -coverprofile=mmctlcover.out $(MMCTL_TESTFLAGS) $(GO) tool cover -html=mmctlcover.out validate-go-version: ## Validates the installed version of go against Mattermost's minimum requirement. diff --git a/server/scripts/test-xprog.sh b/server/scripts/test-xprog.sh deleted file mode 100755 index b9a89a7be2..0000000000 --- a/server/scripts/test-xprog.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -set -e -[[ $1 =~ (github.com.*)/_test ]] && \ - echo Testing ${BASH_REMATCH[1]} -if [[ $1 == *"/enterprise/"* ]]; then - cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" -fi -"$@" diff --git a/server/scripts/test.sh b/server/scripts/test.sh deleted file mode 100755 index c2f5b5b2d1..0000000000 --- a/server/scripts/test.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -set -o pipefail - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -GO=$1 -GOFLAGS=$2 -PACKAGES=$3 -TESTS=$4 -TESTFLAGS=$5 -GOBIN=$6 -TIMEOUT=$7 - -PACKAGES_COMMA=$(echo $PACKAGES | tr ' ' ',') -export MM_SERVER_PATH=$PWD - -echo "Packages to test: $PACKAGES" -echo "GOFLAGS: $GOFLAGS" - -if [[ $GOFLAGS == "-race " && $IS_CI == "true" ]] ; -then - export GOMAXPROCS=4 -fi - -find . -type d -name data -not -path './data' | xargs rm -rf - -$GO test $GOFLAGS -run=$TESTS $TESTFLAGS -v -timeout=$TIMEOUT -exec $DIR/test-xprog.sh $PACKAGES 2>&1 > >( tee output ) -EXIT_STATUS=$? - -cat output | $GOBIN/go-junit-report > report.xml -rm output -rm -f config/*.crt -rm -f config/*.key - -exit $EXIT_STATUS