Gotestsum (#22826)

* disable coverage This reduces runtime of the server test suite from ~30m to ~10m, and as far as I can see: we discarded the coverage output anyway. * allow morph 60s to migrate when running tests * scripts/test.sh: drop COVERMODE Stop generating coverage data when running unit tests. It's likely we'll want this data back at some point, but for now it's unused and removing simplifies invoking tests for developers. * scripts/test.sh: remove cleanup steps * scripts/test.sh: drop TESTS parameter * scripts/test.sh: drop TESTFLAGS parameter * switch to gotestsum
2023-06-15 15:27:52 -03:00
--- a/.github/workflows/mmctl-test-template.yml
+++ b/.github/workflows/mmctl-test-template.yml
@@ -2,22 +2,29 @@ name: mmctl CI
 on:
  workflow_call:
    inputs:
+      name:
+        required: true
+        type: string
      datasource:
        required: true
        type: string
      drivername:
        required: true
        type: string
+      logsartifact:
+        required: true
+        type: string
 env:
  go-version: "1.19.5"
 jobs:
-  run-mmctl-tests:
+  test:
+    name: ${{ inputs.name }}
    runs-on: ubuntu-latest-8-cores
    env:
      COMPOSE_PROJECT_NAME: ghactions
      BUILD_IMAGE: mattermost/mattermost-build-server:20230118_golang-1.19.5
    steps:
-      - name: Checkout mattermost-server
+      - name: Checkout mattermost project
        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
      - name: Setup Go
        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v4.0.0
@@ -26,25 +33,38 @@ jobs:
      - name: Run docker compose
        run: |
          cd server/build
-          docker-compose --no-ansi run --rm start_dependencies
-          cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
-          docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
-          docker-compose --no-ansi ps
-          cd ..
-      - name: Wait for docker compose
-        run: |
-          until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done;
-          docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
-          docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
+          docker-compose --ansi never run --rm start_dependencies
+          cat ../tests/test-data.ldif | docker-compose --ansi never exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+          docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test';
+          docker-compose --ansi never ps
      - name: Run mmctl Tests
        run: |
-          mkdir -p client/plugins
-          cd server/build
          docker run --net ghactions_mm-test \
            --ulimit nofile=8096:8096 \
-            --env-file=dotenv/test.env \
+            --env-file=server/build/dotenv/test.env \
            --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \
-            -v ~/work/mattermost:/mattermost \
-            -w /mattermost/mattermost/server \
+            -v $(go env GOCACHE):/go/cache \
+            -e GOCACHE=/go/cache \
+            -v $PWD:/mattermost \
+            -w /mattermost/server \
            $BUILD_IMAGE \
-            make test-mmctl-coverage BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID MM_SERVER_PATH=/mattermost/mattermost/server
+            make test-mmctl-coverage BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID
+      - name: Stop docker compose
+        run: |
+          cd server/build
+          docker-compose --ansi never stop
+      - name: Archive logs
+        if: ${{ always() }}
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ inputs.logsartifact }}
+          path: |
+            server/gotestsum.json
+            server/report.xml
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@c0e4b81aaa0067314a2d0d06e19b512c9d8af4f5 # v3.7.7
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: server/report.xml
+          check_name: ${{ inputs.name }} (Results)
+          job_name: ${{ inputs.name }}
--- a/.github/workflows/server-ci-template.yml
+++ b/.github/workflows/server-ci-template.yml
@@ -230,29 +230,37 @@ jobs:
    needs: check-mattermost-vet
    uses: ./.github/workflows/server-test-template.yml
    with:
+      name: Postgres with binary parameters
      datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes
      drivername: postgres
+      logsartifact: postgres-binary-server-test-logs
  test-postgres-normal:
    name: Postgres
    needs: check-mattermost-vet
    uses: ./.github/workflows/server-test-template.yml
    with:
+      name: Postgres
      datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
      drivername: postgres
+      logsartifact: postgres-server-test-logs
  test-mysql:
    name: MySQL
    needs: check-mattermost-vet
    uses: ./.github/workflows/server-test-template.yml
    with:
+      name: MySQL
      datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true&maxAllowedPacket=4194304
      drivername: mysql
+      logsartifact: mysql-server-test-logs
  test-mmctl:
    name: Run mmctl tests
    needs: check-mattermost-vet
    uses: ./.github/workflows/mmctl-test-template.yml
    with:
+      name: mmctl
      datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
      drivername: postgres
+      logsartifact: mmctl-test-logs
  build-mattermost-server:
    name: Build mattermost server app
    runs-on: ubuntu-latest-8-cores
--- a/.github/workflows/server-test-template.yml
+++ b/.github/workflows/server-test-template.yml
@@ -2,16 +2,23 @@ name: Server Test Template
 on:
  workflow_call:
    inputs:
+      name:
+        required: true
+        type: string
      datasource:
        required: true
        type: string
      drivername:
        required: true
        type: string
+      logsartifact:
+        required: true
+        type: string
 env:
  go-version: "1.19.5"
 jobs:
  test:
+    name: ${{ inputs.name }}
    runs-on: ubuntu-latest-8-cores
    env:
      COMPOSE_PROJECT_NAME: ghactions
@@ -27,31 +34,44 @@ jobs:
      - name: Run docker compose
        run: |
          cd server/build
-          docker-compose --no-ansi run --rm start_dependencies
-          cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
-          docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
-          docker-compose --no-ansi ps
-          cd ..
-      - name: Wait for docker compose
-        run: |
-          until docker network inspect ghactions_mm-test; do echo "Waiting for Docker Compose Network..."; sleep 1; done;
-          docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
-          docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
+          docker-compose --ansi never run --rm start_dependencies
+          cat ../tests/test-data.ldif | docker-compose --ansi never exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
+          docker-compose --ansi never exec -T minio sh -c 'mkdir -p /data/mattermost-test';
+          docker-compose --ansi never ps
      - name: Run Tests
        run: |
          if [[ ${{ github.ref_name }} == 'master' ]]; then
            export RACE_MODE="-race"
          fi
-          mkdir -p client/plugins
-          cd server/build
          docker run --net ghactions_mm-test \
            --ulimit nofile=8096:8096 \
-            --env-file=dotenv/test.env \
+            --env-file=server/build/dotenv/test.env \
            --env MM_SQLSETTINGS_DRIVERNAME="${{ inputs.drivername }}" \
            --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \
            --env TEST_DATABASE_MYSQL_DSN="${{ inputs.datasource }}" \
            --env TEST_DATABASE_POSTGRESQL_DSN="${{ inputs.datasource }}" \
-            -v ~/work/mattermost:/mattermost \
-            -w /mattermost/mattermost/server \
+            -v $(go env GOCACHE):/go/cache \
+            -e GOCACHE=/go/cache \
+            -v $PWD:/mattermost \
+            -w /mattermost/server \
            $BUILD_IMAGE \
-            make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID TESTFLAGS= TESTFLAGSEE=
+            make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID
+      - name: Stop docker compose
+        run: |
+          cd server/build
+          docker-compose --ansi never stop
+      - name: Archive logs
+        if: ${{ always() }}
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ inputs.logsartifact }}
+          path: |
+            server/gotestsum.json
+            server/report.xml
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@c0e4b81aaa0067314a2d0d06e19b512c9d8af4f5 # v3.7.7
+        if: success() || failure() # always run even if the previous step fails
+        with:
+          report_paths: server/report.xml
+          check_name: ${{ inputs.name }} (Results)
+          job_name: ${{ inputs.name }}
--- a/server/.gitignore
+++ b/server/.gitignore
@@ -113,6 +113,7 @@ cprofile.out
 *.test
 webapp/coverage
 /report.xml
+/gotestsum.json

 .agignore
 .ctags
--- a/server/Makefile
+++ b/server/Makefile
@@ -54,10 +54,17 @@ ifeq ($(BUILD_NUMBER),dev)
 	export MM_FEATUREFLAGS_GRAPHQL = true
 endif

+# Ensure developer invocation and tests are anchored.
+MM_SERVER_PATH ?= $(ROOT)
+
+# Go test sum configuration
+GOTESTSUM_FORMAT ?= testname
+GOTESTSUM_JUNITFILE ?= report.xml
+GOTESTSUM_JSONFILE ?= gotestsum.json
+
 # mmctl
-MM_SERVER_PATH ?= $(PWD)
 MMCTL_BUILD_TAGS =
-MMCTL_TESTFLAGS = -timeout 30m -race -v
+MMCTL_TESTFLAGS = -timeout 30m -race
 MMCTL_PKG = github.com/mattermost/mattermost/server/v8/cmd/mmctl/commands
 LDFLAGS += -X "$(MMCTL_PKG).gitCommit=$(BUILD_HASH)"
 LDFLAGS += -X "$(MMCTL_PKG).gitTreeState=$(GIT_TREESTATE)"
@@ -143,9 +150,6 @@ DIST_PATH_OSX_AMD64=$(DIST_ROOT)/osx_amd64/mattermost
 DIST_PATH_OSX_ARM64=$(DIST_ROOT)/osx_arm64/mattermost
 DIST_PATH_WIN=$(DIST_ROOT)/windows/mattermost

-# Tests
-TESTS=.
-
 # Packages lists
 TE_PACKAGES=$(shell $(GO) list ./... | grep -vE 'server/v8/cmd/mmctl')
 SUITE_PACKAGES=$(shell $(GO) list ./...| grep -vE 'server/v8/cmd/mmctl')
@@ -431,10 +435,10 @@ endif
 check-style: plugin-checker vet golangci-lint ## Runs style/lint checks


-go-junit-report:
-	$(GO) install github.com/jstemmer/go-junit-report@v1.0.0
+gotestsum:
+	$(GO) install gotest.tools/gotestsum@v1.7.0

-test-compile: ## Compile tests.
+test-compile: gotestsum ## Compile tests.
 	@echo COMPILE TESTS

 	for package in $(SUITE_PACKAGES) $(EE_PACKAGES); do \
@@ -451,15 +455,23 @@ modules-tidy:
 		mv channels/imports/imports.go.orig channels/imports/imports.go; \
 	fi;

-test-server-pre: check-prereqs-enterprise start-docker go-junit-report ## Runs tests.
+test-server-pre: check-prereqs-enterprise start-docker gotestsum ## Runs tests.
 ifeq ($(BUILD_ENTERPRISE_READY),true)
 	@echo Running all tests
 else
 	@echo Running only TE tests
 endif

+test-server-race: export MM_SERVER_PATH := $(MM_SERVER_PATH)
+test-server-race: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-server-race: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-server-race: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
 test-server-race: test-server-pre
-	./scripts/test.sh "$(GO)" "-race $(GOFLAGS)" "$(TE_PACKAGES) $(EE_PACKAGES)" "$(TESTS)" "$(TESTFLAGS)" "$(GOBIN)" "90m"
+ifeq ($(IS_CI),true)
+	GOMAXPROCS=4 $(GOBIN)/gotestsum --packages="$(TE_PACKAGES) $(EE_PACKAGES)" -- -race $(GOFLAGS) -timeout=90m
+else
+	$(GOBIN)/gotestsum --packages="$(TE_PACKAGES) $(EE_PACKAGES)" -- -race $(GOFLAGS) -timeout=90m
+endif
 ifneq ($(IS_CI),true)
  ifneq ($(MM_NO_DOCKER),true)
    ifneq ($(TEMP_DOCKER_SERVICES),)
@@ -469,8 +481,12 @@ ifneq ($(IS_CI),true)
  endif
 endif

+test-server: export MM_SERVER_PATH := $(MM_SERVER_PATH)
+test-server: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-server: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-server: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
 test-server: test-server-pre
-	./scripts/test.sh "$(GO)" "$(GOFLAGS)" "$(SUITE_PACKAGES) $(EE_PACKAGES)" "$(TESTS)" "$(TESTFLAGS)" "$(GOBIN)" "90m"
+	$(GOBIN)/gotestsum --packages="$(SUITE_PACKAGES) $(EE_PACKAGES)" -- $(GOFLAGS) -timeout=90m
 ifneq ($(IS_CI),true)
  ifneq ($(MM_NO_DOCKER),true)
    ifneq ($(TEMP_DOCKER_SERVICES),)
@@ -480,17 +496,25 @@ ifneq ($(IS_CI),true)
  endif
 endif

-test-server-ee: check-prereqs-enterprise start-docker go-junit-report ## Runs EE tests.
+test-server-ee: export MM_SERVER_PATH := $(MM_SERVER_PATH)
+test-server-ee: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-server-ee: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-server-ee: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
+test-server-ee: check-prereqs-enterprise start-docker gotestsum ## Runs EE tests.
 	@echo Running only EE tests
-	./scripts/test.sh "$(GO)" "$(GOFLAGS)" "$(EE_PACKAGES)" "$(TESTS)" "$(TESTFLAGS)" "$(GOBIN)" "20m"
+	$(GOBIN)/gotestsum --packages="$(EE_PACKAGES)" -- $(GOFLAGS) -timeout=20m

+test-server-quick: export MM_SERVER_PATH := $(MM_SERVER_PATH)
+test-server-quick: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-server-quick: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-server-quick: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
 test-server-quick: check-prereqs-enterprise ## Runs only quick tests.
 ifeq ($(BUILD_ENTERPRISE_READY),true)
 	@echo Running all tests
-	$(GO) test $(GOFLAGS) -short $(SUITE_PACKAGES) $(EE_PACKAGES)
+	$(GOBIN)/gotestsum --packages="$(SUITE_PACKAGES) $(EE_PACKAGES)" -- $(GOFLAGS) -short
 else
 	@echo Running only TE tests
-	$(GO) test $(GOFLAGS) -short $(SUITE_PACKAGES)
+	$(GOBIN)/gotestsum --packages="$(SUITE_PACKAGES)" -- $(GOFLAGS) -short
 endif

 internal-test-web-client: ## Runs web client tests.
@@ -530,21 +554,37 @@ inject-test-data: # add test data to the local instance.
 	@echo Login with a regular account username=user-1 password=SampleUs@r-1
 	@echo ========================================================================

-test-mmctl-unit:
+test-mmctl-unit: export MM_SERVER_PATH := $(MM_SERVER_PATH)
+test-mmctl-unit: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-mmctl-unit: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-mmctl-unit: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
+test-mmctl-unit: gotestsum
 	@echo Running mmctl unit tests
-	$(GO) test $(MMCTL_TESTFLAGS) -tags 'unit $(MMCTL_BUILD_TAGS)' $(MMCTL_PACKAGES)
+	$(GOBIN)/gotestsum --packages="$(MMCTL_PACKAGES)" -- -tags 'unit $(MMCTL_BUILD_TAGS)' $(MMCTL_TESTFLAGS)

-test-mmctl-e2e: start-docker
+test-mmctl-e2e: export MM_SERVER_PATH := $(MM_SERVER_PATH)
+test-mmctl-e2e: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-mmctl-e2e: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-mmctl-e2e: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
+test-mmctl-e2e: gotestsum start-docker
 	@echo Running mmctl e2e tests
-	MM_SERVER_PATH=$(MM_SERVER_PATH) $(GO) test $(MMCTL_TESTFLAGS) -tags 'e2e $(MMCTL_BUILD_TAGS)' $(MMCTL_PACKAGES)
+	$(GOBIN)/gotestsum --packages="$(MMCTL_PACKAGES)" -- -tags 'e2e $(MMCTL_BUILD_TAGS)' $(MMCTL_TESTFLAGS)

-test-mmctl: start-docker
+test-mmctl: export MM_SERVER_PATH := $(MM_SERVER_PATH)
+test-mmctl-e2e: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-mmctl-e2e: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-mmctl-e2e: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
+test-mmctl: gotestsum start-docker
 	@echo Running all mmctl tests
-	MM_SERVER_PATH=$(MM_SERVER_PATH) $(GO) test $(MMCTL_TESTFLAGS) -tags 'unit e2e $(MMCTL_BUILD_TAGS)' $(MMCTL_PACKAGES)
+	$(GOBIN)/gotestsum --packages="$(MMCTL_PACKAGES)" -- -tags 'unit e2e $(MMCTL_BUILD_TAGS)' $(MMCTL_TESTFLAGS)

-test-mmctl-coverage: start-docker
+test-mmctl-coverage: export MM_SERVER_PATH := $(MM_SERVER_PATH)
+test-mmctl-e2e: export GOTESTSUM_FORMAT := $(GOTESTSUM_FORMAT)
+test-mmctl-e2e: export GOTESTSUM_JUNITFILE := $(GOTESTSUM_JUNITFILE)
+test-mmctl-e2e: export GOTESTSUM_JSONFILE := $(GOTESTSUM_JSONFILE)
+test-mmctl-coverage: gotestsum start-docker
 	@echo Running all mmctl tests with coverage
-	MM_SERVER_PATH=$(MM_SERVER_PATH) $(GO) test $(MMCTL_TESTFLAGS) -tags 'unit e2e $(MMCTL_BUILD_TAGS)' -coverprofile=mmctlcover.out $(MMCTL_PACKAGES)
+	$(GOBIN)/gotestsum --packages="$(MMCTL_PACKAGES)" -- -tags 'unit e2e $(MMCTL_BUILD_TAGS)' -coverprofile=mmctlcover.out $(MMCTL_TESTFLAGS)
 	$(GO) tool cover -html=mmctlcover.out

 validate-go-version: ## Validates the installed version of go against Mattermost's minimum requirement.
--- a/server/scripts/test-xprog.sh
+++ b/server/scripts/test-xprog.sh
@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -e
-[[ $1 =~ (github.com.*)/_test ]] && \
-    echo Testing ${BASH_REMATCH[1]}
-if [[ $1 == *"/enterprise/"* ]]; then
-    cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")"
-fi
-"$@"
--- a/server/scripts/test.sh
+++ b/server/scripts/test.sh
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-set -o pipefail
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-
-GO=$1
-GOFLAGS=$2
-PACKAGES=$3
-TESTS=$4
-TESTFLAGS=$5
-GOBIN=$6
-TIMEOUT=$7
-
-PACKAGES_COMMA=$(echo $PACKAGES | tr ' ' ',')
-export MM_SERVER_PATH=$PWD
-
-echo "Packages to test: $PACKAGES"
-echo "GOFLAGS: $GOFLAGS"
-
-if [[ $GOFLAGS == "-race " && $IS_CI == "true" ]] ;
-then
-	export GOMAXPROCS=4
-fi
-
-find . -type d -name data -not -path './data' | xargs rm -rf
-
-$GO test $GOFLAGS -run=$TESTS $TESTFLAGS -v -timeout=$TIMEOUT -exec $DIR/test-xprog.sh $PACKAGES 2>&1 > >( tee output )
-EXIT_STATUS=$?
-
-cat output | $GOBIN/go-junit-report > report.xml
-rm output
-rm -f config/*.crt
-rm -f config/*.key
-
-exit $EXIT_STATUS