[MM-62408] Server Code Coverage with Fully Parallel Tests (#30078)

* TestPool

* Store infra

* Store tests updates

* Bump maximum concurrent postgres connections

* More infra

* channels/jobs

* channels/app

* channels/api4

* Protect i18n from concurrent access

* Replace some use of os.Setenv

* Remove debug

* Lint fixes

* Fix more linting

* Fix test

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix merge

* [MM-62408] Add CI job to generate test coverage (#30284)

* Add CI job to generate test coverage

* Remove use of Setenv in drafts tests

* Fix flaky TestWebHubCloseConnOnDBFail

* Fix more Setenv usage

* Fix more potential flakyness

* Remove parallelism from flaky test

* Remove conflicting env var

* Fix

* Disable parallelism

* Test atomic covermode

* Disable parallelism

* Enable parallelism

* Add upload coverage step

* Fix codecov.yml

* Add codecov.yml

* Remove redundant workspace field

* Add Parallel() util methods and refactor

* Fix formatting

* More formatting fixes

* Fix reporting

.github/codecov.yml

@@ -0,0 +1,13 @@
+comment:
+  layout: "diff, flags, files"
+  behavior: default
+  require_changes: false  # if true: only post the comment if coverage changes
+coverage:
+  status:
+    changes: false
+    patch: false
+    project:
+      default:
+        threshold: 1.0
+ignore:
+  - ^store/storetest.*

.github/workflows/server-ci-template.yml

@@ -264,6 +264,18 @@ jobs:
       datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4&multiStatements=true&maxAllowedPacket=4194304
       drivername: mysql
       logsartifact: mysql-server-test-logs
+  test-coverage:
+    name: Generate Test Coverage
+    needs: check-mattermost-vet
+    uses: ./.github/workflows/server-test-template.yml
+    secrets: inherit
+    with:
+      name: Generate Test Coverage
+      datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
+      drivername: postgres
+      logsartifact: coverage-server-test-logs
+      fullyparallel: true
+      enablecoverage: true
   test-mmctl:
     name: Run mmctl tests
     needs: check-mattermost-vet

.github/workflows/server-test-template.yml

@@ -14,11 +14,20 @@ on:
       logsartifact:
         required: true
         type: string
+      fullyparallel:
+        required: false
+        type: boolean
+        default: false
+      enablecoverage:
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   test:
     name: ${{ inputs.name }}
     runs-on: ubuntu-latest-8-cores
+    continue-on-error: ${{ inputs.fullyparallel }} # Used to avoid blocking PRs in case of flakiness
     env:
       COMPOSE_PROJECT_NAME: ghactions
     steps:
@@ -60,12 +69,20 @@ jobs:
             --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \
             --env TEST_DATABASE_MYSQL_DSN="${{ inputs.datasource }}" \
             --env TEST_DATABASE_POSTGRESQL_DSN="${{ inputs.datasource }}" \
+            --env ENABLE_FULLY_PARALLEL_TESTS="${{ inputs.fullyparallel }}" \
+            --env ENABLE_COVERAGE="${{ inputs.enablecoverage }}" \
             -v $(go env GOCACHE):/go/cache \
             -e GOCACHE=/go/cache \
             -v $PWD:/mattermost \
             -w /mattermost/server \
             $BUILD_IMAGE \
             make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          disable_search: true
+          files: server/cover.out
       - name: Stop docker compose
         run: |
           cd server/build
@@ -78,5 +95,6 @@ jobs:
           path: |
             server/gotestsum.json
             server/report.xml
+            server/cover.out
             server/test-name
             server/pr-number