MM-52712: Prevent CI cancellation in master (round 2) (#23293)

We discovered that cancel-in-progress only controls
in-progress jobs. Which means that pending jobs will _always_
be cancelled regardless. There is an open discussion:
https://github.com/orgs/community/discussions/5435
which was closed saying this is how the feature is designed.

We try to work around this by refactoring into separate reusable
workflows and having concurrency only for PR workflows.

```release-note
NONE
```

Co-authored-by: Mattermost Build <build@mattermost.com>

.github/workflows/README.md

@@ -0,0 +1,29 @@
+### Background
+
+This document aims to explain the bunch of server and webapp yaml files and their functionality.
+
+The context behind this complexity is that we want new pushes to PR branches to cancel older in-progress and pending CI runs, _but_ we don't want that to happen in master branch. Unfortunately, there is no config knob to control pending workflows and if you set a concurrency group, then pending workflows will _always_ be canceled. Refer to https://github.com/orgs/community/discussions/5435 for discussion.
+
+Therefore, we have a template yaml file which is actually the main CI code. That is then imported by `{server|webapp}-ci-master.yml` and `{server|webapp}-ci-pr.yml`. The `-master.yml` files don't have any concurrency limits, but `-pr.yml` files do.
+
+### Folder structure
+
+server-ci-pr
+|
+---server-ci-template
+	|
+	---server-test-template (common code for postgres and mysql tests)
+
+server-ci-master
+|
+---server-ci-template
+	|
+	---server-test-template (common code for postgres and mysql tests)
+
+webapp-ci-pr
+|
+---webapp-ci-template
+
+webapp-ci-master
+|
+---webapp-ci-template

.github/workflows/artifacts.yml

@@ -1,7 +1,7 @@
 name: Artifacts generation and upload
 on:
   workflow_run:
-    workflows: ["Server CI"]
+    workflows: ["Server CI Master", "Server CI PR"]
     types:
       - completed
 

.github/workflows/codeql-analysis.yml

@@ -1,9 +1,5 @@
 name: "CodeQL"
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
-
 on:
   pull_request:
     # The branches below must be a subset of the branches above

.github/workflows/e2e-tests-ci.yml

@@ -5,9 +5,7 @@ on:
     branches:
       - master
       - mono-repo*
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
 defaults:
   run:
     shell: bash

.github/workflows/scorecards-analysis.yml

@@ -5,10 +5,6 @@ on:
   schedule:
     - cron: '44 6 * * *'
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
-
 # Declare default permissions as read only.
 permissions: read-all
 

.github/workflows/server-ci-master.yml

@@ -0,0 +1,14 @@
+name: Server CI Master
+on:
+  push:
+    branches:
+      - master
+      - cloud
+      - release-*
+      - mono-repo*
+env:
+  go-version: "1.19.5"
+
+jobs:
+  master-ci:
+    uses: ./.github/workflows/server-ci-template.yml

.github/workflows/server-ci-pr.yml

@@ -0,0 +1,19 @@
+name: Server CI PR
+on:
+  pull_request:
+
+env:
+  go-version: "1.19.5"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# This file just imports the template yml
+# and runs it with concurrency. We have to do this in this way
+# because the concurrency label cannot be added conditionally
+# and it _always_ cancels pending workflows. So master CI builds
+# always kept getting canceled.
+
+jobs:
+  pr-ci:
+    uses: ./.github/workflows/server-ci-template.yml

.github/workflows/server-ci-template.yml

@@ -1,17 +1,13 @@
-name: Server CI
+# Base CI template which is called from server-ci-pr.yml
+# and server-ci-master.yml
+
+name: Server CI Template
 on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - cloud
-      - release-*
-      - mono-repo*
+  workflow_call:
+
 env:
   go-version: "1.19.5"
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
 jobs:
   check-mocks:
     name: Check mocks
@@ -190,23 +186,23 @@ jobs:
       - name: Check generated code
         run: if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
   test-postgres-binary:
-    name: Run tests on postgres with binary parameters
+    name: Postgres with binary parameters
     needs: check-mattermost-vet
-    uses: ./.github/workflows/test.yml
+    uses: ./.github/workflows/server-test-template.yml
     with:
       datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes
       drivername: postgres
   test-postgres-normal:
-    name: Run tests on postgres
+    name: Postgres
     needs: check-mattermost-vet
-    uses: ./.github/workflows/test.yml
+    uses: ./.github/workflows/server-test-template.yml
     with:
       datasource: postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10
       drivername: postgres
   test-mysql:
-    name: Run tests on mysql
+    name: MySQL
     needs: check-mattermost-vet
-    uses: ./.github/workflows/test.yml
+    uses: ./.github/workflows/server-test-template.yml
     with:
       datasource: mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8&multiStatements=true
       drivername: mysql

.github/workflows/server-test-template.yml

@@ -1,4 +1,4 @@
-name: Test
+name: Server Test Template
 on:
   workflow_call:
     inputs:
@@ -11,7 +11,7 @@ on:
 env:
   go-version: "1.19.5"
 jobs:
-  run-tests:
+  test:
     runs-on: ubuntu-latest-8-cores
     env:
       COMPOSE_PROJECT_NAME: ghactions

.github/workflows/webapp-ci-master.yml

@@ -0,0 +1,12 @@
+name: Web App CI Master
+on:
+  push:
+    branches:
+      - master
+      - cloud
+      - release-*
+      - mono-repo*
+
+jobs:
+  master-ci:
+    uses: ./.github/workflows/webapp-ci-template.yml

.github/workflows/webapp-ci-pr.yml

@@ -0,0 +1,17 @@
+name: Web App CI PR
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+# This file just imports the template yml
+# and runs it with concurrency. We have to do this in this way
+# because the concurrency label cannot be added conditionally
+# and it _always_ cancels pending workflows. So master CI builds
+# always kept getting canceled.
+
+jobs:
+  pr-ci:
+    uses: ./.github/workflows/webapp-ci-template.yml

.github/workflows/webapp-ci-template.yml

@@ -1,13 +1,10 @@
-name: Web App CI
+# Base CI template which is called from webapp-ci-pr.yml
+# and webapp-ci-master.yml
+
+name: Web App CI Template
 on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - mono-repo*
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+  workflow_call:
+
 defaults:
   run:
     shell: bash