Mono repo -> Master (#22553)

Combines the following repositories into one:

https://github.com/mattermost/mattermost-server
https://github.com/mattermost/mattermost-webapp
https://github.com/mattermost/focalboard
https://github.com/mattermost/mattermost-plugin-playbooks

.github/workflows/channels-ci.yml

@@ -0,0 +1,196 @@
+name: Web App CI
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - mono-repo*
+defaults:
+  run:
+    shell: bash
+jobs:
+  check-lint:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: webapp
+    steps:
+      - name: ci/checkout-repo
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: ci/setup-node
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
+        id: setup_node
+        with:
+          node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: 'webapp/package-lock.json'
+      # - uses: actions/cache@2b8105bdae4b746666ee225970c4a281bbd9d51f # v3.2.4
+      #   id: npm-cache
+      #   with:
+      #     path: |
+      #       '**/node_modules'
+      #       'e2e/playwright/node_modules'
+      #       'e2e/cypress/node_modules'
+      #     key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('e2e/cypress/package-lock.json') }}-${{ hashFiles('e2e/playwright/package-lock.json') }}
+      - name: ci/get-node-modules
+        # if: steps.npm-cache.outputs.cache-hit != 'true'
+        run: |
+          make node_modules
+          # make channels/e2e/playwright/node_modules
+      - name: ci/lint
+        working-directory: webapp/channels
+        run: |
+          npm run check
+  check-i18n:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: webapp
+    steps:
+      - name: ci/checkout-repo
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: ci/setup-node
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
+        id: setup_node
+        with:
+          node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: 'webapp/package-lock.json'
+      # - uses: actions/cache@2b8105bdae4b746666ee225970c4a281bbd9d51f # v3.2.4
+      #   id: npm-cache
+      #   with:
+      #     path: |
+      #       '**/node_modules'
+      #       'e2e/playwright/node_modules'
+      #       'e2e/cypress/node_modules'
+      #     key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('e2e/cypress/package-lock.json') }}-${{ hashFiles('e2e/playwright/package-lock.json') }}
+      - name: ci/get-node-modules
+        # if: steps.npm-cache.outputs.cache-hit != 'true'
+        run: |
+          make node_modules
+          # make channels/e2e/playwright/node_modules
+      - name: ci/lint
+        working-directory: webapp/channels
+        run: |
+          cp src/i18n/en.json /tmp/en.json
+          mkdir -p /tmp/fake-mobile-dir/assets/base/i18n/
+          echo '{}' > /tmp/fake-mobile-dir/assets/base/i18n/en.json
+          npm run mmjstool -- i18n extract-webapp --webapp-dir ./src --mobile-dir /tmp/fake-mobile-dir
+          diff /tmp/en.json src/i18n/en.json
+          # Address weblate behavior which does not remove whole translation item when translation string is set to empty
+          npm run mmjstool -- i18n clean-empty --webapp-dir ./src --mobile-dir /tmp/fake-mobile-dir --check
+          npm run mmjstool -- i18n check-empty-src --webapp-dir ./src --mobile-dir /tmp/fake-mobile-dir
+          rm -rf tmp
+  check-type:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: webapp
+    steps:
+      - name: ci/checkout-repo
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: ci/setup-node
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
+        id: setup_node
+        with:
+          node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: 'webapp/package-lock.json'
+      # - uses: actions/cache@2b8105bdae4b746666ee225970c4a281bbd9d51f # v3.2.4
+      #   id: npm-cache
+      #   with:
+      #     path: |
+      #       '**/node_modules'
+      #       'e2e/playwright/node_modules'
+      #       'e2e/cypress/node_modules'
+      #     key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('e2e/cypress/package-lock.json') }}-${{ hashFiles('e2e/playwright/package-lock.json') }}
+      - name: ci/get-node-modules
+        # if: steps.npm-cache.outputs.cache-hit != 'true'
+        run: |
+          make node_modules
+          # make channels/e2e/playwright/node_modules
+      - name: ci/lint
+        working-directory: webapp/channels
+        run: |
+          npm run check-types
+  tests:
+    runs-on: ubuntu-22.04
+    needs: [check-type, check-i18n, check-lint]
+    permissions:
+      checks: write
+      pull-requests: write
+    defaults:
+      run:
+        working-directory: webapp
+    steps:
+      - name: ci/checkout-repo
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: ci/setup-node
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
+        id: setup_node
+        with:
+          node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: 'webapp/package-lock.json'
+      # - uses: actions/cache@2b8105bdae4b746666ee225970c4a281bbd9d51f # v3.2.4
+      #   id: npm-cache
+      #   with:
+      #     path: |
+      #       '**/node_modules'
+      #       'e2e/playwright/node_modules'
+      #       'e2e/cypress/node_modules'
+      #     key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('e2e/cypress/package-lock.json') }}-${{ hashFiles('e2e/playwright/package-lock.json') }}
+      - name: ci/get-node-modules
+        # if: steps.npm-cache.outputs.cache-hit != 'true'
+        run: |
+          make node_modules
+          # make channels/e2e/playwright/node_modules
+      - name: ci/detect-cpu-core
+        id: cpu-cores
+        uses: SimenB/github-actions-cpu-cores@c508d404ab007faae80a014072fd8c0e17792118 # v1.1.0
+      - name: ci/test
+        working-directory: webapp/channels
+        run: |
+          npm run test-ci -- --maxWorkers=${{ steps.cpu-cores.outputs.count }}
+      - name: ci/publish-test-results
+        uses: EnricoMi/publish-unit-test-result-action@a3caf02865c0604ad3dc1ecfcc5cdec9c41b7936  # v2.3.0
+        if: always()
+        with:
+          junit_files: "webapp/channels/build/**/*.xml"
+          comment_mode: always
+          compare_to_earlier_commit: false
+  build:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: webapp
+    steps:
+      - name: ci/checkout-repo
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: ci/setup-node
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
+        id: setup_node
+        with:
+          node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: 'webapp/package-lock.json'
+      # - uses: actions/cache@2b8105bdae4b746666ee225970c4a281bbd9d51f # v3.2.4
+      #   id: npm-cache
+      #   with:
+      #     path: |
+      #       '**/node_modules'
+      #       'e2e/playwright/node_modules'
+      #       'e2e/cypress/node_modules'
+      #     key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('e2e/cypress/package-lock.json') }}-${{ hashFiles('e2e/playwright/package-lock.json') }}
+      - name: ci/get-node-modules
+        # if: steps.npm-cache.outputs.cache-hit != 'true'
+        run: |
+          make node_modules
+          # make channels/e2e/playwright/node_modules
+      - name: ci/build
+        working-directory: webapp/channels
+        run: |
+          npm run build
+  # run-performance-bechmarks:
+  #   uses: ./.github/workflows/performance-benchmarks.yml
+  #   needs: build

.github/workflows/ci.yml

@@ -6,10 +6,14 @@ on:
       - master
       - cloud
       - release-*
+      - mono-repo*
 jobs:
   check-mocks:
     name: Check mocks
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -20,6 +24,9 @@ jobs:
   check-go-mod-tidy:
     name: Check go mod tidy
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -30,6 +37,9 @@ jobs:
   check-gen-serialized:
     name: Check serialization methods for hot structs
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -40,27 +50,24 @@ jobs:
   check-mattermost-vet:
     name: Check style
     runs-on: ubuntu-latest-8-cores
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: Checkout focalboard
-        run: |
-          cd ..
-          git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git  
-          cd focalboard
-          git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || git checkout rolling-stable
-          echo $(git rev-parse HEAD)
-          cd ../mattermost-server
-          make setup-go-work
       - name: Reset config
         run: make config-reset
       - name: Run plugin-checker
         run: make plugin-checker
       - name: Run mattermost-vet
-        run: make vet BUILD_NUMBER='${GITHUB_HEAD_REF}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
+        run: make vet BUILD_NUMBER='${GITHUB_HEAD_REF}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
   check-migrations:
     name: Check migration files
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -86,6 +93,9 @@ jobs:
   check-generate-work-templates:
     name: Generate work templates
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -96,6 +106,9 @@ jobs:
   check-email-templates:
     name: Generate email templates
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -108,6 +121,9 @@ jobs:
   check-store-layers:
     name: Check store layers
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -118,6 +134,9 @@ jobs:
   check-app-layers:
     name: Check app layers
     runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: server
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
@@ -149,51 +168,13 @@ jobs:
   build-mattermost-server:
     name: Build mattermost-server
     runs-on: ubuntu-latest-8-cores
+    defaults:
+      run:
+        working-directory: server
     needs: check-mattermost-vet
     steps:
       - name: Checkout mattermost-server
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
-      - name: Checkout mattermost-webapp
-        run: |
-          cd ..
-          git clone --depth=1 --no-single-branch https://github.com/mattermost/mattermost-webapp.git
-          cd mattermost-webapp
-          git checkout $GITHUB_HEAD_REF || git checkout master
-          export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
-          echo "$WEBAPP_GIT_COMMIT"
-          FILE_DIST=dist.tar.gz
-          runtime="2 minute"
-          endtime=$(date -ud "$runtime" +%s)
-          while [[ $(date -u +%s) -le $endtime ]]; do
-            if curl -s --max-time 30 -f -o $FILE_DIST https://pr-builds.mattermost.com/mattermost-webapp/commit/$WEBAPP_GIT_COMMIT/mattermost-webapp.tar.gz; then
-              break
-            fi
-            echo "Waiting for webapp git commit $WEBAPP_GIT_COMMIT with sleep 5: `date +%H:%M:%S`"
-            sleep 5
-          done
-          if [[ -f "$FILE_DIST" ]]; then
-            echo "Precompiled version of web app found"
-            mkdir dist && tar -xf $FILE_DIST -C dist --strip-components=1
-          else
-          echo "Building web app from source"
-          make dist
-          fi
-          cd ../mattermost-server
-      - name: Checkout and build focalboard
-        run: |
-          cd ..
-          git clone --depth=1 --no-single-branch https://github.com/mattermost/focalboard.git
-          cd focalboard
-          git checkout $GITHUB_HEAD_REF || git checkout $GITHUB_BASE_REF || git checkout rolling-stable
-          echo $(git rev-parse HEAD)
-          make server-linux
-          echo "Building Boards product for web app"
-          # make prebuild build-product # TODO figure out how to get this to run without bypassing the Makefile
-          make prebuild
-          cd mattermost-plugin/webapp
-          npm run build:product
-          cd ../../../mattermost-server
-          make setup-go-work
       - name: Build
         run: |
           make config-reset
@@ -203,22 +184,18 @@ jobs:
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: server-dist-artifact
-          path: dist/
+          path: server/dist/
           retention-days: 14
       - name: Persist build artifacts
         uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
         with:
           name: server-build-artifact
-          path: build/
+          path: server/build/
           retention-days: 14 
   upload-s3:
     name: Upload to S3 bucket
     runs-on: ubuntu-22.04
-    needs: 
-      - build-mattermost-server
-      - test-mysql
-      - test-postgres-binary
-      - test-postgres-normal 
+    needs: build-mattermost-server
     env:
       REPO_NAME: ${{ github.event.repository.name }}
     steps:
@@ -226,7 +203,7 @@ jobs:
       uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b  # v3.0.2
       with:
         name: server-dist-artifact
-        path: dist/
+        path: server/dist/
     - name: Configure AWS
       uses: aws-actions/configure-aws-credentials@07c2f971bac433df982ccc261983ae443861db49 # v1-node16
       with:
@@ -242,8 +219,8 @@ jobs:
       run: echo "BRANCH_NAME=$(echo ${{ steps.branch.outputs.sanitized-branch-name }} | sed 's/^pull\//PR-/g')" >> $GITHUB_ENV
     - name: ci/artifact-upload
       run: |
-        aws s3 cp dist/ s3://pr-builds.mattermost.com/$REPO_NAME/$BRANCH_NAME/ --acl public-read --cache-control "no-cache" --recursive
-        aws s3 cp dist/ s3://pr-builds.mattermost.com/$REPO_NAME/commit/${{ github.sha }}/ --acl public-read --cache-control "no-cache" --recursive
+        aws s3 cp server/dist/ s3://pr-builds.mattermost.com/$REPO_NAME/$BRANCH_NAME/ --acl public-read --cache-control "no-cache" --recursive
+        aws s3 cp server/dist/ s3://pr-builds.mattermost.com/$REPO_NAME/commit/${{ github.sha }}/ --acl public-read --cache-control "no-cache" --recursive
   build-docker:
     name: Build docker image
     runs-on: ubuntu-22.04
@@ -253,7 +230,7 @@ jobs:
       uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b  # v3.0.2
       with:
         name: server-build-artifact
-        path: build/
+        path: server/build/
     - name: Login to Docker Hub
       uses: docker/login-action@3da7dc6e2b31f99ef2cb9fb4c50fb0971e0d0139 # v2.1.0
       with:
@@ -266,7 +243,7 @@ jobs:
         DOCKER_CLI_EXPERIMENTAL: enabled
       run: |
         export TAG=$(echo "${{ github.event.pull_request.head.sha || github.sha }}" | cut -c1-7)
-        cd build
+        cd server/build
         export DOCKER_CLI_EXPERIMENTAL=enabled
         export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${GITHUB_SHA}/mattermost-team-linux-amd64.tar.gz
         docker buildx build --push --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mm-te-test:${TAG} .

.github/workflows/codeql-analysis.yml

@@ -36,6 +36,7 @@ jobs:
     
     - name: Build
       run: |
+        cd server
         make setup-go-work
         make build-linux-amd64
 

.github/workflows/e2e-ci.yml

@@ -0,0 +1,58 @@
+name: mattermost-e2e
+on:
+  pull_request:
+  push:
+    branches:
+      - master
+      - mono-repo*
+defaults:
+  run:
+    shell: bash
+jobs:
+  cypress-check:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: e2e/cypress
+    steps:
+      - name: ci/checkout-repo
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: ci/setup-node
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
+        id: setup_node
+        with:
+          node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: 'e2e/cypress/package-lock.json'
+      - name: ci/cypress/npm-install
+        run: |
+          npm ci
+      - name: ci/cypress/npm-check
+        run: |
+          npm run check
+  playwright-check:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        working-directory: e2e/playwright
+    steps:
+      - name: ci/checkout-repo
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - name: ci/setup-node
+        uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
+        id: setup_node
+        with:
+          node-version-file: ".nvmrc"
+          cache: npm
+          cache-dependency-path: 'e2e/playwright/package-lock.json'
+      - name: ci/get-webapp-node-modules
+        working-directory: webapp
+        # requires build of client and types
+        run: |
+          make node_modules
+      - name: ci/playwright/npm-install
+        run: |
+          npm ci
+      - name: ci/playwright/npm-check
+        run: |
+          npm run check

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - name: Run docker compose
         run: |
-          cd build
+          cd server/build
           docker-compose --no-ansi run --rm start_dependencies
           cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
           docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
@@ -32,17 +32,19 @@ jobs:
           docker run --net ghactions_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
       - name: Run Tests
         run: |
-          if [[ ${{ github.event_name }} == 'push' ]]; then
+          if [[ ${{ github.ref_name }} == 'master' ]]; then
             export RACE_MODE="-race"
           fi
           mkdir -p client/plugins
-          cd build
+          cd server/build
           docker run --net ghactions_mm-test \
             --ulimit nofile=8096:8096 \
             --env-file=dotenv/test.env \
             --env MM_SQLSETTINGS_DRIVERNAME="${{ inputs.drivername }}" \
             --env MM_SQLSETTINGS_DATASOURCE="${{ inputs.datasource }}" \
+            --env TEST_DATABASE_MYSQL_DSN="${{ inputs.datasource }}" \
+            --env TEST_DATABASE_POSTGRESQL_DSN="${{ inputs.datasource }}" \
             -v ~/work/mattermost-server:/mattermost-server \
-            -w /mattermost-server/mattermost-server \
+            -w /mattermost-server/mattermost-server/server \
             $BUILD_IMAGE \
             make test-server$RACE_MODE BUILD_NUMBER=$GITHUB_HEAD_REF-$GITHUB_RUN_ID TESTFLAGS= TESTFLAGSEE=