CLD-5698 - Add e2e smoketests (#23590)

* Prepare: run E2E smoketests with GitHub actions (#23301)
* Port E2E testing scripts from cypress-ui-automation
* Move server to docker-compose, move E2E images to ecrpublic
* Integrate General channel renaming, fixes
* Add local automation-dashboard
 Add readme
* Add E2E smoketests
* Bump postgres to 12
* Fully rely on mattermostdevelopment images

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
Co-authored-by: Saturnino Abril <saturnino.abril@gmail.com>
Co-authored-by: Antonis Stamatiou <stamatiou.antonis@gmail.com>

server/build/gitlab-dc.common.yml

@@ -1,29 +1,44 @@
 version: '2.4'
 services:
   minio:
-    image: ${CI_REGISTRY}/mattermost/ci/images/minio:RELEASE.2019-10-11T00-38-09Z-1
+    image: mattermostdevelopment/mirrored-minio:RELEASE.2019-10-11T00-38-09Z-1
     command: "server /data"
     environment:
       MINIO_ACCESS_KEY: minioaccesskey
       MINIO_SECRET_KEY: miniosecretkey
       MINIO_SSE_MASTER_KEY: "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574"
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "-w1", "127.0.0.1", "9000" ]
+      interval: 10s
+      timeout: 15s
+      retries: 12
   inbucket:
-    image: ${CI_REGISTRY}/mattermost/ci/images/inbucket:v3.0.0-rc1-2-gc64e7a6-1
+    image: mattermostdevelopment/mirrored-inbucket:3.0.0
     restart: always
     environment:
       INBUCKET_WEB_ADDR: "0.0.0.0:9001"
       INBUCKET_POP3_ADDR: "0.0.0.0:10110"
       INBUCKET_SMTP_ADDR: "0.0.0.0:10025"
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "-w1", "127.0.0.1", "10025" ]
+      interval: 10s
+      timeout: 15s
+      retries: 12
   openldap:
-    image: ${CI_REGISTRY}/mattermost/ci/images/openldap:1.4.0-1
+    image: mattermostdevelopment/mirrored-openldap:1.4.0
     restart: always
     environment:
       LDAP_TLS_VERIFY_CLIENT: "never"
       LDAP_ORGANISATION: "Mattermost Test"
       LDAP_DOMAIN: "mm.test.com"
       LDAP_ADMIN_PASSWORD: "mostest"
+    healthcheck:
+      test: [ "CMD", "bash", "-o", "pipefail", "-c", "ss -ltn 'sport = :636' | grep -qE '^LISTEN'" ]
+      interval: 10s
+      timeout: 15s
+      retries: 12
   elasticsearch:
-    image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-elasticsearch-docker:7.17.3
+    image: mattermost/mattermost-elasticsearch-docker:7.17.3
     environment:
       http.host: "0.0.0.0"
       http.port: 9200
@@ -33,10 +48,20 @@ services:
       http.cors.allow-credentials: "true"
       transport.host: "127.0.0.1"
       ES_JAVA_OPTS: "-Xms512m -Xmx512m"
+    healthcheck:
+      test: [ "CMD", "bash", "-o", "pipefail", "-c", "curl --silent localhost:9200/_cat/health | awk '{ print $$4 }' | grep -qE '^green$$'" ]
+      interval: 10s
+      timeout: 15s
+      retries: 12
   dejavu:
-    image: ${CI_REGISTRY}/mattermost/ci/images/dejavu:3.4.2-1
+    image: mattermostdevelopment/mirrored-dejavu:3.4.2
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "-w1", "127.0.0.1", "1358" ]
+      interval: 10s
+      timeout: 15s
+      retries: 12
   keycloak:
-    image: ${CI_REGISTRY}/mattermost/ci/images/keycloak:10.0.2-1
+    image: mattermostdevelopment/mirrored-keycloak:10.0.2
     restart: always
     environment:
       KEYCLOAK_USER: mmuser
@@ -45,13 +70,29 @@ services:
       KEYCLOAK_IMPORT: /setup/realm.json
     volumes:
      - "./docker/keycloak:/setup"
+    healthcheck:
+      test: [ "CMD", "bash", "-o", "pipefail", "-c", "curl --silent localhost:9990/health | grep -q '\"status\":\"UP\"'" ]
+      interval: 10s
+      timeout: 15s
+      retries: 12
   prometheus:
-    image: ${CI_REGISTRY}/mattermost/ci/images/prometheus:2.27.1-1
+    image: mattermostdevelopment/mirrored-prometheus:v2.27.1
     volumes:
       - "./docker/prometheus-linux.yml:/etc/prometheus/prometheus.yml"
+    healthcheck:
+      test: [ "CMD", "wget", "-q", "-O-", "127.0.0.1:9090/-/ready" ]
+      interval: 10s
+      timeout: 15s
+      retries: 12
+  # wget -q -O- localhost:3000/healthz | grep -q Ok
   grafana:
-    image: ${CI_REGISTRY}/mattermost/ci/images/grafana:8.0.1-1
+    image: mattermostdevelopment/mirrored-grafana:8.0.1
     volumes:
      - "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini"
      - "./docker/grafana/provisioning:/etc/grafana/provisioning"
      - "./docker/grafana/dashboards:/var/lib/grafana/dashboards"
+    healthcheck:
+      test: [ "CMD", "bash", "-o", "pipefail", "-c", "wget -q -O- localhost:3000/healthz | grep -q Ok" ]
+      interval: 10s
+      timeout: 15s
+      retries: 12

server/build/gitlab-dc.mysql.yml

@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   mysql:
-    image: ${CI_REGISTRY}/mattermost/ci/images/mysql:5.7-1
+    image: mattermostdevelopment/mirrored-mysql:5.7.12
     restart: always
     environment:
       MYSQL_ROOT_HOST: "%"
@@ -83,7 +83,7 @@ services:
           - grafana
 
   start_dependencies:
-    image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-wait-for-dep:latest-1
+    image: mattermost/mattermost-wait-for-dep:latest
     depends_on:
       - mysql
       - minio

server/build/gitlab-dc.postgres.yml

@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   postgres:
-    image: ${CI_REGISTRY}/mattermost/ci/images/postgres:10-1
+    image: mattermostdevelopment/mirrored-postgres:12
     restart: always
     environment:
       POSTGRES_USER: mmuser
@@ -12,9 +12,9 @@ services:
      - "./docker/postgres.conf:/etc/postgresql/postgresql.conf"
     healthcheck:
       test: [ "CMD", "pg_isready", "-h", "localhost" ]
-      interval: 5s
-      timeout: 10s
-      retries: 3
+      interval: 10s
+      timeout: 15s
+      retries: 12
     networks:
       default:
         aliases:
@@ -81,7 +81,7 @@ services:
           - grafana
 
   start_dependencies:
-    image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-wait-for-dep:latest-1
+    image: mattermost/mattermost-wait-for-dep:latest
     depends_on:
       - postgres
       - minio
@@ -94,7 +94,6 @@ services:
     networks:
       default:
 
-
 networks:
   default:
     external:

server/build/gitlab-dc.schemamysql.yml

@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   mysql:
-    image: ${CI_REGISTRY}/mattermost/ci/images/mysql:5.7-1
+    image: mattermostdevelopment/mirrored-mysql:5.7.12
     restart: always
     environment:
       MYSQL_ROOT_HOST: "%"
@@ -23,7 +23,7 @@ services:
           - mysql
 
   start_dependencies:
-    image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-wait-for-dep:latest-1
+    image: mattermost/mattermost-wait-for-dep:latest
     depends_on:
       - mysql
     command: mysql:3306

server/build/gitlab-dc.schemapostgres.yml

@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   postgres:
-    image: ${CI_REGISTRY}/mattermost/ci/images/postgres:10-1
+    image: mattermostdevelopment/mirrored-postgres:12
     restart: always
     environment:
       POSTGRES_USER: mmuser
@@ -21,7 +21,7 @@ services:
           - postgres
 
   start_dependencies:
-    image: ${CI_REGISTRY}/mattermost/ci/images/mattermost-wait-for-dep:latest-1
+    image: mattermost/mattermost-wait-for-dep:latest
     depends_on:
       - postgres
     command: postgres:5432

server/scripts/mirror-docker-images.json

@@ -0,0 +1,36 @@
+{
+  "postgres": {
+    "10": "postgres:10@sha256:7a484b11fcabd39596b1bf08780cdb61fa9b0d8bfad0844dbdce3a6922df95d1",
+    "12": "postgres:12@sha256:cc7a021d9aff3aa02788d35c27a5cc32d4790ad92d72232a6be75b76ab7d79db"
+  },
+  "mysql": {
+    "5.7.12": "mysql/mysql-server:5.7.12@sha256:3f0d90736a3298bb04965db697a3a85f1df1480da49eafd256be1ea2b9b5337e"
+  },
+  "minio": {
+    "RELEASE.2019-10-11T00-38-09Z-1": "minio/minio:RELEASE.2019-10-11T00-38-09Z@sha256:0d02f16a1662653f9b961211b21ed7de04bf04492f44c2b7594bacbfcc519eb5"
+  },
+  "inbucket": {
+    "3.0.0": "inbucket/inbucket:3.0.0@sha256:1f10a0efea694592c06799c729aee1d6d71c9a4f72b73031d4a426ef5f26dfc1"
+  },
+  "openldap": {
+    "1.4.0": "osixia/openldap:1.4.0@sha256:d5b2f2b816b25a1b57033b34f5d48c91cc3161a7d041811a9032604030bad9db"
+  },
+  "keycloak": {
+    "10.0.2": "jboss/keycloak:10.0.2@sha256:3720b5ace316b5790a58ce838f46e8cd44cedbdb7e35d3866311ddc5a5e71466"
+  },
+  "dejavu": {
+    "3.4.2": "appbaseio/dejavu:3.4.2@sha256:8f2f4d45565da53c4235495737fff3921d302955daeb2f53a433c7b0e2044951"
+  },
+  "prometheus": {
+    "v2.27.1": "prom/prometheus:v2.27.1@sha256:5accb68b56ba452e449a5e552411acaeabbbe0f087acf19a1157ce3dd10a8bed"
+  },
+  "grafana": {
+    "8.0.1": "grafana/grafana:8.0.1@sha256:1c3e2fc7896adf9e33be5d062c08066087cb556f63b0a95f8aefe92bd37a6f38"
+  },
+  "cypress-browsers-public": {
+    "node16.14.2-slim-chrome100-ff99-edge": "cypress/browsers:node16.14.2-slim-chrome100-ff99-edge@sha256:f8459ee677ce356eff64698095fbdf48eb9ab018fc5eb0d30c07ba23884edace"
+  },
+  "golang": {
+    "1.19.5": "golang:1.19.5@sha256:bb9811fad43a7d6fd2173248d8331b2dcf5ac9af20976b1937ecd214c5b8c383"
+  }
+}

server/scripts/mirror-docker-images.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -eu -o pipefail
+
+# Assert that IMAGE_FILE var is given, and that the file exists
+: ${IMAGES_FILE}
+[ -f ${IMAGES_FILE} ] || {
+  echo "Error: images spec file $IMAGES_FILE does not exist. Aborting." >&2
+  exit 1
+}
+
+DRY_RUN=${DRY_RUN:-yes}
+log () { echo "[$(date -Is)]" $*; }
+get_image_specs_per_line () {
+  jq -c '. as $images | keys[] | . as $image | $images[.] | keys[] | {dst_img_name: $image, dst_img_tag: ., src_img: $images[$image][.]}' <$IMAGES_FILE
+}
+
+log "Pusing images from given spec file: $IMAGES_FILE"
+log "Content of the spec file:"
+cat $IMAGES_FILE
+get_image_specs_per_line | while read IMAGE_SPEC; do
+  DST_IMG_NAME=$(jq -r '.dst_img_name' <<<$IMAGE_SPEC)
+  DST_IMG_TAG=$(jq -r '.dst_img_tag' <<<$IMAGE_SPEC)
+  SOURCE_IMAGE=$(jq -r '.src_img' <<<$IMAGE_SPEC)
+  DESTINATION_IMAGE=mattermostdevelopment/mirrored-${DST_IMG_NAME}:${DST_IMG_TAG}
+  if [ "${DRY_RUN,,}" = "no" ]; then
+    log "Pushing image: $SOURCE_IMAGE ---> $DESTINATION_IMAGE"
+    docker pull $SOURCE_IMAGE
+    docker tag $SOURCE_IMAGE $DESTINATION_IMAGE
+    docker push $DESTINATION_IMAGE
+  else
+    log "Pushing image: $SOURCE_IMAGE ---> $DESTINATION_IMAGE (dry run mode, set the DRY_RUN=no env var to disable)"
+  fi
+done
+log "All images pushed."