docker: prometheus and grafana for development (#16649)

Makefile

@@ -8,6 +8,15 @@ else
 	PLATFORM := $(shell uname)
 endif
 
+# Set an environment variable on Linux used to resolve `docker.host.internal` inconsistencies with
+# docker. This can be reworked once https://github.com/docker/for-linux/issues/264 is resolved
+# satisfactorily.
+ifeq ($(PLATFORM),Linux)
+	export IS_LINUX = -linux
+else
+	export IS_LINUX =
+endif
+
 IS_CI ?= false
 # Build Flags
 BUILD_NUMBER ?= $(BUILD_NUMBER:)

build/docker-compose-generator/main.go

@@ -33,6 +33,8 @@ func main() {
 		"elasticsearch": 9200,
 		"dejavu":        1358,
 		"keycloak":      8080,
+		"prometheus":    9090,
+		"grafana":       3000,
 	}
 	command := []string{}
 	for _, arg := range os.Args[1:] {

build/docker-compose.common.yml

@@ -86,3 +86,17 @@ services:
       - mm-test
     volumes:
      - "./docker/keycloak:/setup"
+  prometheus:
+    image: "prom/prometheus"
+    volumes:
+      - "./docker/prometheus${IS_LINUX}.yml:/etc/prometheus/prometheus.yml"
+    networks:
+      - mm-test
+  grafana:
+    image: "grafana/grafana"
+    volumes:
+     - "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini"
+     - "./docker/grafana/provisioning:/etc/grafana/provisioning"
+     - "./docker/grafana/dashboards:/var/lib/grafana/dashboards"
+    networks:
+      - mm-test

build/docker-compose.yml

@@ -36,6 +36,14 @@ services:
     extends:
         file: docker-compose.common.yml
         service: keycloak
+  prometheus:
+    extends:
+        file: docker-compose.common.yml
+        service: prometheus
+  grafana:
+    extends:
+        file: docker-compose.common.yml
+        service: grafana
 
   start_dependencies:
     image: mattermost/mattermost-wait-for-dep:latest
@@ -48,7 +56,9 @@ services:
       - inbucket
       - openldap
       - elasticsearch
-    command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200
+      - prometheus
+      - grafana
+    command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000
 
 networks:
   mm-test:

build/docker/grafana/dashboards/README.md

@@ -0,0 +1,15 @@
+# Developer Dashboards
+
+This folder contains a modified form of the following Grafana dashboards:
+* [Mattermost Performance Monitoring](https://grafana.com/grafana/dashboards/2542)
+* [Mattermost Performance Monitoring (Bonus Metrics)](https://grafana.com/grafana/dashboards/2545)
+* [Mattermost Performance KPI Metrics](https://grafana.com/grafana/dashboards/2539)
+
+The dashboards are modified from the version available on grafana.com since [Grafana doesn't currently support variables](https://github.com/grafana/grafana/issues/10786) (i.e. `${DS_MATTERMOST}`) and has no way of binding the datasource with the dashboards at the time of provisioning. Instead of falling back to the REST API to effect these changes, the following in-place changes were made to the exported dashboards available above:
+* Remove the top-level `__inputs` key
+* Remove the top-level `__requires` key
+* Replace all instances of `${DS_MATTERMOST}` with `Prometheus`, matching the name of the provisioned datasource.
+
+Upon using the dashboards within Grafana however, it immediately adds various missing fields, presumably due to some internal migration. This results in a spurious prompt to "save" the dashboard. To avoid confusion, these changes were subsequently exported and written back into the provisioned dashboards above.
+
+This entire process above will need to be repeated in the event newer versions of these dashboards are published.

build/docker/grafana/dashboards/home.json

@@ -0,0 +1,129 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 5,
+  "links": [],
+  "panels": [
+    {
+      "datasource": null,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "title": "",
+      "type": "welcome"
+    },
+    {
+      "datasource": null,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 15,
+        "w": 12,
+        "x": 0,
+        "y": 3
+      },
+      "headings": true,
+      "id": 3,
+      "limit": 30,
+      "links": [],
+      "query": "Mattermost",
+      "recent": false,
+      "search": true,
+      "starred": false,
+      "tags": [],
+      "title": "Dashboards",
+      "type": "dashlist"
+    },
+    {
+      "datasource": null,
+      "fieldConfig": {
+        "defaults": {
+          "custom": {}
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 15,
+        "w": 12,
+        "x": 12,
+        "y": 3
+      },
+      "id": 4,
+      "links": [],
+      "options": {
+        "feedUrl": "https://grafana.com/blog/news.xml"
+      },
+      "title": "Latest from the blog",
+      "type": "news"
+    }
+  ],
+  "schemaVersion": 26,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {
+    "hidden": true,
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ],
+    "type": "timepicker"
+  },
+  "timezone": "browser",
+  "title": "Home",
+  "uid": "4yRNKfaGz",
+  "version": 1
+}

build/docker/grafana/grafana.ini

@@ -0,0 +1,9 @@
+[auth]
+disable_login_form = false
+
+[auth.anonymous]
+enabled = true
+org_role = Editor
+
+[dashboards]
+default_home_dashboard_path = /var/lib/grafana/dashboards/home.json

build/docker/grafana/provisioning/dashboards/mattermost.yaml

@@ -0,0 +1,14 @@
+apiVersion: 1
+
+providers:
+  - name: 'mattermost'
+    orgId: 1
+    folder: 'mattermost'
+    type: file
+    disableDeletion: true
+    updateIntervalSeconds: 300
+    allowUiUpdates: true
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: true

build/docker/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: 'prometheus'
+    access: 'proxy'
+    orgId: 1
+    url: 'http://prometheus:9090'
+    isDefault: true
+    version: 1
+    editable: false

build/docker/prometheus-linux.yml

@@ -0,0 +1,8 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 60s
+
+scrape_configs:
+  - job_name: 'mattermost'
+    static_configs:
+      - targets: ['172.17.0.1:8067']

build/docker/prometheus.yml

@@ -0,0 +1,8 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 60s
+
+scrape_configs:
+  - job_name: 'mattermost'
+    static_configs:
+      - targets: ['host.docker.internal:8067']

config.mk

@@ -5,7 +5,7 @@
 # Enable services to be run in docker.
 #
 # Possible options: mysql, postgres, minio, inbucket, openldap, dejavu,
-# keycloak and elasticsearch
+# keycloak, elasticsearch, prometheus, and grafana.
 #
 # Must be space separated names.
 #

docker-compose.makefile.yml

@@ -68,6 +68,22 @@ services:
     extends:
         file: build/docker-compose.common.yml
         service: keycloak
+  prometheus:
+    restart: 'no'
+    container_name: mattermost-prometheus
+    ports:
+      - "9090:9090"
+    extends:
+        file: build/docker-compose.common.yml
+        service: prometheus
+  grafana:
+    restart: 'no'
+    container_name: mattermost-grafana
+    ports:
+      - "3000:3000"
+    extends:
+        file: build/docker-compose.common.yml
+        service: grafana
 
 networks:
   mm-test:

docker-compose.yaml

@@ -60,6 +60,20 @@ services:
     extends:
         file: build/docker-compose.common.yml
         service: keycloak
+  prometheus:
+    container_name: mattermost-prometheus
+    ports:
+      - "9090:9090"
+    extends:
+        file: build/docker-compose.common.yml
+        service: prometheus
+  grafana:
+    container_name: mattermost-grafana
+    ports:
+      - "3000:3000"
+    extends:
+        file: build/docker-compose.common.yml
+        service: grafana
 
   start_dependencies:
     image: mattermost/mattermost-wait-for-dep:latest
@@ -72,7 +86,9 @@ services:
       - inbucket
       - openldap
       - elasticsearch
-    command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200
+      - prometheus
+      - grafana
+    command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000
 
   leader:
     build: