configure Loki for local development (#26997)

* configure Loki for local development

Extend our docker compose setup to include Loki (along with promtail), scraping `logs/*.log` to ingest development logs as well as all logs from running containers.

While we're in here, teach Prometheus to scrape metrics from Docker containers too.

* tweak promtail labels

* document extra services in config.mk

* update home.json

server/build/docker-compose-generator/main.go

@@ -35,6 +35,8 @@ func main() {
 		"keycloak":           8080,
 		"prometheus":         9090,
 		"grafana":            3000,
+		"loki":               3100,
+		"promtail":           3180,
 		"mysql-read-replica": 3306, // FIXME: not recognizing the successfully running service on port 3307.
 	}
 	command := []string{}

server/build/docker-compose.common.yml

@@ -119,15 +119,31 @@ services:
      - "./docker/keycloak:/opt/keycloak/data/import"
   prometheus:
     image: "prom/prometheus:v2.46.0"
+    user: root
     volumes:
       - "./docker/prometheus${IS_LINUX}.yml:/etc/prometheus/prometheus.yml"
+      - "/var/run/docker.sock:/var/run/docker.sock"
     networks:
       - mm-test
   grafana:
-    image: "grafana/grafana:10.0.3"
+    image: "grafana/grafana:10.4.2"
     volumes:
-     - "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini"
+      - "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini"
-     - "./docker/grafana/provisioning:/etc/grafana/provisioning"
+      - "./docker/grafana/provisioning:/etc/grafana/provisioning"
-     - "./docker/grafana/dashboards:/var/lib/grafana/dashboards"
+      - "./docker/grafana/dashboards:/var/lib/grafana/dashboards"
+    networks:
+      - mm-test
+  loki:
+    image: "grafana/loki:3.0.0"
+    networks:
+      - mm-test
+  promtail:
+    image: "grafana/promtail:3.0.0"
+    volumes:
+      - "./docker/promtail/promtail-local-config.yaml:/etc/promtail/docker-config.yaml"
+      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "../logs:/logs"
+    command: -config.file=/etc/promtail/docker-config.yaml
     networks:
       - mm-test

server/build/docker-compose.yml

@@ -44,6 +44,14 @@ services:
     extends:
         file: docker-compose.common.yml
         service: grafana
+  loki:
+    extends:
+        file: docker-compose.common.yml
+        service: loki
+  promtail:
+    extends:
+        file: docker-compose.common.yml
+        service: promtail
 
   start_dependencies:
     image: mattermost/mattermost-wait-for-dep:latest

server/build/docker/grafana/dashboards/home.json

@@ -3,7 +3,10 @@
     "list": [
       {
         "builtIn": 1,
-        "datasource": "-- Grafana --",
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
         "enable": true,
         "hide": true,
         "iconColor": "rgba(0, 211, 255, 1)",
@@ -13,18 +16,15 @@
     ]
   },
   "editable": true,
-  "gnetId": null,
+  "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
   "id": 5,
   "links": [],
   "panels": [
     {
-      "datasource": null,
+      "datasource": {
-      "fieldConfig": {
+        "type": "prometheus",
-        "defaults": {
+        "uid": "PBFA97CFB590B2093"
-          "custom": {}
-        },
-        "overrides": []
       },
       "gridPos": {
         "h": 3,
@@ -33,60 +33,369 @@
         "y": 0
       },
       "id": 1,
-      "title": "",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "refId": "A"
+        }
+      ],
       "type": "welcome"
     },
     {
-      "datasource": null,
+      "datasource": {
-      "fieldConfig": {
+        "type": "prometheus",
-        "defaults": {
+        "uid": "PBFA97CFB590B2093"
-          "custom": {}
-        },
-        "overrides": []
       },
       "gridPos": {
-        "h": 15,
+        "h": 7,
         "w": 12,
         "x": 0,
         "y": 3
       },
-      "headings": true,
       "id": 3,
-      "limit": 30,
+      "options": {
-      "links": [],
+        "includeVars": false,
-      "query": "Mattermost",
+        "keepTime": false,
-      "recent": false,
+        "maxItems": 30,
-      "search": true,
+        "query": "Mattermost",
-      "starred": false,
+        "showHeadings": true,
+        "showRecentlyViewed": false,
+        "showSearch": true,
+        "showStarred": false,
+        "tags": []
+      },
+      "pluginVersion": "10.4.2",
       "tags": [],
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "refId": "A"
+        }
+      ],
       "title": "Dashboards",
       "type": "dashlist"
     },
     {
-      "datasource": null,
+      "datasource": {
+        "type": "datasource",
+        "uid": "-- Mixed --"
+      },
       "fieldConfig": {
         "defaults": {
-          "custom": {}
+          "color": {
+            "mode": "thresholds"
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "red",
+                  "index": 0,
+                  "text": "Offline"
+                }
+              },
+              "type": "value"
+            },
+            {
+              "options": {
+                "from": 0,
+                "result": {
+                  "color": "green",
+                  "index": 1,
+                  "text": "Online"
+                },
+                "to": 99999999999999
+              },
+              "type": "range"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
         },
         "overrides": []
       },
       "gridPos": {
-        "h": 15,
+        "h": 12,
         "w": 12,
         "x": 12,
         "y": 3
       },
-      "id": 4,
+      "id": 7,
-      "links": [],
       "options": {
-        "feedUrl": "https://grafana.com/blog/news.xml"
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "/^(?!Time)/",
+          "values": false
+        },
+        "showPercentChange": false,
+        "textMode": "value_and_name",
+        "wideLayout": true
       },
-      "title": "Latest from the blog",
+      "pluginVersion": "10.4.2",
-      "type": "news"
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({container=\"mattermost-postgres\"}[5m])) or vector(0)",
+          "legendFormat": "Postgres",
+          "queryType": "range",
+          "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({container=\"mattermost-mysql\"}[5m])) or vector(0)",
+          "legendFormat": "MySQL",
+          "queryType": "range",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({container=\"mattermost-inbucket\"}[5m])) or vector(0)",
+          "legendFormat": "Inbucket",
+          "queryType": "range",
+          "refId": "C"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({container=\"mattermost-minio\"}[5m])) or vector(0)",
+          "legendFormat": "MinIO",
+          "queryType": "range",
+          "refId": "D"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({container=\"mattermost-openldap\"}[5m])) or vector(0)",
+          "legendFormat": "OpenLDAP",
+          "queryType": "range",
+          "refId": "E"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({container=\"mattermost-dejavu\"}[5m])) or vector(0)",
+          "legendFormat": "DejaVu",
+          "queryType": "range",
+          "refId": "F"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({container=\"mattermost-keycloak\"}[5m])) or vector(0)",
+          "legendFormat": "Keycloak",
+          "queryType": "range",
+          "refId": "G"
+        },
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "sum(count_over_time({container=\"mattermost-elasticsearch\"}[5m])) or vector(0)",
+          "legendFormat": "Elasticsearch",
+          "queryType": "range",
+          "refId": "H"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(up{container=\"mattermost-prometheus\"}) or vector(0)",
+          "hide": false,
+          "legendFormat": "Prometheus",
+          "range": true,
+          "refId": "I"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(up{container=\"mattermost-grafana\"}) or vector(0)",
+          "hide": false,
+          "legendFormat": "Grafana",
+          "range": true,
+          "refId": "J"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(up{container=\"mattermost-loki\"}) or vector(0)",
+          "hide": false,
+          "legendFormat": "Loki",
+          "range": true,
+          "refId": "K"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(up{container=\"mattermost-promtail\"}) or vector(0)",
+          "hide": false,
+          "legendFormat": "Promtail",
+          "range": true,
+          "refId": "L"
+        }
+      ],
+      "title": "Panel Title",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "description": "Links to common queries.",
+      "gridPos": {
+        "h": 5,
+        "w": 12,
+        "x": 0,
+        "y": 10
+      },
+      "id": 6,
+      "options": {
+        "code": {
+          "language": "plaintext",
+          "showLineNumbers": false,
+          "showMiniMap": false
+        },
+        "content": "* [Mattermost Error Logs](/explore?schemaVersion=1&panes=%7B%22z8g%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22mattermost%5C%22%7D%20%7C%20level%3D%5C%22error%5C%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1)\n* [Plugin Logs](/explore?schemaVersion=1&panes=%7B%22z8g%22:%7B%22datasource%22:%22P8E80F9AEF21F6940%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22expr%22:%22%7Bapp%3D%5C%22mattermost%5C%22%7D%20%7C%20json%20%7C%20plugin_id%20%21%3D%20%5C%22%5C%22%22,%22queryType%22:%22range%22,%22datasource%22:%7B%22type%22:%22loki%22,%22uid%22:%22P8E80F9AEF21F6940%22%7D,%22editorMode%22:%22code%22%7D%5D,%22range%22:%7B%22from%22:%22now-6h%22,%22to%22:%22now%22%7D%7D%7D&orgId=1)",
+        "mode": "markdown"
+      },
+      "pluginVersion": "10.4.2",
+      "title": "Links",
+      "type": "text"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "description": "Logs from the localhost Mattermost instance.",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 15
+      },
+      "id": 4,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "{app=\"mattermost\"}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Mattermost Logs",
+      "type": "logs"
+    },
+    {
+      "datasource": {
+        "type": "loki",
+        "uid": "P8E80F9AEF21F6940"
+      },
+      "description": "Logs from the Docker services running to support Mattermost.",
+      "gridPos": {
+        "h": 10,
+        "w": 24,
+        "x": 0,
+        "y": 25
+      },
+      "id": 5,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "loki",
+            "uid": "P8E80F9AEF21F6940"
+          },
+          "editorMode": "code",
+          "expr": "{job=\"docker\"}",
+          "queryType": "range",
+          "refId": "A"
+        }
+      ],
+      "title": "Docker Logs",
+      "type": "logs"
     }
   ],
-  "schemaVersion": 26,
+  "schemaVersion": 39,
-  "style": "dark",
   "tags": [],
   "templating": {
     "list": []
@@ -125,5 +434,6 @@
   "timezone": "browser",
   "title": "Home",
   "uid": "4yRNKfaGz",
-  "version": 1
+  "version": 1,
+  "weekStart": ""
 }

server/build/docker/grafana/provisioning/datasources/loki.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+  - name:  Loki
+    type: 'loki'
+    access: 'proxy'
+    orgId: 1
+    url: 'http://loki:3100'
+    isDefault: false
+    version: 1
+    editable: false

server/build/docker/prometheus.yml

@@ -6,3 +6,13 @@ scrape_configs:
   - job_name: 'mattermost'
     static_configs:
       - targets: ['host.docker.internal:8067']
+  - job_name: fmetrics_scrape
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'

server/build/docker/promtail/promtail-local-config.yaml

@@ -0,0 +1,46 @@
+server:
+  http_listen_port: 3180
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: mattermost
+    pipeline_stages:
+      - match:
+          selector: '{job="mattermost"}'
+          stages:
+            - json:
+                expressions:
+                  timestamp: timestamp
+                  level: level
+      - labels:
+          level:
+      - timestamp:
+          format: '2006-01-02 15:04:05.999 -07:00'
+          source: timestamp
+    static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: mattermost
+        app: mattermost
+        __path__: /logs/*.log
+  # https://ruanbekker.medium.com/logging-with-docker-promtail-and-grafana-loki-d920fd790ca8
+  - job_name: docker
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'
+    pipeline_stages:
+      - static_labels:
+          job: docker

server/config.mk

@@ -5,7 +5,7 @@
 # Enable services to be run in docker.
 #
 # Possible options: mysql, postgres, minio, inbucket, openldap, dejavu,
-# keycloak, elasticsearch, prometheus, and grafana.
+# keycloak, elasticsearch, prometheus, grafana, loki and promtail.
 #
 # Must be space separated names.
 #

server/docker-compose.makefile.yml

@@ -92,6 +92,20 @@ services:
     extends:
         file: build/docker-compose.common.yml
         service: grafana
+  loki:
+    container_name: mattermost-loki
+    ports:
+      - "3100:3100"
+    extends:
+        file: build/docker-compose.common.yml
+        service: loki
+  promtail:
+    container_name: mattermost-promtail
+    ports:
+      - "3180:3180"
+    extends:
+        file: build/docker-compose.common.yml
+        service: promtail
 
 networks:
   mm-test:

server/docker-compose.yaml

@@ -81,6 +81,20 @@ services:
     extends:
         file: build/docker-compose.common.yml
         service: grafana
+  loki:
+    container_name: mattermost-loki
+    ports:
+      - "3100:3100"
+    extends:
+        file: build/docker-compose.common.yml
+        service: loki
+  promtail:
+    container_name: mattermost-promtail
+    ports:
+      - "3180:3180"
+    extends:
+        file: build/docker-compose.common.yml
+        service: promtail
 
   start_dependencies:
     image: mattermost/mattermost-wait-for-dep:latest
@@ -95,7 +109,9 @@ services:
       - elasticsearch
       - prometheus
       - grafana
-    command: postgres:5432 mysql:3306 minio:9000 inbucket:9001 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000
+      - loki
+      - promtail
+    command: postgres:5432 mysql:3306 minio:9000 inbucket:9001 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000 loki:3100 promtail:3180
 
   leader:
     build: