From d5659adbd72a6dda55b8b72bd268c1784ecf6a1c Mon Sep 17 00:00:00 2001 From: Jesse Hallam Date: Tue, 19 Jan 2021 22:28:22 -0400 Subject: [PATCH] docker: prometheus and grafana for development (#16649) --- Makefile | 9 + build/docker-compose-generator/main.go | 2 + build/docker-compose.common.yml | 14 + build/docker-compose.yml | 12 +- build/docker/grafana/dashboards/README.md | 15 + build/docker/grafana/dashboards/home.json | 129 + ...ttermost-performance-kpi-metrics_rev2.json | 1460 ++++++++++ ...ormance-monitoring-bonus-metrics_rev2.json | 2387 +++++++++++++++++ ...attermost-performance-monitoring_rev2.json | 1571 +++++++++++ build/docker/grafana/grafana.ini | 9 + .../provisioning/dashboards/mattermost.yaml | 14 + .../provisioning/datasources/prometheus.yml | 11 + build/docker/prometheus-linux.yml | 8 + build/docker/prometheus.yml | 8 + config.mk | 2 +- docker-compose.makefile.yml | 16 + docker-compose.yaml | 18 +- 17 files changed, 5682 insertions(+), 3 deletions(-) create mode 100644 build/docker/grafana/dashboards/README.md create mode 100644 build/docker/grafana/dashboards/home.json create mode 100644 build/docker/grafana/dashboards/mattermost/mattermost-performance-kpi-metrics_rev2.json create mode 100644 build/docker/grafana/dashboards/mattermost/mattermost-performance-monitoring-bonus-metrics_rev2.json create mode 100644 build/docker/grafana/dashboards/mattermost/mattermost-performance-monitoring_rev2.json create mode 100644 build/docker/grafana/grafana.ini create mode 100644 build/docker/grafana/provisioning/dashboards/mattermost.yaml create mode 100644 build/docker/grafana/provisioning/datasources/prometheus.yml create mode 100644 build/docker/prometheus-linux.yml create mode 100644 build/docker/prometheus.yml diff --git a/Makefile b/Makefile index b5c498af30..941958e84b 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,15 @@ else PLATFORM := $(shell uname) endif +# Set an environment variable on Linux used to resolve `docker.host.internal` inconsistencies with +# docker. This can be reworked once https://github.com/docker/for-linux/issues/264 is resolved +# satisfactorily. +ifeq ($(PLATFORM),Linux) + export IS_LINUX = -linux +else + export IS_LINUX = +endif + IS_CI ?= false # Build Flags BUILD_NUMBER ?= $(BUILD_NUMBER:) diff --git a/build/docker-compose-generator/main.go b/build/docker-compose-generator/main.go index 3422ae3ded..4533d1d2d1 100644 --- a/build/docker-compose-generator/main.go +++ b/build/docker-compose-generator/main.go @@ -33,6 +33,8 @@ func main() { "elasticsearch": 9200, "dejavu": 1358, "keycloak": 8080, + "prometheus": 9090, + "grafana": 3000, } command := []string{} for _, arg := range os.Args[1:] { diff --git a/build/docker-compose.common.yml b/build/docker-compose.common.yml index d23b6e4f6e..d7f8a16048 100644 --- a/build/docker-compose.common.yml +++ b/build/docker-compose.common.yml @@ -86,3 +86,17 @@ services: - mm-test volumes: - "./docker/keycloak:/setup" + prometheus: + image: "prom/prometheus" + volumes: + - "./docker/prometheus${IS_LINUX}.yml:/etc/prometheus/prometheus.yml" + networks: + - mm-test + grafana: + image: "grafana/grafana" + volumes: + - "./docker/grafana/grafana.ini:/etc/grafana/grafana.ini" + - "./docker/grafana/provisioning:/etc/grafana/provisioning" + - "./docker/grafana/dashboards:/var/lib/grafana/dashboards" + networks: + - mm-test diff --git a/build/docker-compose.yml b/build/docker-compose.yml index 5756a005f8..2065b03aa2 100644 --- a/build/docker-compose.yml +++ b/build/docker-compose.yml @@ -36,6 +36,14 @@ services: extends: file: docker-compose.common.yml service: keycloak + prometheus: + extends: + file: docker-compose.common.yml + service: prometheus + grafana: + extends: + file: docker-compose.common.yml + service: grafana start_dependencies: image: mattermost/mattermost-wait-for-dep:latest @@ -48,7 +56,9 @@ services: - inbucket - openldap - elasticsearch - command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 + - prometheus + - grafana + command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000 networks: mm-test: diff --git a/build/docker/grafana/dashboards/README.md b/build/docker/grafana/dashboards/README.md new file mode 100644 index 0000000000..d2fe7de390 --- /dev/null +++ b/build/docker/grafana/dashboards/README.md @@ -0,0 +1,15 @@ +# Developer Dashboards + +This folder contains a modified form of the following Grafana dashboards: +* [Mattermost Performance Monitoring](https://grafana.com/grafana/dashboards/2542) +* [Mattermost Performance Monitoring (Bonus Metrics)](https://grafana.com/grafana/dashboards/2545) +* [Mattermost Performance KPI Metrics](https://grafana.com/grafana/dashboards/2539) + +The dashboards are modified from the version available on grafana.com since [Grafana doesn't currently support variables](https://github.com/grafana/grafana/issues/10786) (i.e. `${DS_MATTERMOST}`) and has no way of binding the datasource with the dashboards at the time of provisioning. Instead of falling back to the REST API to effect these changes, the following in-place changes were made to the exported dashboards available above: +* Remove the top-level `__inputs` key +* Remove the top-level `__requires` key +* Replace all instances of `${DS_MATTERMOST}` with `Prometheus`, matching the name of the provisioned datasource. + +Upon using the dashboards within Grafana however, it immediately adds various missing fields, presumably due to some internal migration. This results in a spurious prompt to "save" the dashboard. To avoid confusion, these changes were subsequently exported and written back into the provisioned dashboards above. + +This entire process above will need to be repeated in the event newer versions of these dashboards are published. diff --git a/build/docker/grafana/dashboards/home.json b/build/docker/grafana/dashboards/home.json new file mode 100644 index 0000000000..70e636612e --- /dev/null +++ b/build/docker/grafana/dashboards/home.json @@ -0,0 +1,129 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "title": "", + "type": "welcome" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 3 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "query": "Mattermost", + "recent": false, + "search": true, + "starred": false, + "tags": [], + "title": "Dashboards", + "type": "dashlist" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 4, + "links": [], + "options": { + "feedUrl": "https://grafana.com/blog/news.xml" + }, + "title": "Latest from the blog", + "type": "news" + } + ], + "schemaVersion": 26, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Home", + "uid": "4yRNKfaGz", + "version": 1 +} diff --git a/build/docker/grafana/dashboards/mattermost/mattermost-performance-kpi-metrics_rev2.json b/build/docker/grafana/dashboards/mattermost/mattermost-performance-kpi-metrics_rev2.json new file mode 100644 index 0000000000..6efd3f0c45 --- /dev/null +++ b/build/docker/grafana/dashboards/mattermost/mattermost-performance-kpi-metrics_rev2.json @@ -0,0 +1,1460 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 2539, + "graphTooltip": 0, + "id": null, + "iteration": 1532610781587, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " %", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "max(irate(mattermost_process_cpu_seconds_total{instance=~\"$server\"}[1m])* 100)", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "", + "metric": "mattermost_process_cpu_seconds_total", + "refId": "A", + "step": 5 + } + ], + "thresholds": "50,80", + "title": "Max CPU utilization rate (%)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(mattermost_post_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Number of messages / min", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(mattermost_http_errors_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "1m", + "intervalFactor": 1, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Number of API errors / min", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 1000, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 0 + }, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(mattermost_http_websockets_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "refId": "A", + "step": 5 + } + ], + "thresholds": "500, 800", + "title": "Number of connected devices", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 0 + }, + "id": 17, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(mattermost_db_master_connections_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "refId": "A", + "step": 5 + } + ], + "thresholds": "50, 80", + "title": "Number of master DB conns", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 1000, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": " MB", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(go_memstats_alloc_bytes{instance=~\"$server\"})/1024/1024", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "refId": "A", + "step": 5 + } + ], + "thresholds": "500, 800", + "title": "Server memory usage (in MB)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_post_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_post_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_post_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_post_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of Messages per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 5000 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C", + "1m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "name": "Number of API Errors per Second alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_http_errors_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_http_errors_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_http_errors_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_http_errors_total", + "refId": "B", + "step": 5 + }, + { + "expr": "irate(mattermost_http_errors_total[1m])", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "C" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 5000 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Number of API Errors per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 1000 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "1m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "name": "Mean API Request Time (in milliseconds) alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1000.0 * rate(mattermost_http_request_duration_seconds_sum{instance=~\"$server\"}[1m]) / rate(mattermost_http_request_duration_seconds_count{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_http_request_duration_seconds_sum", + "refId": "A", + "step": 5 + }, + { + "expr": "1000.0 * rate(mattermost_http_request_duration_seconds_sum[1m]) / rate(mattermost_http_request_duration_seconds_count[1m])", + "format": "time_series", + "hide": true, + "intervalFactor": 1, + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 1000 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Mean API Request Time (in milliseconds)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds (ms)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_db_master_connections_total{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_db_master_connections_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_db_master_connections_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_db_master_connections_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of Connections to Master Database", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1000.0 * rate(mattermost_cluster_cluster_request_duration_seconds_sum{instance=~\"$server\"}[1m]) / rate(mattermost_cluster_cluster_request_duration_seconds_count{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_cluster_cluster_request_duration_seconds_sum", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Mean Cluster Request Time (in milliseconds)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds (ms)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_db_read_replica_connections_total{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{job}}", + "metric": "mattermost_db_read_replica_connections_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_db_read_replica_connections_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_db_read_replica_connections_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of Connections to Read Replica Databases", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 25 + }, + "id": 22, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_process_cpu_seconds_total{instance=~\"$server\"}[1m])* 100", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_process_cpu_seconds_total", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Utilization Rate (%)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Utilization Rate (%)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 25 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "go_goroutines", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(go_goroutines{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "go_goroutines", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Number of Go Routines", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "mattermost", + "mattermost-perf" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": {}, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "server", + "multi": true, + "name": "server", + "options": [], + "query": "label_values(instance)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Mattermost Performance KPI Metrics", + "uid": "000000010", + "version": 41 +} diff --git a/build/docker/grafana/dashboards/mattermost/mattermost-performance-monitoring-bonus-metrics_rev2.json b/build/docker/grafana/dashboards/mattermost/mattermost-performance-monitoring-bonus-metrics_rev2.json new file mode 100644 index 0000000000..27fd1032ad --- /dev/null +++ b/build/docker/grafana/dashboards/mattermost/mattermost-performance-monitoring-bonus-metrics_rev2.json @@ -0,0 +1,2387 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 2545, + "graphTooltip": 0, + "iteration": 1610119732885, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_login_logins_total{instance=~\"$server\"}/(mattermost_login_logins_total{instance=~\"$server\"}+mattermost_login_logins_fail_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_login_logins_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_login_logins_total{instance=~\"$server\"})/(sum(mattermost_login_logins_total{instance=~\"$server\"})+sum(mattermost_login_logins_fail_total{instance=~\"$server\"}))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "All", + "metric": "mattermost_login_logins_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Successful Logins / Number of Login Attempts", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Login Success Rate", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_login_logins_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "1s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_login_logins_total", + "refId": "A", + "step": 2 + }, + { + "expr": "sum(irate(mattermost_login_logins_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "1s", + "intervalFactor": 1, + "legendFormat": "Total", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Successful Logins per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_post_emails_sent_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_post_broadcasts_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_post_emails_sent_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_post_broadcasts_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Emails Sent per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_post_pushes_sent_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_post_pushes_sent_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Mobile Push Notifications Sent per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_post_file_attachments_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_post_file_attachments_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_post_file_attachments_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_post_file_attachments_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of File Attachments per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_search_posts_searches_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + }, + { + "expr": "sum(irate(mattermost_search_posts_searches_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Search Queries for Posts per Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 53 + }, + "hiddenSeries": false, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1000.0 * rate(mattermost_search_posts_searches_duration_seconds_sum{instance=~\"$server\"}[1m]) / rate(mattermost_search_posts_searches_duration_seconds_count{instance=~\"$server\"}[1m])", + "format": "time_series", + "instant": false, + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mean Search Query Time for Posts (in milliseconds)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds (ms)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 61 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_websocket_event_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{type}}", + "metric": "mattermost_websocket_event_total", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Websocket Events per Second", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 73 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_websocket_broadcasts_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "metric": "mattermost_websocket_broadcasts_total", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Websocket Broadcasts per Second", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 85 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_http_requests_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_http_requests_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_http_requests_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_http_requests_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of API Requests per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 94 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_cluster_cluster_requests_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_cluster_cluster_requests_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_cluster_cluster_requests_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_cluster_cluster_requests_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Cluster Requests per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 104 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_cluster_cluster_event_type_totals{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "metric": "mattermost_websocket_broadcasts_total", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Cluster Events per Second", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 114 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_gc_duration_seconds{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}} {{quantile}}", + "metric": "go_gc_duration_seconds", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Garbage Collection Duration by Quantile (in seconds)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Seconds", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 123 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_heap_objects{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "go_memstats_heap_objects", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(go_memstats_heap_objects{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "go_memstats_heap_objects", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Objects on the Heap", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 132 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "mattermost_cache_etag_hit_total{instance=~\"$server\"}/(mattermost_cache_etag_miss_total{instance=~\"$server\"} + mattermost_cache_etag_hit_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "{{route}}", + "metric": "mattermost_cache_etag_hit_total", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Etag Hit Rate", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Hit Rate", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 141 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_cache_mem_hit_total{instance=~\"$server\"}/(mattermost_cache_mem_miss_total{instance=~\"$server\"} + mattermost_cache_mem_hit_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "metric": "mattermost_cache_mem_hit_total", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Cache Hit Rate", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Hit Rate", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 150 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_cache_etag_hit_total{instance=~\"$server\"}/(mattermost_cache_etag_miss_total{instance=~\"$server\"} + mattermost_cache_etag_hit_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "{{route}}", + "metric": "mattermost_cache_etag_hit_total", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mean Etag Hit Rate (for each server)", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Hit Rate", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 159 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_cache_mem_hit_total{instance=~\"$server\"}/(mattermost_cache_mem_miss_total{instance=~\"$server\"} + mattermost_cache_mem_hit_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "30s", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "metric": "mattermost_cache_etag_hit_total", + "refId": "A", + "step": 30 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mean Memory Cache Hit Rate (for each server)", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "series", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Hit Rate", + "logBase": 1, + "max": "1", + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 171 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_process_resident_memory_bytes{instance=~\"$server\"}/1024/1024", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_process_resident_memory_bytes", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_process_resident_memory_bytes{instance=~\"$server\"}/1024/1024)", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_process_resident_memory_bytes", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Resident Memory Size (in MB)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "MB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 180 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_process_virtual_memory_bytes{instance=~\"$server\"}/1024/1024", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_process_virtual_memory_bytes", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_process_virtual_memory_bytes{instance=~\"$server\"}/1024/1024)", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_process_virtual_memory_bytes", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Virtual Memory Size (in MB)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "MB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 189 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_process_open_fds{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_process_open_fds", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_process_open_fds{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_process_open_fds", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Open File Descriptors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "mattermost", + "mattermost-perf" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "host.docker.internal:8067", + "value": "host.docker.internal:8067" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "server", + "multi": true, + "name": "server", + "options": [], + "query": "label_values(instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Mattermost Performance Monitoring (Bonus Metrics)", + "uid": "000000013", + "version": 1 +} diff --git a/build/docker/grafana/dashboards/mattermost/mattermost-performance-monitoring_rev2.json b/build/docker/grafana/dashboards/mattermost/mattermost-performance-monitoring_rev2.json new file mode 100644 index 0000000000..27ca40e54e --- /dev/null +++ b/build/docker/grafana/dashboards/mattermost/mattermost-performance-monitoring_rev2.json @@ -0,0 +1,1571 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": 2542, + "graphTooltip": 0, + "iteration": 1610119520865, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_post_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_post_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_post_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_post_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Messages per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_http_errors_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_http_errors_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_http_errors_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_http_errors_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of API Errors per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 36000 + ], + "type": "gt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "10s", + "now" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "#grafana-alert", + "name": "Number of Connected Devices (WebSocket Connections) alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_http_websockets_total{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_http_websockets_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_http_websockets_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt", + "value": 36000 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Connected Devices (WebSocket Connections)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "hiddenSeries": false, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_db_master_connections_total{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_db_master_connections_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_db_master_connections_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_db_master_connections_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Connections to Master Database", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 36 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_db_read_replica_connections_total{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_db_read_replica_connections_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_db_read_replica_connections_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_db_read_replica_connections_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Connections to Read Replica Databases", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 45 + }, + "hiddenSeries": false, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "mattermost_db_search_replica_connections_total{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_db_search_replica_connections_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(mattermost_db_search_replica_connections_total{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_db_search_replica_connections_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Connections to Search Replica Databases", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 54 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(mattermost_post_webhooks_total[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Webhooks", + "metric": "mattermost_post_webhooks_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_post_total[1m])*10)-sum(irate(mattermost_post_webhooks_total[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Users", + "metric": "mattermost_post_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Messages Per Second by Type", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 63 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_post_broadcasts_total{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_post_broadcasts_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(irate(mattermost_post_broadcasts_total{instance=~\"$server\"}[1m]))", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "mattermost_post_broadcasts_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Broadcasts per Second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 72 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(mattermost_process_cpu_seconds_total{instance=~\"$server\"}[1m])* 100", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU Utilization Rate (%)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Utilization Rate (%)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 81 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1000.0 * rate(mattermost_http_request_duration_seconds_sum{instance=~\"$server\"}[1m]) / rate(mattermost_http_request_duration_seconds_count{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "mattermost_http_request_duration_seconds_sum", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mean API Request Time (in milliseconds)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds (ms)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 90 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1000.0 * rate(mattermost_cluster_cluster_request_duration_seconds_sum{instance=~\"$server\"}[1m]) / rate(mattermost_cluster_cluster_request_duration_seconds_count{instance=~\"$server\"}[1m])", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Mean Cluster Request Time (in milliseconds)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Milliseconds (ms)", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 99 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{instance=~\"$server\"}", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "go_goroutines", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(go_goroutines{instance=~\"$server\"})", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "go_goroutines", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of Go Routines", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Count", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 108 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.6", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_alloc_bytes{instance=~\"$server\"}/1024/1024", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "go_memstats_alloc_bytes_total", + "refId": "A", + "step": 5 + }, + { + "expr": "sum(go_memstats_alloc_bytes{instance=~\"$server\"})/1024/1024", + "format": "time_series", + "interval": "5s", + "intervalFactor": 1, + "legendFormat": "Total", + "metric": "go_memstats_alloc_bytes_total", + "refId": "B", + "step": 5 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory Usage (in MB)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "MB", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "mattermost", + "mattermost-perf" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": false, + "text": "host.docker.internal:8067", + "value": "host.docker.internal:8067" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "server", + "multi": true, + "name": "server", + "options": [], + "query": "label_values(instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Mattermost Performance Monitoring", + "uid": "000000011", + "version": 1 +} diff --git a/build/docker/grafana/grafana.ini b/build/docker/grafana/grafana.ini new file mode 100644 index 0000000000..88701d64a4 --- /dev/null +++ b/build/docker/grafana/grafana.ini @@ -0,0 +1,9 @@ +[auth] +disable_login_form = false + +[auth.anonymous] +enabled = true +org_role = Editor + +[dashboards] +default_home_dashboard_path = /var/lib/grafana/dashboards/home.json diff --git a/build/docker/grafana/provisioning/dashboards/mattermost.yaml b/build/docker/grafana/provisioning/dashboards/mattermost.yaml new file mode 100644 index 0000000000..b464cbbea4 --- /dev/null +++ b/build/docker/grafana/provisioning/dashboards/mattermost.yaml @@ -0,0 +1,14 @@ +apiVersion: 1 + +providers: + - name: 'mattermost' + orgId: 1 + folder: 'mattermost' + type: file + disableDeletion: true + updateIntervalSeconds: 300 + allowUiUpdates: true + options: + # path to dashboard files on disk. Required when using the 'file' type + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true diff --git a/build/docker/grafana/provisioning/datasources/prometheus.yml b/build/docker/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000000..bdce542f27 --- /dev/null +++ b/build/docker/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: 'prometheus' + access: 'proxy' + orgId: 1 + url: 'http://prometheus:9090' + isDefault: true + version: 1 + editable: false diff --git a/build/docker/prometheus-linux.yml b/build/docker/prometheus-linux.yml new file mode 100644 index 0000000000..7a575ca959 --- /dev/null +++ b/build/docker/prometheus-linux.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 5s + evaluation_interval: 60s + +scrape_configs: + - job_name: 'mattermost' + static_configs: + - targets: ['172.17.0.1:8067'] diff --git a/build/docker/prometheus.yml b/build/docker/prometheus.yml new file mode 100644 index 0000000000..58f8858c16 --- /dev/null +++ b/build/docker/prometheus.yml @@ -0,0 +1,8 @@ +global: + scrape_interval: 5s + evaluation_interval: 60s + +scrape_configs: + - job_name: 'mattermost' + static_configs: + - targets: ['host.docker.internal:8067'] diff --git a/config.mk b/config.mk index f68f9d9c16..f54403b3a5 100644 --- a/config.mk +++ b/config.mk @@ -5,7 +5,7 @@ # Enable services to be run in docker. # # Possible options: mysql, postgres, minio, inbucket, openldap, dejavu, -# keycloak and elasticsearch +# keycloak, elasticsearch, prometheus, and grafana. # # Must be space separated names. # diff --git a/docker-compose.makefile.yml b/docker-compose.makefile.yml index dbe62f77ff..c66a6867c4 100644 --- a/docker-compose.makefile.yml +++ b/docker-compose.makefile.yml @@ -68,6 +68,22 @@ services: extends: file: build/docker-compose.common.yml service: keycloak + prometheus: + restart: 'no' + container_name: mattermost-prometheus + ports: + - "9090:9090" + extends: + file: build/docker-compose.common.yml + service: prometheus + grafana: + restart: 'no' + container_name: mattermost-grafana + ports: + - "3000:3000" + extends: + file: build/docker-compose.common.yml + service: grafana networks: mm-test: diff --git a/docker-compose.yaml b/docker-compose.yaml index ecebf1f63d..cccf264c46 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -60,6 +60,20 @@ services: extends: file: build/docker-compose.common.yml service: keycloak + prometheus: + container_name: mattermost-prometheus + ports: + - "9090:9090" + extends: + file: build/docker-compose.common.yml + service: prometheus + grafana: + container_name: mattermost-grafana + ports: + - "3000:3000" + extends: + file: build/docker-compose.common.yml + service: grafana start_dependencies: image: mattermost/mattermost-wait-for-dep:latest @@ -72,7 +86,9 @@ services: - inbucket - openldap - elasticsearch - command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 + - prometheus + - grafana + command: postgres:5432 mysql:3306 minio:9000 inbucket:10080 openldap:389 elasticsearch:9200 prometheus:9090 grafana:3000 leader: build: