From aa6cd39b8c63dec1d0f6fbd1c786704bb10345d7 Mon Sep 17 00:00:00 2001 From: rezayaghobi-dev Date: Sat, 3 Oct 2026 19:47:36 +0330 Subject: [PATCH] Add monitoring stack: Prometheus, Alertmanager, Grafana --- monitoring/alertmanager/alertmanager.yml | 12 ++++ monitoring/compose.yml | 63 +++++++++++++++++++ monitoring/grafana/dashboards/webapp.json | 35 +++++++++++ .../provisioning/dashboards/dashboards.yml | 7 +++ .../provisioning/datasources/prometheus.yml | 8 +++ monitoring/prometheus/alert_rules.yml | 29 +++++++++ monitoring/prometheus/prometheus.yml | 21 +++++++ 7 files changed, 175 insertions(+) create mode 100644 monitoring/alertmanager/alertmanager.yml create mode 100644 monitoring/compose.yml create mode 100644 monitoring/grafana/dashboards/webapp.json create mode 100644 monitoring/grafana/provisioning/dashboards/dashboards.yml create mode 100644 monitoring/grafana/provisioning/datasources/prometheus.yml create mode 100644 monitoring/prometheus/alert_rules.yml create mode 100644 monitoring/prometheus/prometheus.yml diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml new file mode 100644 index 0000000..8da3391 --- /dev/null +++ b/monitoring/alertmanager/alertmanager.yml @@ -0,0 +1,12 @@ +route: + receiver: logger + group_by: ["alertname"] + group_wait: 10s + group_interval: 1m + repeat_interval: 1h + +receivers: + - name: logger + webhook_configs: + - url: http://alert-logger:8080/ + send_resolved: true \ No newline at end of file diff --git a/monitoring/compose.yml b/monitoring/compose.yml new file mode 100644 index 0000000..38ffece --- /dev/null +++ b/monitoring/compose.yml @@ -0,0 +1,63 @@ +services: + prometheus: + image: prom/prometheus:v2.53.0 + container_name: prometheus + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=7d + - --web.enable-lifecycle + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ./prometheus/alert_rules.yml:/etc/prometheus/alert_rules.yml:ro + - prometheus-data:/prometheus + ports: + - "9090:9090" + networks: + - monitoring-net + + alertmanager: + image: prom/alertmanager:v0.27.0 + container_name: alertmanager + restart: unless-stopped + command: + - --config.file=/etc/alertmanager/alertmanager.yml + volumes: + - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - alertmanager-data:/alertmanager + ports: + - "9093:9093" + networks: + - monitoring-net + + alert-logger: + image: mendhak/http-https-echo:latest + container_name: alert-logger + restart: unless-stopped + networks: + - monitoring-net + + grafana: + image: grafana/grafana:11.1.0 + container_name: grafana + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin} + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + ports: + - "3000:3000" + networks: + - monitoring-net + +volumes: + prometheus-data: + alertmanager-data: + grafana-data: + +networks: + monitoring-net: + external: true \ No newline at end of file diff --git a/monitoring/grafana/dashboards/webapp.json b/monitoring/grafana/dashboards/webapp.json new file mode 100644 index 0000000..ecc836e --- /dev/null +++ b/monitoring/grafana/dashboards/webapp.json @@ -0,0 +1,35 @@ +{ + "title": "Webapp Overview", + "uid": "webapp-overview", + "schemaVersion": 39, + "refresh": "10s", + "time": { "from": "now-15m", "to": "now" }, + "panels": [ + { + "id": 1, "type": "stat", "title": "Webapp up (1 = yes)", + "gridPos": { "x": 0, "y": 0, "w": 6, "h": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ { "refId": "A", "expr": "up{job=\"webapp\"}" } ] + }, + { + "id": 2, "type": "timeseries", "title": "Request rate by path (req/s)", + "gridPos": { "x": 6, "y": 0, "w": 18, "h": 6 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "targets": [ { "refId": "A", "expr": "sum by (path) (rate(http_requests_total[1m]))", "legendFormat": "{{path}}" } ] + }, + { + "id": 3, "type": "timeseries", "title": "5xx error ratio", + "gridPos": { "x": 0, "y": 6, "w": 12, "h": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "sum(rate(http_requests_total{status=~\"5..\"}[1m])) / sum(rate(http_requests_total[1m]))", "legendFormat": "5xx ratio" } ] + }, + { + "id": 4, "type": "timeseries", "title": "p95 latency (s)", + "gridPos": { "x": 12, "y": 6, "w": 12, "h": 8 }, + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, + "targets": [ { "refId": "A", "expr": "histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m])))", "legendFormat": "p95" } ] + } + ] +} \ No newline at end of file diff --git a/monitoring/grafana/provisioning/dashboards/dashboards.yml b/monitoring/grafana/provisioning/dashboards/dashboards.yml new file mode 100644 index 0000000..81b8c37 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboards.yml @@ -0,0 +1,7 @@ +apiVersion: 1 +providers: + - name: default + folder: "" + type: file + options: + path: /var/lib/grafana/dashboards \ No newline at end of file diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..3c1e815 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,8 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true \ No newline at end of file diff --git a/monitoring/prometheus/alert_rules.yml b/monitoring/prometheus/alert_rules.yml new file mode 100644 index 0000000..915f4e0 --- /dev/null +++ b/monitoring/prometheus/alert_rules.yml @@ -0,0 +1,29 @@ +groups: + - name: webapp + rules: + - alert: WebappDown + expr: up{job="webapp"} == 0 + for: 30s + labels: + severity: critical + annotations: + summary: "Webapp is down" + description: "Prometheus cannot scrape the webapp for more than 30 seconds." + + - alert: HighErrorRate + expr: sum(rate(http_requests_total{status=~"5.."}[1m])) / sum(rate(http_requests_total[1m])) > 0.05 + for: 1m + labels: + severity: warning + annotations: + summary: "More than 5% of requests are failing" + description: "The 5xx ratio has been above 5% for 1 minute." + + - alert: HighLatencyP95 + expr: histogram_quantile(0.95, sum by (le) (rate(http_request_duration_seconds_bucket[5m]))) > 0.5 + for: 2m + labels: + severity: warning + annotations: + summary: "p95 latency above 500ms" + description: "95% of requests take longer than 0.5s." \ No newline at end of file diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..efdd457 --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,21 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +alerting: + alertmanagers: + - static_configs: + - targets: ["alertmanager:9093"] + +rule_files: + - /etc/prometheus/alert_rules.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + - job_name: webapp + metrics_path: /metrics + static_configs: + - targets: ["webapp:8000"] \ No newline at end of file