added alerting rules for grafana

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/.env.example

@@ -0,0 +1,7 @@
+# Grafana SMTP configuration for alert emails
+# Generate a Gmail App Password at: https://myaccount.google.com/apppasswords
+GMAIL_SMTP_USER=your-email@gmail.com
+GMAIL_APP_PASSWORD=CHANGE_ME
+
+# Alerting contact point email (must match your Grafana login)
+ALERT_EMAIL=your-email@gmail.com

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/docker-compose.yml

@@ -33,10 +33,20 @@ services:
     container_name: grafana
     restart: unless-stopped
     ports:
-      - "3000:3000"                    # keep for now
+      - "3000:3000"
+    env_file:
+      - .env
+    environment:
+      - GF_SMTP_ENABLED=true
+      - GF_SMTP_HOST=smtp.gmail.com:587
+      - GF_SMTP_USER=${GMAIL_SMTP_USER}
+      - GF_SMTP_PASSWORD=${GMAIL_APP_PASSWORD}
+      - GF_SMTP_FROM_ADDRESS=${GMAIL_SMTP_USER}
+      - GF_SMTP_FROM_NAME=Grafana Alerts
     volumes:
       - ./grafana/data:/var/lib/grafana
       - ./grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
+      - ./grafana/provisioning/alerting:/etc/grafana/provisioning/alerting:ro
     networks:
       - monitoring
       - web_net

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/grafana/provisioning/alerting/contactpoints.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+contactPoints:
+  - orgId: 1
+    name: gmail-alerts
+    receivers:
+      - uid: gmail-email-receiver
+        type: email
+        settings:
+          addresses: YOUR_EMAIL@example.com
+          singleEmail: true

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/grafana/provisioning/alerting/policies.yaml

@@ -0,0 +1,8 @@
+apiVersion: 1
+policies:
+  - orgId: 1
+    receiver: gmail-alerts
+    group_by: ['alertname', 'instance']
+    group_wait: 30s
+    group_interval: 5m
+    repeat_interval: 4h

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/grafana/provisioning/alerting/rules.yaml

@@ -0,0 +1,93 @@
+apiVersion: 1
+groups:
+  - orgId: 1
+    name: infra-alerts
+    folder: Infra
+    interval: 1m
+    rules:
+      - uid: node-down
+        title: Node exporter target down
+        condition: C
+        data:
+          - refId: A
+            datasourceUid: efck3im1w4pvkf
+            relativeTimeRange:
+              from: 300
+              to: 0
+            model:
+              expr: up{job="node_exporter"}
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator:
+                    type: lt
+                    params: [1]
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Node exporter target is down"
+
+      - uid: high-cpu
+        title: High CPU usage
+        condition: C
+        data:
+          - refId: A
+            datasourceUid: efck3im1w4pvkf
+            relativeTimeRange:
+              from: 300
+              to: 0
+            model:
+              expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)
+              instant: true
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [85]
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "CPU usage above 85% on {{ $labels.instance }}"
+
+      - uid: loki-error-spike
+        title: Error log rate spike
+        condition: C
+        data:
+          - refId: A
+            datasourceUid: P8E80F9AEF21F6940
+            relativeTimeRange:
+              from: 300
+              to: 0
+            model:
+              expr: 'sum(count_over_time({job=~".+"} |= "error" [5m])) or vector(0)'
+              instant: true
+          - refId: B
+            datasourceUid: __expr__
+            model:
+              type: reduce
+              expression: A
+              reducer: last
+          - refId: C
+            datasourceUid: __expr__
+            model:
+              type: threshold
+              expression: B
+              conditions:
+                - evaluator:
+                    type: gt
+                    params: [20]
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Error log rate is elevated"

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/grafana/provisioning/datasources/loki.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+datasources:
+  - name: Loki
+    type: loki
+    uid: P8E80F9AEF21F6940
+    access: proxy
+    url: http://loki:3100
+    isDefault: false
+    editable: false

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+datasources:
+  - name: prometheus
+    type: prometheus
+    uid: efck3im1w4pvkf
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false