updated monitoring stack for k3s and added kube dashboards

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/alloy/config.alloy

@@ -0,0 +1,84 @@
+// Discover all running Docker containers
+discovery.docker "containers" {
+  host = "unix:///var/run/docker.sock"
+}
+
+// Extract a clean "container" label from the container name
+discovery.relabel "containers" {
+  targets = discovery.docker.containers.targets
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container"
+  }
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex         = "/(.*)"
+    target_label  = "container_name"
+  }
+
+  rule {
+    target_label = "instance"
+    replacement  = constants.hostname
+  }
+}
+
+// Tail logs from every discovered container
+loki.source.docker "containers" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.relabel.containers.output
+  forward_to = [loki.write.default.receiver]
+}
+
+// Ship to the local Loki instance
+loki.write "default" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}
+// Discover K3s pods via the Kubernetes API
+discovery.kubernetes "k3s_pods" {
+  role = "pod"
+  api_server = "https://192.168.100.6:6443"
+  bearer_token_file = "/etc/alloy/k3s-token"
+  tls_config {
+    insecure_skip_verify = true
+  }
+}
+
+// Label pod logs with namespace/pod/container for clean Grafana filtering
+discovery.relabel "k3s_pods" {
+  targets = discovery.kubernetes.k3s_pods.targets
+  rule {
+    source_labels = ["__meta_kubernetes_namespace"]
+    target_label  = "namespace"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_name"]
+    target_label  = "pod"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_container_name"]
+    target_label  = "container"
+  }
+  rule {
+    target_label = "cluster"
+    replacement  = "k3s-homeserver"
+  }
+}
+
+// Tail logs from every discovered K3s pod
+loki.source.kubernetes "k3s_pods" {
+  targets    = discovery.relabel.k3s_pods.output
+  forward_to = [loki.write.default.receiver]
+
+  client {
+    api_server = "https://192.168.100.6:6443"
+    bearer_token_file = "/etc/alloy/k3s-token"
+    tls_config {
+      insecure_skip_verify = true
+    }
+  }
+}

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/config.alloy

@@ -1,40 +0,0 @@
-// Discover all running Docker containers
-discovery.docker "containers" {
-  host = "unix:///var/run/docker.sock"
-}
-
-// Extract a clean "container" label from the container name
-discovery.relabel "containers" {
-  targets = discovery.docker.containers.targets
-
-  rule {
-    source_labels = ["__meta_docker_container_name"]
-    regex         = "/(.*)"
-    target_label  = "container"
-  }
-
-  rule {
-    source_labels = ["__meta_docker_container_name"]
-    regex         = "/(.*)"
-    target_label  = "container_name"
-  }
-
-  rule {
-    target_label = "instance"
-    replacement  = constants.hostname
-  }
-}
-
-// Tail logs from every discovered container
-loki.source.docker "containers" {
-  host       = "unix:///var/run/docker.sock"
-  targets    = discovery.relabel.containers.output
-  forward_to = [loki.write.default.receiver]
-}
-
-// Ship to the local Loki instance
-loki.write "default" {
-  endpoint {
-    url = "http://loki:3100/loki/api/v1/push"
-  }
-}

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/docker-compose.yml

@@ -5,15 +5,13 @@ services:
     restart: unless-stopped
     ports:
       - "9090:9090"
-
     volumes:
       - ./prometheus:/etc/prometheus:ro
       - ./data/prometheus:/prometheus
-
     command:
       - "--config.file=/etc/prometheus/prometheus.yml"
       - "--storage.tsdb.path=/prometheus"
-
+      - "--web.enable-lifecycle"
     networks:
       - monitoring
 
@@ -64,6 +62,7 @@ services:
     ports:
       - "3100:3100"
     volumes:
+      - ./loki/local-config.yaml:/etc/loki/local-config.yaml:ro    
       - ./data/loki:/loki
     mem_limit: 400m
     cpus: 1
@@ -83,6 +82,7 @@ services:
       - "12345:12345"
     volumes:
       - ./alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - ./alloy/k3s-token:/etc/alloy/k3s-token:ro
       - /var/run/docker.sock:/var/run/docker.sock:ro
       - ./data/alloy:/var/lib/alloy/data
     mem_limit: 300m

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/loki/local-config.yaml

@@ -0,0 +1,44 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /loki
+  storage:
+    filesystem:
+      chunks_directory: /loki/chunks
+      rules_directory: /loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+ruler:
+  alertmanager_url: http://localhost:9093
+limits_config:
+  reject_old_samples: false
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

docker-compose/monitoring-grafana-promethues-cadvisor-node-exporter/prometheus/prometheus.yml

@@ -21,3 +21,16 @@ scrape_configs:
   - job_name: 'gitlab-runner'
     static_configs:
       - targets: ['192.168.100.6:9252'] # Or your runner container name/IP
+
+  - job_name: 'kube-state-metrics'
+    static_configs:
+      - targets: ['192.168.100.6:30900']
+
+  - job_name: 'k3s-kubelet-cadvisor'
+    scheme: https
+    metrics_path: /metrics/cadvisor
+    tls_config:
+      insecure_skip_verify: true
+    bearer_token_file: /etc/prometheus/k3s-token
+    static_configs:
+      - targets: ['192.168.100.6:10250']