From 36ad36848b23ec9c14da4cd9a3b8e07bafefef83 Mon Sep 17 00:00:00 2001 From: rezayaghobi-dev Date: Sun, 26 Jul 2026 09:10:53 +0330 Subject: [PATCH] added elk stack and updated the docs accordingly --- README.md | 19 ++++++-- docker-compose/ELK/.env.example | 6 +++ docker-compose/ELK/docker-compose.yml | 68 +++++++++++++++++++++++++++ docs/ElasticStack.md | 60 +++++++++++++++++++++++ docs/Home.md | 3 ++ 5 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 docker-compose/ELK/.env.example create mode 100644 docker-compose/ELK/docker-compose.yml create mode 100644 docs/ElasticStack.md diff --git a/README.md b/README.md index 690f416..cf8e3f1 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,14 @@ This is a real, running homelab — not a demo. It's deployed on a home Ubuntu s | **Semaphore** | `docker-compose/semaphore/` | Self-hosted Ansible UI (alternative to AWX) — see [Semaphore Guide](docs/Semaphore.md) | | **Ansible playbooks** | `ansible/` | Shared update role (pull → recreate → health-check → prune) reused across every stack, run via Semaphore — see [Ansible Playbooks](docs/AnsiblePlaybooks.md) | +## Learning labs (not always-on) + +Not everything in this repo runs permanently. Some folders are kept purely as reference from one-off learning exercises for my DevOps cert coursework — the compose files work, but they're not part of the day-to-day stack above. + +| Lab | Folder | Purpose | +|---|---|---| +| **Elastic Stack (ELK)** | `docker-compose/elk/` | Elasticsearch + Kibana, security-enabled, sized for this hardware — a modern-workflow (Elastic Agent/Fleet) alternative to an older Filebeat/Metricbeat-based course reference. Not run continuously since Prometheus/Grafana already cover monitoring here — see [Elastic Stack (ELK)](docs/ElasticStack.md) | + ## Repository structure ``` @@ -72,9 +80,12 @@ homelab/ │ │ └── docker-compose.yaml │ ├── nginx-proxy-manager/ │ │ └── docker-compose.yaml -│ └── semaphore/ +│ ├── semaphore/ +│ │ ├── docker-compose.yml +│ │ └── README.md +│ └── elk/ │ ├── docker-compose.yml -│ └── README.md +│ └── .env.example ├── ansible/ │ └── playbooks/ │ ├── roles/ @@ -110,6 +121,7 @@ homelab/ │ ├── Semaphore.md │ ├── DockScope.md │ ├── AnsiblePlaybooks.md +│ ├── ElasticStack.md │ └── images/ └── .gitignore ``` @@ -126,7 +138,7 @@ docker compose up -d ## Updating a service -Rather than pulling and recreating manually, every stack has a matching Ansible playbook that pulls the latest images, recreates containers, verifies they're healthy, and prunes unused images — run through Semaphore's web UI. See [Ansible Playbooks](docs/AnsiblePlaybooks.md) for details on how the shared role works. +Rather than pulling and recreating manually, every always-on stack has a matching Ansible playbook that pulls the latest images, recreates containers, verifies they're healthy, and prunes unused images — run through Semaphore's web UI. See [Ansible Playbooks](docs/AnsiblePlaybooks.md) for details on how the shared role works. ## Security notes @@ -143,6 +155,7 @@ Rather than pulling and recreating manually, every stack has a matching Ansible - Free-tier cloud AI APIs enforce regional access restrictions automatically — worth checking a provider's terms before building around "free tier" as an assumption. See [AI Coding Agent](docs/AI-Agent.md). - Wildcard TLS certs (`*.home`) are rejected by browsers on single-label local domains — certificates for local reverse proxies need explicit hostnames listed instead. See [Reverse Proxy](docs/Reverse-Proxy.md). - Grouping all Compose stacks under one `docker-compose/` folder (rather than flat at repo root) kept the structure readable once Ansible playbooks were added as a sibling — a flat repo mixing deployment configs and automation code got confusing fast. +- Newer Docker images can silently raise the minimum CPU generation required (base-OS changes, not anything in the compose file itself) — hit this running Elasticsearch 9.x on older hardware. See [Elastic Stack (ELK)](docs/ElasticStack.md). ## Roadmap diff --git a/docker-compose/ELK/.env.example b/docker-compose/ELK/.env.example new file mode 100644 index 0000000..df26d89 --- /dev/null +++ b/docker-compose/ELK/.env.example @@ -0,0 +1,6 @@ +# Rename to .env before running docker compose up. +# Generate your own values rather than reusing these if you ever bring this back up. + +ELASTIC_PASSWORD=sSBdnyE9WupLYHyZ +KIBANA_PASSWORD=49c1VjcqgfDGNRFA +KIBANA_ENCRYPTION_KEY=625fef0e09b89fc810269149b5584f17 diff --git a/docker-compose/ELK/docker-compose.yml b/docker-compose/ELK/docker-compose.yml new file mode 100644 index 0000000..4013af3 --- /dev/null +++ b/docker-compose/ELK/docker-compose.yml @@ -0,0 +1,68 @@ +services: + elasticsearch: + image: elasticsearch:8.19.19 + container_name: elasticsearch + environment: + - node.name=elasticsearch + - discovery.type=single-node + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=false + - xpack.security.transport.ssl.enabled=false + - xpack.ml.enabled=false + - ES_JAVA_OPTS=-Xms1g -Xmx1g + mem_limit: 1500m + cpus: 1.5 + volumes: + - es-data:/usr/share/elasticsearch/data + ports: + - "9200:9200" + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "curl -s -u elastic:${ELASTIC_PASSWORD} http://localhost:9200 | grep -q cluster_name"] + interval: 10s + timeout: 10s + retries: 30 + + kibana-setup: + image: elasticsearch:8.19.19 + container_name: kibana-setup + depends_on: + elasticsearch: + condition: service_healthy + restart: "no" + entrypoint: ["curl"] + command: + - "-s" + - "-u" + - "elastic:${ELASTIC_PASSWORD}" + - "-X" + - "POST" + - "http://elasticsearch:9200/_security/user/kibana_system/_password" + - "-H" + - "Content-Type: application/json" + - "-d" + - "{\"password\":\"${KIBANA_PASSWORD}\"}" + + kibana: + image: kibana:8.19.19 + container_name: kibana + depends_on: + kibana-setup: + condition: service_completed_successfully + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + - ELASTICSEARCH_USERNAME=kibana_system + - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD} + - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${KIBANA_ENCRYPTION_KEY} + - XPACK_SECURITY_ENCRYPTIONKEY=${KIBANA_ENCRYPTION_KEY} + - XPACK_REPORTING_ENCRYPTIONKEY=${KIBANA_ENCRYPTION_KEY} + - NODE_OPTIONS=--max-old-space-size=1024 + mem_limit: 1200m + cpus: 1 + ports: + - "5601:5601" + restart: unless-stopped + +volumes: + es-data: diff --git a/docs/ElasticStack.md b/docs/ElasticStack.md new file mode 100644 index 0000000..a250bf5 --- /dev/null +++ b/docs/ElasticStack.md @@ -0,0 +1,60 @@ +# Elastic Stack (ELK) + +A hands-on lab for the observability chapter of my DevOps cert — not a permanent part of this homelab's monitoring, since Prometheus + Grafana + cAdvisor + node-exporter already cover that role here. + +## Why + +My course's reference material for this chapter is a few years old — it walks through a basic ELK setup (Elasticsearch + Logstash + Kibana) with security disabled and metrics wired in by hand through separate Beats agents. That workflow has since been superseded: the current recommended approach centers on a single **Elastic Agent**, centrally managed through **Fleet** in Kibana, with pre-built **Integrations** replacing hand-configured Filebeat/Metricbeat setups. This lab was about running the current version of that workflow rather than the older one, purely to understand it for the cert — not to replace the monitoring stack already documented in [Monitoring Stack](Monitoring-Stack.md). + +## What actually got deployed + +Elasticsearch + Kibana, secured (not the "turn off X-Pack security" pattern from older tutorials), sized for this hardware. Fleet/Elastic Agent integrations were browsed (System, Docker) but not fully enrolled before the lab was torn down — see [Semaphore](Semaphore.md) and [Ansible Playbooks](AnsiblePlaybooks.md) for the pattern this homelab actually uses for ongoing service management; ELK wasn't kept running long enough to need either. + +```yaml +services: + elasticsearch: + image: elasticsearch:8.19.19 + environment: + - discovery.type=single-node + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=false + - xpack.ml.enabled=false + - ES_JAVA_OPTS=-Xms1g -Xmx1g + mem_limit: 1500m + + kibana: + image: kibana:8.19.19 + environment: + - NODE_OPTIONS=--max-old-space-size=1024 + mem_limit: 1200m +``` + +(Full compose file with the setup container that provisions the `kibana_system` password is in `docker-compose/elk/`.) + +## Hardware roadblocks — the actually useful part of this lab + +Two CPU-specific failures came up, both worth documenting since they're the kind of thing that doesn't show up until you try running current-generation software on older hardware: + +**1. `Fatal glibc error: CPU does not support x86-64-v2`** + +Elasticsearch 9.x's Docker image failed outright on this box's CPU (AMD Phenom II, 2009-era). This turned out to be a base-OS change on Elastic's side — their 9.x images switched base images in a way that assumes a CPU baseline (`x86-64-v2`: SSE4.2, CMPXCHG16B, POPCNT) that this CPU predates by about two years. Fix: pinned to the **8.19.x branch**, which still targets an older baseline and started fine. Still a current, actively patched release (supported through mid-2027) — not a step back to something outdated. + +**2. `Failure running machine learning native code`** + +Elasticsearch's ML module ships a native binary that also failed to run on this CPU. ML wasn't needed for this lab (it's for anomaly detection, unrelated to basic log/metric ingestion), so it was disabled outright with `xpack.ml.enabled=false` rather than chased further. + +**3. Kibana `JavaScript heap out of memory` crash loop** + +Not CPU-related this time — Kibana 8.19 initializes 172 plugins on startup, and Node.js's auto-sized heap (based on the container's memory limit) wasn't enough to get through that without crashing. Fixed by raising Kibana's container memory limit and explicitly setting `NODE_OPTIONS=--max-old-space-size=1024` rather than relying on Node's auto-detection. + +## Why it's not staying up + +Between Elasticsearch's heap and Kibana's heap, this pushed total memory usage close to the ceiling of what's actually free on this box alongside the rest of the stack (see [Architecture and Hardware](Architecture-and-Hardware.md)) — workable for a lab session, not something worth running 24/7 next to a monitoring stack that already does the job. The compose file stays in this repo as a reference for re-running the lab, not as an always-on service. + +## Lessons learned + +- "Modern ELK" today means Elastic Agent + Fleet integrations replacing individually-configured Beats — worth knowing conceptually even without a permanently running deployment to point at. +- Docker image base-OS changes can silently raise the minimum CPU generation required, independent of anything in your own compose file — this will likely resurface on other images down the line on this hardware, not just Elasticsearch. +- Default resource auto-detection (Node.js heap sizing, in this case) doesn't always get it right inside a memory-constrained container — explicit sizing beats trusting the default when things are already tight. + +[← Back to Home](Home.md) diff --git a/docs/Home.md b/docs/Home.md index 3022688..bc40d18 100644 --- a/docs/Home.md +++ b/docs/Home.md @@ -16,6 +16,7 @@ Documentation for a self-hosted homelab running entirely on Docker Compose. This - [Semaphore](Semaphore.md) - [DockScope](DockScope.md) - [Ansible Playbooks](AnsiblePlaybooks.md) +- [Elastic Stack (ELK) — Lab](ElasticStack.md) ## At a glance @@ -31,3 +32,5 @@ Documentation for a self-hosted homelab running entirely on Docker Compose. This | **AI Agent** | OpenHands + Ollama | This is a running system, not a demo — screenshots throughout this wiki are from the live dashboards. + +Not everything in this repo runs permanently — see [Elastic Stack (ELK) — Lab](ElasticStack.md) for a compose stack kept as a reference from a one-off learning exercise, not part of the always-on services above.