Merge pull request #32 from beatz174-bit/codex/create-infrastructure-diagrams-for-docker/traefik-xvw0xz

Add README and architecture overview for Docker + Traefik homelab stack

.gitignore

@@ -21,6 +21,9 @@ apps/searxng/*
 venv/
 core/authelia/users_database.yml
 monitoring/influxdb/*
+!monitoring/influxdb/docker-compose.yml
 secrets/*
 !secrets/.env.secrets.example
 !.env.example
+core/traefik/certs/*
+!core/traefik/certs/.gitkeep

README.md

@@ -0,0 +1,46 @@
+# Docker + Traefik Homelab Stack
+
+This repository defines a multi-compose Docker environment with Traefik as ingress, app workloads, and a monitoring/alerting plane.
+
+## High-Level Architecture
+
+```mermaid
+flowchart TB
+  Internet((Internet Clients)) -->|HTTPS 443 / HTTP 80| Traefik[Traefik Ingress\nACME TLS + Security Middlewares]
+
+  subgraph DockerHost[Primary Docker Host]
+    Traefik
+    Authelia[Authelia SSO / ForwardAuth]
+    CrowdSec[CrowdSec + Traefik Bouncer]
+    ErrPages[Error Pages Fallback]
+
+    subgraph Apps[Business / User Applications]
+      Nextcloud[Nextcloud]
+      Passbolt[Passbolt]
+      Gitea[Gitea]
+      FamilyTree[Gramps Web]
+      Searxng[SearXNG]
+    end
+
+    subgraph Ops[Operations & Monitoring]
+      Grafana[Grafana]
+      Prometheus[Prometheus]
+      InfluxDB[InfluxDB]
+      NodeRED[Node-RED]
+      Portainer[Portainer]
+      UptimeKuma[Uptime Kuma]
+      Gotify[Gotify Notifications]
+    end
+  end
+
+  Traefik --> Apps
+  Traefik --> Ops
+  Traefik -->|ForwardAuth for selected routes| Authelia
+  Traefik -->|Threat decisions| CrowdSec
+  Traefik -->|4xx/5xx fallback| ErrPages
+
+  Prometheus --> Grafana
+  Prometheus --> Gotify
+```
+
+For a request-flow/network view and architecture notes, see [docs/architecture.md](docs/architecture.md).

core/authelia/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   authelia:
-    profiles: ["core","all","traefik"]
+    profiles: ["core","all","authelia", "traefik"]
     image: authelia/authelia
     restart: always
     build:

core/crowdsec/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   crowdsec:
+    profiles: ["core","all","crowdsec", "traefik"]
 #    image: crowdsecurity/crowdsec:latest
-    profiles: ["core","all","traefik"]
     build: ${PROJECT_ROOT}/core/crowdsec
     container_name: crowdsec
     restart: always

core/docker-compose.yml

@@ -1 +0,0 @@
-services: {}

core/error-pages/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   error-pages:
-    profiles: ["core","all","traefik"]
+    profiles: ["core","all","error-pages", "traefik"]
     image: tarampampam/error-pages:3
     restart: always
     container_name: error-pages

core/test/docker-compose.yml

@@ -2,7 +2,7 @@ services:
   update-test:
     image: nginx:1.28.1
     container_name: update-test
-    profiles: ["test"]
+    profiles: ["core","all","test"]
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost"]  # returns 0 if Nginx is up
       interval: 5s
@@ -11,7 +11,7 @@ services:
       start_period: 2s
 
   docker-update-exporter-test:
-    profiles: ["test"]
+    profiles: ["core","all","test"]
     build:
       context: ${PROJECT_ROOT}/core/test
     container_name: docker-update-exporter-test

core/traefik/MTLS.md

@@ -0,0 +1,38 @@
+# Private-admin mTLS for Traefik
+
+`private-admin` routers are configured to require client certificates via the Traefik TLS option `mtls-private-admin@file`.
+
+## Certificate paths
+
+- Trusted client CA bundle expected by Traefik:
+  - `core/traefik/certs/ca/clients-ca.crt`
+- CA private key (keep secret, never commit):
+  - `core/traefik/certs/ca/clients-ca.key`
+- Issued client certs:
+  - `core/traefik/certs/clients/<client-name>/`
+
+## Bootstrap
+
+From repository root:
+
+```bash
+./core/traefik/scripts/init-mtls-ca.sh
+./core/traefik/scripts/issue-mtls-client-cert.sh admin-laptop
+```
+
+The second command exports a PKCS#12 bundle (`.p12`) for browser import and also leaves PEM `.crt`/`.key` artifacts for CLI usage.
+
+## Revocation workflow
+
+Because Traefik is configured with `clientAuth.caFiles`, revoked cert serials are not enforced by default.
+
+- Use `./core/traefik/scripts/revoke-mtls-client-cert.sh <client-name>` to quarantine a client cert bundle.
+- For strict revocation, rotate the CA (`init-mtls-ca.sh` after removing old CA) and re-issue all trusted client certs.
+
+## Deploy
+
+After CA/certs are in place, restart Traefik to ensure updated files are loaded:
+
+```bash
+docker compose -f core/traefik/docker-compose.yml up -d traefik
+```

core/traefik/docker-compose.yml

@@ -24,6 +24,7 @@ services:
     volumes:
       - ${PROJECT_ROOT}/core/traefik/data/letsencrypt:/letsencrypt
       - ${PROJECT_ROOT}/core/traefik/data/logs:/logs
+      - ${PROJECT_ROOT}/core/traefik/certs:/etc/traefik/certs:ro
       - ${PROJECT_ROOT}/core/traefik/dynamic.yml:/etc/traefik/dynamic.yml:ro
       - ${PROJECT_ROOT}/core/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
       - ${PROJECT_ROOT}/core/traefik/data/plugins:/plugins-storage
@@ -37,6 +38,7 @@ services:
       - "traefik.http.routers.traefik.service=api@internal"
       - "traefik.http.routers.traefik.entrypoints=websecure"
       - "traefik.http.routers.traefik.tls.certresolver=myresolver"
+      - "traefik.http.routers.traefik.tls.options=mtls-private-admin@file"
       - "traefik.http.routers.traefik.middlewares=authelia"
       - "io.portainer.accesscontrol.public"
       - "traefik.docker.network=core_traefik"

core/traefik/dynamic.yml

@@ -31,3 +31,13 @@ http:
           - crowdsec@file
 #          - tracing-middleware@file
           - error-pages-middleware@docker
+
+tls:
+  options:
+    mtls-private-admin:
+      minVersion: VersionTLS12
+      sniStrict: true
+      clientAuth:
+        caFiles:
+          - /etc/traefik/certs/ca/clients-ca.crt
+        clientAuthType: RequireAndVerifyClientCert

core/traefik/scripts/init-mtls-ca.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TRAEFIK_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+CA_DIR="${TRAEFIK_ROOT}/certs/ca"
+
+CA_KEY="${CA_DIR}/clients-ca.key"
+CA_CERT="${CA_DIR}/clients-ca.crt"
+CA_SERIAL="${CA_DIR}/clients-ca.srl"
+
+DAYS="${DAYS:-3650}"
+SUBJECT="${SUBJECT:-/CN=Traefik Private Admin Client CA/O=Homelab}"
+
+mkdir -p "${CA_DIR}"
+chmod 700 "${CA_DIR}"
+
+if [[ -f "${CA_KEY}" || -f "${CA_CERT}" ]]; then
+  echo "Refusing to overwrite existing CA material in ${CA_DIR}."
+  echo "Delete existing files first if you intentionally want to rotate the CA."
+  exit 1
+fi
+
+openssl genrsa -out "${CA_KEY}" 4096
+chmod 600 "${CA_KEY}"
+
+openssl req -x509 -new -nodes \
+  -key "${CA_KEY}" \
+  -sha256 \
+  -days "${DAYS}" \
+  -subj "${SUBJECT}" \
+  -out "${CA_CERT}"
+chmod 644 "${CA_CERT}"
+
+rm -f "${CA_SERIAL}"
+
+echo "Created mTLS client CA: ${CA_CERT}"
+echo "Use issue-mtls-client-cert.sh to issue client certificates signed by this CA."

core/traefik/scripts/issue-mtls-client-cert.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <client-name> [days]"
+  exit 1
+fi
+
+CLIENT_NAME="$1"
+DAYS="${2:-825}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TRAEFIK_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+CA_DIR="${TRAEFIK_ROOT}/certs/ca"
+CLIENT_DIR="${TRAEFIK_ROOT}/certs/clients/${CLIENT_NAME}"
+
+CA_KEY="${CA_DIR}/clients-ca.key"
+CA_CERT="${CA_DIR}/clients-ca.crt"
+CA_SERIAL="${CA_DIR}/clients-ca.srl"
+
+CLIENT_KEY="${CLIENT_DIR}/${CLIENT_NAME}.key"
+CLIENT_CSR="${CLIENT_DIR}/${CLIENT_NAME}.csr"
+CLIENT_CERT="${CLIENT_DIR}/${CLIENT_NAME}.crt"
+CLIENT_P12="${CLIENT_DIR}/${CLIENT_NAME}.p12"
+OPENSSL_EXT="${CLIENT_DIR}/client.ext"
+
+if [[ ! -f "${CA_KEY}" || ! -f "${CA_CERT}" ]]; then
+  echo "Missing CA material. Run init-mtls-ca.sh first."
+  exit 1
+fi
+
+if [[ -d "${CLIENT_DIR}" ]]; then
+  echo "Client directory already exists (${CLIENT_DIR}); refusing to overwrite."
+  exit 1
+fi
+
+mkdir -p "${CLIENT_DIR}"
+chmod 700 "${CLIENT_DIR}"
+
+cat > "${OPENSSL_EXT}" <<EXT
+basicConstraints=CA:FALSE
+keyUsage = digitalSignature, keyEncipherment
+extendedKeyUsage = clientAuth
+subjectAltName = DNS:${CLIENT_NAME}
+EXT
+
+openssl genrsa -out "${CLIENT_KEY}" 2048
+chmod 600 "${CLIENT_KEY}"
+
+openssl req -new -key "${CLIENT_KEY}" -subj "/CN=${CLIENT_NAME}" -out "${CLIENT_CSR}"
+
+openssl x509 -req \
+  -in "${CLIENT_CSR}" \
+  -CA "${CA_CERT}" \
+  -CAkey "${CA_KEY}" \
+  -CAcreateserial \
+  -out "${CLIENT_CERT}" \
+  -days "${DAYS}" \
+  -sha256 \
+  -extfile "${OPENSSL_EXT}"
+chmod 644 "${CLIENT_CERT}"
+
+openssl pkcs12 -export \
+  -inkey "${CLIENT_KEY}" \
+  -in "${CLIENT_CERT}" \
+  -certfile "${CA_CERT}" \
+  -name "${CLIENT_NAME}" \
+  -out "${CLIENT_P12}"
+chmod 600 "${CLIENT_P12}"
+
+rm -f "${CLIENT_CSR}" "${OPENSSL_EXT}"
+
+echo "Issued client certificate for ${CLIENT_NAME}."
+echo "CRT: ${CLIENT_CERT}"
+echo "KEY: ${CLIENT_KEY}"
+echo "P12: ${CLIENT_P12}"

core/traefik/scripts/revoke-mtls-client-cert.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -lt 1 ]]; then
+  echo "Usage: $0 <client-name>"
+  exit 1
+fi
+
+CLIENT_NAME="$1"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TRAEFIK_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+CLIENT_DIR="${TRAEFIK_ROOT}/certs/clients/${CLIENT_NAME}"
+REVOKED_DIR="${TRAEFIK_ROOT}/certs/revoked"
+
+if [[ ! -d "${CLIENT_DIR}" ]]; then
+  echo "No certificate directory found for client '${CLIENT_NAME}'."
+  exit 1
+fi
+
+mkdir -p "${REVOKED_DIR}"
+
+STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+mv "${CLIENT_DIR}" "${REVOKED_DIR}/${CLIENT_NAME}-${STAMP}"
+
+echo "Moved client certificate material to ${REVOKED_DIR}/${CLIENT_NAME}-${STAMP}."
+echo "Note: Traefik clientAuth with a CA file does not enforce revocation lists by default."
+echo "For immediate hard revocation, rotate the client CA and re-issue trusted client certificates."

docs/architecture.md

@@ -0,0 +1,109 @@
+# Architecture Summary
+
+## Overview
+
+This stack uses **Traefik v3** as the internet-facing ingress for application and operations UIs. Service routing is primarily label-driven from Docker Compose files, with a shared `traefik` bridge network for reverse-proxied traffic and a `monitor` network for internal telemetry components.
+
+TLS is terminated at Traefik using ACME HTTP challenge (`myresolver`), with additional hardening via:
+
+- a default middleware chain (security headers, CrowdSec bouncer, error pages),
+- Authelia forward-auth middleware on selected routes,
+- mTLS TLS options (`mtls-private-admin`) on private-admin endpoints.
+
+## Network / Request Flow
+
+```mermaid
+flowchart LR
+  C[Internet Client] -->|80/443| T[Traefik Ingress]
+  T -->|HTTP->HTTPS redirect| T
+  T -->|ACME HTTP challenge| LE[Let's Encrypt ACME]
+
+  subgraph TraefikNet["Docker network: traefik (172.21.0.0 slash 16)"]
+    A[Authelia]
+    CS[CrowdSec LAPI]
+    EP[Error Pages]
+
+    NC[Nextcloud]
+    PB[Passbolt]
+    GT[Gitea]
+    GW[Gramps Web]
+    SX[SearXNG]
+
+    GF[Grafana]
+    PR[Prometheus]
+    NR[Node-RED]
+    PT[Portainer]
+    UK[Uptime Kuma]
+    IF[InfluxDB]
+    GO[Gotify]
+  end
+
+  T -->|forwardAuth for selected services| A
+  T -->|plugin decisions| CS
+  T -->|4xx/5xx middleware| EP
+
+  T --> NC
+  T --> PB
+  T --> GT
+  T --> GW
+  T --> SX
+
+  T --> GF
+  T --> PR
+  T --> NR
+  T --> PT
+  T --> UK
+  T --> IF
+  T --> GO
+
+  subgraph MonitorNet[Docker network: monitor]
+    NE[Node Exporter]
+    TE[Telegraf]
+    DE[Docker Update Exporter]
+    PE[Pi-hole Exporter]
+    DSP[Docker Socket Proxy]
+  end
+
+  PR --> NE
+  PR --> TE
+  PR --> DE
+  PR --> PE
+  PR --> UK
+  PR -->|remote scrape| RH[Remote Hosts]
+  TE --> DSP
+  NR --> DSP
+  PT --> DSP
+  T --> DSP
+```
+
+## Key Components
+
+- **Ingress & security plane:** Traefik, Authelia, CrowdSec, Error Pages.
+- **User-facing applications:** Nextcloud, Passbolt, Gitea, Gramps Web (Family Tree), SearXNG.
+- **Monitoring/ops:** Prometheus, Grafana, InfluxDB, Node-RED, Uptime Kuma, Portainer, Gotify.
+- **Support plane:** Docker Socket Proxy (shared Docker API gateway for Traefik/automation/ops tools).
+
+## Remote Hosts Observed
+
+Prometheus scrape targets indicate additional infrastructure outside the local Compose deployment, including hostnames for:
+
+- `raspberrypi.tail13f623.ts.net`
+- `pve.sweet.home`
+- `pbs.sweet.home`
+- `pihole`
+- `server`
+- `nix-cache`
+- `kuma.lan.ddnsgeek.com`
+
+## Assumptions / Unknowns
+
+The repository provides enough detail to infer **container-level architecture**, but not full **Proxmox host/VM topology**.
+
+Unknowns (left intentionally as placeholders):
+
+- **Proxmox physical hosts:** _unknown from repo contents._
+- **VM/LXC inventory and placement:** _unknown from repo contents._
+- **Which services run on which Proxmox node(s):** _unknown from repo contents._
+- **Inter-host VLAN/subnet layout beyond Docker bridges:** _unknown from repo contents._
+
+If you want, this section can be replaced with a concrete Proxmox topology once you add an inventory source (e.g., Terraform, Ansible inventory, or a diagram export).

monitoring/docker-exporter/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   docker-update-exporter:
-    profiles: ["monitoring","all","prometheus-exporters"]
+    profiles: ["monitoring","all","docker-exporter", "prometheus"]
     build:
       context: ${PROJECT_ROOT}/monitoring/docker-exporter
     container_name: docker-update-exporter

monitoring/docker-socket-proxy/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   docker-socket-proxy:
-    profiles: ["monitoring","all","prometheus","prometheus-exporters"]
+    profiles: ["monitoring","all","docker-socket-proxy", "core", "traefik", "prometheus"]
     image: tecnativa/docker-socket-proxy:latest
     container_name: docker-socket-proxy
     hostname: docker-socket-proxy

monitoring/gotify/docker-compose.yml

@@ -23,4 +23,5 @@ services:
       - "traefik.http.routers.gotify.rule=Host(`gotify.lan.ddnsgeek.com`)"
       - "traefik.http.routers.gotify.entrypoints=websecure"
       - "traefik.http.routers.gotify.tls.certresolver=myresolver"
+      - "traefik.http.routers.gotify.tls.options=mtls-private-admin@file"
       - "traefik.http.services.gotify.loadbalancer.server.port=80"

monitoring/grafana/docker-compose.yml

@@ -18,6 +18,7 @@ services:
       - "traefik.enable=true"
       - "traefik.http.routers.grafana.entrypoints=websecure"
       - "traefik.http.routers.grafana.tls.certresolver=myresolver"
+      - "traefik.http.routers.grafana.tls.options=mtls-private-admin@file"
       - "io.portainer.accesscontrol.public"
       - "traefik.http.services.grafana.loadbalancer.server.port=3000"
       - "traefik.docker.network=core_traefik"

monitoring/influxdb/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   influxdb:
-    profiles: ["monitoring","all","prometheus"]
+    profiles: ["monitoring","all","influxdb", "prometheus"]
     image: influxdb:2.7
     container_name: influxdb
     restart: unless-stopped
@@ -26,6 +26,7 @@ services:
        - "traefik.enable=true"
        - "traefik.http.routers.influxdb.entrypoints=websecure"
        - "traefik.http.routers.influxdb.tls.certresolver=myresolver"
+       - "traefik.http.routers.influxdb.tls.options=mtls-private-admin@file"
        - "io.portainer.accesscontrol.public"
        - "traefik.http.services.influxdb.loadbalancer.server.port=8086"
        - "traefik.http.routers.influxdb.middlewares=authelia"

monitoring/mtls-bridge/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY app.py .
+
+EXPOSE 8080
+
+CMD ["python", "app.py"]

monitoring/mtls-bridge/README.md

@@ -0,0 +1,38 @@
+# mTLS Bridge Service
+
+Internal HTTP-to-mTLS bridge for services that cannot present client certificates directly (for example, Grafana webhooks).
+
+## How it works
+
+1. Accepts plain HTTP requests inside the Docker network.
+2. Forwards requests to an HTTPS upstream.
+3. Presents a client certificate/key pair for mTLS authentication.
+
+## Environment variables
+
+- `TARGET_URL` (required): HTTPS upstream base URL.
+- `CLIENT_CERT` (default `/certs/client.crt`): client certificate path.
+- `CLIENT_KEY` (default `/certs/client.key`): client private key path.
+- `CA_CERT` (default `/certs/ca.crt`): CA certificate bundle used to verify upstream TLS.
+- `TIMEOUT` (default `5`): request timeout in seconds.
+- `LOG_LEVEL` (default `INFO`): Python logging level.
+
+## Endpoints
+
+- `GET /health` returns `200 OK` for container health checks.
+- `/*` proxies requests to `${TARGET_URL}` with method/body/headers preserved.
+
+## Compose integration
+
+This repository includes `monitoring/mtls-bridge/docker-compose.yml`:
+
+- No public port exposure.
+- Read-only cert mount (`${PROJECT_ROOT}/core/traefik/certs:/certs:ro`).
+- Joined to internal monitoring/traefik networks.
+
+## Example test
+
+```bash
+curl http://mtls-bridge:8080/health
+curl -X POST http://mtls-bridge:8080
+```

monitoring/mtls-bridge/app.py

@@ -0,0 +1,240 @@
+import logging
+import os
+import time
+
+import requests
+from flask import Flask, Response, g, request
+
+app = Flask(__name__)
+
+logging.basicConfig(
+    level=os.environ.get("LOG_LEVEL", "INFO"),
+    format="%(asctime)s %(levelname)s %(message)s",
+)
+logger = logging.getLogger("mtls-bridge")
+logging.getLogger("werkzeug").setLevel(logging.WARNING)
+
+# Config via env
+TARGET_URL = os.environ.get("TARGET_URL")
+CLIENT_CERT = os.environ.get("CLIENT_CERT", "/certs/client.crt")
+CLIENT_KEY = os.environ.get("CLIENT_KEY", "/certs/client.key")
+UPSTREAM_CA_CERT = os.environ.get("UPSTREAM_CA_CERT", os.environ.get("CA_CERT", "")).strip()
+# Backward-compat alias: keep CA_CERT defined so legacy code paths/log statements don't crash.
+CA_CERT = UPSTREAM_CA_CERT
+TIMEOUT = int(os.environ.get("TIMEOUT", "5"))
+
+logger.info(
+    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s ca=%s log_level=%s",
+    TARGET_URL,
+    TIMEOUT,
+    CLIENT_CERT,
+    CLIENT_KEY,
+    CA_CERT,
+    os.environ.get("LOG_LEVEL", "INFO"),
+)
+
+
+def get_verify_setting():
+    if not UPSTREAM_CA_CERT:
+        return True
+
+    lowered = UPSTREAM_CA_CERT.lower()
+    if lowered in {"false", "0", "no"}:
+        logger.warning("TLS verification for upstream is disabled via UPSTREAM_CA_CERT=%s", UPSTREAM_CA_CERT)
+        return False
+
+    if not os.path.exists(UPSTREAM_CA_CERT):
+        logger.warning(
+            "Configured UPSTREAM_CA_CERT path does not exist: %s (falling back to system CA bundle)",
+            UPSTREAM_CA_CERT,
+        )
+        return True
+
+    return UPSTREAM_CA_CERT
+
+
+VERIFY_SETTING = get_verify_setting()
+
+logger.info(
+    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s log_level=%s",
+    TARGET_URL,
+    TIMEOUT,
+    CLIENT_CERT,
+    CLIENT_KEY,
+    VERIFY_SETTING,
+    os.environ.get("LOG_LEVEL", "INFO"),
+)
+
+
+def get_verify_setting():
+    if not UPSTREAM_CA_CERT:
+        return True
+
+    lowered = UPSTREAM_CA_CERT.lower()
+    if lowered in {"false", "0", "no"}:
+        logger.warning("TLS verification for upstream is disabled via UPSTREAM_CA_CERT=%s", UPSTREAM_CA_CERT)
+        return False
+
+    if not os.path.exists(UPSTREAM_CA_CERT):
+        logger.warning(
+            "Configured UPSTREAM_CA_CERT path does not exist: %s (falling back to system CA bundle)",
+            UPSTREAM_CA_CERT,
+        )
+        return True
+
+    return UPSTREAM_CA_CERT
+
+
+VERIFY_SETTING = get_verify_setting()
+
+logger.info(
+    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s log_level=%s",
+    TARGET_URL,
+    TIMEOUT,
+    CLIENT_CERT,
+    CLIENT_KEY,
+    VERIFY_SETTING,
+    os.environ.get("LOG_LEVEL", "INFO"),
+)
+
+if TARGET_URL and TARGET_URL.lower().startswith("http://"):
+    logger.warning(
+        "TARGET_URL uses http://; upstream may redirect to https:// and change request behavior: %s",
+        TARGET_URL,
+    )
+
+
+def get_verify_setting():
+    if not UPSTREAM_CA_CERT:
+        return True
+
+    lowered = UPSTREAM_CA_CERT.lower()
+    if lowered in {"false", "0", "no"}:
+        logger.warning("TLS verification for upstream is disabled via UPSTREAM_CA_CERT=%s", UPSTREAM_CA_CERT)
+        return False
+
+    if not os.path.exists(UPSTREAM_CA_CERT):
+        logger.warning(
+            "Configured UPSTREAM_CA_CERT path does not exist: %s (falling back to system CA bundle)",
+            UPSTREAM_CA_CERT,
+        )
+        return True
+
+    return UPSTREAM_CA_CERT
+
+
+VERIFY_SETTING = get_verify_setting()
+
+logger.info(
+    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s log_level=%s",
+    TARGET_URL,
+    TIMEOUT,
+    CLIENT_CERT,
+    CLIENT_KEY,
+    VERIFY_SETTING,
+    os.environ.get("LOG_LEVEL", "INFO"),
+)
+
+if TARGET_URL and TARGET_URL.lower().startswith("http://"):
+    logger.warning(
+        "TARGET_URL uses http://; upstream may redirect to https:// and change request behavior: %s",
+        TARGET_URL,
+    )
+
+
+@app.route("/health", methods=["GET"])
+def health():
+    logger.debug("healthcheck request from %s", request.remote_addr)
+    return "OK", 200
+
+
+@app.before_request
+def before_request():
+    g.request_start = time.time()
+
+
+@app.after_request
+def after_request(response):
+    elapsed_ms = int((time.time() - g.request_start) * 1000)
+    if request.path != "/health":
+        logger.info(
+            "request complete method=%s path=%s status=%s elapsed_ms=%s",
+            request.method,
+            request.path,
+            response.status_code,
+            elapsed_ms,
+        )
+    return response
+
+
+@app.route(
+    "/",
+    defaults={"path": ""},
+    methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"],
+    provide_automatic_options=False,
+)
+@app.route(
+    "/<path:path>",
+    methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"],
+    provide_automatic_options=False,
+)
+def proxy(path):
+    request_path = f"/{path}" if path else "/"
+    request_size = len(request.get_data(cache=True))
+    logger.info(
+        "incoming request method=%s path=%s query=%s remote=%s bytes=%s",
+        request.method,
+        request_path,
+        request.query_string.decode("utf-8", "ignore"),
+        request.remote_addr,
+        request_size,
+    )
+
+    if not TARGET_URL:
+        logger.error("TARGET_URL is not set; cannot proxy request")
+        return Response("TARGET_URL is not set", status=500)
+
+    try:
+        url = f"{TARGET_URL.rstrip('/')}/{path}".rstrip("/")
+        start_time = time.time()
+        headers = {k: v for k, v in request.headers if k.lower() != "host"}
+        headers["X-Forwarded-By"] = "mtls-bridge"
+
+        logger.debug("forwarding request to upstream url=%s headers=%s", url, headers)
+
+        resp = requests.request(
+            method=request.method,
+            url=url,
+            headers=headers,
+            data=request.get_data(cache=True),
+            cookies=request.cookies,
+            cert=(CLIENT_CERT, CLIENT_KEY),
+            verify=VERIFY_SETTING,
+            timeout=TIMEOUT,
+        )
+
+        elapsed_ms = int((time.time() - start_time) * 1000)
+        logger.info(
+            "upstream response status=%s url=%s elapsed_ms=%s response_bytes=%s",
+            resp.status_code,
+            url,
+            elapsed_ms,
+            len(resp.content),
+        )
+
+        excluded_headers = ["content-encoding", "content-length", "transfer-encoding", "connection"]
+        response_headers = [
+            (k, v)
+            for k, v in resp.raw.headers.items()
+            if k.lower() not in excluded_headers
+        ]
+
+        return Response(resp.content, resp.status_code, response_headers)
+
+    except Exception as e:
+        logger.exception("proxy request failed")
+        return Response(str(e), status=500)
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=8080)

monitoring/mtls-bridge/docker-compose.yml

@@ -0,0 +1,25 @@
+services:
+  mtls-bridge:
+    profiles: ["monitoring", "all", "mtls-bridge"]
+    build:
+      context: ${PROJECT_ROOT}/monitoring/mtls-bridge
+    container_name: mtls-bridge
+    restart: unless-stopped
+    environment:
+      - TARGET_URL=https://node-red.lan.ddnsgeek.com/docker-update-lockouts/clear
+      - CLIENT_CERT=/certs/clients/office-pc/office-pc.crt
+      - CLIENT_KEY=/certs/clients/office-pc/office-pc.key
+      - TIMEOUT=5
+      - LOG_LEVEL=${MTLS_BRIDGE_LOG_LEVEL:-INFO}
+      - UPSTREAM_CA_CERT=${MTLS_BRIDGE_UPSTREAM_CA_CERT:-}
+    volumes:
+      - ${PROJECT_ROOT}/core/traefik/certs:/certs:ro
+    healthcheck:
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health', timeout=3).read()"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+    networks:
+      - monitor
+      - traefik

monitoring/mtls-bridge/requirements.txt

@@ -0,0 +1,2 @@
+flask
+requests

monitoring/node-exporter/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   node-exporter:
-    profiles: ["monitoring","all","prometheus-exporters"]
+    profiles: ["monitoring","all","node-exporter", "prometheus"]
     image: prom/node-exporter:latest
     container_name: node-exporter
     pid: host

monitoring/node-red/data/.flows.json.backup

@@ -276,7 +276,7 @@
         "type": "function",
         "z": "00b02bbd01c91485",
         "name": "Build Deploy Command",
-        "func": "const container = msg.container;\nconst image = msg.image;\nconst host = msg.host;\nmsg.payload = `PROJECT_ROOT=\"/compose/${host} /compose/${host}/services-up.sh --profile all up -d ${container}`;\nmsg.image = image;\nmsg.container = container;\nmsg.host = host;\nnode.log(`Test Successful\n    Container: ${container}\n    Image: ${msg.image}\n    Host: ${host}`\n    )\nreturn msg;",
+        "func": "const container = msg.container;\nconst image = msg.image;\nconst host = msg.host;\nmsg.payload = `PROJECT_ROOT=\"/compose/${host}\" /compose/${host}/services-up.sh --profile all up -d ${container}`;\nmsg.image = image;\nmsg.container = container;\nmsg.host = host;\nnode.log(`Test Successful\n    Container: ${container}\n    Image: ${msg.image}\n    Host: ${host}`\n    )\nreturn msg;",
         "outputs": 1,
         "timeout": "",
         "noerr": 0,
@@ -827,7 +827,7 @@
         "method": "POST",
         "ret": "obj",
         "paytoqs": "ignore",
-        "url": "https://gotify.lan.ddnsgeek.com/message?token=ATSMpSKrdNjKXeT",
+        "url": "http://gotify/message?token=ATSMpSKrdNjKXeT",
         "tls": "",
         "persist": false,
         "proxy": "",

monitoring/node-red/data/.flows_cred.json.backup

@@ -1,3 +1,3 @@
 {
-    "$": "3a4824a80803ad539950f337d67e528bZGBrGLreu0SvnxrH7vE1YhegoHsDWV1xaC+K2ZKHKVkduSQVcOTDerb6wuXV"
+    "$": "0875f76c323e5c597b616b8d0802ec74qpYy2kDXF8kqK4a8NneEiGhCLy5Lp+vI1FkwjR0umSjQBv2nQKyxpVC2eFxg"
 }

monitoring/node-red/data/context/00b02bbd01c91485/flow.json

@@ -1,3 +1,10 @@
 {
-    "dockerUpdateAttempts": {}
+    "dockerUpdateAttempts": {
+        "telegraf|telegraf:latest|docker": {
+            "time": 1776048224012,
+            "status": "test_failed",
+            "failedAt": 1776048287524,
+            "notified": true
+        }
+    }
 }

monitoring/node-red/data/flows.json

@@ -827,7 +827,7 @@
         "method": "POST",
         "ret": "obj",
         "paytoqs": "ignore",
-        "url": "https://gotify.lan.ddnsgeek.com/message?token=ATSMpSKrdNjKXeT",
+        "url": "http://gotify/message?token=ATSMpSKrdNjKXeT",
         "tls": "",
         "persist": false,
         "proxy": "",
@@ -876,7 +876,7 @@
         "initialize": "",
         "finalize": "",
         "libs": [],
-        "x": 500,
+        "x": 480,
         "y": 100,
         "wires": [
             [
@@ -916,7 +916,7 @@
         "initialize": "",
         "finalize": "",
         "libs": [],
-        "x": 490,
+        "x": 710,
         "y": 200,
         "wires": [
             [
@@ -952,7 +952,7 @@
         "initialize": "",
         "finalize": "",
         "libs": [],
-        "x": 490,
+        "x": 710,
         "y": 300,
         "wires": [
             [
@@ -988,7 +988,7 @@
         "initialize": "",
         "finalize": "",
         "libs": [],
-        "x": 490,
+        "x": 790,
         "y": 380,
         "wires": [
             [
@@ -1024,7 +1024,7 @@
         "initialize": "",
         "finalize": "",
         "libs": [],
-        "x": 520,
+        "x": 800,
         "y": 460,
         "wires": [
             [
@@ -1060,7 +1060,7 @@
         "initialize": "",
         "finalize": "",
         "libs": [],
-        "x": 490,
+        "x": 750,
         "y": 520,
         "wires": [
             [
@@ -1111,7 +1111,7 @@
         "initialize": "",
         "finalize": "",
         "libs": [],
-        "x": 480,
+        "x": 740,
         "y": 580,
         "wires": [
             [
@@ -1273,7 +1273,7 @@
         "initialize": "",
         "finalize": "",
         "libs": [],
-        "x": 470,
+        "x": 460,
         "y": 580,
         "wires": [
             [

monitoring/node-red/data/flows_cred.json

@@ -1,3 +1,3 @@
 {
-    "$": "793e894c453e963b28f6b2f601eb1089FMPGL4pXtRVHmXNPWy9/lr/WoWBaGVA="
+    "$": "b5f1756334f073b04b89cd6319c4ce1aBB8dAVwKD38fNXCo2L9NODemuEpJ4T4="
 }

monitoring/node-red/data/update-events.ndjson

@@ -104,3 +104,8 @@
 {"ts":"2026-04-12T23:23:55.515Z","flow":"docker-updates","event":"completed","container":"update-test","project":"unknown","host":"docker","status":"locked","success":0,"failed":1,"duration_ms":0,"code":0,"error":""}
 {"ts":"2026-04-12T23:37:38.907Z","flow":"docker-updates","event":"completed","container":"update-test","project":"unknown","host":"docker","status":"success","success":1,"failed":0,"duration_ms":0,"code":0,"error":""}
 {"ts":"2026-04-12T23:38:32.835Z","flow":"docker-updates","event":"completed","container":"telegraf","project":"unknown","host":"docker","status":"failed","success":0,"failed":1,"duration_ms":0,"code":0,"error":""}
+{"ts":"2026-04-13T02:43:53.730Z","flow":"docker-updates","event":"completed","container":"update-test","project":"unknown","host":"docker","status":"success","success":1,"failed":0,"duration_ms":0,"code":0,"error":""}
+{"ts":"2026-04-13T02:44:47.524Z","flow":"docker-updates","event":"completed","container":"telegraf","project":"unknown","host":"docker","status":"failed","success":0,"failed":1,"duration_ms":0,"code":0,"error":""}
+{"ts":"2026-04-13T02:44:47.524Z","flow":"docker-updates","event":"completed","container":"telegraf","project":"unknown","host":"docker","status":"locked","success":0,"failed":1,"duration_ms":0,"code":0,"error":""}
+{"ts":"2026-04-13T02:46:35.255Z","flow":"docker-updates","event":"completed","container":"update-test","project":"unknown","host":"docker","status":"success","success":1,"failed":0,"duration_ms":0,"code":0,"error":""}
+{"ts":"2026-04-13T02:58:03.503Z","flow":"docker-updates","event":"completed","container":"update-test","project":"unknown","host":"docker","status":"success","success":1,"failed":0,"duration_ms":0,"code":0,"error":""}

monitoring/node-red/docker-compose.yml

@@ -1,10 +1,10 @@
 services:
   node-red:
+    profiles: ["monitoring","all","node-red"]
 #    image: nodered/node-red:latest
     build:
       context: ${PROJECT_ROOT}/monitoring/node-red
     container_name: node-red
-    profiles: ["monitoring","all"]
     restart: unless-stopped
     depends_on:
       - docker-socket-proxy
@@ -56,6 +56,7 @@ services:
 #      - "traefik.http.routers.node-red.service=api@internal"
       - "traefik.http.routers.node-red.entrypoints=websecure"
       - "traefik.http.routers.node-red.tls.certresolver=myresolver"
+      - "traefik.http.routers.node-red.tls.options=mtls-private-admin@file"
       - "traefik.http.routers.node-red.middlewares=authelia"
       - "io.portainer.accesscontrol.public"
       - "traefik.docker.network=core_traefik"

monitoring/pihole-exporter/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   pihole-exporter:
-    profiles: ["monitoring","all","prometheus-exporters"]
+    profiles: ["monitoring","all","pihole-exporter", "prometheus"]
     image: ekofr/pihole-exporter:latest
     container_name: pihole-exporter
 #    env_file:

monitoring/portainer/docker-compose.yml

@@ -20,6 +20,7 @@ services:
       - traefik.http.routers.portainer.entrypoints=websecure
       - traefik.http.routers.portainer.tls=true
       - traefik.http.routers.portainer.tls.certresolver=myresolver
+      - traefik.http.routers.portainer.tls.options=mtls-private-admin@file
       - io.portainer.accesscontrol.public
       # Service -> Portainer listens on 9000 inside the container
       - traefik.http.services.portainer.loadbalancer.server.port=9000

monitoring/prometheus/docker-compose.yml

@@ -30,6 +30,7 @@ services:
        - "traefik.enable=true"
        - "traefik.http.routers.prometheus.entrypoints=websecure"
        - "traefik.http.routers.prometheus.tls.certresolver=myresolver"
+       - "traefik.http.routers.prometheus.tls.options=mtls-private-admin@file"
        - "io.portainer.accesscontrol.public"
        - "traefik.http.services.prometheus.loadbalancer.server.port=9090"
        - "traefik.http.routers.prometheus.middlewares=authelia"

monitoring/telegraf/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   telegraf:
-    profiles: ["monitoring","all","prometheus"]
+    profiles: ["monitoring","all","telegraf", "prometheus"]
     image: telegraf:latest
     container_name: telegraf
     restart: unless-stopped

monitoring/uptime-kuma/docker-compose.yml

@@ -20,6 +20,7 @@ services:
       - traefik.http.routers.monitor.entrypoints=websecure
       - traefik.http.routers.monitor.tls=true
       - traefik.http.routers.monitor.tls.certresolver=myresolver
+      - traefik.http.routers.monitor.tls.options=mtls-private-admin@file
       - io.portainer.accesscontrol.public
       - traefik.docker.network=core_traefik
       # Service -> container port

service-access-policy.md

@@ -0,0 +1,78 @@
+# Service Access Policy and External Exposure Hardening
+
+## 1) Service classification
+
+| Service/Host | Classification | Rationale |
+|---|---|---|
+| `auth.lan.ddnsgeek.com` | `authenticated-public` | Public identity/login entrypoint; internet-accessible but requires user authentication. |
+| `nextcloud.lan.ddnsgeek.com` | `authenticated-public` | Internet-facing collaboration app that must remain reachable to authenticated users. |
+| `passbolt.lan.ddnsgeek.com` | `authenticated-public` | Public password-management portal with strong authentication controls. |
+| `gitea.lan.ddnsgeek.com` | `authenticated-public` | Public developer endpoint with account-based access. |
+| `searxng.lan.ddnsgeek.com` | `public` | Intended anonymous/search access endpoint. |
+| `familytree.lan.ddnsgeek.com` | `authenticated-public` | End-user app; externally reachable but login-protected. |
+| `shifts.lan.ddnsgeek.com` | `authenticated-public` | End-user app; externally reachable but login-protected. |
+| `stockfill.lan.ddnsgeek.com` | `authenticated-public` | End-user app; externally reachable but login-protected. |
+| `gotify.lan.ddnsgeek.com` | `private-admin` | Admin/ops notification backend; should not be internet reachable. |
+| `grafana.lan.ddnsgeek.com` | `private-admin` | Infrastructure admin/observability console. |
+| `prometheus.lan.ddnsgeek.com` | `private-admin` | Monitoring datastore/query interface. |
+| `node-red.lan.ddnsgeek.com` | `private-admin` | Automation runtime and flow editor. |
+| `traefik.lan.ddnsgeek.com` | `private-admin` | Reverse-proxy admin/dashboard surface. |
+| `portainer.lan.ddnsgeek.com` | `private-admin` | Container management plane. |
+| `influxdb.lan.ddnsgeek.com` | `private-admin` | Metrics datastore admin/API surface. |
+| `kuma.lan.ddnsgeek.com` | `private-admin` | Monitoring admin surface. |
+| `monitor-kuma.lan.ddnsgeek.com` | `private-admin` | Monitoring admin surface. |
+| `edge.lan.ddnsgeek.com` | `private-admin` | Edge/network administration plane. |
+
+## 2) Required controls for `private-admin`
+
+Apply **at least one** trusted-path control (preferably layered):
+
+- Private network only (no public DNS / no internet route).
+- WireGuard/Tailscale/OpenVPN access gate.
+- mTLS client certificate requirement at reverse proxy.
+- Source IP allowlist at firewall and reverse proxy.
+
+Minimum target state for all `private-admin` hosts:
+
+- Public internet: connection refused/timeout, or immediate `403` for untrusted source.
+- Trusted path (VPN/mTLS/allowlisted IP): normal authenticated access.
+
+## 3) Gateway auth hardening
+
+For all `public` and `authenticated-public` services:
+
+- Keep SSO and MFA enforcement at the identity gateway.
+- Enforce lockout/backoff on `/login`, `/oauth/*`, `/auth/*`, `/api/auth/*`.
+- Rate-limit by source IP + account identifier to deter credential stuffing.
+
+Suggested baseline:
+
+- Soft limit: `10 req/min` per IP for auth endpoints.
+- Burst: `20`.
+- Temporary block: `15 min` after repeated failures.
+- Account lockout: `5-10` consecutive failed attempts (with secure unlock flow).
+
+## 4) WAF / reverse-proxy protections
+
+Deploy one of:
+
+- WAF managed rules for bot/credential-stuffing signatures.
+- Reverse-proxy failed-auth throttling and tarpit/delay policy.
+
+Implement logging + alerting thresholds:
+
+- High failed-auth rate from one IP/CIDR.
+- Password spray pattern across many usernames.
+- Geo/ASN anomalies for sensitive apps.
+
+## 5) External re-test procedure
+
+Re-test from a non-trusted external network and record outcomes.
+
+Success criteria:
+
+- Every `private-admin` host is inaccessible without VPN/mTLS/allowlisted source.
+- `public` and `authenticated-public` hosts remain reachable.
+- Auth endpoints trigger rate-limit/lockout controls under failed-attempt simulation.
+
+Use `./scripts/retest-external-access.sh` for a repeatable external validation pass.