From 46abc20577136f458861c26df07db9f04db4c635 Mon Sep 17 00:00:00 2001 From: git Date: Sun, 12 Jul 2026 19:15:22 +1000 Subject: [PATCH] migrated to beszel --- core/traefik/dynamic.yml | 57 ++ core/traefik/traefik.yml | 2 +- default-network.yml | 2 +- monitoring/beszel/docker-compose.yml | 58 ++ monitoring/docker-exporter/Dockerfile | 15 - monitoring/docker-exporter/Dockerfile.old | 8 - monitoring/docker-exporter/TEST_EXAMPLE.md | 24 - monitoring/docker-exporter/docker-compose.yml | 59 -- monitoring/docker-exporter/exporter.py | 579 ------------------ monitoring/docker-exporter/exporter.py.old | 514 ---------------- .../docker-socket-proxy/docker-compose.yml | 6 +- monitoring/grafana/docker-compose.yml | 55 -- monitoring/influxdb/docker-compose.yml | 50 -- monitoring/mtls-bridge/Dockerfile | 12 - monitoring/mtls-bridge/README.md | 60 -- monitoring/mtls-bridge/app.py | 208 ------- monitoring/mtls-bridge/docker-compose.yml | 49 -- monitoring/mtls-bridge/requirements.txt | 2 - monitoring/node-exporter/docker-compose.yml | 23 - monitoring/node-red/Dockerfile | 7 - monitoring/node-red/UPDATE_LOGGING_GRAFANA.md | 160 ----- monitoring/node-red/docker-compose.yml | 67 -- monitoring/pihole-exporter/docker-compose.yml | 17 - monitoring/prometheus/docker-compose.yml | 56 -- monitoring/prometheus/prometheus.yml | 166 ----- monitoring/prometheus/prometheus.yml.old | 192 ------ monitoring/telegraf/docker-compose.yml | 26 - monitoring/telegraf/telegraf.conf | 45 -- monitoring/uptime-kuma/docker-compose.yml | 2 +- 29 files changed, 122 insertions(+), 2399 deletions(-) create mode 100644 monitoring/beszel/docker-compose.yml delete mode 100644 monitoring/docker-exporter/Dockerfile delete mode 100644 monitoring/docker-exporter/Dockerfile.old delete mode 100644 monitoring/docker-exporter/TEST_EXAMPLE.md delete mode 100644 monitoring/docker-exporter/docker-compose.yml delete mode 100644 monitoring/docker-exporter/exporter.py delete mode 100644 monitoring/docker-exporter/exporter.py.old delete mode 100644 monitoring/grafana/docker-compose.yml delete mode 100644 monitoring/influxdb/docker-compose.yml delete mode 100644 monitoring/mtls-bridge/Dockerfile delete mode 100644 monitoring/mtls-bridge/README.md delete mode 100644 monitoring/mtls-bridge/app.py delete mode 100644 monitoring/mtls-bridge/docker-compose.yml delete mode 100644 monitoring/mtls-bridge/requirements.txt delete mode 100644 monitoring/node-exporter/docker-compose.yml delete mode 100644 monitoring/node-red/Dockerfile delete mode 100644 monitoring/node-red/UPDATE_LOGGING_GRAFANA.md delete mode 100644 monitoring/node-red/docker-compose.yml delete mode 100644 monitoring/pihole-exporter/docker-compose.yml delete mode 100644 monitoring/prometheus/docker-compose.yml delete mode 100644 monitoring/prometheus/prometheus.yml delete mode 100644 monitoring/prometheus/prometheus.yml.old delete mode 100644 monitoring/telegraf/docker-compose.yml delete mode 100644 monitoring/telegraf/telegraf.conf diff --git a/core/traefik/dynamic.yml b/core/traefik/dynamic.yml index d62086f..f18c00f 100644 --- a/core/traefik/dynamic.yml +++ b/core/traefik/dynamic.yml @@ -32,6 +32,63 @@ http: # - tracing-middleware@file - error-pages-middleware@docker + serversTransports: + insecureTransport: + insecureSkipVerify: true + + routers: + pve: + rule: "Host(`pve.lan.ddnsgeek.com`)" + entryPoints: + - websecure + service: pve + middlewares: + - authelia@docker + tls: + certResolver: myresolver + options: mtls-private-admin@file + + pbs: + rule: "Host(`pbs.lan.ddnsgeek.com`)" + entryPoints: + - websecure + service: pbs + middlewares: + - authelia@docker + tls: + certResolver: myresolver + options: mtls-private-admin@file + + pdm: + rule: "Host(`pdm.lan.ddnsgeek.com`)" + entryPoints: + - websecure + service: pdm + middlewares: + - authelia@docker + tls: + certResolver: myresolver + options: mtls-private-admin@file + + services: + pve: + loadBalancer: + serversTransport: insecureTransport + servers: + - url: "https://pve.sweet.home:8006" + + pbs: + loadBalancer: + serversTransport: insecureTransport + servers: + - url: "https://pbs.sweet.home:8007" + + pdm: + loadBalancer: + serversTransport: insecureTransport + servers: + - url: "https://pdm.sweet.home:8443" + tls: options: mtls-private-admin: diff --git a/core/traefik/traefik.yml b/core/traefik/traefik.yml index 2f0bbd1..637a869 100644 --- a/core/traefik/traefik.yml +++ b/core/traefik/traefik.yml @@ -1,5 +1,5 @@ log: - level: INFO + level: DEBUG accessLog: filePath: /logs/access.log diff --git a/default-network.yml b/default-network.yml index 8dacf6c..219b56c 100644 --- a/default-network.yml +++ b/default-network.yml @@ -4,4 +4,4 @@ networks: ipam: config: - subnet: 172.21.0.0/16 - monitor: +# monitor: diff --git a/monitoring/beszel/docker-compose.yml b/monitoring/beszel/docker-compose.yml new file mode 100644 index 0000000..8452587 --- /dev/null +++ b/monitoring/beszel/docker-compose.yml @@ -0,0 +1,58 @@ +services: + beszel: + profiles: ["monitoring","all","beszel"] + image: henrygd/beszel:latest + container_name: beszel + restart: always + environment: + APP_URL: https://beszel.lan.ddnsgeek.com + ports: + - 8090:8090 + volumes: + - ${DOCKER_VOLUMES}/beszel/data:/beszel_data + - ${DOCKER_VOLUMES}/beszel/socket:/beszel_socket + healthcheck: + test: ['CMD', '/beszel', 'health', '--url', 'http://localhost:8090'] + interval: 120s + start_period: 10s + timeout: 5s + networks: + - traefik + # - beszel + # - monitor + labels: + - "traefik.enable=true" + - "traefik.http.routers.beszel.rule=Host(`beszel.lan.ddnsgeek.com`)" + - "traefik.http.routers.beszel.entrypoints=websecure" + - "traefik.http.routers.beszel.tls=true" + - "traefik.http.routers.beszel.tls.certresolver=myresolver" + - "traefik.http.services.beszel.loadbalancer.server.port=8090" + - "io.portainer.accesscontrol.public" + - "traefik.docker.network=core_traefik" + - "traefik.http.routers.beszel.tls.options=mtls-private-admin@file" + + beszel-agent: + profiles: ["monitoring","all","beszel"] + image: henrygd/beszel-agent:latest + container_name: beszel-agent + restart: always + network_mode: host + volumes: + - ./beszel/agent:/var/lib/beszel-agent + - ./beszel/socket:/beszel_socket +# - /var/run/docker.sock:/var/run/docker.sock:ro + environment: + LISTEN: /beszel_socket/beszel.sock + HUB_URL: http://localhost:8090 + KEY: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG' + TOKEN: ee2f2df3-d79b-4695-8840-bc936dd7b121 + DOCKER_HOST: tcp://localhost:2375 + healthcheck: + test: ['CMD', '/agent', 'health'] + interval: 120s +# networks: +# - beszel +# - monitor + +#networks: +# beszel: diff --git a/monitoring/docker-exporter/Dockerfile b/monitoring/docker-exporter/Dockerfile deleted file mode 100644 index 7d741b7..0000000 --- a/monitoring/docker-exporter/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.11-slim - -#RUN groupadd -g 1000 appuser || true && \ -# useradd -m -u 1000 -g 1000 -s /bin/bash appuser -#RUN groupadd -g 999 docker || true && usermod -aG docker appuser -WORKDIR /app -COPY exporter.py . - -RUN mkdir -p /data - -RUN pip install --no-cache-dir docker prometheus_client requests pyyaml - -#USER appuser - -CMD ["python", "-u", "exporter.py"] diff --git a/monitoring/docker-exporter/Dockerfile.old b/monitoring/docker-exporter/Dockerfile.old deleted file mode 100644 index 9981774..0000000 --- a/monitoring/docker-exporter/Dockerfile.old +++ /dev/null @@ -1,8 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app -COPY exporter.py . - -RUN pip install docker prometheus_client requests pyyaml - -CMD ["python", "exporter.py"] diff --git a/monitoring/docker-exporter/TEST_EXAMPLE.md b/monitoring/docker-exporter/TEST_EXAMPLE.md deleted file mode 100644 index d6af05b..0000000 --- a/monitoring/docker-exporter/TEST_EXAMPLE.md +++ /dev/null @@ -1,24 +0,0 @@ -# Exporter image-mapping test example - -Run the exporter in dry-run mode to print the `service -> image:tag` mapping without starting the metrics loop: - -```bash -SERVICES_UP_SCRIPT=/workspace/docker/services-up.sh python monitoring/docker-exporter/exporter.py --dry-run -``` - -Example output excerpt: - -```json -{ - "crowdsec": "crowdsecurity/crowdsec:latest", - "docker-update-exporter": "python:3.11-slim", - "nextcloud-webapp": "nextcloud:production", - "node-red": "nodered/node-red:latest", - "prometheus": "prom/prometheus:latest", - "traefik": "traefik:3" -} -``` - -This confirms the exporter now reports images for both: -- services with explicit `image:` values, and -- services using `build:` contexts. diff --git a/monitoring/docker-exporter/docker-compose.yml b/monitoring/docker-exporter/docker-compose.yml deleted file mode 100644 index 284c3f8..0000000 --- a/monitoring/docker-exporter/docker-compose.yml +++ /dev/null @@ -1,59 +0,0 @@ -services: - docker-update-exporter: - profiles: ["monitoring","all","docker-exporter", "prometheus"] - build: - context: ${PROJECT_ROOT}/monitoring/docker-exporter - container_name: docker-update-exporter -# volumes: -# - /var/run/docker.sock:/var/run/docker.sock -# - ${PROJECT_ROOT}/monitoring/docker-exporter/data:/data:rw -# - ${PROJECT_ROOT}/services-up.sh:/app/services-up.sh:ro - environment: - LOG_LEVEL: ${DOCKER_EXPORTER_LOG_LEVEL} - DOCKER_HOST: ${DOCKER_SOCKET_PROXY_HOST} - depends_on: - - docker-socket-proxy - - volumes: - - ~/.docker/config.json:/root/.docker/config.json:ro -# - ${PROJECT_ROOT}/monitoring/docker-exporter/data:/data:rw - - ${DOCKER_VOLUMES}/docker-update-exporter-data:/data:rw - - ${PROJECT_ROOT}:/compose:ro -# - ${PROJECT_ROOT}/default-environment.env:/compose/default-environment.env:ro -# - ${PROJECT_ROOT}/default-network.yml:/compose/default-network.yml:ro -# - ${PROJECT_ROOT}/core/docker-compose.yml:/compose/core/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/prometheus/docker-compose.yml:/compose/monitoring/prometheus/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/gotify/docker-compose.yml:/compose/monitoring/gotify/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/grafana/docker-compose.yml:/compose/monitoring/grafana/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/portainer/docker-compose.yml:/compose/monitoring/portainer/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/uptime-kuma/docker-compose.yml:/compose/monitoring/uptime-kuma/docker-compose.yml:> -# - ${PROJECT_ROOT}/apps/gitea/docker-compose.yml:/compose/apps/gitea/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/gramps/docker-compose.yml:/compose/apps/gramps/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/nextcloud/docker-compose.yml:/compose/apps/nextcloud/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/passbolt/docker-compose.yml:/compose/apps/passbolt/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/searxng/docker-compose.yml:/compose/apps/searxng/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/shift-recorder/docker-compose.yml:/compose/apps/shift-recorder/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/stockfill/docker-compose.yml:/compose/apps/stockfill/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/node-red/docker-compose.yml:/compose/monitoring/node-red/docker-compose.yml:ro -# - ${PROJECT_ROOT}/core/test/docker-compose.yml:/compose/core/test/docker-compose.yml:ro - - -# ports: -# - "9105:9105" - restart: unless-stopped - cap_drop: - - ALL - security_opt: - - no-new-privileges:true - networks: -# - edge - - monitor - healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9105/metrics')"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 10s -#volumes: -# docker-update-exporter-data: -# external: true diff --git a/monitoring/docker-exporter/exporter.py b/monitoring/docker-exporter/exporter.py deleted file mode 100644 index 2e2d55a..0000000 --- a/monitoring/docker-exporter/exporter.py +++ /dev/null @@ -1,579 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import os -import re -import time -import json -import logging -import docker -import yaml -from prometheus_client import Gauge, start_http_server - -# --- Logging --- -LOG_LEVEL = os.getenv("LOG_LEVEL", "DEBUG").upper() -logging.basicConfig( - level=getattr(logging, LOG_LEVEL, logging.DEBUG), - format="%(asctime)s [%(levelname)s] %(message)s" -) -logger = logging.getLogger("docker-update-exporter") - -# --- Config --- -EXPORTER_PORT = 9105 -CHECK_INTERVAL = 3600 -CACHE_TTL = int(os.getenv("CACHE_TTL", "300")) -SERVICES_UP_SCRIPT = os.getenv("SERVICES_UP_SCRIPT", "/compose/services-up.sh") -CACHE_FILE = os.getenv("CACHE_FILE", "/data/remote_digest_cache.json") -DRY_RUN = os.getenv("DRY_RUN", "false").lower() in ("1", "true", "yes") - -try: - client = docker.from_env() -except Exception as e: - logger.warning(f"Docker client unavailable at startup: {e}") - client = None - -# --- Metrics --- -CONTAINER_UPDATE = Gauge( - "docker_container_update_available", - "1 if container image is out of date (compose drift or registry), 0 otherwise", - ["container", "compose_image", "running_image", "com_docker_compose_project"] -) - -LAST_CHECK = Gauge( - "docker_image_update_last_check_timestamp", - "Last time the update check ran (unix timestamp)" -) - - -def set_container_update_metric(container_name, compose_image, running_image, project_name, update_flag): - """Set update metric for a container and log the emitted metric payload.""" - metric_labels = { - "container": container_name, - "compose_image": compose_image or "unknown", - "running_image": running_image, - "com_docker_compose_project": project_name, - } - CONTAINER_UPDATE.labels(**metric_labels).set(update_flag) - logger.info( - "Metric emitted: docker_container_update_available=%s labels=%s", - update_flag, - metric_labels, - ) - - -def set_last_check_metric(): - """Set and log the timestamp for the most recent check cycle.""" - ts = time.time() - LAST_CHECK.set(ts) - logger.info("Metric emitted: docker_image_update_last_check_timestamp=%s", ts) - -# --- Persistent Cache --- -def load_cache(): - if not os.path.exists(CACHE_FILE): - logger.info(f"Cache file does not exist yet: {CACHE_FILE}") - return {} - try: - with open(CACHE_FILE, "r") as f: - cache = json.load(f) - logger.info(f"Loaded {len(cache)} cached remote digests") - return cache - except Exception as e: - logger.error(f"Failed to load cache: {e}") - return {} - -def save_cache(): - try: - os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True) - with open(CACHE_FILE, "w") as f: - json.dump(REMOTE_DIGEST_CACHE, f) - logger.debug(f"Saved {len(REMOTE_DIGEST_CACHE)} remote digests to cache") - except Exception as e: - logger.error(f"Failed to save cache: {e}") - -REMOTE_DIGEST_CACHE = load_cache() -now = time.time() -REMOTE_DIGEST_CACHE = { - image: (digest, ts) - for image, (digest, ts) in REMOTE_DIGEST_CACHE.items() - if now - ts < CACHE_TTL -} - -# --- Helpers --- -def get_project_prefix_from_script(script_path): - prefix = "core-" - if not os.path.exists(script_path): - return prefix - try: - with open(script_path) as f: - for line in f: - m = re.match(r'PROJECT\s*=\s*["\']?([^"\']+)', line) - if m: - return m.group(1) + "-" - except Exception as e: - logger.warning(f"Failed reading project prefix: {e}") - return prefix - -def get_local_digest(image_name): - """ - Return the local digest for the specific image reference. - """ - if client is None: - return None - - try: - img = client.images.get(image_name) - digests = img.attrs.get("RepoDigests", []) - - logger.debug(f"RepoDigests for {image_name}: {digests}") - - for entry in digests: - if "@" in entry: - digest = entry.split("@", 1)[1] - logger.debug(f"Local digest for {image_name}: {digest}") - return digest - - logger.debug(f"No RepoDigest found for {image_name}") - - except Exception as e: - logger.debug(f"Could not get local digest for {image_name}: {e}") - - return None - -def get_remote_digest(image_name): - """ - Return the upstream digest for the exact platform-specific image that Docker - would pull on this host. This avoids false positives with multi-arch images - where the registry manifest-list digest differs from the pulled image digest. - """ - now = time.time() - - cached = REMOTE_DIGEST_CACHE.get(image_name) - if cached: - digest, ts = cached - if now - ts < CACHE_TTL: - logger.debug(f"Using cached remote digest for {image_name}: {digest}") - return digest - - if client is None: - return None - - try: - registry_data = client.images.get_registry_data(image_name) - - digest = None - - # docker SDK versions differ; try the common fields in order - if hasattr(registry_data, "id") and registry_data.id: - digest = registry_data.id - elif hasattr(registry_data, "attrs"): - digest = ( - registry_data.attrs.get("Descriptor", {}).get("digest") - or registry_data.attrs.get("digest") - ) - - if digest: - REMOTE_DIGEST_CACHE[image_name] = (digest, now) - save_cache() - logger.debug(f"Remote digest for {image_name}: {digest}") - return digest - - logger.warning(f"No remote digest found for {image_name}") - return None - except Exception as e: - logger.debug(f"Error fetching remote digest for {image_name}: {e}") - - return None - -# --- Dockerfile Image Extraction --- -def parse_dockerfile_for_image(dockerfile_path): - if not os.path.exists(dockerfile_path): - return None - - try: - arg_defaults = {} - last_from = None - with open(dockerfile_path) as df: - for line in df: - line = line.strip() - if not line or line.startswith("#"): - continue - - if line.upper().startswith("ARG "): - arg_body = line[4:].strip() - if "=" in arg_body: - key, value = arg_body.split("=", 1) - arg_defaults[key.strip()] = value.strip() - continue - - # Prefer LABEL with image if present. - if "LABEL" in line and "image=" in line: - match = re.search(r'image=["\']?([^"\']+)["\']?', line) - if match: - image_name = normalize_image_name(substitute_dockerfile_args(match.group(1), arg_defaults)) - logger.debug(f"Found LABEL image={image_name} in {dockerfile_path}") - return image_name - - if line.upper().startswith("FROM "): - from_clause = line[5:].strip() - if from_clause.startswith("--"): - split_clause = from_clause.split(None, 1) - if len(split_clause) < 2: - continue - from_clause = split_clause[1] - - parts = from_clause.split() - if not parts: - continue - - candidate = substitute_dockerfile_args(parts[0], arg_defaults) - if candidate and candidate.lower() != "scratch": - last_from = normalize_image_name(candidate) - - if last_from: - logger.debug(f"Found base FROM {last_from} in {dockerfile_path}") - return last_from - except Exception as e: - logger.debug(f"Error reading Dockerfile {dockerfile_path}: {e}") - - return None - -def normalize_image_name(image_name): - if not image_name: - return None - if "@" in image_name: - return image_name - if ":" in image_name.rsplit("/", 1)[-1]: - return image_name - return f"{image_name}:latest" - -def is_compose_build_placeholder(image_name, project_name): - if not image_name: - return False - candidate = str(image_name) - project_prefix = f"{project_name}-" - if candidate.startswith(project_prefix): - return True - # Keep backward-compatible behavior for historical default project prefix. - return candidate.startswith("core-") - -def substitute_dockerfile_args(value, arg_defaults): - if not value: - return value - - pattern = re.compile(r"\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)") - - def replacer(match): - expr = match.group(1) - simple = match.group(2) - if simple: - return arg_defaults.get(simple, "") - - if ":-" in expr: - var_name, default_value = expr.split(":-", 1) - return arg_defaults.get(var_name, default_value) - if "-" in expr: - var_name, default_value = expr.split("-", 1) - return arg_defaults.get(var_name, default_value) - return arg_defaults.get(expr, "") - - return pattern.sub(replacer, value) - -def expand_compose_path(path_value, project_root): - raw = str(path_value) - raw = raw.replace("${PROJECT_ROOT}", project_root).replace("$PROJECT_ROOT", project_root) - return os.path.expandvars(raw) - -def get_project_root_from_script(script_path): - if not script_path: - return os.getcwd() - return os.path.dirname(os.path.abspath(script_path)) - -# --- Compose parsing --- -def get_compose_files_from_script(script_path): - files = [] - if not os.path.exists(script_path): - return files - base_dir = get_project_root_from_script(script_path) - - def _clean_compose_path(raw_path): - cleaned = str(raw_path).strip().strip(",") - if (cleaned.startswith('"') and cleaned.endswith('"')) or ( - cleaned.startswith("'") and cleaned.endswith("'") - ): - cleaned = cleaned[1:-1] - expanded = expand_compose_path(cleaned, base_dir) - if os.path.isabs(expanded): - return os.path.normpath(expanded) - return os.path.normpath(os.path.join(base_dir, expanded)) - - try: - with open(script_path) as f: - content = f.read() - match = re.search(r'FILES\s*=\s*\((.*?)\)', content, re.DOTALL) - if match: - for line in match.group(1).splitlines(): - line = line.strip() - if line.startswith("-f"): - path = line[2:].strip() - if path: - full = _clean_compose_path(path) - files.append(full) - - # services-up.sh can append many compose files at runtime via: - # FILES+=(-f "$file") done < <(find "$PROJECT_ROOT/apps" ...) - # Mirror that behavior here so we can map service->compose image. - root_dirs = [] - find_match = re.search(r'find\s+(.*?)\s+\\\s*\n', content) - if find_match: - for token in re.findall(r'"([^"]+)"|\'([^\']+)\'', find_match.group(1)): - candidate = token[0] or token[1] - if candidate: - root_dirs.append(_clean_compose_path(candidate)) - else: - root_dirs = [ - os.path.join(base_dir, "apps"), - os.path.join(base_dir, "monitoring"), - os.path.join(base_dir, "core"), - ] - - for root_dir in root_dirs: - if not os.path.isdir(root_dir): - continue - for candidate in sorted(os.listdir(root_dir)): - svc_dir = os.path.join(root_dir, candidate) - if not os.path.isdir(svc_dir): - continue - for compose_name in ("docker-compose.yml", "docker-compose.yaml"): - compose_path = os.path.join(svc_dir, compose_name) - if os.path.exists(compose_path): - files.append(compose_path) - - # Preserve order while removing duplicates. - deduped = [] - seen = set() - for path in files: - if path in seen: - continue - seen.add(path) - deduped.append(path) - files = deduped - except Exception as e: - logger.warning(f"Failed parsing services-up.sh: {e}") - return files - -def parse_project_name_from_script(script_path): - project = "core" - if not os.path.exists(script_path): - return project - try: - with open(script_path) as f: - for line in f: - m = re.match(r'PROJECT\s*=\s*["\']?([^"\']+)', line) - if m: - project = m.group(1) - break - except Exception as e: - logger.warning(f"Failed reading project name: {e}") - return project - -def resolve_local_build_image(service_name, project_name): - if client is None: - return None - try: - images = client.images.list(filters={"label": f"com.docker.compose.service={service_name}"}) - for image in images: - labels = image.attrs.get("Config", {}).get("Labels", {}) or {} - if labels.get("com.docker.compose.project") != project_name: - continue - for tag in image.tags: - if tag and "" not in tag: - logger.debug(f"Resolved local compose image for {service_name}: {tag}") - return normalize_image_name(tag) - except Exception as e: - logger.debug(f"Could not inspect local build metadata for {service_name}: {e}") - return None - -def parse_compose_services(compose_files, project_name, project_root): - svc_map = {} - for f in compose_files: - if not os.path.exists(f): - logger.warning(f"Compose file from services-up.sh is missing: {f}") - continue - try: - with open(f) as stream: - data = yaml.safe_load(stream) or {} - for svc_name, svc_def in data.get("services", {}).items(): - image = normalize_image_name(svc_def.get("image")) - profiles = svc_def.get("profiles", []) - build_ctx = svc_def.get("build") - dockerfile_path = None - from_dockerfile = None - local_built_image = None - - if build_ctx: - if isinstance(build_ctx, dict): - context = build_ctx.get("context", ".") - dockerfile = build_ctx.get("dockerfile", "Dockerfile") - else: - context = build_ctx - dockerfile = "Dockerfile" - - compose_dir = os.path.dirname(f) - context_expanded = expand_compose_path(context, project_root) - if os.path.isabs(context_expanded): - context_path = context_expanded - else: - context_path = os.path.normpath(os.path.join(compose_dir, context_expanded)) - dockerfile_expanded = expand_compose_path(dockerfile, project_root) - dockerfile_path = os.path.normpath(os.path.join(context_path, dockerfile_expanded)) - from_dockerfile = normalize_image_name(parse_dockerfile_for_image(dockerfile_path)) - local_built_image = resolve_local_build_image(svc_name, project_name) - - placeholder_image = is_compose_build_placeholder(image, project_name) or is_compose_build_placeholder(local_built_image, project_name) - if placeholder_image: - resolved_image = from_dockerfile or image or local_built_image or f"{project_name}-{svc_name}:latest" - else: - resolved_image = image or local_built_image or from_dockerfile or f"{project_name}-{svc_name}:latest" - - svc_map[svc_name] = { - "image": resolved_image, - "profiles": profiles, - "build_context": build_ctx, - "compose_file": f, - "dockerfile": dockerfile_path - } - except Exception as e: - logger.warning(f"Failed parsing {f}: {e}") - - logger.debug(f"Service image mapping: {svc_map}") - return svc_map - -# --- Main check --- -def check_containers(): - if client is None: - logger.error("Docker client is unavailable; skipping check cycle") - return - - set_last_check_metric() - CONTAINER_UPDATE.clear() - - project_name = parse_project_name_from_script(SERVICES_UP_SCRIPT) - project_root = get_project_root_from_script(SERVICES_UP_SCRIPT) - compose_files = get_compose_files_from_script(SERVICES_UP_SCRIPT) - svc_map = parse_compose_services(compose_files, project_name, project_root) - - containers = client.containers.list() - pending_metrics = [] - remote_targets = set() - - for container in containers: - proj = container.labels.get("com.docker.compose.project") - if not proj: - continue - - svc = container.labels.get("com.docker.compose.service") - running = container.attrs["Config"]["Image"] - - compose_image = None - if svc in svc_map: - compose_image = svc_map[svc]["image"] - - local_digest = get_local_digest(running) - remote_target = compose_image or running - - # If we cannot determine a local digest, we cannot compare and should - # avoid spending a registry lookup for this container. - if local_digest: - remote_targets.add(remote_target) - - pending_metrics.append({ - "container_name": container.name, - "service": svc, - "compose_image": compose_image, - "running_image": running, - "project_name": proj, - "remote_target": remote_target, - "local_digest": local_digest, - }) - - remote_digests = {target: get_remote_digest(target) for target in remote_targets} - - for payload in pending_metrics: - local_digest = payload["local_digest"] - remote_target = payload["remote_target"] - remote_digest = remote_digests.get(remote_target) - update_flag = 1 if (local_digest and remote_digest and local_digest != remote_digest) else 0 - - logger.info( - "Digest comparison: container=%s service=%s running=%s target=%s local=%s remote=%s", - payload["container_name"], - payload["service"], - payload["running_image"], - remote_target, - local_digest, - remote_digest, - ) - - set_container_update_metric( - container_name=payload["container_name"], - compose_image=payload["compose_image"], - running_image=payload["running_image"], - project_name=payload["project_name"], - update_flag=update_flag, - ) - -def dump_service_image_mapping(): - project_name = parse_project_name_from_script(SERVICES_UP_SCRIPT) - project_root = get_project_root_from_script(SERVICES_UP_SCRIPT) - compose_files = get_compose_files_from_script(SERVICES_UP_SCRIPT) - svc_map = parse_compose_services(compose_files, project_name, project_root) - mapping = {name: data["image"] for name, data in sorted(svc_map.items())} - logger.info("Service to image mapping:") - logger.info(json.dumps(mapping, indent=2, sort_keys=True)) - return mapping - -# --- Runner --- -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Docker image update exporter") - parser.add_argument("--dry-run", action="store_true", help="Only print service->image mapping and exit") - parser.add_argument( - "--services-up-script", - default=SERVICES_UP_SCRIPT, - help=f"Path to services-up script (default: {SERVICES_UP_SCRIPT})", - ) - parser.add_argument( - "--cache-file", - default=CACHE_FILE, - help=f"Path to digest cache file (default: {CACHE_FILE})", - ) - parser.add_argument( - "--log-level", - default=LOG_LEVEL, - help=f"Logging level (default: {LOG_LEVEL})", - ) - args = parser.parse_args() - - effective_log_level = str(args.log_level).upper() - logging.getLogger().setLevel(getattr(logging, effective_log_level, logging.DEBUG)) - logger.setLevel(getattr(logging, effective_log_level, logging.DEBUG)) - - SERVICES_UP_SCRIPT = args.services_up_script - CACHE_FILE = args.cache_file - REMOTE_DIGEST_CACHE = load_cache() - now = time.time() - REMOTE_DIGEST_CACHE = { - image: (digest, ts) - for image, (digest, ts) in REMOTE_DIGEST_CACHE.items() - if now - ts < CACHE_TTL - } - - if DRY_RUN or args.dry_run: - dump_service_image_mapping() - raise SystemExit(0) - - start_http_server(EXPORTER_PORT) - while True: - try: - check_containers() - except Exception as e: - logger.exception(f"update check failed: {e}") - time.sleep(CHECK_INTERVAL) diff --git a/monitoring/docker-exporter/exporter.py.old b/monitoring/docker-exporter/exporter.py.old deleted file mode 100644 index 3d0c5f3..0000000 --- a/monitoring/docker-exporter/exporter.py.old +++ /dev/null @@ -1,514 +0,0 @@ -#!/usr/bin/env python3 -import os -import re -import time -import json -import logging -import docker -import requests -import yaml -from prometheus_client import Gauge, start_http_server - -# --- Logging --- -LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper() - -logging.basicConfig( - level=getattr(logging, LOG_LEVEL, logging.INFO), - format="%(asctime)s [%(levelname)s] %(message)s" -) - -logger = logging.getLogger("docker-update-exporter") - -# --- Config --- -EXPORTER_PORT = 9105 -CHECK_INTERVAL = 60 -CACHE_TTL = 6 * 3600 -SERVICES_UP_SCRIPT = "/compose/services-up.sh" -CACHE_FILE = "/data/remote_digest_cache.json" - -client = docker.from_env() - -# --- Metrics --- -CONTAINER_UPDATE = Gauge( - "docker_container_update_available", - "1 if container image is out of date (compose drift or registry), 0 otherwise", - ["container", "compose_image", "running_image", "com_docker_compose_project"] -) - -LAST_CHECK = Gauge( - "docker_image_update_last_check_timestamp", - "Last time the update check ran (unix timestamp)" -) - -# --- Persistent Cache --- - -def load_cache(): - if not os.path.exists(CACHE_FILE): - logger.info(f"Cache file does not exist yet: {CACHE_FILE}") - return {} - - try: - with open(CACHE_FILE, "r") as f: - cache = json.load(f) - logger.info(f"Loaded {len(cache)} cached remote digests") - logger.debug(f"Cache contents: {cache}") - return cache - except Exception as e: - logger.error(f"Failed to load cache from {CACHE_FILE}: {e}") - return {} - -def save_cache(): - try: - os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True) - with open(CACHE_FILE, "w") as f: - json.dump(REMOTE_DIGEST_CACHE, f) - - logger.debug( - f"Saved {len(REMOTE_DIGEST_CACHE)} entries to cache file {CACHE_FILE}" - ) - except Exception as e: - logger.error(f"Failed to save cache to {CACHE_FILE}: {e}") - -REMOTE_DIGEST_CACHE = load_cache() - -# --- Helpers --- - -def get_project_prefix_from_script(script_path): - project_prefix = "core-" # fallback - - if not os.path.exists(script_path): - logger.warning( - f"services-up script not found at {script_path}, using fallback project prefix {project_prefix}" - ) - return project_prefix - - try: - with open(script_path, "r") as f: - for line in f: - line = line.strip() - m = re.match(r'PROJECT\s*=\s*["\']?([^"\']+)["\']?', line) - if m: - project_prefix = m.group(1) + "-" - logger.debug( - f"Detected compose project prefix from script: {project_prefix}" - ) - break - except Exception as e: - logger.error(f"Failed reading project prefix from {script_path}: {e}") - - return project_prefix - -def get_local_digest(image_name): - try: - img = client.images.get(image_name) - digests = img.attrs.get("RepoDigests", []) - - logger.debug(f"Local RepoDigests for {image_name}: {digests}") - - if digests: - digest = digests[0].split("@")[1] - logger.debug(f"Local digest for {image_name}: {digest}") - return digest - - logger.info(f"No local digest found for image {image_name}") - - except Exception as e: - logger.warning(f"Failed to retrieve local digest for {image_name}: {e}") - - return None - -def get_remote_digest(image_name): - now = time.time() - original = image_name - - # Cache hit - if original in REMOTE_DIGEST_CACHE: - digest, ts = REMOTE_DIGEST_CACHE[original] - age = now - ts - - if age < CACHE_TTL: - logger.debug( - f"Using cached remote digest for {original} " - f"(age={int(age)}s, ttl={CACHE_TTL}s): {digest}" - ) - return digest - - logger.info( - f"Cache entry expired for {original} " - f"(age={int(age)}s > ttl={CACHE_TTL}s)" - ) - - try: - if "/" not in image_name: - registry = "docker.io" - repo = "library/" + image_name - else: - parts = image_name.split("/") - if "." in parts[0] or ":" in parts[0]: - registry = parts[0] - repo = "/".join(parts[1:]) - else: - registry = "docker.io" - repo = image_name - - if ":" in repo: - repo, tag = repo.rsplit(":", 1) - else: - tag = "latest" - - logger.debug( - f"Resolving remote digest for {original}: " - f"registry={registry}, repo={repo}, tag={tag}" - ) - - token = None - manifest_url = None - - if registry in ["docker.io", "registry-1.docker.io"]: - logger.debug(f"Requesting Docker Hub token for {repo}") - - token_res = requests.get( - "https://auth.docker.io/token", - params={ - "service": "registry.docker.io", - "scope": f"repository:{repo}:pull" - }, - timeout=10 - ) - - logger.debug( - f"Docker Hub token response for {repo}: " - f"status={token_res.status_code}" - ) - - token = token_res.json().get("token") - manifest_url = ( - f"https://registry-1.docker.io/v2/{repo}/manifests/{tag}" - ) - - elif registry == "ghcr.io": - logger.debug(f"Requesting GHCR token for {repo}") - - token_res = requests.get( - "https://ghcr.io/token", - params={ - "service": "ghcr.io", - "scope": f"repository:{repo}:pull" - }, - timeout=10 - ) - - logger.debug( - f"GHCR token response for {repo}: " - f"status={token_res.status_code}" - ) - - token = token_res.json().get("token") - manifest_url = f"https://ghcr.io/v2/{repo}/manifests/{tag}" - - else: - logger.warning( - f"Unsupported registry '{registry}' for image {original}" - ) - return None - - if not token: - logger.warning( - f"No authentication token returned for {original}" - ) - return None - - logger.debug(f"Requesting manifest for {original}: {manifest_url}") - - res = requests.get( - manifest_url, - headers={ - "Authorization": f"Bearer {token}", - "Accept": "application/vnd.docker.distribution.manifest.v2+json" - }, - timeout=10 - ) - - logger.debug( - f"Manifest response for {original}: " - f"status={res.status_code}" - ) - - if res.status_code == 200: - digest = res.headers.get("Docker-Content-Digest") - - logger.info( - f"Fetched remote digest for {original}: {digest}" - ) - - REMOTE_DIGEST_CACHE[original] = (digest, now) - save_cache() - - logger.debug( - f"Cached remote digest for {original}: {digest}" - ) - - return digest - - if res.status_code == 429: - logger.warning( - f"Registry rate limit hit while fetching {original}" - ) - elif res.status_code in [401, 403]: - logger.warning( - f"Authentication failed while fetching {original}: " - f"status={res.status_code}" - ) - else: - logger.warning( - f"Unexpected manifest response for {original}: " - f"status={res.status_code}, body={res.text[:250]}" - ) - - except Exception as e: - logger.error(f"Failed to fetch remote digest for {original}: {e}") - - return None - -def get_compose_files_from_script(script_path): - files = [] - - if not os.path.exists(script_path): - logger.error(f"services-up script not found: {script_path}") - return files - - base_dir = os.path.dirname(script_path) - - try: - with open(script_path, "r") as f: - content = f.read() - - match = re.search(r'FILES\s*=\s*\((.*?)\)', content, re.DOTALL) - - if not match: - logger.warning( - f"No FILES=(...) block found in {script_path}" - ) - return files - - lines = match.group(1).splitlines() - - for line in lines: - line = line.strip() - - if line.startswith("-f"): - rel_path = line[2:].strip() - - if rel_path: - full_path = os.path.normpath( - os.path.join(base_dir, rel_path) - ) - - logger.debug( - f"Resolved compose file: {rel_path} -> {full_path}" - ) - - files.append(full_path) - - logger.info(f"Found {len(files)} compose files") - - except Exception as e: - logger.error(f"Failed parsing compose files from {script_path}: {e}") - - return files - -def parse_compose_files(compose_files): - service_to_image = {} - - for f in compose_files: - if not os.path.exists(f): - logger.warning(f"Compose file missing: {f}") - continue - - try: - with open(f, "r") as stream: - data = yaml.safe_load(stream) or {} - services = data.get("services", {}) - - logger.debug( - f"Parsing {len(services)} services from compose file {f}" - ) - - for service_name, service_def in services.items(): - image = service_def.get("image") - is_built = False - - if not image: - is_built = True - build_ctx = service_def.get("build") - - logger.debug( - f"Service {service_name} is build-based, build config={build_ctx}" - ) - - if isinstance(build_ctx, dict): - context_path = build_ctx.get("context", ".") - dockerfile_path = os.path.join( - context_path, - build_ctx.get("dockerfile", "Dockerfile") - ) - elif isinstance(build_ctx, str): - context_path = build_ctx - dockerfile_path = os.path.join( - context_path, "Dockerfile" - ) - else: - dockerfile_path = None - - if dockerfile_path and os.path.exists(dockerfile_path): - try: - with open(dockerfile_path, "r") as df: - for line in df: - line = line.strip() - if ( - line.upper().startswith("LABEL") - and "image=" in line - ): - m = re.search( - r'image=["\']?([^"\']+)["\']?', - line - ) - if m: - image = m.group(1) - logger.debug( - f"Found upstream image label for {service_name}: {image}" - ) - break - except Exception as e: - logger.warning( - f"Failed reading Dockerfile {dockerfile_path}: {e}" - ) - - if not image: - image = f"{service_name}:latest" - logger.info( - f"No image label found for build service {service_name}, " - f"defaulting to {image}" - ) - - service_to_image[service_name] = (image, is_built) - - except Exception as e: - logger.error(f"Failed parsing compose file {f}: {e}") - - logger.info(f"Mapped {len(service_to_image)} compose services to images") - logger.debug(f"Service/image mapping: {service_to_image}") - - return service_to_image - -def check_containers(): - logger.info("Starting container update check") - - CONTAINER_UPDATE.clear() - - project_prefix = get_project_prefix_from_script(SERVICES_UP_SCRIPT) - compose_files = get_compose_files_from_script(SERVICES_UP_SCRIPT) - service_to_image = parse_compose_files(compose_files) - - containers = client.containers.list() - logger.info(f"Checking {len(containers)} running containers") - - for container in containers: - project_label = container.labels.get("com.docker.compose.project") - - if not project_label: - logger.debug( - f"Skipping non-compose container {container.name}" - ) - continue - - service_label = container.labels.get("com.docker.compose.service") - running_image = container.attrs["Config"]["Image"] - - logger.debug( - f"Evaluating container={container.name}, " - f"service={service_label}, project={project_label}, " - f"running_image={running_image}" - ) - - compose_image = None - is_built = False - - if service_label and service_label in service_to_image: - compose_image, is_built = service_to_image[service_label] - - if is_built: - compose_image_name, _, _ = compose_image.partition(":") - compose_image = f"{project_prefix}{compose_image_name}" - - update_flag = 0 - - if is_built: - if running_image != compose_image: - logger.info( - f"Update detected for build-based container {container.name}: " - f"running image {running_image} != expected {compose_image}" - ) - update_flag = 1 - else: - local_digest = get_local_digest(running_image) - remote_digest = get_remote_digest( - service_to_image[service_label][0] - ) - - if local_digest and remote_digest and local_digest != remote_digest: - logger.info( - f"Remote image update available for {container.name}: " - f"{local_digest} != {remote_digest}" - ) - update_flag = 1 - else: - if running_image != compose_image: - logger.info( - f"Compose drift detected for {container.name}: " - f"running image {running_image} != compose image {compose_image}" - ) - update_flag = 1 - else: - local_digest = get_local_digest(running_image) - remote_digest = get_remote_digest(running_image) - - if local_digest and remote_digest and local_digest != remote_digest: - logger.info( - f"Registry update available for {container.name}: " - f"{local_digest} != {remote_digest}" - ) - update_flag = 1 - - CONTAINER_UPDATE.labels( - container=container.name, - compose_image=compose_image if compose_image else "unknown", - running_image=running_image, - com_docker_compose_project=project_label - ).set(update_flag) - - logger.info( - f"Container {container.name}: " - f"running={running_image}, " - f"compose={compose_image}, " - f"update_available={update_flag}" - ) - - LAST_CHECK.set(time.time()) - logger.info("Container update check complete") - -if __name__ == "__main__": - logger.info( - f"Starting Docker update exporter on port {EXPORTER_PORT} " - f"with LOG_LEVEL={LOG_LEVEL}" - ) - - start_http_server(EXPORTER_PORT) - - while True: - try: - check_containers() - except Exception as e: - logger.exception(f"Unhandled error during update check: {e}") - - time.sleep(CHECK_INTERVAL) diff --git a/monitoring/docker-socket-proxy/docker-compose.yml b/monitoring/docker-socket-proxy/docker-compose.yml index 5bb4982..e589af3 100644 --- a/monitoring/docker-socket-proxy/docker-compose.yml +++ b/monitoring/docker-socket-proxy/docker-compose.yml @@ -1,6 +1,6 @@ services: docker-socket-proxy: - profiles: ["monitoring","all","docker-socket-proxy", "core", "traefik", "prometheus"] + profiles: ["monitoring","all","docker-socket-proxy", "core", "traefik", "uptime-kuma", "beszel"] image: tecnativa/docker-socket-proxy:latest container_name: docker-socket-proxy hostname: docker-socket-proxy @@ -43,5 +43,7 @@ services: security_opt: - no-new-privileges:true networks: - - monitor +# - monitor - traefik + ports: + - 2375:2375 diff --git a/monitoring/grafana/docker-compose.yml b/monitoring/grafana/docker-compose.yml deleted file mode 100644 index ec5b127..0000000 --- a/monitoring/grafana/docker-compose.yml +++ /dev/null @@ -1,55 +0,0 @@ -services: - grafana: - profiles: ["monitoring","all","grafana"] - image: grafana/grafana:latest - container_name: grafana - restart: unless-stopped - environment: - - GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL} - volumes: -# - ${PROJECT_ROOT}/monitoring/grafana/data:/var/lib/grafana - - ${DOCKER_VOLUMES}/grafana-data:/var/lib/grafana - networks: -# - traefik_reverse_proxy -# - prometheus_edge - - traefik - - monitor - labels: - - "traefik.http.routers.grafana.rule=Host(`grafana.lan.ddnsgeek.com`)" - - "traefik.enable=true" - - "traefik.http.routers.grafana.entrypoints=websecure" - - "traefik.http.routers.grafana.tls.certresolver=myresolver" - - "traefik.http.routers.grafana.tls.options=mtls-private-admin@file" - - "io.portainer.accesscontrol.public" - - "traefik.http.services.grafana.loadbalancer.server.port=3000" - - "traefik.docker.network=core_traefik" - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - -# tempo: -# image: grafana/tempo:latest -# container_name: tempo -# command: -# - "-config.file=/etc/tempo/config.yaml" -# volumes: -# - ./tempo/config.yaml:/etc/tempo/config.yaml -# - ./tempo/data:/var/lib/tempo -# ports: -# - "4317:4317" # OTLP gRPC endpoint for Traefik -# - "3200:3200" # optional: HTTP endpoint -# networks: -# - prometheus_edge - -#networks: -# traefik_reverse_proxy: -# external: true -# prometheus_edge: -# external: true - -#volumes: -# grafana-data: -# external: true diff --git a/monitoring/influxdb/docker-compose.yml b/monitoring/influxdb/docker-compose.yml deleted file mode 100644 index ad78629..0000000 --- a/monitoring/influxdb/docker-compose.yml +++ /dev/null @@ -1,50 +0,0 @@ -services: - influxdb: - profiles: ["monitoring","all","influxdb", "prometheus"] - image: influxdb:2 - container_name: influxdb - restart: unless-stopped -# env_file: -# - ${PROJECT_ROOT}/secrets/stack-secrets.env - volumes: -# - ${PROJECT_ROOT}/monitoring/influxdb:/var/lib/influxdb2 - - ${DOCKER_DATABASES}/influxdb-data:/var/lib/influxdb2 - environment: - DOCKER_INFLUXDB_INIT_MODE: ${INFLUXDB_INIT_MODE} - DOCKER_INFLUXDB_INIT_USERNAME: ${INFLUXDB_INIT_USERNAME} - DOCKER_INFLUXDB_INIT_PASSWORD_FILE: /run/secrets/influxdb_init_password - DOCKER_INFLUXDB_INIT_ORG: ${INFLUXDB_INIT_ORG} - DOCKER_INFLUXDB_INIT_BUCKET: ${INFLUXDB_INIT_BUCKET} - secrets: - - influxdb_init_password - networks: -# - edge -# - traefik_reverse_proxy - - traefik - - monitor - ports: - - 8086:8086 - labels: - - "traefik.http.routers.influxdb.rule=Host(`influxdb.lan.ddnsgeek.com`)" - - "traefik.enable=true" - - "traefik.http.routers.influxdb.entrypoints=websecure" - - "traefik.http.routers.influxdb.tls.certresolver=myresolver" - - "traefik.http.routers.influxdb.tls.options=mtls-private-admin@file" - - "io.portainer.accesscontrol.public" - - "traefik.http.services.influxdb.loadbalancer.server.port=8086" - - "traefik.http.routers.influxdb.middlewares=authelia" - - "traefik.docker.network=core_traefik" - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:8086/health || exit 1"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 10s - -secrets: - influxdb_init_password: - file: ${PROJECT_ROOT}/secrets/influxdb_init_password.txt - -#volumes: -# influxdb-data: -# external: true diff --git a/monitoring/mtls-bridge/Dockerfile b/monitoring/mtls-bridge/Dockerfile deleted file mode 100644 index cef343b..0000000 --- a/monitoring/mtls-bridge/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM python:3.11-slim - -WORKDIR /app - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -COPY app.py . - -EXPOSE 8080 - -CMD ["python", "app.py"] diff --git a/monitoring/mtls-bridge/README.md b/monitoring/mtls-bridge/README.md deleted file mode 100644 index 3dfe5e3..0000000 --- a/monitoring/mtls-bridge/README.md +++ /dev/null @@ -1,60 +0,0 @@ -# mTLS Bridge Service - -Internal HTTP-to-mTLS bridge for services that cannot present client certificates directly (for example, Grafana webhooks). - -## How it works - -1. Accepts plain HTTP requests inside the Docker network. -2. Forwards requests to an upstream base URL. -3. Preserves the incoming request path/method/body/query string. -4. Presents a client certificate/key pair for mTLS authentication. - -## Environment variables - -- `TARGET_URL` (required): upstream base URL (for example `http://node-red:1880`). -- `CLIENT_CERT` (default `/certs/client.crt`): client certificate path. -- `CLIENT_KEY` (default `/certs/client.key`): client private key path. -- `UPSTREAM_CA_CERT` (optional, alias: `CA_CERT`): CA bundle path to verify upstream TLS. Use `false`/`0`/`no` to disable verification. -- `TIMEOUT` (default `5`): request timeout in seconds. -- `LOG_LEVEL` (default `INFO`): Python logging level. -- `HEALTH_ENDPOINT` (default `/_mtls_bridge/health`): local container health endpoint path. -- `ALLOWED_PATHS_FILE` (optional): file path containing one allowed endpoint path per line (for example `/health`). Blank lines and `#` comments are ignored. If unset, all paths are allowed. -- `MTLS_BRIDGE_BASIC_AUTH_USERS` (required for Traefik auth): value for `traefik.http.middlewares.*.basicauth.users` (e.g. `user:$$apr1$$...`). -- `MTLS_BRIDGE_CORS_ALLOW_ORIGIN` (default `https://grafana.lan.ddnsgeek.com`): origin allowed for browser-based panel actions. - -## Endpoints - -- `GET /_mtls_bridge/health` returns `200 OK` for container health checks. -- `/*` proxies requests to `${TARGET_URL}/*` with method/body/headers/query string preserved (subject to optional allow-list checks). - -Examples with `TARGET_URL=http://node-red:1880`: - -- `https://mtls-bridge.../docker-update-lockouts/clear` -> `http://node-red:1880/docker-update-lockouts/clear` -- `https://mtls-bridge.../health` -> `http://node-red:1880/health` -- `https://mtls-bridge.../uptime-kuma` -> `http://node-red:1880/uptime-kuma` - -## Compose integration - -This repository includes `monitoring/mtls-bridge/docker-compose.yml`: - -- No public port exposure. -- Read-only cert mount (`${PROJECT_ROOT}/core/traefik/certs:/certs:ro`). -- Joined to internal monitoring/traefik networks. - -## Example test - -```bash -curl http://mtls-bridge:8080/_mtls_bridge/health -curl -X POST http://mtls-bridge:8080/docker-update-lockouts/clear -``` - -## Allow-list file example - -```text -# one path per line -/docker-update-lockouts/clear -/health -/uptime-kuma -``` - -When `ALLOWED_PATHS_FILE` is set, any path not listed returns `403 Endpoint not allowed`. diff --git a/monitoring/mtls-bridge/app.py b/monitoring/mtls-bridge/app.py deleted file mode 100644 index dca29b7..0000000 --- a/monitoring/mtls-bridge/app.py +++ /dev/null @@ -1,208 +0,0 @@ -import logging -import os -import time -from urllib.parse import urljoin - -import requests -from flask import Flask, Response, g, request - -app = Flask(__name__) - -logging.basicConfig( - level=os.environ.get("LOG_LEVEL", "INFO"), - format="%(asctime)s %(levelname)s %(message)s", -) -logger = logging.getLogger("mtls-bridge") -logging.getLogger("werkzeug").setLevel(logging.WARNING) - -# Config via env -TARGET_URL = (os.environ.get("TARGET_URL") or "").strip() -CLIENT_CERT = os.environ.get("CLIENT_CERT", "/certs/client.crt") -CLIENT_KEY = os.environ.get("CLIENT_KEY", "/certs/client.key") -UPSTREAM_CA_CERT = os.environ.get("UPSTREAM_CA_CERT", os.environ.get("CA_CERT", "")).strip() -TIMEOUT = int(os.environ.get("TIMEOUT", "5")) -HEALTH_ENDPOINT = os.environ.get("HEALTH_ENDPOINT", "/_mtls_bridge/health") -ALLOWED_PATHS_FILE = (os.environ.get("ALLOWED_PATHS_FILE") or "").strip() - - -def normalize_path(path: str) -> str: - if not path or path == "/": - return "/" - return f"/{path.lstrip('/')}" - - -def load_allowed_paths() -> set[str]: - if not ALLOWED_PATHS_FILE: - return set() - - if not os.path.exists(ALLOWED_PATHS_FILE): - logger.warning("ALLOWED_PATHS_FILE does not exist: %s (allow-list disabled)", ALLOWED_PATHS_FILE) - return set() - - allowed_paths = set() - with open(ALLOWED_PATHS_FILE, encoding="utf-8") as f: - for line in f: - entry = line.strip() - if not entry or entry.startswith("#"): - continue - allowed_paths.add(normalize_path(entry)) - - logger.info("loaded %s allowed path(s) from %s", len(allowed_paths), ALLOWED_PATHS_FILE) - return allowed_paths - - -def get_verify_setting(): - if not UPSTREAM_CA_CERT: - return True - - lowered = UPSTREAM_CA_CERT.lower() - if lowered in {"false", "0", "no"}: - logger.warning("TLS verification for upstream is disabled via UPSTREAM_CA_CERT=%s", UPSTREAM_CA_CERT) - return False - - if not os.path.exists(UPSTREAM_CA_CERT): - logger.warning( - "Configured UPSTREAM_CA_CERT path does not exist: %s (falling back to system CA bundle)", - UPSTREAM_CA_CERT, - ) - return True - - return UPSTREAM_CA_CERT - - -VERIFY_SETTING = get_verify_setting() -ALLOWED_PATHS = load_allowed_paths() - -if TARGET_URL and TARGET_URL.lower().startswith("http://"): - logger.warning("TARGET_URL uses http:// (plaintext): %s", TARGET_URL) - -logger.info( - "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s health_endpoint=%s allow_list_file=%s allow_list_entries=%s log_level=%s", - TARGET_URL, - TIMEOUT, - CLIENT_CERT, - CLIENT_KEY, - VERIFY_SETTING, - HEALTH_ENDPOINT, - ALLOWED_PATHS_FILE, - len(ALLOWED_PATHS), - os.environ.get("LOG_LEVEL", "INFO"), -) - - -def build_upstream_url(path: str) -> str: - """Map incoming path directly onto TARGET_URL origin/base path.""" - if not TARGET_URL: - raise ValueError("TARGET_URL is not set") - - normalized_target = TARGET_URL.rstrip("/") + "/" - normalized_path = path.lstrip("/") - upstream_url = urljoin(normalized_target, normalized_path) - - if request.query_string: - upstream_url = f"{upstream_url}?{request.query_string.decode('utf-8', 'ignore')}" - - return upstream_url - - -def is_path_allowed(request_path: str) -> bool: - if not ALLOWED_PATHS: - return True - return request_path in ALLOWED_PATHS - - -@app.route(HEALTH_ENDPOINT, methods=["GET"]) -def health(): - logger.debug("healthcheck request from %s", request.remote_addr) - return "OK", 200 - - -@app.before_request -def before_request(): - g.request_start = time.time() - - -@app.after_request -def after_request(response): - elapsed_ms = int((time.time() - g.request_start) * 1000) - if request.path != HEALTH_ENDPOINT: - logger.info( - "request complete method=%s path=%s status=%s elapsed_ms=%s", - request.method, - request.path, - response.status_code, - elapsed_ms, - ) - return response - - -@app.route( - "/", - defaults={"path": ""}, - methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"], - provide_automatic_options=False, -) -@app.route( - "/", - methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"], - provide_automatic_options=False, -) -def proxy(path): - request_path = normalize_path(path) - request_size = len(request.get_data(cache=True)) - logger.info( - "incoming request method=%s path=%s query=%s remote=%s bytes=%s", - request.method, - request_path, - request.query_string.decode("utf-8", "ignore"), - request.remote_addr, - request_size, - ) - - if not is_path_allowed(request_path): - logger.warning("request blocked by allow-list path=%s", request_path) - return Response("Endpoint not allowed", status=403) - - try: - upstream_url = build_upstream_url(path) - - headers = {k: v for k, v in request.headers if k.lower() != "host"} - headers["X-Forwarded-By"] = "mtls-bridge" - - start_time = time.time() - resp = requests.request( - method=request.method, - url=upstream_url, - headers=headers, - data=request.get_data(cache=True), - cookies=request.cookies, - cert=(CLIENT_CERT, CLIENT_KEY), - verify=VERIFY_SETTING, - timeout=TIMEOUT, - allow_redirects=False, - ) - - elapsed_ms = int((time.time() - start_time) * 1000) - logger.info( - "upstream response status=%s url=%s elapsed_ms=%s response_bytes=%s", - resp.status_code, - upstream_url, - elapsed_ms, - len(resp.content), - ) - - excluded_headers = {"content-encoding", "content-length", "transfer-encoding", "connection"} - response_headers = [(k, v) for k, v in resp.headers.items() if k.lower() not in excluded_headers] - - return Response(resp.content, resp.status_code, response_headers) - - except ValueError as exc: - logger.error("proxy request failed: %s", exc) - return Response(str(exc), status=500) - except Exception as exc: # noqa: BLE001 - logger.exception("proxy request failed") - return Response(str(exc), status=500) - - -if __name__ == "__main__": - app.run(host="0.0.0.0", port=8080) diff --git a/monitoring/mtls-bridge/docker-compose.yml b/monitoring/mtls-bridge/docker-compose.yml deleted file mode 100644 index 7d8938d..0000000 --- a/monitoring/mtls-bridge/docker-compose.yml +++ /dev/null @@ -1,49 +0,0 @@ -services: - mtls-bridge: - profiles: ["monitoring", "all", "mtls-bridge"] - build: - context: ${PROJECT_ROOT}/monitoring/mtls-bridge - container_name: mtls-bridge - hostname: mtls-bridge.lan.ddnsgeek.com - restart: unless-stopped - environment: - - TARGET_URL=${MTLS_BRIDGE_TARGET_URL} - - CLIENT_CERT=${MTLS_BRIDGE_CLIENT_CERT} - - CLIENT_KEY=${MTLS_BRIDGE_CLIENT_KEY} - - TIMEOUT=${MTLS_BRIDGE_TIMEOUT} - - LOG_LEVEL=${MTLS_BRIDGE_LOG_LEVEL:-INFO} - - UPSTREAM_CA_CERT=${MTLS_BRIDGE_UPSTREAM_CA_CERT:-} - - ALLOWED_PATHS_FILE=${MTLS_BRIDGE_ALLOWED_PATHS_FILE:-} - volumes: - - ${PROJECT_ROOT}/core/traefik/certs:/certs:ro - labels: - - "traefik.http.routers.mtls-bridge.rule=Host(`mtls-bridge.lan.ddnsgeek.com`)" - - "traefik.enable=true" - - "traefik.http.routers.mtls-bridge.entrypoints=websecure" - - "traefik.http.routers.mtls-bridge.tls.certresolver=myresolver" - - "traefik.http.routers.mtls-bridge.middlewares=mtls-bridge-auth,mtls-bridge-cors" - - "traefik.http.middlewares.mtls-bridge-auth.basicauth.users=${MTLS_BRIDGE_BASIC_AUTH_USERS}" - - "traefik.http.routers.mtls-bridge-preflight.rule=Host(`mtls-bridge.lan.ddnsgeek.com`) && Method(`OPTIONS`)" - - "traefik.http.routers.mtls-bridge-preflight.entrypoints=websecure" - - "traefik.http.routers.mtls-bridge-preflight.tls.certresolver=myresolver" - - "traefik.http.routers.mtls-bridge-preflight.middlewares=mtls-bridge-cors" - - "traefik.http.routers.mtls-bridge-preflight.priority=100" - - "traefik.http.routers.mtls-bridge-preflight.service=mtls-bridge" - - "traefik.http.middlewares.mtls-bridge-cors.headers.accesscontrolalloworiginlist=${MTLS_BRIDGE_CORS_ALLOW_ORIGIN}" - - "traefik.http.middlewares.mtls-bridge-cors.headers.accesscontrolallowmethods=GET,POST,PUT,PATCH,DELETE,OPTIONS" - - "traefik.http.middlewares.mtls-bridge-cors.headers.accesscontrolallowheaders=authorization,content-type,x-grafana-action,x-grafana-device-id" - - "traefik.http.middlewares.mtls-bridge-cors.headers.accesscontrolallowcredentials=true" - - "traefik.http.middlewares.mtls-bridge-cors.headers.addvaryheader=true" - - "io.portainer.accesscontrol.public" -# - "traefik.http.routers.searxng.middlewares=crowdsec@file,secHeaders@file,error-pages-middleware" - - "traefik.http.services.mtls-bridge.loadbalancer.server.port=8080" - - "traefik.docker.network=core_traefik" - healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/_mtls_bridge/health', timeout=3).read()"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 10s - networks: - - monitor - - traefik diff --git a/monitoring/mtls-bridge/requirements.txt b/monitoring/mtls-bridge/requirements.txt deleted file mode 100644 index 30692b7..0000000 --- a/monitoring/mtls-bridge/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -flask -requests diff --git a/monitoring/node-exporter/docker-compose.yml b/monitoring/node-exporter/docker-compose.yml deleted file mode 100644 index f7f769f..0000000 --- a/monitoring/node-exporter/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -services: - node-exporter: - profiles: ["monitoring","all","node-exporter", "prometheus"] - image: prom/node-exporter:latest - container_name: node-exporter - pid: host - volumes: - - /proc:/host/proc:ro - - /sys:/host/sys:ro - - /:/rootfs:ro - command: - - "--path.procfs=/host/proc" - - "--path.sysfs=/host/sys" - - "--path.rootfs=/rootfs" - restart: unless-stopped - networks: -# - edge - - monitor - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:9100/metrics"] - interval: 30s - timeout: 10s - retries: 3 diff --git a/monitoring/node-red/Dockerfile b/monitoring/node-red/Dockerfile deleted file mode 100644 index 0fb8220..0000000 --- a/monitoring/node-red/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM nodered/node-red:latest - -USER root -RUN apk add --no-cache docker-cli docker-cli-compose -#RUN addgroup -g 131 -S docker && addgroup node-red docker - -USER node-red diff --git a/monitoring/node-red/UPDATE_LOGGING_GRAFANA.md b/monitoring/node-red/UPDATE_LOGGING_GRAFANA.md deleted file mode 100644 index 51c8fc5..0000000 --- a/monitoring/node-red/UPDATE_LOGGING_GRAFANA.md +++ /dev/null @@ -1,160 +0,0 @@ -# Node-RED update logging for Grafana - -This guide adds structured update-event logging to your existing Node-RED + Telegraf + Prometheus + Grafana stack without introducing Loki. - -## Goal - -Track and surface (in Grafana) the latest update attempts from Node-RED, including: - -- when an update attempt started, -- target container/project, -- success/failure, -- optional failure reason, -- elapsed duration. - -## 1) Add a reusable logger function in Node-RED - -Create a **Function** node named `Build update log event` and use: - -```javascript -const nowIso = new Date().toISOString(); -const startedAt = msg.update_started_at || Date.now(); -const durationMs = Math.max(0, Date.now() - startedAt); - -const payload = msg.payload || {}; -const labels = payload.labels || {}; - -const status = (msg.update_status || payload.status || "unknown").toString().toLowerCase(); -const success = status === "success" ? 1 : 0; -const failed = status === "failed" ? 1 : 0; - -msg.payload = { - ts: nowIso, - flow: "docker-updates", - event: msg.update_event || "attempt", - container: msg.container || labels.container || "unknown", - project: labels.com_docker_compose_project || msg.project || "unknown", - host: msg.host || "unknown", - status, - success, - failed, - duration_ms: durationMs, - code: Number.isFinite(Number(payload.code)) ? Number(payload.code) : 0, - error: (msg.update_error || payload.error || "").toString().slice(0, 300) -}; - -// one JSON line per event for file output -msg.payload = JSON.stringify(msg.payload); -return msg; -``` - -### Wiring recommendation - -Use the same logger function in these branches: - -- before a pull/update command (`update_status=started`, `update_event=attempt`), -- success path (`update_status=success`, `update_event=completed`), -- failure path (`update_status=failed`, `update_event=completed`, and include `msg.update_error`). - -Then route each branch into a **File** node configured as: - -- Filename: `/data/update-events.ndjson` -- Action: append to file -- Add newline: enabled - -## 2) Make update state explicit in existing update flow - -In your current update flow (already present in `flows.json`), add/change **Change** nodes around your shell/docker nodes: - -- At update start: - - `msg.update_started_at = $millis()` - - `msg.update_status = "started"` - - `msg.update_event = "attempt"` -- At success: - - `msg.update_status = "success"` - - `msg.update_event = "completed"` -- At failure: - - `msg.update_status = "failed"` - - `msg.update_event = "completed"` - - `msg.update_error = msg.payload.stderr` (or equivalent error field) - -## 3) Let Telegraf ingest Node-RED event logs - -Append this to `monitoring/telegraf/telegraf.conf`: - -```toml -[[inputs.tail]] - files = ["/var/log/node-red/update-events.ndjson"] - from_beginning = false - name_override = "node_red_update_event" - data_format = "json_v2" - - [[inputs.tail.json_v2]] - measurement_name = "node_red_update_event" - - [[inputs.tail.json_v2.tag]] - path = "flow" - [[inputs.tail.json_v2.tag]] - path = "event" - [[inputs.tail.json_v2.tag]] - path = "container" - [[inputs.tail.json_v2.tag]] - path = "project" - [[inputs.tail.json_v2.tag]] - path = "host" - [[inputs.tail.json_v2.tag]] - path = "status" - - [[inputs.tail.json_v2.field]] - path = "success" - type = "int" - [[inputs.tail.json_v2.field]] - path = "failed" - type = "int" - [[inputs.tail.json_v2.field]] - path = "duration_ms" - type = "int" - [[inputs.tail.json_v2.field]] - path = "code" - type = "int" -``` - -And mount the Node-RED data directory into Telegraf (read-only) in `monitoring/prometheus/docker-compose.yml` under `telegraf.volumes`: - -```yaml - - ${PROJECT_ROOT}/monitoring/node-red/data:/var/log/node-red:ro -``` - -## 4) Prometheus scrape (already in place) - -No Prometheus scrape change is required as long as it already scrapes Telegraf (`telegraf:9273`). - -## 5) Grafana queries to start with - -Use your Prometheus data source and try: - -- Latest success/failure by container: - - `last_over_time(node_red_update_event_success[24h])` - - `last_over_time(node_red_update_event_failed[24h])` -- Failed updates in the last 24h: - - `sum by (container, project) (increase(node_red_update_event_failed[24h]))` -- Average update duration in last 24h: - - `avg by (container, project) (avg_over_time(node_red_update_event_duration_ms[24h]))` - -Recommended panels: - -- **Table**: container, project, status (last value), duration_ms (last value) -- **Time series**: failed count over time -- **Stat**: total failed updates in last 24h - -## 6) Validation checklist - -1. Trigger a known update path (including one failure if possible). -2. Check Node-RED log file: - - `tail -n 20 monitoring/node-red/data/update-events.ndjson` -3. Check Telegraf metrics endpoint for `node_red_update_event_` metrics. -4. Confirm Grafana panel values match the latest Node-RED run. - -## Optional next step - -If you want searchable raw log text and richer log UX, add Loki + Promtail later. Keep this structured metrics path for high-signal alerting even after adding logs. diff --git a/monitoring/node-red/docker-compose.yml b/monitoring/node-red/docker-compose.yml deleted file mode 100644 index fbdbf8f..0000000 --- a/monitoring/node-red/docker-compose.yml +++ /dev/null @@ -1,67 +0,0 @@ -services: - node-red: - profiles: ["monitoring","all","node-red"] -# image: nodered/node-red:latest - build: - context: ${PROJECT_ROOT}/monitoring/node-red - container_name: node-red - restart: unless-stopped - depends_on: - - docker-socket-proxy - environment: - DOCKER_HOST: ${DOCKER_SOCKET_PROXY_HOST} - TZ: ${TZ} - PROJECT_ROOT: ${NODE_COMPOSE_ROOT} - cap_drop: - - ALL - security_opt: - - no-new-privileges:true -# ports: -# - "1880:1880" - volumes: -# - ${PROJECT_ROOT}/monitoring/node-red/data:/data - - ${DOCKER_VOLUMES}/node-red-data:/data - - ${PROJECT_ROOT}:/compose/docker:ro - - /home/nixos/raspi:/compose/raspi:ro -# - ${PROJECT_ROOT}:/usr/src/node-red:ro - -# - ${PROJECT_ROOT}/default-environment.env:/usr/src/node-red/default-environment.env:ro -# - ${PROJECT_ROOT}/default-network.yml:/usr/src/node-red/default-network.yml:ro -# - ${PROJECT_ROOT}/core/docker-compose.yml:/usr/src/node-red/core/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/prometheus/docker-compose.yml:/usr/src/node-red/monitoring/prometheus/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/gotify/docker-compose.yml:/usr/src/node-red/monitoring/gotify/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/grafana/docker-compose.yml:/usr/src/node-red/monitoring/grafana/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/portainer/docker-compose.yml:/usr/src/node-red/monitoring/portainer/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/uptime-kuma/docker-compose.yml:/usr/src/node-red/monitoring/uptime-kuma/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/gitea/docker-compose.yml:/usr/src/node-red/apps/gitea/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/gramps/docker-compose.yml:/usr/src/node-red/apps/gramps/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/nextcloud/docker-compose.yml:/usr/src/node-red/apps/nextcloud/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/passbolt/docker-compose.yml:/usr/src/node-red/apps/passbolt/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/searxng/docker-compose.yml:/usr/src/node-red/apps/searxng/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/shift-recorder/docker-compose.yml:/usr/src/node-red/apps/shift-recorder/docker-compose.yml:ro -# - ${PROJECT_ROOT}/apps/stockfill/docker-compose.yml:/usr/src/node-red/apps/stockfill/docker-compose.yml:ro -# - ${PROJECT_ROOT}/monitoring/node-red/docker-compose.yml:/usr/src/node-red/monitoring/node-red/docker-compose.yml:ro -# - ${PROJECT_ROOT}/core/test/docker-compose.yml:/usr/src/node-red/core/test/docker-compose.yml:ro -# - ${PROJECT_ROOT}/secrets/stack-secrets.env:/usr/src/node-red/secrets/stack-secrets.env:ro - -# - /run/current-system/sw/bin/docker:/usr/bin/docker:ro -# depends_on: -# - mosquitto -# - influxdb - networks: - - monitor - - traefik - labels: - - "traefik.enable=true" - - "traefik.http.routers.node-red.rule=Host(`node-red.lan.ddnsgeek.com`)" -# - "traefik.http.routers.node-red.service=api@internal" - - "traefik.http.routers.node-red.entrypoints=websecure" - - "traefik.http.routers.node-red.tls.certresolver=myresolver" - - "traefik.http.routers.node-red.tls.options=mtls-private-admin@file" - - "traefik.http.routers.node-red.middlewares=authelia" - - "io.portainer.accesscontrol.public" - - "traefik.docker.network=core_traefik" - - "traefik.http.services.node-red.loadbalancer.server.port=1880" -#volumes: -# node-red-data: -# external: true diff --git a/monitoring/pihole-exporter/docker-compose.yml b/monitoring/pihole-exporter/docker-compose.yml deleted file mode 100644 index c681841..0000000 --- a/monitoring/pihole-exporter/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -services: - pihole-exporter: - profiles: ["monitoring","all","pihole-exporter", "prometheus"] - image: ekofr/pihole-exporter:latest - container_name: pihole-exporter -# env_file: -# - ${PROJECT_ROOT}/secrets/stack-secrets.env - environment: - PIHOLE_HOSTNAME: ${PIHOLE_HOSTNAME} - PIHOLE_PASSWORD: ${PIHOLE_PASSWORD} - PORT: ${PIHOLE_EXPORTER_PORT} - ports: - - "${PIHOLE_EXPORTER_PORT}:${PIHOLE_EXPORTER_PORT}" - restart: unless-stopped - networks: -# - edge - - monitor diff --git a/monitoring/prometheus/docker-compose.yml b/monitoring/prometheus/docker-compose.yml deleted file mode 100644 index 5dc7fed..0000000 --- a/monitoring/prometheus/docker-compose.yml +++ /dev/null @@ -1,56 +0,0 @@ -services: - prometheus: - profiles: ["monitoring","all","prometheus"] - image: prom/prometheus:latest -# env_file: -# - ${PROJECT_ROOT}/secrets/stack-secrets.env - container_name: prometheus - depends_on: -# - alertmanager - - telegraf - - influxdb - - node-exporter - - docker-update-exporter - - pihole-exporter - command: - - "--config.file=/etc/prometheus/prometheus.yml" - - "--storage.tsdb.path=/prometheus" - - "--storage.tsdb.retention.time=15d" -# build: -# context: ${PROJECT_ROOT}/monitoring/prometheus - volumes: - - ${PROJECT_ROOT}/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro -# - ${PROJECT_ROOT}/monitoring/prometheus/data:/prometheus -# - ${PROJECT_ROOT}/monitoring/prometheus/rules:/etc/prometheus/rules:ro - - ${DOCKER_VOLUMES}/prometheus/data:/prometheus - - ${DOCKER_VOLUMES}/prometheus/rules:/etc/prometheus/rules:ro - - ${PROJECT_ROOT}/secrets/prometheus_kuma_basic_auth_password.txt:/run/secrets/prometheus_kuma_basic_auth_password:ro - - restart: unless-stopped - labels: - - "traefik.http.routers.prometheus.rule=Host(`prometheus.lan.ddnsgeek.com`)" - - "traefik.enable=true" - - "traefik.http.routers.prometheus.entrypoints=websecure" - - "traefik.http.routers.prometheus.tls.certresolver=myresolver" - - "traefik.http.routers.prometheus.tls.options=mtls-private-admin@file" - - "io.portainer.accesscontrol.public" - - "traefik.http.services.prometheus.loadbalancer.server.port=9090" - - "traefik.http.routers.prometheus.middlewares=authelia" - - "traefik.docker.network=core_traefik" - networks: -# - edge -# - traefik_reverse_proxy - - traefik - - monitor - healthcheck: - test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - -#volumes: -# prometheus-data: -# external: true -# prometheus-rules: -# external: true diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml deleted file mode 100644 index 6085a7b..0000000 --- a/monitoring/prometheus/prometheus.yml +++ /dev/null @@ -1,166 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -#alerting: -# alertmanagers: -# - static_configs: -# - targets: -# - alertmanager:9093 - -scrape_configs: - - # Prometheus itself - - job_name: "prometheus" - static_configs: - - targets: ["prometheus:9090"] - labels: - role: prometheus - # ========================= - # Node Exporters (ALL hosts) - # ========================= - - job_name: "node" - static_configs: - - targets: - - node-exporter:9100 - labels: - role: docker - - - targets: - - raspberrypi.tail13f623.ts.net:9100 - labels: - role: raspberrypi - - - targets: - - pve.sweet.home:9100 - labels: - role: proxmox - - - targets: - - pbs.sweet.home:9100 - labels: - role: backup - - - targets: - - pihole:9100 - labels: - role: pihole - - - targets: - - server:9100 - labels: - role: server - - - targets: - - nix-cache:9100 - labels: - role: cache - - # ========================= - # Telegraf (Docker metrics) - # ========================= - - job_name: "telegraf" - static_configs: - - targets: - - telegraf:9273 - - raspberrypi.tail13f623.ts.net:9273 - labels: - role: docker - - # ========================= - # Traefik (all instances) - # ========================= - - job_name: "traefik" - static_configs: - - targets: - - traefik.lan.ddnsgeek.com:8080 - labels: - role: docker - - - targets: - - raspberrypi.tail13f623.ts.net:8080 - labels: - role: raspberrypi - - metric_relabel_configs: - - source_labels: [service] - regex: '(.+)@.+' - target_label: service - replacement: '$1' - - # ========================= - # Uptime Kuma (separate due to auth) - # ========================= - - job_name: "kuma" - metrics_path: /metrics - scrape_interval: 30s - - basic_auth: - username: wayne.bennett@live.com - password_file: /run/secrets/prometheus_kuma_basic_auth_password -# password: '4vjCco?[%{=+,t`):C' - static_configs: - - targets: - - monitor-kuma:3001 - labels: - role: docker - - - targets: - - kuma.lan.ddnsgeek.com - labels: - role: raspberrypi - tls_config: -# ca_file: /prometheus/clients-ca.crt - cert_file: /prometheus/office-pc.crt - key_file: /prometheus/office-pc.key -# server_name: kuma.lan.ddnsgeek.com - - # ========================= - # Proxmox Storage Exporters - # ========================= - - job_name: "proxmox-storage" - metrics_path: /metrics - static_configs: - - targets: - - pve.sweet.home:9101 - labels: - role: proxmox - storage: lvm - -# - targets: -# - pbs.sweet.home:9102 -# labels: -# role: backup -# storage: datastore - - # ========================= - # Docker Updates Exporter - # ========================= - - - - job_name: "container-updates" - static_configs: - - targets: - - docker-update-exporter:9105 - labels: - role: docker - - targets: - - raspberrypi.tail13f623.ts.net:9105 - labels: - role: raspberrypi - - # ========================= - # pihole Exporter - # ========================= - - - - job_name: "pihole" - static_configs: - - targets: - - pihole-exporter:9617 - labels: - role: pihole - - -#rule_files: -# - /etc/prometheus/rules/*.yml diff --git a/monitoring/prometheus/prometheus.yml.old b/monitoring/prometheus/prometheus.yml.old deleted file mode 100644 index 37dbcbd..0000000 --- a/monitoring/prometheus/prometheus.yml.old +++ /dev/null @@ -1,192 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -alerting: - alertmanagers: - - static_configs: - - targets: - - alertmanager:9093 - -scrape_configs: - - job_name: "prometheus" - static_configs: - - targets: ["prometheus:9090"] - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - job_name: "docker-node" - static_configs: - - targets: ["node-exporter:9100"] - labels: - type: "virtual" - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - -# - job_name: "cadvisor" -# static_configs: -# - targets: ["cadvisor:8080"] -# labels: -# type: "container" -# relabel_configs: -# - source_labels: [__address__] -# regex: '([^:]+):.*' -# target_label: instance - - job_name: "raspberrypi-node" - static_configs: - - targets: - - "raspberrypi.tail13f623.ts.net:9100" - labels: - type: "physical" - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - job_name: "proxmox-node" - static_configs: - - targets: - - "pve.sweet.home:9100" - labels: - type: "physical" - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - job_name: "proxmox-backup-server" - static_configs: - - targets: - - "pbs.sweet.home:9100" - labels: - type: "virtual" - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - job_name: "pihole" - static_configs: - - targets: - - "pihole:9100" - labels: - type: "virtual" - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - job_name: "server" - static_configs: - - targets: - - "server:9100" - labels: - type: "virtual" - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - job_name: "nix-cache" - static_configs: - - targets: - - "nix-cache:9100" - labels: - type: "virtual" - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - job_name: docker - static_configs: - - targets: ['telegraf:9273'] - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - - job_name: 'traefik' - static_configs: - - targets: ['traefik.lan.ddnsgeek.com:8080'] # replace with your Traefik host:port - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - metric_relabel_configs: - - source_labels: [service] - regex: '(.+)@.+' - target_label: service - replacement: '$1' - - - job_name: 'raspi-traefik' - static_configs: - - targets: ['raspberrypi.tail13f623.ts.net:8080'] # replace with your Traefik host:port - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - metric_relabel_configs: - - source_labels: [service] - regex: '(.+)@.+' - target_label: service - replacement: '$1' - - - job_name: 'raspi-kuma' - metrics_path: /metrics - scrape_interval: 30s - - basic_auth: - username: wayne.bennett@live.com - password: '4vjCco?[%{=+,t`):C' - - static_configs: - - targets: - - kuma.lan.ddnsgeek.com - - - job_name: 'docker-kuma' - metrics_path: /metrics - scrape_interval: 30s - - basic_auth: - username: wayne.bennett@live.com - password: '4vjCco?[%{=+,t`):C' - - static_configs: - - targets: - - uptime-kuma:3001 - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - - job_name: 'proxmox LVM storage' - static_configs: - - targets: ['pve.sweet.home:9101'] - metrics_path: /metrics - scheme: http - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - - job_name: 'proxmox backup storage' - static_configs: - - targets: ['pbs.sweet.home:9102'] - metrics_path: /metrics - scheme: http - relabel_configs: - - source_labels: [__address__] - regex: '([^:]+):.*' - target_label: instance - - -rule_files: - - /etc/prometheus/rules/*.yml - diff --git a/monitoring/telegraf/docker-compose.yml b/monitoring/telegraf/docker-compose.yml deleted file mode 100644 index 3c5f7d1..0000000 --- a/monitoring/telegraf/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -services: - telegraf: - profiles: ["monitoring","all","telegraf", "prometheus"] - image: telegraf:latest - container_name: telegraf - restart: unless-stopped - depends_on: - - docker-socket-proxy -# cap_drop: -# - ALL - security_opt: - - no-new-privileges:true - volumes: - - ${PROJECT_ROOT}/monitoring/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro -# - ${PROJECT_ROOT}/monitoring/node-red/data:/var/log/node-red:ro - - - ${DOCKER_VOLUMES}/node-red-data:/var/log/node-red:ro - networks: -# - edge - - monitor - healthcheck: - test: ["CMD-SHELL", "curl -f http://localhost:9273/metrics || exit 1"] - interval: 30s - timeout: 5s - retries: 3 - start_period: 10s diff --git a/monitoring/telegraf/telegraf.conf b/monitoring/telegraf/telegraf.conf deleted file mode 100644 index 2e3956e..0000000 --- a/monitoring/telegraf/telegraf.conf +++ /dev/null @@ -1,45 +0,0 @@ -[agent] - interval = "10s" - -[[inputs.docker]] - endpoint = "tcp://docker-socket-proxy:2375" - gather_services = false - -[[outputs.prometheus_client]] - listen = ":9273" - -# Node-RED update-event logs (structured NDJSON) -> Prometheus metrics for Grafana -[[inputs.tail]] - files = ["/var/log/node-red/update-events.ndjson"] - from_beginning = false - name_override = "node_red_update_event" - data_format = "json_v2" - - [[inputs.tail.json_v2]] - measurement_name = "node_red_update_event" - - [[inputs.tail.json_v2.tag]] - path = "flow" - [[inputs.tail.json_v2.tag]] - path = "event" - [[inputs.tail.json_v2.tag]] - path = "container" - [[inputs.tail.json_v2.tag]] - path = "project" - [[inputs.tail.json_v2.tag]] - path = "host" - [[inputs.tail.json_v2.tag]] - path = "status" - - [[inputs.tail.json_v2.field]] - path = "success" - type = "int" - [[inputs.tail.json_v2.field]] - path = "failed" - type = "int" - [[inputs.tail.json_v2.field]] - path = "duration_ms" - type = "int" - [[inputs.tail.json_v2.field]] - path = "code" - type = "int" diff --git a/monitoring/uptime-kuma/docker-compose.yml b/monitoring/uptime-kuma/docker-compose.yml index 6631d56..12e24dc 100644 --- a/monitoring/uptime-kuma/docker-compose.yml +++ b/monitoring/uptime-kuma/docker-compose.yml @@ -29,7 +29,7 @@ services: networks: # - traefik_reverse_proxy - traefik - - monitor +# - monitor #networks: # traefik_reverse_proxy: