diff --git a/archive/dynamic-system/monitoring/docker-exporter/Dockerfile b/archive/dynamic-system/monitoring/docker-exporter/Dockerfile new file mode 100644 index 0000000..7d741b7 --- /dev/null +++ b/archive/dynamic-system/monitoring/docker-exporter/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.11-slim + +#RUN groupadd -g 1000 appuser || true && \ +# useradd -m -u 1000 -g 1000 -s /bin/bash appuser +#RUN groupadd -g 999 docker || true && usermod -aG docker appuser +WORKDIR /app +COPY exporter.py . + +RUN mkdir -p /data + +RUN pip install --no-cache-dir docker prometheus_client requests pyyaml + +#USER appuser + +CMD ["python", "-u", "exporter.py"] diff --git a/archive/dynamic-system/monitoring/docker-exporter/Dockerfile.old b/archive/dynamic-system/monitoring/docker-exporter/Dockerfile.old new file mode 100644 index 0000000..9981774 --- /dev/null +++ b/archive/dynamic-system/monitoring/docker-exporter/Dockerfile.old @@ -0,0 +1,8 @@ +FROM python:3.11-slim + +WORKDIR /app +COPY exporter.py . + +RUN pip install docker prometheus_client requests pyyaml + +CMD ["python", "exporter.py"] diff --git a/archive/dynamic-system/monitoring/docker-exporter/TEST_EXAMPLE.md b/archive/dynamic-system/monitoring/docker-exporter/TEST_EXAMPLE.md new file mode 100644 index 0000000..d6af05b --- /dev/null +++ b/archive/dynamic-system/monitoring/docker-exporter/TEST_EXAMPLE.md @@ -0,0 +1,24 @@ +# Exporter image-mapping test example + +Run the exporter in dry-run mode to print the `service -> image:tag` mapping without starting the metrics loop: + +```bash +SERVICES_UP_SCRIPT=/workspace/docker/services-up.sh python monitoring/docker-exporter/exporter.py --dry-run +``` + +Example output excerpt: + +```json +{ + "crowdsec": "crowdsecurity/crowdsec:latest", + "docker-update-exporter": "python:3.11-slim", + "nextcloud-webapp": "nextcloud:production", + "node-red": "nodered/node-red:latest", + "prometheus": "prom/prometheus:latest", + "traefik": "traefik:3" +} +``` + +This confirms the exporter now reports images for both: +- services with explicit `image:` values, and +- services using `build:` contexts. diff --git a/archive/dynamic-system/monitoring/docker-exporter/docker-compose.yml b/archive/dynamic-system/monitoring/docker-exporter/docker-compose.yml new file mode 100644 index 0000000..284c3f8 --- /dev/null +++ b/archive/dynamic-system/monitoring/docker-exporter/docker-compose.yml @@ -0,0 +1,59 @@ +services: + docker-update-exporter: + profiles: ["monitoring","all","docker-exporter", "prometheus"] + build: + context: ${PROJECT_ROOT}/monitoring/docker-exporter + container_name: docker-update-exporter +# volumes: +# - /var/run/docker.sock:/var/run/docker.sock +# - ${PROJECT_ROOT}/monitoring/docker-exporter/data:/data:rw +# - ${PROJECT_ROOT}/services-up.sh:/app/services-up.sh:ro + environment: + LOG_LEVEL: ${DOCKER_EXPORTER_LOG_LEVEL} + DOCKER_HOST: ${DOCKER_SOCKET_PROXY_HOST} + depends_on: + - docker-socket-proxy + + volumes: + - ~/.docker/config.json:/root/.docker/config.json:ro +# - ${PROJECT_ROOT}/monitoring/docker-exporter/data:/data:rw + - ${DOCKER_VOLUMES}/docker-update-exporter-data:/data:rw + - ${PROJECT_ROOT}:/compose:ro +# - ${PROJECT_ROOT}/default-environment.env:/compose/default-environment.env:ro +# - ${PROJECT_ROOT}/default-network.yml:/compose/default-network.yml:ro +# - ${PROJECT_ROOT}/core/docker-compose.yml:/compose/core/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/prometheus/docker-compose.yml:/compose/monitoring/prometheus/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/gotify/docker-compose.yml:/compose/monitoring/gotify/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/grafana/docker-compose.yml:/compose/monitoring/grafana/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/portainer/docker-compose.yml:/compose/monitoring/portainer/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/uptime-kuma/docker-compose.yml:/compose/monitoring/uptime-kuma/docker-compose.yml:> +# - ${PROJECT_ROOT}/apps/gitea/docker-compose.yml:/compose/apps/gitea/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/gramps/docker-compose.yml:/compose/apps/gramps/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/nextcloud/docker-compose.yml:/compose/apps/nextcloud/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/passbolt/docker-compose.yml:/compose/apps/passbolt/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/searxng/docker-compose.yml:/compose/apps/searxng/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/shift-recorder/docker-compose.yml:/compose/apps/shift-recorder/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/stockfill/docker-compose.yml:/compose/apps/stockfill/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/node-red/docker-compose.yml:/compose/monitoring/node-red/docker-compose.yml:ro +# - ${PROJECT_ROOT}/core/test/docker-compose.yml:/compose/core/test/docker-compose.yml:ro + + +# ports: +# - "9105:9105" + restart: unless-stopped + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + networks: +# - edge + - monitor + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:9105/metrics')"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s +#volumes: +# docker-update-exporter-data: +# external: true diff --git a/archive/dynamic-system/monitoring/docker-exporter/exporter.py b/archive/dynamic-system/monitoring/docker-exporter/exporter.py new file mode 100644 index 0000000..2e2d55a --- /dev/null +++ b/archive/dynamic-system/monitoring/docker-exporter/exporter.py @@ -0,0 +1,579 @@ +#!/usr/bin/env python3 +import argparse +import os +import re +import time +import json +import logging +import docker +import yaml +from prometheus_client import Gauge, start_http_server + +# --- Logging --- +LOG_LEVEL = os.getenv("LOG_LEVEL", "DEBUG").upper() +logging.basicConfig( + level=getattr(logging, LOG_LEVEL, logging.DEBUG), + format="%(asctime)s [%(levelname)s] %(message)s" +) +logger = logging.getLogger("docker-update-exporter") + +# --- Config --- +EXPORTER_PORT = 9105 +CHECK_INTERVAL = 3600 +CACHE_TTL = int(os.getenv("CACHE_TTL", "300")) +SERVICES_UP_SCRIPT = os.getenv("SERVICES_UP_SCRIPT", "/compose/services-up.sh") +CACHE_FILE = os.getenv("CACHE_FILE", "/data/remote_digest_cache.json") +DRY_RUN = os.getenv("DRY_RUN", "false").lower() in ("1", "true", "yes") + +try: + client = docker.from_env() +except Exception as e: + logger.warning(f"Docker client unavailable at startup: {e}") + client = None + +# --- Metrics --- +CONTAINER_UPDATE = Gauge( + "docker_container_update_available", + "1 if container image is out of date (compose drift or registry), 0 otherwise", + ["container", "compose_image", "running_image", "com_docker_compose_project"] +) + +LAST_CHECK = Gauge( + "docker_image_update_last_check_timestamp", + "Last time the update check ran (unix timestamp)" +) + + +def set_container_update_metric(container_name, compose_image, running_image, project_name, update_flag): + """Set update metric for a container and log the emitted metric payload.""" + metric_labels = { + "container": container_name, + "compose_image": compose_image or "unknown", + "running_image": running_image, + "com_docker_compose_project": project_name, + } + CONTAINER_UPDATE.labels(**metric_labels).set(update_flag) + logger.info( + "Metric emitted: docker_container_update_available=%s labels=%s", + update_flag, + metric_labels, + ) + + +def set_last_check_metric(): + """Set and log the timestamp for the most recent check cycle.""" + ts = time.time() + LAST_CHECK.set(ts) + logger.info("Metric emitted: docker_image_update_last_check_timestamp=%s", ts) + +# --- Persistent Cache --- +def load_cache(): + if not os.path.exists(CACHE_FILE): + logger.info(f"Cache file does not exist yet: {CACHE_FILE}") + return {} + try: + with open(CACHE_FILE, "r") as f: + cache = json.load(f) + logger.info(f"Loaded {len(cache)} cached remote digests") + return cache + except Exception as e: + logger.error(f"Failed to load cache: {e}") + return {} + +def save_cache(): + try: + os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True) + with open(CACHE_FILE, "w") as f: + json.dump(REMOTE_DIGEST_CACHE, f) + logger.debug(f"Saved {len(REMOTE_DIGEST_CACHE)} remote digests to cache") + except Exception as e: + logger.error(f"Failed to save cache: {e}") + +REMOTE_DIGEST_CACHE = load_cache() +now = time.time() +REMOTE_DIGEST_CACHE = { + image: (digest, ts) + for image, (digest, ts) in REMOTE_DIGEST_CACHE.items() + if now - ts < CACHE_TTL +} + +# --- Helpers --- +def get_project_prefix_from_script(script_path): + prefix = "core-" + if not os.path.exists(script_path): + return prefix + try: + with open(script_path) as f: + for line in f: + m = re.match(r'PROJECT\s*=\s*["\']?([^"\']+)', line) + if m: + return m.group(1) + "-" + except Exception as e: + logger.warning(f"Failed reading project prefix: {e}") + return prefix + +def get_local_digest(image_name): + """ + Return the local digest for the specific image reference. + """ + if client is None: + return None + + try: + img = client.images.get(image_name) + digests = img.attrs.get("RepoDigests", []) + + logger.debug(f"RepoDigests for {image_name}: {digests}") + + for entry in digests: + if "@" in entry: + digest = entry.split("@", 1)[1] + logger.debug(f"Local digest for {image_name}: {digest}") + return digest + + logger.debug(f"No RepoDigest found for {image_name}") + + except Exception as e: + logger.debug(f"Could not get local digest for {image_name}: {e}") + + return None + +def get_remote_digest(image_name): + """ + Return the upstream digest for the exact platform-specific image that Docker + would pull on this host. This avoids false positives with multi-arch images + where the registry manifest-list digest differs from the pulled image digest. + """ + now = time.time() + + cached = REMOTE_DIGEST_CACHE.get(image_name) + if cached: + digest, ts = cached + if now - ts < CACHE_TTL: + logger.debug(f"Using cached remote digest for {image_name}: {digest}") + return digest + + if client is None: + return None + + try: + registry_data = client.images.get_registry_data(image_name) + + digest = None + + # docker SDK versions differ; try the common fields in order + if hasattr(registry_data, "id") and registry_data.id: + digest = registry_data.id + elif hasattr(registry_data, "attrs"): + digest = ( + registry_data.attrs.get("Descriptor", {}).get("digest") + or registry_data.attrs.get("digest") + ) + + if digest: + REMOTE_DIGEST_CACHE[image_name] = (digest, now) + save_cache() + logger.debug(f"Remote digest for {image_name}: {digest}") + return digest + + logger.warning(f"No remote digest found for {image_name}") + return None + except Exception as e: + logger.debug(f"Error fetching remote digest for {image_name}: {e}") + + return None + +# --- Dockerfile Image Extraction --- +def parse_dockerfile_for_image(dockerfile_path): + if not os.path.exists(dockerfile_path): + return None + + try: + arg_defaults = {} + last_from = None + with open(dockerfile_path) as df: + for line in df: + line = line.strip() + if not line or line.startswith("#"): + continue + + if line.upper().startswith("ARG "): + arg_body = line[4:].strip() + if "=" in arg_body: + key, value = arg_body.split("=", 1) + arg_defaults[key.strip()] = value.strip() + continue + + # Prefer LABEL with image if present. + if "LABEL" in line and "image=" in line: + match = re.search(r'image=["\']?([^"\']+)["\']?', line) + if match: + image_name = normalize_image_name(substitute_dockerfile_args(match.group(1), arg_defaults)) + logger.debug(f"Found LABEL image={image_name} in {dockerfile_path}") + return image_name + + if line.upper().startswith("FROM "): + from_clause = line[5:].strip() + if from_clause.startswith("--"): + split_clause = from_clause.split(None, 1) + if len(split_clause) < 2: + continue + from_clause = split_clause[1] + + parts = from_clause.split() + if not parts: + continue + + candidate = substitute_dockerfile_args(parts[0], arg_defaults) + if candidate and candidate.lower() != "scratch": + last_from = normalize_image_name(candidate) + + if last_from: + logger.debug(f"Found base FROM {last_from} in {dockerfile_path}") + return last_from + except Exception as e: + logger.debug(f"Error reading Dockerfile {dockerfile_path}: {e}") + + return None + +def normalize_image_name(image_name): + if not image_name: + return None + if "@" in image_name: + return image_name + if ":" in image_name.rsplit("/", 1)[-1]: + return image_name + return f"{image_name}:latest" + +def is_compose_build_placeholder(image_name, project_name): + if not image_name: + return False + candidate = str(image_name) + project_prefix = f"{project_name}-" + if candidate.startswith(project_prefix): + return True + # Keep backward-compatible behavior for historical default project prefix. + return candidate.startswith("core-") + +def substitute_dockerfile_args(value, arg_defaults): + if not value: + return value + + pattern = re.compile(r"\$\{([^}]+)\}|\$([A-Za-z_][A-Za-z0-9_]*)") + + def replacer(match): + expr = match.group(1) + simple = match.group(2) + if simple: + return arg_defaults.get(simple, "") + + if ":-" in expr: + var_name, default_value = expr.split(":-", 1) + return arg_defaults.get(var_name, default_value) + if "-" in expr: + var_name, default_value = expr.split("-", 1) + return arg_defaults.get(var_name, default_value) + return arg_defaults.get(expr, "") + + return pattern.sub(replacer, value) + +def expand_compose_path(path_value, project_root): + raw = str(path_value) + raw = raw.replace("${PROJECT_ROOT}", project_root).replace("$PROJECT_ROOT", project_root) + return os.path.expandvars(raw) + +def get_project_root_from_script(script_path): + if not script_path: + return os.getcwd() + return os.path.dirname(os.path.abspath(script_path)) + +# --- Compose parsing --- +def get_compose_files_from_script(script_path): + files = [] + if not os.path.exists(script_path): + return files + base_dir = get_project_root_from_script(script_path) + + def _clean_compose_path(raw_path): + cleaned = str(raw_path).strip().strip(",") + if (cleaned.startswith('"') and cleaned.endswith('"')) or ( + cleaned.startswith("'") and cleaned.endswith("'") + ): + cleaned = cleaned[1:-1] + expanded = expand_compose_path(cleaned, base_dir) + if os.path.isabs(expanded): + return os.path.normpath(expanded) + return os.path.normpath(os.path.join(base_dir, expanded)) + + try: + with open(script_path) as f: + content = f.read() + match = re.search(r'FILES\s*=\s*\((.*?)\)', content, re.DOTALL) + if match: + for line in match.group(1).splitlines(): + line = line.strip() + if line.startswith("-f"): + path = line[2:].strip() + if path: + full = _clean_compose_path(path) + files.append(full) + + # services-up.sh can append many compose files at runtime via: + # FILES+=(-f "$file") done < <(find "$PROJECT_ROOT/apps" ...) + # Mirror that behavior here so we can map service->compose image. + root_dirs = [] + find_match = re.search(r'find\s+(.*?)\s+\\\s*\n', content) + if find_match: + for token in re.findall(r'"([^"]+)"|\'([^\']+)\'', find_match.group(1)): + candidate = token[0] or token[1] + if candidate: + root_dirs.append(_clean_compose_path(candidate)) + else: + root_dirs = [ + os.path.join(base_dir, "apps"), + os.path.join(base_dir, "monitoring"), + os.path.join(base_dir, "core"), + ] + + for root_dir in root_dirs: + if not os.path.isdir(root_dir): + continue + for candidate in sorted(os.listdir(root_dir)): + svc_dir = os.path.join(root_dir, candidate) + if not os.path.isdir(svc_dir): + continue + for compose_name in ("docker-compose.yml", "docker-compose.yaml"): + compose_path = os.path.join(svc_dir, compose_name) + if os.path.exists(compose_path): + files.append(compose_path) + + # Preserve order while removing duplicates. + deduped = [] + seen = set() + for path in files: + if path in seen: + continue + seen.add(path) + deduped.append(path) + files = deduped + except Exception as e: + logger.warning(f"Failed parsing services-up.sh: {e}") + return files + +def parse_project_name_from_script(script_path): + project = "core" + if not os.path.exists(script_path): + return project + try: + with open(script_path) as f: + for line in f: + m = re.match(r'PROJECT\s*=\s*["\']?([^"\']+)', line) + if m: + project = m.group(1) + break + except Exception as e: + logger.warning(f"Failed reading project name: {e}") + return project + +def resolve_local_build_image(service_name, project_name): + if client is None: + return None + try: + images = client.images.list(filters={"label": f"com.docker.compose.service={service_name}"}) + for image in images: + labels = image.attrs.get("Config", {}).get("Labels", {}) or {} + if labels.get("com.docker.compose.project") != project_name: + continue + for tag in image.tags: + if tag and "" not in tag: + logger.debug(f"Resolved local compose image for {service_name}: {tag}") + return normalize_image_name(tag) + except Exception as e: + logger.debug(f"Could not inspect local build metadata for {service_name}: {e}") + return None + +def parse_compose_services(compose_files, project_name, project_root): + svc_map = {} + for f in compose_files: + if not os.path.exists(f): + logger.warning(f"Compose file from services-up.sh is missing: {f}") + continue + try: + with open(f) as stream: + data = yaml.safe_load(stream) or {} + for svc_name, svc_def in data.get("services", {}).items(): + image = normalize_image_name(svc_def.get("image")) + profiles = svc_def.get("profiles", []) + build_ctx = svc_def.get("build") + dockerfile_path = None + from_dockerfile = None + local_built_image = None + + if build_ctx: + if isinstance(build_ctx, dict): + context = build_ctx.get("context", ".") + dockerfile = build_ctx.get("dockerfile", "Dockerfile") + else: + context = build_ctx + dockerfile = "Dockerfile" + + compose_dir = os.path.dirname(f) + context_expanded = expand_compose_path(context, project_root) + if os.path.isabs(context_expanded): + context_path = context_expanded + else: + context_path = os.path.normpath(os.path.join(compose_dir, context_expanded)) + dockerfile_expanded = expand_compose_path(dockerfile, project_root) + dockerfile_path = os.path.normpath(os.path.join(context_path, dockerfile_expanded)) + from_dockerfile = normalize_image_name(parse_dockerfile_for_image(dockerfile_path)) + local_built_image = resolve_local_build_image(svc_name, project_name) + + placeholder_image = is_compose_build_placeholder(image, project_name) or is_compose_build_placeholder(local_built_image, project_name) + if placeholder_image: + resolved_image = from_dockerfile or image or local_built_image or f"{project_name}-{svc_name}:latest" + else: + resolved_image = image or local_built_image or from_dockerfile or f"{project_name}-{svc_name}:latest" + + svc_map[svc_name] = { + "image": resolved_image, + "profiles": profiles, + "build_context": build_ctx, + "compose_file": f, + "dockerfile": dockerfile_path + } + except Exception as e: + logger.warning(f"Failed parsing {f}: {e}") + + logger.debug(f"Service image mapping: {svc_map}") + return svc_map + +# --- Main check --- +def check_containers(): + if client is None: + logger.error("Docker client is unavailable; skipping check cycle") + return + + set_last_check_metric() + CONTAINER_UPDATE.clear() + + project_name = parse_project_name_from_script(SERVICES_UP_SCRIPT) + project_root = get_project_root_from_script(SERVICES_UP_SCRIPT) + compose_files = get_compose_files_from_script(SERVICES_UP_SCRIPT) + svc_map = parse_compose_services(compose_files, project_name, project_root) + + containers = client.containers.list() + pending_metrics = [] + remote_targets = set() + + for container in containers: + proj = container.labels.get("com.docker.compose.project") + if not proj: + continue + + svc = container.labels.get("com.docker.compose.service") + running = container.attrs["Config"]["Image"] + + compose_image = None + if svc in svc_map: + compose_image = svc_map[svc]["image"] + + local_digest = get_local_digest(running) + remote_target = compose_image or running + + # If we cannot determine a local digest, we cannot compare and should + # avoid spending a registry lookup for this container. + if local_digest: + remote_targets.add(remote_target) + + pending_metrics.append({ + "container_name": container.name, + "service": svc, + "compose_image": compose_image, + "running_image": running, + "project_name": proj, + "remote_target": remote_target, + "local_digest": local_digest, + }) + + remote_digests = {target: get_remote_digest(target) for target in remote_targets} + + for payload in pending_metrics: + local_digest = payload["local_digest"] + remote_target = payload["remote_target"] + remote_digest = remote_digests.get(remote_target) + update_flag = 1 if (local_digest and remote_digest and local_digest != remote_digest) else 0 + + logger.info( + "Digest comparison: container=%s service=%s running=%s target=%s local=%s remote=%s", + payload["container_name"], + payload["service"], + payload["running_image"], + remote_target, + local_digest, + remote_digest, + ) + + set_container_update_metric( + container_name=payload["container_name"], + compose_image=payload["compose_image"], + running_image=payload["running_image"], + project_name=payload["project_name"], + update_flag=update_flag, + ) + +def dump_service_image_mapping(): + project_name = parse_project_name_from_script(SERVICES_UP_SCRIPT) + project_root = get_project_root_from_script(SERVICES_UP_SCRIPT) + compose_files = get_compose_files_from_script(SERVICES_UP_SCRIPT) + svc_map = parse_compose_services(compose_files, project_name, project_root) + mapping = {name: data["image"] for name, data in sorted(svc_map.items())} + logger.info("Service to image mapping:") + logger.info(json.dumps(mapping, indent=2, sort_keys=True)) + return mapping + +# --- Runner --- +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Docker image update exporter") + parser.add_argument("--dry-run", action="store_true", help="Only print service->image mapping and exit") + parser.add_argument( + "--services-up-script", + default=SERVICES_UP_SCRIPT, + help=f"Path to services-up script (default: {SERVICES_UP_SCRIPT})", + ) + parser.add_argument( + "--cache-file", + default=CACHE_FILE, + help=f"Path to digest cache file (default: {CACHE_FILE})", + ) + parser.add_argument( + "--log-level", + default=LOG_LEVEL, + help=f"Logging level (default: {LOG_LEVEL})", + ) + args = parser.parse_args() + + effective_log_level = str(args.log_level).upper() + logging.getLogger().setLevel(getattr(logging, effective_log_level, logging.DEBUG)) + logger.setLevel(getattr(logging, effective_log_level, logging.DEBUG)) + + SERVICES_UP_SCRIPT = args.services_up_script + CACHE_FILE = args.cache_file + REMOTE_DIGEST_CACHE = load_cache() + now = time.time() + REMOTE_DIGEST_CACHE = { + image: (digest, ts) + for image, (digest, ts) in REMOTE_DIGEST_CACHE.items() + if now - ts < CACHE_TTL + } + + if DRY_RUN or args.dry_run: + dump_service_image_mapping() + raise SystemExit(0) + + start_http_server(EXPORTER_PORT) + while True: + try: + check_containers() + except Exception as e: + logger.exception(f"update check failed: {e}") + time.sleep(CHECK_INTERVAL) diff --git a/archive/dynamic-system/monitoring/docker-exporter/exporter.py.old b/archive/dynamic-system/monitoring/docker-exporter/exporter.py.old new file mode 100644 index 0000000..3d0c5f3 --- /dev/null +++ b/archive/dynamic-system/monitoring/docker-exporter/exporter.py.old @@ -0,0 +1,514 @@ +#!/usr/bin/env python3 +import os +import re +import time +import json +import logging +import docker +import requests +import yaml +from prometheus_client import Gauge, start_http_server + +# --- Logging --- +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper() + +logging.basicConfig( + level=getattr(logging, LOG_LEVEL, logging.INFO), + format="%(asctime)s [%(levelname)s] %(message)s" +) + +logger = logging.getLogger("docker-update-exporter") + +# --- Config --- +EXPORTER_PORT = 9105 +CHECK_INTERVAL = 60 +CACHE_TTL = 6 * 3600 +SERVICES_UP_SCRIPT = "/compose/services-up.sh" +CACHE_FILE = "/data/remote_digest_cache.json" + +client = docker.from_env() + +# --- Metrics --- +CONTAINER_UPDATE = Gauge( + "docker_container_update_available", + "1 if container image is out of date (compose drift or registry), 0 otherwise", + ["container", "compose_image", "running_image", "com_docker_compose_project"] +) + +LAST_CHECK = Gauge( + "docker_image_update_last_check_timestamp", + "Last time the update check ran (unix timestamp)" +) + +# --- Persistent Cache --- + +def load_cache(): + if not os.path.exists(CACHE_FILE): + logger.info(f"Cache file does not exist yet: {CACHE_FILE}") + return {} + + try: + with open(CACHE_FILE, "r") as f: + cache = json.load(f) + logger.info(f"Loaded {len(cache)} cached remote digests") + logger.debug(f"Cache contents: {cache}") + return cache + except Exception as e: + logger.error(f"Failed to load cache from {CACHE_FILE}: {e}") + return {} + +def save_cache(): + try: + os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True) + with open(CACHE_FILE, "w") as f: + json.dump(REMOTE_DIGEST_CACHE, f) + + logger.debug( + f"Saved {len(REMOTE_DIGEST_CACHE)} entries to cache file {CACHE_FILE}" + ) + except Exception as e: + logger.error(f"Failed to save cache to {CACHE_FILE}: {e}") + +REMOTE_DIGEST_CACHE = load_cache() + +# --- Helpers --- + +def get_project_prefix_from_script(script_path): + project_prefix = "core-" # fallback + + if not os.path.exists(script_path): + logger.warning( + f"services-up script not found at {script_path}, using fallback project prefix {project_prefix}" + ) + return project_prefix + + try: + with open(script_path, "r") as f: + for line in f: + line = line.strip() + m = re.match(r'PROJECT\s*=\s*["\']?([^"\']+)["\']?', line) + if m: + project_prefix = m.group(1) + "-" + logger.debug( + f"Detected compose project prefix from script: {project_prefix}" + ) + break + except Exception as e: + logger.error(f"Failed reading project prefix from {script_path}: {e}") + + return project_prefix + +def get_local_digest(image_name): + try: + img = client.images.get(image_name) + digests = img.attrs.get("RepoDigests", []) + + logger.debug(f"Local RepoDigests for {image_name}: {digests}") + + if digests: + digest = digests[0].split("@")[1] + logger.debug(f"Local digest for {image_name}: {digest}") + return digest + + logger.info(f"No local digest found for image {image_name}") + + except Exception as e: + logger.warning(f"Failed to retrieve local digest for {image_name}: {e}") + + return None + +def get_remote_digest(image_name): + now = time.time() + original = image_name + + # Cache hit + if original in REMOTE_DIGEST_CACHE: + digest, ts = REMOTE_DIGEST_CACHE[original] + age = now - ts + + if age < CACHE_TTL: + logger.debug( + f"Using cached remote digest for {original} " + f"(age={int(age)}s, ttl={CACHE_TTL}s): {digest}" + ) + return digest + + logger.info( + f"Cache entry expired for {original} " + f"(age={int(age)}s > ttl={CACHE_TTL}s)" + ) + + try: + if "/" not in image_name: + registry = "docker.io" + repo = "library/" + image_name + else: + parts = image_name.split("/") + if "." in parts[0] or ":" in parts[0]: + registry = parts[0] + repo = "/".join(parts[1:]) + else: + registry = "docker.io" + repo = image_name + + if ":" in repo: + repo, tag = repo.rsplit(":", 1) + else: + tag = "latest" + + logger.debug( + f"Resolving remote digest for {original}: " + f"registry={registry}, repo={repo}, tag={tag}" + ) + + token = None + manifest_url = None + + if registry in ["docker.io", "registry-1.docker.io"]: + logger.debug(f"Requesting Docker Hub token for {repo}") + + token_res = requests.get( + "https://auth.docker.io/token", + params={ + "service": "registry.docker.io", + "scope": f"repository:{repo}:pull" + }, + timeout=10 + ) + + logger.debug( + f"Docker Hub token response for {repo}: " + f"status={token_res.status_code}" + ) + + token = token_res.json().get("token") + manifest_url = ( + f"https://registry-1.docker.io/v2/{repo}/manifests/{tag}" + ) + + elif registry == "ghcr.io": + logger.debug(f"Requesting GHCR token for {repo}") + + token_res = requests.get( + "https://ghcr.io/token", + params={ + "service": "ghcr.io", + "scope": f"repository:{repo}:pull" + }, + timeout=10 + ) + + logger.debug( + f"GHCR token response for {repo}: " + f"status={token_res.status_code}" + ) + + token = token_res.json().get("token") + manifest_url = f"https://ghcr.io/v2/{repo}/manifests/{tag}" + + else: + logger.warning( + f"Unsupported registry '{registry}' for image {original}" + ) + return None + + if not token: + logger.warning( + f"No authentication token returned for {original}" + ) + return None + + logger.debug(f"Requesting manifest for {original}: {manifest_url}") + + res = requests.get( + manifest_url, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.docker.distribution.manifest.v2+json" + }, + timeout=10 + ) + + logger.debug( + f"Manifest response for {original}: " + f"status={res.status_code}" + ) + + if res.status_code == 200: + digest = res.headers.get("Docker-Content-Digest") + + logger.info( + f"Fetched remote digest for {original}: {digest}" + ) + + REMOTE_DIGEST_CACHE[original] = (digest, now) + save_cache() + + logger.debug( + f"Cached remote digest for {original}: {digest}" + ) + + return digest + + if res.status_code == 429: + logger.warning( + f"Registry rate limit hit while fetching {original}" + ) + elif res.status_code in [401, 403]: + logger.warning( + f"Authentication failed while fetching {original}: " + f"status={res.status_code}" + ) + else: + logger.warning( + f"Unexpected manifest response for {original}: " + f"status={res.status_code}, body={res.text[:250]}" + ) + + except Exception as e: + logger.error(f"Failed to fetch remote digest for {original}: {e}") + + return None + +def get_compose_files_from_script(script_path): + files = [] + + if not os.path.exists(script_path): + logger.error(f"services-up script not found: {script_path}") + return files + + base_dir = os.path.dirname(script_path) + + try: + with open(script_path, "r") as f: + content = f.read() + + match = re.search(r'FILES\s*=\s*\((.*?)\)', content, re.DOTALL) + + if not match: + logger.warning( + f"No FILES=(...) block found in {script_path}" + ) + return files + + lines = match.group(1).splitlines() + + for line in lines: + line = line.strip() + + if line.startswith("-f"): + rel_path = line[2:].strip() + + if rel_path: + full_path = os.path.normpath( + os.path.join(base_dir, rel_path) + ) + + logger.debug( + f"Resolved compose file: {rel_path} -> {full_path}" + ) + + files.append(full_path) + + logger.info(f"Found {len(files)} compose files") + + except Exception as e: + logger.error(f"Failed parsing compose files from {script_path}: {e}") + + return files + +def parse_compose_files(compose_files): + service_to_image = {} + + for f in compose_files: + if not os.path.exists(f): + logger.warning(f"Compose file missing: {f}") + continue + + try: + with open(f, "r") as stream: + data = yaml.safe_load(stream) or {} + services = data.get("services", {}) + + logger.debug( + f"Parsing {len(services)} services from compose file {f}" + ) + + for service_name, service_def in services.items(): + image = service_def.get("image") + is_built = False + + if not image: + is_built = True + build_ctx = service_def.get("build") + + logger.debug( + f"Service {service_name} is build-based, build config={build_ctx}" + ) + + if isinstance(build_ctx, dict): + context_path = build_ctx.get("context", ".") + dockerfile_path = os.path.join( + context_path, + build_ctx.get("dockerfile", "Dockerfile") + ) + elif isinstance(build_ctx, str): + context_path = build_ctx + dockerfile_path = os.path.join( + context_path, "Dockerfile" + ) + else: + dockerfile_path = None + + if dockerfile_path and os.path.exists(dockerfile_path): + try: + with open(dockerfile_path, "r") as df: + for line in df: + line = line.strip() + if ( + line.upper().startswith("LABEL") + and "image=" in line + ): + m = re.search( + r'image=["\']?([^"\']+)["\']?', + line + ) + if m: + image = m.group(1) + logger.debug( + f"Found upstream image label for {service_name}: {image}" + ) + break + except Exception as e: + logger.warning( + f"Failed reading Dockerfile {dockerfile_path}: {e}" + ) + + if not image: + image = f"{service_name}:latest" + logger.info( + f"No image label found for build service {service_name}, " + f"defaulting to {image}" + ) + + service_to_image[service_name] = (image, is_built) + + except Exception as e: + logger.error(f"Failed parsing compose file {f}: {e}") + + logger.info(f"Mapped {len(service_to_image)} compose services to images") + logger.debug(f"Service/image mapping: {service_to_image}") + + return service_to_image + +def check_containers(): + logger.info("Starting container update check") + + CONTAINER_UPDATE.clear() + + project_prefix = get_project_prefix_from_script(SERVICES_UP_SCRIPT) + compose_files = get_compose_files_from_script(SERVICES_UP_SCRIPT) + service_to_image = parse_compose_files(compose_files) + + containers = client.containers.list() + logger.info(f"Checking {len(containers)} running containers") + + for container in containers: + project_label = container.labels.get("com.docker.compose.project") + + if not project_label: + logger.debug( + f"Skipping non-compose container {container.name}" + ) + continue + + service_label = container.labels.get("com.docker.compose.service") + running_image = container.attrs["Config"]["Image"] + + logger.debug( + f"Evaluating container={container.name}, " + f"service={service_label}, project={project_label}, " + f"running_image={running_image}" + ) + + compose_image = None + is_built = False + + if service_label and service_label in service_to_image: + compose_image, is_built = service_to_image[service_label] + + if is_built: + compose_image_name, _, _ = compose_image.partition(":") + compose_image = f"{project_prefix}{compose_image_name}" + + update_flag = 0 + + if is_built: + if running_image != compose_image: + logger.info( + f"Update detected for build-based container {container.name}: " + f"running image {running_image} != expected {compose_image}" + ) + update_flag = 1 + else: + local_digest = get_local_digest(running_image) + remote_digest = get_remote_digest( + service_to_image[service_label][0] + ) + + if local_digest and remote_digest and local_digest != remote_digest: + logger.info( + f"Remote image update available for {container.name}: " + f"{local_digest} != {remote_digest}" + ) + update_flag = 1 + else: + if running_image != compose_image: + logger.info( + f"Compose drift detected for {container.name}: " + f"running image {running_image} != compose image {compose_image}" + ) + update_flag = 1 + else: + local_digest = get_local_digest(running_image) + remote_digest = get_remote_digest(running_image) + + if local_digest and remote_digest and local_digest != remote_digest: + logger.info( + f"Registry update available for {container.name}: " + f"{local_digest} != {remote_digest}" + ) + update_flag = 1 + + CONTAINER_UPDATE.labels( + container=container.name, + compose_image=compose_image if compose_image else "unknown", + running_image=running_image, + com_docker_compose_project=project_label + ).set(update_flag) + + logger.info( + f"Container {container.name}: " + f"running={running_image}, " + f"compose={compose_image}, " + f"update_available={update_flag}" + ) + + LAST_CHECK.set(time.time()) + logger.info("Container update check complete") + +if __name__ == "__main__": + logger.info( + f"Starting Docker update exporter on port {EXPORTER_PORT} " + f"with LOG_LEVEL={LOG_LEVEL}" + ) + + start_http_server(EXPORTER_PORT) + + while True: + try: + check_containers() + except Exception as e: + logger.exception(f"Unhandled error during update check: {e}") + + time.sleep(CHECK_INTERVAL) diff --git a/archive/dynamic-system/monitoring/grafana/docker-compose.yml b/archive/dynamic-system/monitoring/grafana/docker-compose.yml new file mode 100644 index 0000000..ec5b127 --- /dev/null +++ b/archive/dynamic-system/monitoring/grafana/docker-compose.yml @@ -0,0 +1,55 @@ +services: + grafana: + profiles: ["monitoring","all","grafana"] + image: grafana/grafana:latest + container_name: grafana + restart: unless-stopped + environment: + - GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL} + volumes: +# - ${PROJECT_ROOT}/monitoring/grafana/data:/var/lib/grafana + - ${DOCKER_VOLUMES}/grafana-data:/var/lib/grafana + networks: +# - traefik_reverse_proxy +# - prometheus_edge + - traefik + - monitor + labels: + - "traefik.http.routers.grafana.rule=Host(`grafana.lan.ddnsgeek.com`)" + - "traefik.enable=true" + - "traefik.http.routers.grafana.entrypoints=websecure" + - "traefik.http.routers.grafana.tls.certresolver=myresolver" + - "traefik.http.routers.grafana.tls.options=mtls-private-admin@file" + - "io.portainer.accesscontrol.public" + - "traefik.http.services.grafana.loadbalancer.server.port=3000" + - "traefik.docker.network=core_traefik" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + +# tempo: +# image: grafana/tempo:latest +# container_name: tempo +# command: +# - "-config.file=/etc/tempo/config.yaml" +# volumes: +# - ./tempo/config.yaml:/etc/tempo/config.yaml +# - ./tempo/data:/var/lib/tempo +# ports: +# - "4317:4317" # OTLP gRPC endpoint for Traefik +# - "3200:3200" # optional: HTTP endpoint +# networks: +# - prometheus_edge + +#networks: +# traefik_reverse_proxy: +# external: true +# prometheus_edge: +# external: true + +#volumes: +# grafana-data: +# external: true diff --git a/archive/dynamic-system/monitoring/influxdb/docker-compose.yml b/archive/dynamic-system/monitoring/influxdb/docker-compose.yml new file mode 100644 index 0000000..ad78629 --- /dev/null +++ b/archive/dynamic-system/monitoring/influxdb/docker-compose.yml @@ -0,0 +1,50 @@ +services: + influxdb: + profiles: ["monitoring","all","influxdb", "prometheus"] + image: influxdb:2 + container_name: influxdb + restart: unless-stopped +# env_file: +# - ${PROJECT_ROOT}/secrets/stack-secrets.env + volumes: +# - ${PROJECT_ROOT}/monitoring/influxdb:/var/lib/influxdb2 + - ${DOCKER_DATABASES}/influxdb-data:/var/lib/influxdb2 + environment: + DOCKER_INFLUXDB_INIT_MODE: ${INFLUXDB_INIT_MODE} + DOCKER_INFLUXDB_INIT_USERNAME: ${INFLUXDB_INIT_USERNAME} + DOCKER_INFLUXDB_INIT_PASSWORD_FILE: /run/secrets/influxdb_init_password + DOCKER_INFLUXDB_INIT_ORG: ${INFLUXDB_INIT_ORG} + DOCKER_INFLUXDB_INIT_BUCKET: ${INFLUXDB_INIT_BUCKET} + secrets: + - influxdb_init_password + networks: +# - edge +# - traefik_reverse_proxy + - traefik + - monitor + ports: + - 8086:8086 + labels: + - "traefik.http.routers.influxdb.rule=Host(`influxdb.lan.ddnsgeek.com`)" + - "traefik.enable=true" + - "traefik.http.routers.influxdb.entrypoints=websecure" + - "traefik.http.routers.influxdb.tls.certresolver=myresolver" + - "traefik.http.routers.influxdb.tls.options=mtls-private-admin@file" + - "io.portainer.accesscontrol.public" + - "traefik.http.services.influxdb.loadbalancer.server.port=8086" + - "traefik.http.routers.influxdb.middlewares=authelia" + - "traefik.docker.network=core_traefik" + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:8086/health || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + +secrets: + influxdb_init_password: + file: ${PROJECT_ROOT}/secrets/influxdb_init_password.txt + +#volumes: +# influxdb-data: +# external: true diff --git a/archive/dynamic-system/monitoring/mtls-bridge/Dockerfile b/archive/dynamic-system/monitoring/mtls-bridge/Dockerfile new file mode 100644 index 0000000..cef343b --- /dev/null +++ b/archive/dynamic-system/monitoring/mtls-bridge/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app.py . + +EXPOSE 8080 + +CMD ["python", "app.py"] diff --git a/archive/dynamic-system/monitoring/mtls-bridge/README.md b/archive/dynamic-system/monitoring/mtls-bridge/README.md new file mode 100644 index 0000000..3dfe5e3 --- /dev/null +++ b/archive/dynamic-system/monitoring/mtls-bridge/README.md @@ -0,0 +1,60 @@ +# mTLS Bridge Service + +Internal HTTP-to-mTLS bridge for services that cannot present client certificates directly (for example, Grafana webhooks). + +## How it works + +1. Accepts plain HTTP requests inside the Docker network. +2. Forwards requests to an upstream base URL. +3. Preserves the incoming request path/method/body/query string. +4. Presents a client certificate/key pair for mTLS authentication. + +## Environment variables + +- `TARGET_URL` (required): upstream base URL (for example `http://node-red:1880`). +- `CLIENT_CERT` (default `/certs/client.crt`): client certificate path. +- `CLIENT_KEY` (default `/certs/client.key`): client private key path. +- `UPSTREAM_CA_CERT` (optional, alias: `CA_CERT`): CA bundle path to verify upstream TLS. Use `false`/`0`/`no` to disable verification. +- `TIMEOUT` (default `5`): request timeout in seconds. +- `LOG_LEVEL` (default `INFO`): Python logging level. +- `HEALTH_ENDPOINT` (default `/_mtls_bridge/health`): local container health endpoint path. +- `ALLOWED_PATHS_FILE` (optional): file path containing one allowed endpoint path per line (for example `/health`). Blank lines and `#` comments are ignored. If unset, all paths are allowed. +- `MTLS_BRIDGE_BASIC_AUTH_USERS` (required for Traefik auth): value for `traefik.http.middlewares.*.basicauth.users` (e.g. `user:$$apr1$$...`). +- `MTLS_BRIDGE_CORS_ALLOW_ORIGIN` (default `https://grafana.lan.ddnsgeek.com`): origin allowed for browser-based panel actions. + +## Endpoints + +- `GET /_mtls_bridge/health` returns `200 OK` for container health checks. +- `/*` proxies requests to `${TARGET_URL}/*` with method/body/headers/query string preserved (subject to optional allow-list checks). + +Examples with `TARGET_URL=http://node-red:1880`: + +- `https://mtls-bridge.../docker-update-lockouts/clear` -> `http://node-red:1880/docker-update-lockouts/clear` +- `https://mtls-bridge.../health` -> `http://node-red:1880/health` +- `https://mtls-bridge.../uptime-kuma` -> `http://node-red:1880/uptime-kuma` + +## Compose integration + +This repository includes `monitoring/mtls-bridge/docker-compose.yml`: + +- No public port exposure. +- Read-only cert mount (`${PROJECT_ROOT}/core/traefik/certs:/certs:ro`). +- Joined to internal monitoring/traefik networks. + +## Example test + +```bash +curl http://mtls-bridge:8080/_mtls_bridge/health +curl -X POST http://mtls-bridge:8080/docker-update-lockouts/clear +``` + +## Allow-list file example + +```text +# one path per line +/docker-update-lockouts/clear +/health +/uptime-kuma +``` + +When `ALLOWED_PATHS_FILE` is set, any path not listed returns `403 Endpoint not allowed`. diff --git a/archive/dynamic-system/monitoring/mtls-bridge/app.py b/archive/dynamic-system/monitoring/mtls-bridge/app.py new file mode 100644 index 0000000..dca29b7 --- /dev/null +++ b/archive/dynamic-system/monitoring/mtls-bridge/app.py @@ -0,0 +1,208 @@ +import logging +import os +import time +from urllib.parse import urljoin + +import requests +from flask import Flask, Response, g, request + +app = Flask(__name__) + +logging.basicConfig( + level=os.environ.get("LOG_LEVEL", "INFO"), + format="%(asctime)s %(levelname)s %(message)s", +) +logger = logging.getLogger("mtls-bridge") +logging.getLogger("werkzeug").setLevel(logging.WARNING) + +# Config via env +TARGET_URL = (os.environ.get("TARGET_URL") or "").strip() +CLIENT_CERT = os.environ.get("CLIENT_CERT", "/certs/client.crt") +CLIENT_KEY = os.environ.get("CLIENT_KEY", "/certs/client.key") +UPSTREAM_CA_CERT = os.environ.get("UPSTREAM_CA_CERT", os.environ.get("CA_CERT", "")).strip() +TIMEOUT = int(os.environ.get("TIMEOUT", "5")) +HEALTH_ENDPOINT = os.environ.get("HEALTH_ENDPOINT", "/_mtls_bridge/health") +ALLOWED_PATHS_FILE = (os.environ.get("ALLOWED_PATHS_FILE") or "").strip() + + +def normalize_path(path: str) -> str: + if not path or path == "/": + return "/" + return f"/{path.lstrip('/')}" + + +def load_allowed_paths() -> set[str]: + if not ALLOWED_PATHS_FILE: + return set() + + if not os.path.exists(ALLOWED_PATHS_FILE): + logger.warning("ALLOWED_PATHS_FILE does not exist: %s (allow-list disabled)", ALLOWED_PATHS_FILE) + return set() + + allowed_paths = set() + with open(ALLOWED_PATHS_FILE, encoding="utf-8") as f: + for line in f: + entry = line.strip() + if not entry or entry.startswith("#"): + continue + allowed_paths.add(normalize_path(entry)) + + logger.info("loaded %s allowed path(s) from %s", len(allowed_paths), ALLOWED_PATHS_FILE) + return allowed_paths + + +def get_verify_setting(): + if not UPSTREAM_CA_CERT: + return True + + lowered = UPSTREAM_CA_CERT.lower() + if lowered in {"false", "0", "no"}: + logger.warning("TLS verification for upstream is disabled via UPSTREAM_CA_CERT=%s", UPSTREAM_CA_CERT) + return False + + if not os.path.exists(UPSTREAM_CA_CERT): + logger.warning( + "Configured UPSTREAM_CA_CERT path does not exist: %s (falling back to system CA bundle)", + UPSTREAM_CA_CERT, + ) + return True + + return UPSTREAM_CA_CERT + + +VERIFY_SETTING = get_verify_setting() +ALLOWED_PATHS = load_allowed_paths() + +if TARGET_URL and TARGET_URL.lower().startswith("http://"): + logger.warning("TARGET_URL uses http:// (plaintext): %s", TARGET_URL) + +logger.info( + "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s health_endpoint=%s allow_list_file=%s allow_list_entries=%s log_level=%s", + TARGET_URL, + TIMEOUT, + CLIENT_CERT, + CLIENT_KEY, + VERIFY_SETTING, + HEALTH_ENDPOINT, + ALLOWED_PATHS_FILE, + len(ALLOWED_PATHS), + os.environ.get("LOG_LEVEL", "INFO"), +) + + +def build_upstream_url(path: str) -> str: + """Map incoming path directly onto TARGET_URL origin/base path.""" + if not TARGET_URL: + raise ValueError("TARGET_URL is not set") + + normalized_target = TARGET_URL.rstrip("/") + "/" + normalized_path = path.lstrip("/") + upstream_url = urljoin(normalized_target, normalized_path) + + if request.query_string: + upstream_url = f"{upstream_url}?{request.query_string.decode('utf-8', 'ignore')}" + + return upstream_url + + +def is_path_allowed(request_path: str) -> bool: + if not ALLOWED_PATHS: + return True + return request_path in ALLOWED_PATHS + + +@app.route(HEALTH_ENDPOINT, methods=["GET"]) +def health(): + logger.debug("healthcheck request from %s", request.remote_addr) + return "OK", 200 + + +@app.before_request +def before_request(): + g.request_start = time.time() + + +@app.after_request +def after_request(response): + elapsed_ms = int((time.time() - g.request_start) * 1000) + if request.path != HEALTH_ENDPOINT: + logger.info( + "request complete method=%s path=%s status=%s elapsed_ms=%s", + request.method, + request.path, + response.status_code, + elapsed_ms, + ) + return response + + +@app.route( + "/", + defaults={"path": ""}, + methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"], + provide_automatic_options=False, +) +@app.route( + "/", + methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"], + provide_automatic_options=False, +) +def proxy(path): + request_path = normalize_path(path) + request_size = len(request.get_data(cache=True)) + logger.info( + "incoming request method=%s path=%s query=%s remote=%s bytes=%s", + request.method, + request_path, + request.query_string.decode("utf-8", "ignore"), + request.remote_addr, + request_size, + ) + + if not is_path_allowed(request_path): + logger.warning("request blocked by allow-list path=%s", request_path) + return Response("Endpoint not allowed", status=403) + + try: + upstream_url = build_upstream_url(path) + + headers = {k: v for k, v in request.headers if k.lower() != "host"} + headers["X-Forwarded-By"] = "mtls-bridge" + + start_time = time.time() + resp = requests.request( + method=request.method, + url=upstream_url, + headers=headers, + data=request.get_data(cache=True), + cookies=request.cookies, + cert=(CLIENT_CERT, CLIENT_KEY), + verify=VERIFY_SETTING, + timeout=TIMEOUT, + allow_redirects=False, + ) + + elapsed_ms = int((time.time() - start_time) * 1000) + logger.info( + "upstream response status=%s url=%s elapsed_ms=%s response_bytes=%s", + resp.status_code, + upstream_url, + elapsed_ms, + len(resp.content), + ) + + excluded_headers = {"content-encoding", "content-length", "transfer-encoding", "connection"} + response_headers = [(k, v) for k, v in resp.headers.items() if k.lower() not in excluded_headers] + + return Response(resp.content, resp.status_code, response_headers) + + except ValueError as exc: + logger.error("proxy request failed: %s", exc) + return Response(str(exc), status=500) + except Exception as exc: # noqa: BLE001 + logger.exception("proxy request failed") + return Response(str(exc), status=500) + + +if __name__ == "__main__": + app.run(host="0.0.0.0", port=8080) diff --git a/archive/dynamic-system/monitoring/mtls-bridge/docker-compose.yml b/archive/dynamic-system/monitoring/mtls-bridge/docker-compose.yml new file mode 100644 index 0000000..7d8938d --- /dev/null +++ b/archive/dynamic-system/monitoring/mtls-bridge/docker-compose.yml @@ -0,0 +1,49 @@ +services: + mtls-bridge: + profiles: ["monitoring", "all", "mtls-bridge"] + build: + context: ${PROJECT_ROOT}/monitoring/mtls-bridge + container_name: mtls-bridge + hostname: mtls-bridge.lan.ddnsgeek.com + restart: unless-stopped + environment: + - TARGET_URL=${MTLS_BRIDGE_TARGET_URL} + - CLIENT_CERT=${MTLS_BRIDGE_CLIENT_CERT} + - CLIENT_KEY=${MTLS_BRIDGE_CLIENT_KEY} + - TIMEOUT=${MTLS_BRIDGE_TIMEOUT} + - LOG_LEVEL=${MTLS_BRIDGE_LOG_LEVEL:-INFO} + - UPSTREAM_CA_CERT=${MTLS_BRIDGE_UPSTREAM_CA_CERT:-} + - ALLOWED_PATHS_FILE=${MTLS_BRIDGE_ALLOWED_PATHS_FILE:-} + volumes: + - ${PROJECT_ROOT}/core/traefik/certs:/certs:ro + labels: + - "traefik.http.routers.mtls-bridge.rule=Host(`mtls-bridge.lan.ddnsgeek.com`)" + - "traefik.enable=true" + - "traefik.http.routers.mtls-bridge.entrypoints=websecure" + - "traefik.http.routers.mtls-bridge.tls.certresolver=myresolver" + - "traefik.http.routers.mtls-bridge.middlewares=mtls-bridge-auth,mtls-bridge-cors" + - "traefik.http.middlewares.mtls-bridge-auth.basicauth.users=${MTLS_BRIDGE_BASIC_AUTH_USERS}" + - "traefik.http.routers.mtls-bridge-preflight.rule=Host(`mtls-bridge.lan.ddnsgeek.com`) && Method(`OPTIONS`)" + - "traefik.http.routers.mtls-bridge-preflight.entrypoints=websecure" + - "traefik.http.routers.mtls-bridge-preflight.tls.certresolver=myresolver" + - "traefik.http.routers.mtls-bridge-preflight.middlewares=mtls-bridge-cors" + - "traefik.http.routers.mtls-bridge-preflight.priority=100" + - "traefik.http.routers.mtls-bridge-preflight.service=mtls-bridge" + - "traefik.http.middlewares.mtls-bridge-cors.headers.accesscontrolalloworiginlist=${MTLS_BRIDGE_CORS_ALLOW_ORIGIN}" + - "traefik.http.middlewares.mtls-bridge-cors.headers.accesscontrolallowmethods=GET,POST,PUT,PATCH,DELETE,OPTIONS" + - "traefik.http.middlewares.mtls-bridge-cors.headers.accesscontrolallowheaders=authorization,content-type,x-grafana-action,x-grafana-device-id" + - "traefik.http.middlewares.mtls-bridge-cors.headers.accesscontrolallowcredentials=true" + - "traefik.http.middlewares.mtls-bridge-cors.headers.addvaryheader=true" + - "io.portainer.accesscontrol.public" +# - "traefik.http.routers.searxng.middlewares=crowdsec@file,secHeaders@file,error-pages-middleware" + - "traefik.http.services.mtls-bridge.loadbalancer.server.port=8080" + - "traefik.docker.network=core_traefik" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/_mtls_bridge/health', timeout=3).read()"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s + networks: + - monitor + - traefik diff --git a/archive/dynamic-system/monitoring/mtls-bridge/requirements.txt b/archive/dynamic-system/monitoring/mtls-bridge/requirements.txt new file mode 100644 index 0000000..30692b7 --- /dev/null +++ b/archive/dynamic-system/monitoring/mtls-bridge/requirements.txt @@ -0,0 +1,2 @@ +flask +requests diff --git a/archive/dynamic-system/monitoring/node-exporter/docker-compose.yml b/archive/dynamic-system/monitoring/node-exporter/docker-compose.yml new file mode 100644 index 0000000..f7f769f --- /dev/null +++ b/archive/dynamic-system/monitoring/node-exporter/docker-compose.yml @@ -0,0 +1,23 @@ +services: + node-exporter: + profiles: ["monitoring","all","node-exporter", "prometheus"] + image: prom/node-exporter:latest + container_name: node-exporter + pid: host + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + restart: unless-stopped + networks: +# - edge + - monitor + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:9100/metrics"] + interval: 30s + timeout: 10s + retries: 3 diff --git a/archive/dynamic-system/monitoring/node-red/Dockerfile b/archive/dynamic-system/monitoring/node-red/Dockerfile new file mode 100644 index 0000000..0fb8220 --- /dev/null +++ b/archive/dynamic-system/monitoring/node-red/Dockerfile @@ -0,0 +1,7 @@ +FROM nodered/node-red:latest + +USER root +RUN apk add --no-cache docker-cli docker-cli-compose +#RUN addgroup -g 131 -S docker && addgroup node-red docker + +USER node-red diff --git a/archive/dynamic-system/monitoring/node-red/UPDATE_LOGGING_GRAFANA.md b/archive/dynamic-system/monitoring/node-red/UPDATE_LOGGING_GRAFANA.md new file mode 100644 index 0000000..51c8fc5 --- /dev/null +++ b/archive/dynamic-system/monitoring/node-red/UPDATE_LOGGING_GRAFANA.md @@ -0,0 +1,160 @@ +# Node-RED update logging for Grafana + +This guide adds structured update-event logging to your existing Node-RED + Telegraf + Prometheus + Grafana stack without introducing Loki. + +## Goal + +Track and surface (in Grafana) the latest update attempts from Node-RED, including: + +- when an update attempt started, +- target container/project, +- success/failure, +- optional failure reason, +- elapsed duration. + +## 1) Add a reusable logger function in Node-RED + +Create a **Function** node named `Build update log event` and use: + +```javascript +const nowIso = new Date().toISOString(); +const startedAt = msg.update_started_at || Date.now(); +const durationMs = Math.max(0, Date.now() - startedAt); + +const payload = msg.payload || {}; +const labels = payload.labels || {}; + +const status = (msg.update_status || payload.status || "unknown").toString().toLowerCase(); +const success = status === "success" ? 1 : 0; +const failed = status === "failed" ? 1 : 0; + +msg.payload = { + ts: nowIso, + flow: "docker-updates", + event: msg.update_event || "attempt", + container: msg.container || labels.container || "unknown", + project: labels.com_docker_compose_project || msg.project || "unknown", + host: msg.host || "unknown", + status, + success, + failed, + duration_ms: durationMs, + code: Number.isFinite(Number(payload.code)) ? Number(payload.code) : 0, + error: (msg.update_error || payload.error || "").toString().slice(0, 300) +}; + +// one JSON line per event for file output +msg.payload = JSON.stringify(msg.payload); +return msg; +``` + +### Wiring recommendation + +Use the same logger function in these branches: + +- before a pull/update command (`update_status=started`, `update_event=attempt`), +- success path (`update_status=success`, `update_event=completed`), +- failure path (`update_status=failed`, `update_event=completed`, and include `msg.update_error`). + +Then route each branch into a **File** node configured as: + +- Filename: `/data/update-events.ndjson` +- Action: append to file +- Add newline: enabled + +## 2) Make update state explicit in existing update flow + +In your current update flow (already present in `flows.json`), add/change **Change** nodes around your shell/docker nodes: + +- At update start: + - `msg.update_started_at = $millis()` + - `msg.update_status = "started"` + - `msg.update_event = "attempt"` +- At success: + - `msg.update_status = "success"` + - `msg.update_event = "completed"` +- At failure: + - `msg.update_status = "failed"` + - `msg.update_event = "completed"` + - `msg.update_error = msg.payload.stderr` (or equivalent error field) + +## 3) Let Telegraf ingest Node-RED event logs + +Append this to `monitoring/telegraf/telegraf.conf`: + +```toml +[[inputs.tail]] + files = ["/var/log/node-red/update-events.ndjson"] + from_beginning = false + name_override = "node_red_update_event" + data_format = "json_v2" + + [[inputs.tail.json_v2]] + measurement_name = "node_red_update_event" + + [[inputs.tail.json_v2.tag]] + path = "flow" + [[inputs.tail.json_v2.tag]] + path = "event" + [[inputs.tail.json_v2.tag]] + path = "container" + [[inputs.tail.json_v2.tag]] + path = "project" + [[inputs.tail.json_v2.tag]] + path = "host" + [[inputs.tail.json_v2.tag]] + path = "status" + + [[inputs.tail.json_v2.field]] + path = "success" + type = "int" + [[inputs.tail.json_v2.field]] + path = "failed" + type = "int" + [[inputs.tail.json_v2.field]] + path = "duration_ms" + type = "int" + [[inputs.tail.json_v2.field]] + path = "code" + type = "int" +``` + +And mount the Node-RED data directory into Telegraf (read-only) in `monitoring/prometheus/docker-compose.yml` under `telegraf.volumes`: + +```yaml + - ${PROJECT_ROOT}/monitoring/node-red/data:/var/log/node-red:ro +``` + +## 4) Prometheus scrape (already in place) + +No Prometheus scrape change is required as long as it already scrapes Telegraf (`telegraf:9273`). + +## 5) Grafana queries to start with + +Use your Prometheus data source and try: + +- Latest success/failure by container: + - `last_over_time(node_red_update_event_success[24h])` + - `last_over_time(node_red_update_event_failed[24h])` +- Failed updates in the last 24h: + - `sum by (container, project) (increase(node_red_update_event_failed[24h]))` +- Average update duration in last 24h: + - `avg by (container, project) (avg_over_time(node_red_update_event_duration_ms[24h]))` + +Recommended panels: + +- **Table**: container, project, status (last value), duration_ms (last value) +- **Time series**: failed count over time +- **Stat**: total failed updates in last 24h + +## 6) Validation checklist + +1. Trigger a known update path (including one failure if possible). +2. Check Node-RED log file: + - `tail -n 20 monitoring/node-red/data/update-events.ndjson` +3. Check Telegraf metrics endpoint for `node_red_update_event_` metrics. +4. Confirm Grafana panel values match the latest Node-RED run. + +## Optional next step + +If you want searchable raw log text and richer log UX, add Loki + Promtail later. Keep this structured metrics path for high-signal alerting even after adding logs. diff --git a/archive/dynamic-system/monitoring/node-red/docker-compose.yml b/archive/dynamic-system/monitoring/node-red/docker-compose.yml new file mode 100644 index 0000000..fbdbf8f --- /dev/null +++ b/archive/dynamic-system/monitoring/node-red/docker-compose.yml @@ -0,0 +1,67 @@ +services: + node-red: + profiles: ["monitoring","all","node-red"] +# image: nodered/node-red:latest + build: + context: ${PROJECT_ROOT}/monitoring/node-red + container_name: node-red + restart: unless-stopped + depends_on: + - docker-socket-proxy + environment: + DOCKER_HOST: ${DOCKER_SOCKET_PROXY_HOST} + TZ: ${TZ} + PROJECT_ROOT: ${NODE_COMPOSE_ROOT} + cap_drop: + - ALL + security_opt: + - no-new-privileges:true +# ports: +# - "1880:1880" + volumes: +# - ${PROJECT_ROOT}/monitoring/node-red/data:/data + - ${DOCKER_VOLUMES}/node-red-data:/data + - ${PROJECT_ROOT}:/compose/docker:ro + - /home/nixos/raspi:/compose/raspi:ro +# - ${PROJECT_ROOT}:/usr/src/node-red:ro + +# - ${PROJECT_ROOT}/default-environment.env:/usr/src/node-red/default-environment.env:ro +# - ${PROJECT_ROOT}/default-network.yml:/usr/src/node-red/default-network.yml:ro +# - ${PROJECT_ROOT}/core/docker-compose.yml:/usr/src/node-red/core/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/prometheus/docker-compose.yml:/usr/src/node-red/monitoring/prometheus/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/gotify/docker-compose.yml:/usr/src/node-red/monitoring/gotify/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/grafana/docker-compose.yml:/usr/src/node-red/monitoring/grafana/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/portainer/docker-compose.yml:/usr/src/node-red/monitoring/portainer/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/uptime-kuma/docker-compose.yml:/usr/src/node-red/monitoring/uptime-kuma/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/gitea/docker-compose.yml:/usr/src/node-red/apps/gitea/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/gramps/docker-compose.yml:/usr/src/node-red/apps/gramps/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/nextcloud/docker-compose.yml:/usr/src/node-red/apps/nextcloud/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/passbolt/docker-compose.yml:/usr/src/node-red/apps/passbolt/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/searxng/docker-compose.yml:/usr/src/node-red/apps/searxng/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/shift-recorder/docker-compose.yml:/usr/src/node-red/apps/shift-recorder/docker-compose.yml:ro +# - ${PROJECT_ROOT}/apps/stockfill/docker-compose.yml:/usr/src/node-red/apps/stockfill/docker-compose.yml:ro +# - ${PROJECT_ROOT}/monitoring/node-red/docker-compose.yml:/usr/src/node-red/monitoring/node-red/docker-compose.yml:ro +# - ${PROJECT_ROOT}/core/test/docker-compose.yml:/usr/src/node-red/core/test/docker-compose.yml:ro +# - ${PROJECT_ROOT}/secrets/stack-secrets.env:/usr/src/node-red/secrets/stack-secrets.env:ro + +# - /run/current-system/sw/bin/docker:/usr/bin/docker:ro +# depends_on: +# - mosquitto +# - influxdb + networks: + - monitor + - traefik + labels: + - "traefik.enable=true" + - "traefik.http.routers.node-red.rule=Host(`node-red.lan.ddnsgeek.com`)" +# - "traefik.http.routers.node-red.service=api@internal" + - "traefik.http.routers.node-red.entrypoints=websecure" + - "traefik.http.routers.node-red.tls.certresolver=myresolver" + - "traefik.http.routers.node-red.tls.options=mtls-private-admin@file" + - "traefik.http.routers.node-red.middlewares=authelia" + - "io.portainer.accesscontrol.public" + - "traefik.docker.network=core_traefik" + - "traefik.http.services.node-red.loadbalancer.server.port=1880" +#volumes: +# node-red-data: +# external: true diff --git a/archive/dynamic-system/monitoring/pihole-exporter/docker-compose.yml b/archive/dynamic-system/monitoring/pihole-exporter/docker-compose.yml new file mode 100644 index 0000000..c681841 --- /dev/null +++ b/archive/dynamic-system/monitoring/pihole-exporter/docker-compose.yml @@ -0,0 +1,17 @@ +services: + pihole-exporter: + profiles: ["monitoring","all","pihole-exporter", "prometheus"] + image: ekofr/pihole-exporter:latest + container_name: pihole-exporter +# env_file: +# - ${PROJECT_ROOT}/secrets/stack-secrets.env + environment: + PIHOLE_HOSTNAME: ${PIHOLE_HOSTNAME} + PIHOLE_PASSWORD: ${PIHOLE_PASSWORD} + PORT: ${PIHOLE_EXPORTER_PORT} + ports: + - "${PIHOLE_EXPORTER_PORT}:${PIHOLE_EXPORTER_PORT}" + restart: unless-stopped + networks: +# - edge + - monitor diff --git a/archive/dynamic-system/monitoring/prometheus/docker-compose.yml b/archive/dynamic-system/monitoring/prometheus/docker-compose.yml new file mode 100644 index 0000000..5dc7fed --- /dev/null +++ b/archive/dynamic-system/monitoring/prometheus/docker-compose.yml @@ -0,0 +1,56 @@ +services: + prometheus: + profiles: ["monitoring","all","prometheus"] + image: prom/prometheus:latest +# env_file: +# - ${PROJECT_ROOT}/secrets/stack-secrets.env + container_name: prometheus + depends_on: +# - alertmanager + - telegraf + - influxdb + - node-exporter + - docker-update-exporter + - pihole-exporter + command: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus" + - "--storage.tsdb.retention.time=15d" +# build: +# context: ${PROJECT_ROOT}/monitoring/prometheus + volumes: + - ${PROJECT_ROOT}/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro +# - ${PROJECT_ROOT}/monitoring/prometheus/data:/prometheus +# - ${PROJECT_ROOT}/monitoring/prometheus/rules:/etc/prometheus/rules:ro + - ${DOCKER_VOLUMES}/prometheus/data:/prometheus + - ${DOCKER_VOLUMES}/prometheus/rules:/etc/prometheus/rules:ro + - ${PROJECT_ROOT}/secrets/prometheus_kuma_basic_auth_password.txt:/run/secrets/prometheus_kuma_basic_auth_password:ro + + restart: unless-stopped + labels: + - "traefik.http.routers.prometheus.rule=Host(`prometheus.lan.ddnsgeek.com`)" + - "traefik.enable=true" + - "traefik.http.routers.prometheus.entrypoints=websecure" + - "traefik.http.routers.prometheus.tls.certresolver=myresolver" + - "traefik.http.routers.prometheus.tls.options=mtls-private-admin@file" + - "io.portainer.accesscontrol.public" + - "traefik.http.services.prometheus.loadbalancer.server.port=9090" + - "traefik.http.routers.prometheus.middlewares=authelia" + - "traefik.docker.network=core_traefik" + networks: +# - edge +# - traefik_reverse_proxy + - traefik + - monitor + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + +#volumes: +# prometheus-data: +# external: true +# prometheus-rules: +# external: true diff --git a/archive/dynamic-system/monitoring/prometheus/prometheus.yml b/archive/dynamic-system/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..6085a7b --- /dev/null +++ b/archive/dynamic-system/monitoring/prometheus/prometheus.yml @@ -0,0 +1,166 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +#alerting: +# alertmanagers: +# - static_configs: +# - targets: +# - alertmanager:9093 + +scrape_configs: + + # Prometheus itself + - job_name: "prometheus" + static_configs: + - targets: ["prometheus:9090"] + labels: + role: prometheus + # ========================= + # Node Exporters (ALL hosts) + # ========================= + - job_name: "node" + static_configs: + - targets: + - node-exporter:9100 + labels: + role: docker + + - targets: + - raspberrypi.tail13f623.ts.net:9100 + labels: + role: raspberrypi + + - targets: + - pve.sweet.home:9100 + labels: + role: proxmox + + - targets: + - pbs.sweet.home:9100 + labels: + role: backup + + - targets: + - pihole:9100 + labels: + role: pihole + + - targets: + - server:9100 + labels: + role: server + + - targets: + - nix-cache:9100 + labels: + role: cache + + # ========================= + # Telegraf (Docker metrics) + # ========================= + - job_name: "telegraf" + static_configs: + - targets: + - telegraf:9273 + - raspberrypi.tail13f623.ts.net:9273 + labels: + role: docker + + # ========================= + # Traefik (all instances) + # ========================= + - job_name: "traefik" + static_configs: + - targets: + - traefik.lan.ddnsgeek.com:8080 + labels: + role: docker + + - targets: + - raspberrypi.tail13f623.ts.net:8080 + labels: + role: raspberrypi + + metric_relabel_configs: + - source_labels: [service] + regex: '(.+)@.+' + target_label: service + replacement: '$1' + + # ========================= + # Uptime Kuma (separate due to auth) + # ========================= + - job_name: "kuma" + metrics_path: /metrics + scrape_interval: 30s + + basic_auth: + username: wayne.bennett@live.com + password_file: /run/secrets/prometheus_kuma_basic_auth_password +# password: '4vjCco?[%{=+,t`):C' + static_configs: + - targets: + - monitor-kuma:3001 + labels: + role: docker + + - targets: + - kuma.lan.ddnsgeek.com + labels: + role: raspberrypi + tls_config: +# ca_file: /prometheus/clients-ca.crt + cert_file: /prometheus/office-pc.crt + key_file: /prometheus/office-pc.key +# server_name: kuma.lan.ddnsgeek.com + + # ========================= + # Proxmox Storage Exporters + # ========================= + - job_name: "proxmox-storage" + metrics_path: /metrics + static_configs: + - targets: + - pve.sweet.home:9101 + labels: + role: proxmox + storage: lvm + +# - targets: +# - pbs.sweet.home:9102 +# labels: +# role: backup +# storage: datastore + + # ========================= + # Docker Updates Exporter + # ========================= + + + - job_name: "container-updates" + static_configs: + - targets: + - docker-update-exporter:9105 + labels: + role: docker + - targets: + - raspberrypi.tail13f623.ts.net:9105 + labels: + role: raspberrypi + + # ========================= + # pihole Exporter + # ========================= + + + - job_name: "pihole" + static_configs: + - targets: + - pihole-exporter:9617 + labels: + role: pihole + + +#rule_files: +# - /etc/prometheus/rules/*.yml diff --git a/archive/dynamic-system/monitoring/prometheus/prometheus.yml.old b/archive/dynamic-system/monitoring/prometheus/prometheus.yml.old new file mode 100644 index 0000000..37dbcbd --- /dev/null +++ b/archive/dynamic-system/monitoring/prometheus/prometheus.yml.old @@ -0,0 +1,192 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +scrape_configs: + - job_name: "prometheus" + static_configs: + - targets: ["prometheus:9090"] + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + - job_name: "docker-node" + static_configs: + - targets: ["node-exporter:9100"] + labels: + type: "virtual" + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + +# - job_name: "cadvisor" +# static_configs: +# - targets: ["cadvisor:8080"] +# labels: +# type: "container" +# relabel_configs: +# - source_labels: [__address__] +# regex: '([^:]+):.*' +# target_label: instance + - job_name: "raspberrypi-node" + static_configs: + - targets: + - "raspberrypi.tail13f623.ts.net:9100" + labels: + type: "physical" + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + - job_name: "proxmox-node" + static_configs: + - targets: + - "pve.sweet.home:9100" + labels: + type: "physical" + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + - job_name: "proxmox-backup-server" + static_configs: + - targets: + - "pbs.sweet.home:9100" + labels: + type: "virtual" + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + - job_name: "pihole" + static_configs: + - targets: + - "pihole:9100" + labels: + type: "virtual" + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + - job_name: "server" + static_configs: + - targets: + - "server:9100" + labels: + type: "virtual" + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + - job_name: "nix-cache" + static_configs: + - targets: + - "nix-cache:9100" + labels: + type: "virtual" + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + - job_name: docker + static_configs: + - targets: ['telegraf:9273'] + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + + - job_name: 'traefik' + static_configs: + - targets: ['traefik.lan.ddnsgeek.com:8080'] # replace with your Traefik host:port + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + metric_relabel_configs: + - source_labels: [service] + regex: '(.+)@.+' + target_label: service + replacement: '$1' + + - job_name: 'raspi-traefik' + static_configs: + - targets: ['raspberrypi.tail13f623.ts.net:8080'] # replace with your Traefik host:port + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + metric_relabel_configs: + - source_labels: [service] + regex: '(.+)@.+' + target_label: service + replacement: '$1' + + - job_name: 'raspi-kuma' + metrics_path: /metrics + scrape_interval: 30s + + basic_auth: + username: wayne.bennett@live.com + password: '4vjCco?[%{=+,t`):C' + + static_configs: + - targets: + - kuma.lan.ddnsgeek.com + + - job_name: 'docker-kuma' + metrics_path: /metrics + scrape_interval: 30s + + basic_auth: + username: wayne.bennett@live.com + password: '4vjCco?[%{=+,t`):C' + + static_configs: + - targets: + - uptime-kuma:3001 + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + + - job_name: 'proxmox LVM storage' + static_configs: + - targets: ['pve.sweet.home:9101'] + metrics_path: /metrics + scheme: http + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + - job_name: 'proxmox backup storage' + static_configs: + - targets: ['pbs.sweet.home:9102'] + metrics_path: /metrics + scheme: http + relabel_configs: + - source_labels: [__address__] + regex: '([^:]+):.*' + target_label: instance + + +rule_files: + - /etc/prometheus/rules/*.yml + diff --git a/archive/dynamic-system/monitoring/telegraf/docker-compose.yml b/archive/dynamic-system/monitoring/telegraf/docker-compose.yml new file mode 100644 index 0000000..3c5f7d1 --- /dev/null +++ b/archive/dynamic-system/monitoring/telegraf/docker-compose.yml @@ -0,0 +1,26 @@ +services: + telegraf: + profiles: ["monitoring","all","telegraf", "prometheus"] + image: telegraf:latest + container_name: telegraf + restart: unless-stopped + depends_on: + - docker-socket-proxy +# cap_drop: +# - ALL + security_opt: + - no-new-privileges:true + volumes: + - ${PROJECT_ROOT}/monitoring/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro +# - ${PROJECT_ROOT}/monitoring/node-red/data:/var/log/node-red:ro + + - ${DOCKER_VOLUMES}/node-red-data:/var/log/node-red:ro + networks: +# - edge + - monitor + healthcheck: + test: ["CMD-SHELL", "curl -f http://localhost:9273/metrics || exit 1"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s diff --git a/archive/dynamic-system/monitoring/telegraf/telegraf.conf b/archive/dynamic-system/monitoring/telegraf/telegraf.conf new file mode 100644 index 0000000..2e3956e --- /dev/null +++ b/archive/dynamic-system/monitoring/telegraf/telegraf.conf @@ -0,0 +1,45 @@ +[agent] + interval = "10s" + +[[inputs.docker]] + endpoint = "tcp://docker-socket-proxy:2375" + gather_services = false + +[[outputs.prometheus_client]] + listen = ":9273" + +# Node-RED update-event logs (structured NDJSON) -> Prometheus metrics for Grafana +[[inputs.tail]] + files = ["/var/log/node-red/update-events.ndjson"] + from_beginning = false + name_override = "node_red_update_event" + data_format = "json_v2" + + [[inputs.tail.json_v2]] + measurement_name = "node_red_update_event" + + [[inputs.tail.json_v2.tag]] + path = "flow" + [[inputs.tail.json_v2.tag]] + path = "event" + [[inputs.tail.json_v2.tag]] + path = "container" + [[inputs.tail.json_v2.tag]] + path = "project" + [[inputs.tail.json_v2.tag]] + path = "host" + [[inputs.tail.json_v2.tag]] + path = "status" + + [[inputs.tail.json_v2.field]] + path = "success" + type = "int" + [[inputs.tail.json_v2.field]] + path = "failed" + type = "int" + [[inputs.tail.json_v2.field]] + path = "duration_ms" + type = "int" + [[inputs.tail.json_v2.field]] + path = "code" + type = "int" diff --git a/monitoring/beszel/docker-compose.yml b/monitoring/beszel/docker-compose.yml index 9a54d05..9b64da2 100644 --- a/monitoring/beszel/docker-compose.yml +++ b/monitoring/beszel/docker-compose.yml @@ -38,8 +38,8 @@ services: restart: always network_mode: host volumes: - - ./beszel/agent:/var/lib/beszel-agent - - ./beszel/socket:/beszel_socket + - ${DOCKER_VOLUMES}/beszel/agent:/var/lib/beszel-agent + - ${DOCKER_VOLUMES}/beszel/socket:/beszel_socket # - /var/run/docker.sock:/var/run/docker.sock:ro # - /mnt/docker/volumes/.beszel:/extra-filesystems/docker-volumes:ro environment: diff --git a/tree.out b/tree.out deleted file mode 100644 index 15900bd..0000000 --- a/tree.out +++ /dev/null @@ -1,1895 +0,0 @@ -. -├── apps -│   ├── gitea -│   │   ├── data -│   │   │   ├── git -│   │   │   ├── gitea -│   │   │   ├── gitea.db -│   │   │   ├── log -│   │   │   └── ssh -│   │   └── docker-compose.yml -│   ├── gramps -│   │   ├── data -│   │   │   ├── cache -│   │   │   ├── db -│   │   │   ├── media -│   │   │   └── users -│   │   ├── db -│   │   │   ├── 18 -│   │   │   ├── base -│   │   │   ├── data -│   │   │   ├── global -│   │   │   ├── pg_commit_ts -│   │   │   ├── pg_dynshmem -│   │   │   ├── pg_hba.conf -│   │   │   ├── pg_ident.conf -│   │   │   ├── pg_logical -│   │   │   ├── pg_multixact -│   │   │   ├── pg_notify -│   │   │   ├── pg_replslot -│   │   │   ├── pg_serial -│   │   │   ├── pg_snapshots -│   │   │   ├── pg_stat -│   │   │   ├── pg_stat_tmp -│   │   │   ├── pg_subtrans -│   │   │   ├── pg_tblspc -│   │   │   ├── pg_twophase -│   │   │   ├── PG_VERSION -│   │   │   ├── pg_wal -│   │   │   ├── pg_xact -│   │   │   ├── postgresql.auto.conf -│   │   │   ├── postgresql.conf -│   │   │   └── postmaster.opts -│   │   └── docker-compose.yml -│   ├── nextcloud -│   │   ├── config -│   │   │   ├── apache-pretty-urls.config.php -│   │   │   ├── apcu.config.php -│   │   │   ├── apps.config.php -│   │   │   ├── config.php -│   │   │   ├── config.php.bak -│   │   │   ├── config.sample.php -│   │   │   ├── .htaccess -│   │   │   ├── redis.config.php -│   │   │   ├── reverse-proxy.config.php -│   │   │   ├── s3.config.php -│   │   │   ├── smtp.config.php -│   │   │   ├── swift.config.php -│   │   │   └── upgrade-disable-web.config.php -│   │   ├── data [error opening dir] -│   │   ├── database -│   │   │   ├── aria_log.00000001 -│   │   │   ├── aria_log_control -│   │   │   ├── binlog.000013 -│   │   │   ├── binlog.000014 -│   │   │   ├── binlog.000015 -│   │   │   ├── binlog.000016 -│   │   │   ├── binlog.000017 -│   │   │   ├── binlog.000018 -│   │   │   ├── binlog.000018.idx -│   │   │   ├── binlog.000019 -│   │   │   ├── binlog.000019.idx -│   │   │   ├── binlog.000020 -│   │   │   ├── binlog.000020.idx -│   │   │   ├── binlog.000021 -│   │   │   ├── binlog.000021.idx -│   │   │   ├── binlog.000022 -│   │   │   ├── binlog.000022.idx -│   │   │   ├── binlog.000023 -│   │   │   ├── binlog.000023.idx -│   │   │   ├── binlog.000024 -│   │   │   ├── binlog.000024.idx -│   │   │   ├── binlog.000025 -│   │   │   ├── binlog.000025.idx -│   │   │   ├── binlog.000026 -│   │   │   ├── binlog.000026.idx -│   │   │   ├── binlog.000027 -│   │   │   ├── binlog.000027.idx -│   │   │   ├── binlog.000028 -│   │   │   ├── binlog.000028.idx -│   │   │   ├── binlog.000029 -│   │   │   ├── binlog.000029.idx -│   │   │   ├── binlog.000030 -│   │   │   ├── binlog.000030.idx -│   │   │   ├── binlog.000031 -│   │   │   ├── binlog.000031.idx -│   │   │   ├── binlog.000032 -│   │   │   ├── binlog.000032.idx -│   │   │   ├── binlog.000033 -│   │   │   ├── binlog.000033.idx -│   │   │   ├── binlog.000034 -│   │   │   ├── binlog.000034.idx -│   │   │   ├── binlog.000035 -│   │   │   ├── binlog.000035.idx -│   │   │   ├── binlog.000036 -│   │   │   ├── binlog.000036.idx -│   │   │   ├── binlog.000037 -│   │   │   ├── binlog.000037.idx -│   │   │   ├── binlog.000038 -│   │   │   ├── binlog.000038.idx -│   │   │   ├── binlog.000039 -│   │   │   ├── binlog.000039.idx -│   │   │   ├── binlog.000040 -│   │   │   ├── binlog.000040.idx -│   │   │   ├── binlog.000041 -│   │   │   ├── binlog.000041.idx -│   │   │   ├── binlog.000042 -│   │   │   ├── binlog.000042.idx -│   │   │   ├── binlog.000043 -│   │   │   ├── binlog.000043.idx -│   │   │   ├── binlog.000044 -│   │   │   ├── binlog.000044.idx -│   │   │   ├── binlog.000045 -│   │   │   ├── binlog.000045.idx -│   │   │   ├── binlog.000046 -│   │   │   ├── binlog.000046.idx -│   │   │   ├── binlog.000047 -│   │   │   ├── binlog.000047.idx -│   │   │   ├── binlog.000048 -│   │   │   ├── binlog.000048.idx -│   │   │   ├── binlog.000049 -│   │   │   ├── binlog.000049.idx -│   │   │   ├── binlog.000050 -│   │   │   ├── binlog.000050.idx -│   │   │   ├── binlog.000051 -│   │   │   ├── binlog.000051.idx -│   │   │   ├── binlog.000052 -│   │   │   ├── binlog.000052.idx -│   │   │   ├── binlog.000053 -│   │   │   ├── binlog.000053.idx -│   │   │   ├── binlog.000054 -│   │   │   ├── binlog.000054.idx -│   │   │   ├── binlog.000055 -│   │   │   ├── binlog.000055.idx -│   │   │   ├── binlog.000056 -│   │   │   ├── binlog.000056.idx -│   │   │   ├── binlog.000057 -│   │   │   ├── binlog.000057.idx -│   │   │   ├── binlog.000058 -│   │   │   ├── binlog.000058.idx -│   │   │   ├── binlog.000059 -│   │   │   ├── binlog.000059.idx -│   │   │   ├── binlog.000060 -│   │   │   ├── binlog.000060.idx -│   │   │   ├── binlog.000061 -│   │   │   ├── binlog.000061.idx -│   │   │   ├── binlog.000062 -│   │   │   ├── binlog.000062.idx -│   │   │   ├── binlog.000063 -│   │   │   ├── binlog.000063.idx -│   │   │   ├── binlog.000064 -│   │   │   ├── binlog.000064.idx -│   │   │   ├── binlog.000065 -│   │   │   ├── binlog.000065.idx -│   │   │   ├── binlog.000066 -│   │   │   ├── binlog.000066.idx -│   │   │   ├── binlog.000067 -│   │   │   ├── binlog.000067.idx -│   │   │   ├── binlog.000068 -│   │   │   ├── binlog.000068.idx -│   │   │   ├── binlog.000069 -│   │   │   ├── binlog.000069.idx -│   │   │   ├── binlog.000070 -│   │   │   ├── binlog.000070.idx -│   │   │   ├── binlog.000071 -│   │   │   ├── binlog.000071.idx -│   │   │   ├── binlog.000072 -│   │   │   ├── binlog.000072.idx -│   │   │   ├── binlog.000073 -│   │   │   ├── binlog.000073.idx -│   │   │   ├── binlog.000074 -│   │   │   ├── binlog.000074.idx -│   │   │   ├── binlog.000075 -│   │   │   ├── binlog.000075.idx -│   │   │   ├── binlog.000076 -│   │   │   ├── binlog.000076.idx -│   │   │   ├── binlog.000077 -│   │   │   ├── binlog.000077.idx -│   │   │   ├── binlog.000078 -│   │   │   ├── binlog.000078.idx -│   │   │   ├── binlog.000079 -│   │   │   ├── binlog.000079.idx -│   │   │   ├── binlog.000080 -│   │   │   ├── binlog.000080.idx -│   │   │   ├── binlog.000081 -│   │   │   ├── binlog.000081.idx -│   │   │   ├── binlog.000082 -│   │   │   ├── binlog.000082.idx -│   │   │   ├── binlog.000083 -│   │   │   ├── binlog.000083.idx -│   │   │   ├── binlog.000084 -│   │   │   ├── binlog.000084.idx -│   │   │   ├── binlog.000085 -│   │   │   ├── binlog.000085.idx -│   │   │   ├── binlog.000086 -│   │   │   ├── binlog.000086.idx -│   │   │   ├── binlog.000087 -│   │   │   ├── binlog.000087.idx -│   │   │   ├── binlog.000088 -│   │   │   ├── binlog.000088.idx -│   │   │   ├── binlog.000089 -│   │   │   ├── binlog.000089.idx -│   │   │   ├── binlog.000090 -│   │   │   ├── binlog.000090.idx -│   │   │   ├── binlog.000091 -│   │   │   ├── binlog.000091.idx -│   │   │   ├── binlog.000092 -│   │   │   ├── binlog.000092.idx -│   │   │   ├── binlog.000093 -│   │   │   ├── binlog.000093.idx -│   │   │   ├── binlog.000094 -│   │   │   ├── binlog.000094.idx -│   │   │   ├── binlog.000095 -│   │   │   ├── binlog.000095.idx -│   │   │   ├── binlog.000096 -│   │   │   ├── binlog.000096.idx -│   │   │   ├── binlog.000097 -│   │   │   ├── binlog.000097.idx -│   │   │   ├── binlog.000098 -│   │   │   ├── binlog.000098.idx -│   │   │   ├── binlog.000099 -│   │   │   ├── binlog.000099.idx -│   │   │   ├── binlog.000100 -│   │   │   ├── binlog.000100.idx -│   │   │   ├── binlog.000101 -│   │   │   ├── binlog.000101.idx -│   │   │   ├── binlog.000102 -│   │   │   ├── binlog.000102.idx -│   │   │   ├── binlog.000103 -│   │   │   ├── binlog.000103.idx -│   │   │   ├── binlog.000104 -│   │   │   ├── binlog.000104.idx -│   │   │   ├── binlog.000105 -│   │   │   ├── binlog.000105.idx -│   │   │   ├── binlog.000106 -│   │   │   ├── binlog.000106.idx -│   │   │   ├── binlog.000107 -│   │   │   ├── binlog.000107.idx -│   │   │   ├── binlog.000108 -│   │   │   ├── binlog.000108.idx -│   │   │   ├── binlog.000109 -│   │   │   ├── binlog.000109.idx -│   │   │   ├── binlog.000110 -│   │   │   ├── binlog.000110.idx -│   │   │   ├── binlog.000111 -│   │   │   ├── binlog.000111.idx -│   │   │   ├── binlog.000112 -│   │   │   ├── binlog.000112.idx -│   │   │   ├── binlog.000113 -│   │   │   ├── binlog.000113.idx -│   │   │   ├── binlog.000114 -│   │   │   ├── binlog.000114.idx -│   │   │   ├── binlog.000115 -│   │   │   ├── binlog.000115.idx -│   │   │   ├── binlog.000116 -│   │   │   ├── binlog.000116.idx -│   │   │   ├── binlog.000117 -│   │   │   ├── binlog.000117.idx -│   │   │   ├── binlog.000118 -│   │   │   ├── binlog.000118.idx -│   │   │   ├── binlog.000119 -│   │   │   ├── binlog.000119.idx -│   │   │   ├── binlog.000120 -│   │   │   ├── binlog.000120.idx -│   │   │   ├── binlog.000121 -│   │   │   ├── binlog.000121.idx -│   │   │   ├── binlog.000122 -│   │   │   ├── binlog.000122.idx -│   │   │   ├── binlog.000123 -│   │   │   ├── binlog.000123.idx -│   │   │   ├── binlog.000124 -│   │   │   ├── binlog.000124.idx -│   │   │   ├── binlog.000125 -│   │   │   ├── binlog.000125.idx -│   │   │   ├── binlog.000126 -│   │   │   ├── binlog.000126.idx -│   │   │   ├── binlog.000127 -│   │   │   ├── binlog.000127.idx -│   │   │   ├── binlog.000128 -│   │   │   ├── binlog.000128.idx -│   │   │   ├── binlog.000129 -│   │   │   ├── binlog.000129.idx -│   │   │   ├── binlog.000130 -│   │   │   ├── binlog.000130.idx -│   │   │   ├── binlog.000131 -│   │   │   ├── binlog.000131.idx -│   │   │   ├── binlog.000132 -│   │   │   ├── binlog.000132.idx -│   │   │   ├── binlog.000133 -│   │   │   ├── binlog.000133.idx -│   │   │   ├── binlog.000134 -│   │   │   ├── binlog.000134.idx -│   │   │   ├── binlog.000135 -│   │   │   ├── binlog.000135.idx -│   │   │   ├── binlog.000136 -│   │   │   ├── binlog.000136.idx -│   │   │   ├── binlog.000137 -│   │   │   ├── binlog.000137.idx -│   │   │   ├── binlog.000138 -│   │   │   ├── binlog.000138.idx -│   │   │   ├── binlog.000139 -│   │   │   ├── binlog.000139.idx -│   │   │   ├── binlog.000140 -│   │   │   ├── binlog.000140.idx -│   │   │   ├── binlog.000141 -│   │   │   ├── binlog.000141.idx -│   │   │   ├── binlog.000142 -│   │   │   ├── binlog.000142.idx -│   │   │   ├── binlog.000143 -│   │   │   ├── binlog.000143.idx -│   │   │   ├── binlog.000144 -│   │   │   ├── binlog.000144.idx -│   │   │   ├── binlog.000145 -│   │   │   ├── binlog.000145.idx -│   │   │   ├── binlog.000146 -│   │   │   ├── binlog.000146.idx -│   │   │   ├── binlog.000147 -│   │   │   ├── binlog.000147.idx -│   │   │   ├── binlog.000148 -│   │   │   ├── binlog.000148.idx -│   │   │   ├── binlog.000149 -│   │   │   ├── binlog.000149.idx -│   │   │   ├── binlog.000150 -│   │   │   ├── binlog.000150.idx -│   │   │   ├── binlog.000151 -│   │   │   ├── binlog.000151.idx -│   │   │   ├── binlog.000152 -│   │   │   ├── binlog.000152.idx -│   │   │   ├── binlog.000153 -│   │   │   ├── binlog.000153.idx -│   │   │   ├── binlog.000154 -│   │   │   ├── binlog.000154.idx -│   │   │   ├── binlog.000155 -│   │   │   ├── binlog.000155.idx -│   │   │   ├── binlog.000156 -│   │   │   ├── binlog.000156.idx -│   │   │   ├── binlog.000157 -│   │   │   ├── binlog.000157.idx -│   │   │   ├── binlog.000158 -│   │   │   ├── binlog.000158.idx -│   │   │   ├── binlog.000159 -│   │   │   ├── binlog.000159.idx -│   │   │   ├── binlog.000160 -│   │   │   ├── binlog.000160.idx -│   │   │   ├── binlog.000161 -│   │   │   ├── binlog.000161.idx -│   │   │   ├── binlog.000162 -│   │   │   ├── binlog.000162.idx -│   │   │   ├── binlog.000163 -│   │   │   ├── binlog.000163.idx -│   │   │   ├── binlog.000164 -│   │   │   ├── binlog.000164.idx -│   │   │   ├── binlog.000165 -│   │   │   ├── binlog.000165.idx -│   │   │   ├── binlog.000166 -│   │   │   ├── binlog.000166.idx -│   │   │   ├── binlog.000167 -│   │   │   ├── binlog.000167.idx -│   │   │   ├── binlog.000168 -│   │   │   ├── binlog.000168.idx -│   │   │   ├── binlog.000169 -│   │   │   ├── binlog.000169.idx -│   │   │   ├── binlog.000170 -│   │   │   ├── binlog.000170.idx -│   │   │   ├── binlog.000171 -│   │   │   ├── binlog.000171.idx -│   │   │   ├── binlog.000172 -│   │   │   ├── binlog.000172.idx -│   │   │   ├── binlog.000173 -│   │   │   ├── binlog.000173.idx -│   │   │   ├── binlog.000174 -│   │   │   ├── binlog.000174.idx -│   │   │   ├── binlog.000175 -│   │   │   ├── binlog.000175.idx -│   │   │   ├── binlog.000176 -│   │   │   ├── binlog.000176.idx -│   │   │   ├── binlog.000177 -│   │   │   ├── binlog.000177.idx -│   │   │   ├── binlog.000178 -│   │   │   ├── binlog.000178.idx -│   │   │   ├── binlog.000179 -│   │   │   ├── binlog.000179.idx -│   │   │   ├── binlog.000180 -│   │   │   ├── binlog.000180.idx -│   │   │   ├── binlog.000181 -│   │   │   ├── binlog.000181.idx -│   │   │   ├── binlog.000182 -│   │   │   ├── binlog.000182.idx -│   │   │   ├── binlog.000183 -│   │   │   ├── binlog.000183.idx -│   │   │   ├── binlog.000184 -│   │   │   ├── binlog.000184.idx -│   │   │   ├── binlog.000185 -│   │   │   ├── binlog.000185.idx -│   │   │   ├── binlog.000186 -│   │   │   ├── binlog.000186.idx -│   │   │   ├── binlog.000187 -│   │   │   ├── binlog.000187.idx -│   │   │   ├── binlog.000188 -│   │   │   ├── binlog.000188.idx -│   │   │   ├── binlog.000189 -│   │   │   ├── binlog.000189.idx -│   │   │   ├── binlog.000190 -│   │   │   ├── binlog.000190.idx -│   │   │   ├── binlog.000191 -│   │   │   ├── binlog.000191.idx -│   │   │   ├── binlog.000192 -│   │   │   ├── binlog.000192.idx -│   │   │   ├── binlog.000193 -│   │   │   ├── binlog.000193.idx -│   │   │   ├── binlog.000194 -│   │   │   ├── binlog.000194.idx -│   │   │   ├── binlog.000195 -│   │   │   ├── binlog.000195.idx -│   │   │   ├── binlog.000196 -│   │   │   ├── binlog.000196.idx -│   │   │   ├── binlog.000197 -│   │   │   ├── binlog.000197.idx -│   │   │   ├── binlog.000198 -│   │   │   ├── binlog.000198.idx -│   │   │   ├── binlog.000199 -│   │   │   ├── binlog.000199.idx -│   │   │   ├── binlog.000200 -│   │   │   ├── binlog.000200.idx -│   │   │   ├── binlog.000201 -│   │   │   ├── binlog.000201.idx -│   │   │   ├── binlog.000202 -│   │   │   ├── binlog.000202.idx -│   │   │   ├── binlog.000203 -│   │   │   ├── binlog.000203.idx -│   │   │   ├── binlog.000204 -│   │   │   ├── binlog.000204.idx -│   │   │   ├── binlog.000205 -│   │   │   ├── binlog.000205.idx -│   │   │   ├── binlog.000206 -│   │   │   ├── binlog.000206.idx -│   │   │   ├── binlog.000207 -│   │   │   ├── binlog.000207.idx -│   │   │   ├── binlog.000208 -│   │   │   ├── binlog.000208.idx -│   │   │   ├── binlog.000209 -│   │   │   ├── binlog.000209.idx -│   │   │   ├── binlog.000210 -│   │   │   ├── binlog.000210.idx -│   │   │   ├── #binlog_cache_files -│   │   │   ├── binlog.index -│   │   │   ├── ddl_recovery-backup.log -│   │   │   ├── ddl_recovery.log -│   │   │   ├── ib_buffer_pool -│   │   │   ├── ibdata1 -│   │   │   ├── ib_logfile0 -│   │   │   ├── ibtmp1 -│   │   │   ├── mariadb_upgrade_info -│   │   │   ├── multi-master.info -│   │   │   ├── .my-healthcheck.cnf -│   │   │   ├── mysql -│   │   │   ├── nextcloud -│   │   │   ├── performance_schema -│   │   │   ├── sys -│   │   │   ├── system_mysql_backup_11.3.2-MariaDB.sql.zst -│   │   │   ├── system_mysql_backup_11.4.2-MariaDB.sql.zst -│   │   │   ├── system_mysql_backup_11.5.2-MariaDB.sql.zst -│   │   │   ├── system_mysql_backup_11.6.2-MariaDB.sql.zst -│   │   │   ├── system_mysql_backup_11.7.2-MariaDB.sql.zst -│   │   │   ├── system_mysql_backup_11.8.2-MariaDB.sql.zst -│   │   │   ├── system_mysql_backup_12.0.2-MariaDB.sql.zst -│   │   │   ├── undo001 -│   │   │   ├── undo002 -│   │   │   └── undo003 -│   │   ├── docker-compose.yml -│   │   ├── Dockerfile -│   │   └── .env -│   ├── passbolt -│   │   ├── data -│   │   │   ├── database -│   │   │   ├── gpg -│   │   │   └── jwt -│   │   ├── docker-compose.yml -│   │   └── Dockerfile -│   ├── searxng -│   │   ├── AUTHORS.rst -│   │   ├── babel.cfg -│   │   ├── CHANGELOG.rst -│   │   ├── CONTRIBUTING.md -│   │   ├── docker-compose.yml -│   │   ├── Dockerfile -│   │   ├── dockerfiles -│   │   │   ├── docker-entrypoint.sh -│   │   │   └── uwsgi.ini -│   │   ├── docker-health-alert -│   │   ├── docs -│   │   │   ├── admin -│   │   │   ├── build-templates -│   │   │   ├── conf.py -│   │   │   ├── dev -│   │   │   ├── index.rst -│   │   │   ├── own-instance.rst -│   │   │   ├── src -│   │   │   ├── _themes -│   │   │   ├── user -│   │   │   └── utils -│   │   ├── examples -│   │   │   └── basic_engine.py -│   │   ├── LICENSE -│   │   ├── Makefile -│   │   ├── manage -│   │   ├── package.json -│   │   ├── PULL_REQUEST_TEMPLATE.md -│   │   ├── pyrightconfig-ci.json -│   │   ├── pyrightconfig.json -│   │   ├── README.rst -│   │   ├── requirements-dev.txt -│   │   ├── requirements.txt -│   │   ├── searx -│   │   │   ├── answerers -│   │   │   ├── autocomplete.py -│   │   │   ├── babel_extract.py -│   │   │   ├── botdetection -│   │   │   ├── compat.py -│   │   │   ├── data -│   │   │   ├── enginelib -│   │   │   ├── engines -│   │   │   ├── exceptions.py -│   │   │   ├── external_bang.py -│   │   │   ├── external_urls.py -│   │   │   ├── flaskfix.py -│   │   │   ├── infopage -│   │   │   ├── __init__.py -│   │   │   ├── locales.py -│   │   │   ├── metrics -│   │   │   ├── network -│   │   │   ├── plugins -│   │   │   ├── preferences.py -│   │   │   ├── __pycache__ -│   │   │   ├── query.py -│   │   │   ├── redisdb.py -│   │   │   ├── redislib.py -│   │   │   ├── results.py -│   │   │   ├── search -│   │   │   ├── searxng.msg -│   │   │   ├── settings_defaults.py -│   │   │   ├── settings_loader.py -│   │   │   ├── settings.yml -│   │   │   ├── static -│   │   │   ├── sxng_locales.py -│   │   │   ├── templates -│   │   │   ├── tools -│   │   │   ├── translations -│   │   │   ├── unixthreadname.py -│   │   │   ├── utils.py -│   │   │   ├── version.py -│   │   │   ├── webadapter.py -│   │   │   ├── webapp.py -│   │   │   └── webutils.py -│   │   ├── searxng_extra -│   │   │   ├── docs_prebuild -│   │   │   ├── __init__.py -│   │   │   ├── standalone_searx.py -│   │   │   └── update -│   │   ├── SECURITY.md -│   │   ├── setup.py -│   │   ├── src -│   │   │   └── brand -│   │   ├── tests -│   │   │   ├── __init__.py -│   │   │   ├── robot -│   │   │   └── unit -│   │   └── utils -│   │   ├── brand.env -│   │   ├── build_env.py -│   │   ├── filtron.sh -│   │   ├── lib_go.sh -│   │   ├── lib_nvm.sh -│   │   ├── lib_redis.sh -│   │   ├── lib.sh -│   │   ├── lib_sxng_data.sh -│   │   ├── lib_sxng_node.sh -│   │   ├── lib_sxng_static.sh -│   │   ├── lib_sxng_test.sh -│   │   ├── lib_sxng_themes.sh -│   │   ├── lib_sxng_weblate.sh -│   │   ├── lxc-searxng.env -│   │   ├── lxc.sh -│   │   ├── makefile.include -│   │   ├── makefile.lxc -│   │   ├── morty.sh -│   │   ├── searxng_check.py -│   │   ├── searxng.sh -│   │   ├── searx.sh -│   │   └── templates -│   ├── shift-recorder -│   │   ├── AGENTS.md -│   │   ├── app -│   │   │   ├── build.gradle -│   │   │   └── src -│   │   ├── app-release-bundle.aab -│   │   ├── app-release-signed.apk -│   │   ├── app-release-signed.apk.idsig -│   │   ├── app-release-unsigned-aligned.apk -│   │   ├── build.gradle -│   │   ├── CHANGELOG.md -│   │   ├── deploy -│   │   │   └── nginx.conf -│   │   ├── docker-compose.yml -│   │   ├── Dockerfile -│   │   ├── .dockerignore -│   │   ├── eslint.config.js -│   │   ├── .git -│   │   │   ├── config -│   │   │   ├── description -│   │   │   ├── FETCH_HEAD -│   │   │   ├── HEAD -│   │   │   ├── hooks -│   │   │   ├── index -│   │   │   ├── info -│   │   │   ├── logs -│   │   │   ├── objects -│   │   │   ├── ORIG_HEAD -│   │   │   ├── packed-refs -│   │   │   └── refs -│   │   ├── .gitignore -│   │   ├── gradle -│   │   │   └── wrapper -│   │   ├── gradle.properties -│   │   ├── gradlew -│   │   ├── gradlew.bat -│   │   ├── index.html -│   │   ├── manifest-checksum.txt -│   │   ├── package.json -│   │   ├── package-lock.json -│   │   ├── packages -│   │   │   └── tax-engine -│   │   ├── playwright.config.ts -│   │   ├── postcss.config.js -│   │   ├── .prettierrc -│   │   ├── public -│   │   │   ├── apple-touch-icon.svg -│   │   │   ├── chrona-logo.svg -│   │   │   ├── favicon.ico -│   │   │   ├── favicon.svg -│   │   │   ├── pwa-icon-maskable.svg -│   │   │   ├── pwa-icon.svg -│   │   │   ├── robots.txt -│   │   │   └── .well-known -│   │   ├── README.md -│   │   ├── scripts -│   │   │   └── install-playwright.mjs -│   │   ├── settings.gradle -│   │   ├── src -│   │   │   ├── app -│   │   │   ├── sw-notifications.ts -│   │   │   ├── sw.ts -│   │   │   ├── tests -│   │   │   └── vite-env.d.ts -│   │   ├── store_icon.png -│   │   ├── tailwind.config.js -│   │   ├── tsconfig.json -│   │   ├── tsconfig.node.json -│   │   ├── twa-manifest.json -│   │   ├── vite.config.ts -│   │   ├── vitest.setup.ts -│   │   └── .vscode -│   │   ├── launch.json -│   │   └── tasks.json -│   └── stockfill -│   ├── Agents.md -│   ├── android.keystore -│   ├── app -│   │   ├── build.gradle -│   │   └── src -│   ├── app-release-bundle.aab -│   ├── app-release-signed.apk -│   ├── app-release-signed.apk.idsig -│   ├── app-release-unsigned-aligned.apk -│   ├── build.gradle -│   ├── CODE_OF_CONDUCT.md -│   ├── .codex_playwright_version -│   ├── CONTRIBUTING.md -│   ├── docker-compose.yml -│   ├── Dockerfile -│   ├── .dockerignore -│   ├── docs -│   │   └── seed -│   ├── e2e -│   │   ├── fixtures.ts -│   │   ├── picklist.spec.ts -│   │   ├── product-filter.spec.ts -│   │   ├── test-helpers.ts -│   │   ├── unit -│   │   └── utils -│   ├── eslint.config.js -│   ├── .git -│   │   ├── COMMIT_EDITMSG -│   │   ├── config -│   │   ├── description -│   │   ├── FETCH_HEAD -│   │   ├── HEAD -│   │   ├── hooks -│   │   ├── index -│   │   ├── info -│   │   ├── logs -│   │   ├── objects -│   │   ├── ORIG_HEAD -│   │   ├── packed-refs -│   │   └── refs -│   ├── .gitignore -│   ├── gradle -│   │   └── wrapper -│   ├── gradle.properties -│   ├── gradlew -│   ├── gradlew.bat -│   ├── index.html -│   ├── LICENSE -│   ├── maintenance.sh -│   ├── manifest-checksum.txt -│   ├── nginx.conf -│   ├── package.json -│   ├── package-lock.json -│   ├── playwright.config.ts -│   ├── public -│   │   ├── icons -│   │   ├── manifest.json -│   │   ├── service-worker.js -│   │   ├── templates -│   │   └── .well-known -│   ├── README.md -│   ├── scripts -│   │   ├── merge-lcov.cjs -│   │   ├── normalize-lcov.cjs -│   │   ├── playwright-collect-coverage.cjs -│   │   └── remap-and-report.cjs -│   ├── SECURITY.md -│   ├── settings.gradle -│   ├── setup.sh -│   ├── src -│   │   ├── App.tsx -│   │   ├── components -│   │   ├── context -│   │   ├── db -│   │   ├── hooks -│   │   ├── main.tsx -│   │   ├── models -│   │   ├── modules -│   │   ├── platform -│   │   ├── pwa -│   │   ├── screens -│   │   ├── services -│   │   ├── test -│   │   ├── testUtils -│   │   └── utils -│   ├── store_icon.png -│   ├── TESTING.md -│   ├── tsconfig.json -│   ├── tsconfig.node.json -│   ├── twa-manifest.json -│   ├── vite.config.ts -│   ├── vitest.config.ts -│   └── .vscode -│   ├── launch.json -│   ├── settings.json -│   └── tasks.json -├── archive -│   ├── autossh -│   │   ├── docker-compose.yml -│   │   └── Dockerfile -│   ├── dnscrypt-proxy -│   │   └── docker-compose.yml -│   ├── docker-compose-no-tz.yaml -│   ├── docker-compose.yaml -│   ├── docuseal -│   │   └── docker-compose.yml -│   ├── doh -│   │   └── docker-compose.yml -│   ├── doods -│   │   ├── config.yaml -│   │   ├── docker-compose.yml -│   │   ├── Dockerfile -│   │   └── fetch_models.sh -│   ├── dynu -│   │   └── dynu.sh -│   ├── email-alerts -│   │   └── Dockerfile -│   ├── esphome -│   │   ├── create-default-config.sh -│   │   ├── data -│   │   │   ├── esphome-garage.yaml -│   │   │   ├── esphome-waynes-room.yaml -│   │   │   └── .gitignore -│   │   ├── default_configuration.yaml -│   │   ├── docker-compose.yml -│   │   ├── Dockerfile -│   │   └── remote_transmitter-codes.yaml -│   ├── graylog -│   │   └── docker-compose.yml -│   ├── hass -│   │   └── docker-compose.yml -│   ├── lfs -│   │   ├── cross-toolchain.sh -│   │   └── Dockerfile -│   ├── mqtt -│   │   ├── config.yaml -│   │   └── docker-compose.yml -│   ├── netdata -│   │   └── docker-compose.yml -│   ├── office365 -│   │   └── Dockerfile -│   ├── pihole -│   │   └── docker-compose.yml -│   ├── portainer -│   │   └── docker-compose.yml -│   ├── portainer-agent-stack.yml -│   ├── portainer-compose-files -│   │   ├── haproxy.yaml -│   │   ├── keeweb.yaml -│   │   ├── registry.yaml -│   │   ├── searxng.yaml -│   │   └── webdav.yaml -│   ├── portainer-compose.yaml -│   ├── recreate-containers.sh -│   ├── recreate-nextcloud.sh -│   ├── recreate-passbolt.sh -│   ├── traccar -│   │   └── docker-compose.yml -│   ├── ubuntu-gui -│   │   ├── docker-compose.yml -│   │   └── Dockerfile -│   ├── update-containers.sh -│   └── webdav -│   └── docker-compose.yml -├── check_last_run.py -├── core -│   ├── authelia -│   │   ├── configuration.yml -│   │   ├── data -│   │   │   ├── db.sqlite3 -│   │   │   └── notification.txt -│   │   └── users_database.yml -│   ├── crowdsec -│   │   ├── config -│   │   │   ├── acquis.d -│   │   │   ├── acquis.yaml -│   │   │   ├── collections -│   │   │   ├── config.yaml -│   │   │   ├── console.yaml -│   │   │   ├── contexts -│   │   │   ├── dev.yaml -│   │   │   ├── hub -│   │   │   ├── local_api_credentials.yaml -│   │   │   ├── notifications -│   │   │   ├── online_api_credentials.yaml -│   │   │   ├── parsers -│   │   │   ├── patterns -│   │   │   ├── postoverflows -│   │   │   ├── profiles.yaml -│   │   │   ├── scenarios -│   │   │   ├── simulation.yaml -│   │   │   └── user.yaml -│   │   ├── data -│   │   │   ├── admin_interfaces.txt -│   │   │   ├── backdoors.txt -│   │   │   ├── bad_user_agents.regex.txt -│   │   │   ├── cloudflare_ip6s.txt -> /staging/var/lib/crowdsec/data/cloudflare_ip6s.txt -│   │   │   ├── cloudflare_ips.txt -> /staging/var/lib/crowdsec/data/cloudflare_ips.txt -│   │   │   ├── crowdsec.db -│   │   │   ├── detect.yaml -> /staging/var/lib/crowdsec/data/detect.yaml -│   │   │   ├── GeoLite2-ASN.mmdb -│   │   │   ├── GeoLite2-City.mmdb -│   │   │   ├── http_path_traversal.txt -│   │   │   ├── ip_seo_bots.txt -> /staging/var/lib/crowdsec/data/ip_seo_bots.txt -│   │   │   ├── jira_cve_2021-26086.txt -│   │   │   ├── log4j2_cve_2021_44228.txt -│   │   │   ├── rdns_seo_bots.regex -> /staging/var/lib/crowdsec/data/rdns_seo_bots.regex -│   │   │   ├── rdns_seo_bots.txt -> /staging/var/lib/crowdsec/data/rdns_seo_bots.txt -│   │   │   ├── sensitive_data.txt -│   │   │   ├── sqli_probe_patterns.txt -│   │   │   ├── thinkphp_cve_2018-20062.txt -│   │   │   ├── trace -> /staging/var/lib/crowdsec/data/trace -│   │   │   ├── trendy_cves_uris.json -│   │   │   └── xss_probe_patterns.txt -│   │   ├── Dockerfile -│   │   └── logs -│   ├── docker-compose.yml -│   ├── test -│   │   ├── docker-compose.yml -│   │   ├── Dockerfile -│   │   └── exporter.py -│   └── traefik -│   ├── data -│   │   ├── dynamic.yaml -│   │   ├── letsencrypt -│   │   ├── logs -│   │   ├── plugins -│   │   └── plugins.yaml -│   ├── dynamic.yml -│   └── traefik.yml -├── default-environment.env -├── default-network.yml -├── docker -> /mnt/docker-persistent-data/docker -├── .git -│   ├── COMMIT_EDITMSG -│   ├── config -│   ├── description -│   ├── FETCH_HEAD -│   ├── HEAD -│   ├── hooks -│   │   ├── applypatch-msg.sample -│   │   ├── commit-msg.sample -│   │   ├── fsmonitor-watchman.sample -│   │   ├── post-update.sample -│   │   ├── pre-applypatch.sample -│   │   ├── pre-commit.sample -│   │   ├── pre-merge-commit.sample -│   │   ├── prepare-commit-msg.sample -│   │   ├── pre-push.sample -│   │   ├── pre-rebase.sample -│   │   ├── pre-receive.sample -│   │   ├── push-to-checkout.sample -│   │   ├── sendemail-validate.sample -│   │   └── update.sample -│   ├── index -│   ├── info -│   │   └── exclude -│   ├── logs -│   │   ├── HEAD -│   │   └── refs -│   │   ├── heads -│   │   └── remotes -│   ├── objects -│   │   ├── 00 -│   │   │   ├── 40b60e747422049af7563b8cfd52c0545487be -│   │   │   ├── 6d24a122c7abdbc152362d9a7ec64a623f5b12 -│   │   │   ├── f208b1761162ffaffde42763c3005d5309ce7a -│   │   │   └── f86e2da8519a2e87abb03c191ecd236d8d53f3 -│   │   ├── 01 -│   │   │   ├── 203ac506791bb4061ffe542afce71b58788c1c -│   │   │   └── 86adc586c99c5fb2d6190af0ffa29d95ccfd45 -│   │   ├── 02 -│   │   │   ├── 0bf689b687f345af7afb24bfb2bc6343c1ebe7 -│   │   │   └── e282d5f9504224c7f7534d3f6315336fbebdef -│   │   ├── 03 -│   │   │   ├── 36d880b1905be16195f94e42958cc5bf77ef55 -│   │   │   └── 9aadb91142b5a3d015c3b8d8580f76bf10f5d4 -│   │   ├── 04 -│   │   │   ├── 73b5e4aac29c8d5ae45ed88a3fafa953056176 -│   │   │   ├── c8fab0a7b412afb382b799192f527e59a7b5dc -│   │   │   ├── ce8507480ca64775211d515fbbed7c0d5b5e16 -│   │   │   └── d25b9d37156ac70f4ba25ea46b16eb3810b8d3 -│   │   ├── 05 -│   │   │   ├── 44d697f1a20000afe0d72005aad2694bdd6b33 -│   │   │   └── 9dd7c8711100344908f63069eb14eabcccd8c9 -│   │   ├── 06 -│   │   │   ├── 03a45647b68ae6cbb981036b85db6937a2c02c -│   │   │   ├── 591e39ddb29c3009e8ed73b291d1bba91f79f1 -│   │   │   ├── 92d4a7a45fffd5b74d50210394257d3f96c238 -│   │   │   └── 9be9057182ebc61eb30982658311d9274e26cd -│   │   ├── 07 -│   │   │   └── 870f044fbf65ef1b67b21ef1f8b8f03233bed6 -│   │   ├── 08 -│   │   │   ├── 1ed4ca8e9a7b2c9839a19c493177e22e4b3bb5 -│   │   │   ├── 22f4ac8628cd245dfddfdd8c8af6df59d01e11 -│   │   │   ├── 433fe8dfc8411f2a835a823ed551e42f7d8e26 -│   │   │   ├── 946c7d2f2fce83ad03179efccf3d4cd1ea28e1 -│   │   │   ├── 97b8dcaac094bb2706fb2658ab2f0f33b73e6d -│   │   │   ├── add5371d18c7421905d31a5a8cd6128eb6a565 -│   │   │   ├── e58b989777eec6479d5c0ec3e5d552ffd0d597 -│   │   │   ├── e6d982fc74ac2655582c3dbd8341e4b5d050d9 -│   │   │   └── f56bbe75d5d0da1fcf7be457456dbc6f15e449 -│   │   ├── 0a -│   │   │   ├── 47274af71b0facd0ec906cb3d640b302637ed3 -│   │   │   └── 78a00cc4ad15271fc47bb3600ecb00cf1ac710 -│   │   ├── 0b -│   │   │   ├── 2f3a6896be68f8eb5af0e252655c4d2ac51539 -│   │   │   └── c7b621ba5d63c8bb440513321284ec9133fdba -│   │   ├── 0c -│   │   │   ├── 23702161daac2b799e0844b679aae55e4fb9f0 -│   │   │   └── 5306b713d13c7845adf6dde2d9bb13782150d0 -│   │   ├── 0d -│   │   │   ├── 555fdc091c48b2d1342653d96dee010a22369b -│   │   │   └── 6ddff4a1f652449b7612caca34f83816ce711f -│   │   ├── 0f -│   │   │   ├── 2c46439201418513b4bd6e5c2d6ba8a248d62d -│   │   │   ├── 5be0671b329070ada170702ab2c48b793e86c3 -│   │   │   └── deacec2f79e7ffc6d784ad79d048bd45ae2f9e -│   │   ├── 10 -│   │   │   └── 0a267e175a403da4c6121a30134baf852b7ff2 -│   │   ├── 11 -│   │   │   ├── 3e9cd3c73227ce84176f0b92bc4759524d5e46 -│   │   │   ├── 55b04f76732aaa0cf04a9074eea0c39c1f495e -│   │   │   └── cb8d4e4556e14e351d533c094a119c20fb5045 -│   │   ├── 12 -│   │   │   ├── 066f3d75b41abb6f3690c5187fa913925c6e68 -│   │   │   ├── 1319eeb42182a9a06cf398b53455cb148dcf8a -│   │   │   └── 6e7537495c98b49d0715426536f6c42296f4bc -│   │   ├── 13 -│   │   │   ├── 90de456ba24d78b14861c53e7509f700757bf7 -│   │   │   ├── d004322a9d37f2523fbfd259b069d76dcc502c -│   │   │   └── f077cb152896180b7faa70f2ac76df56c18cfb -│   │   ├── 14 -│   │   │   └── 5ba7b0c482a7dfbdcd98a9361c60b49162f745 -│   │   ├── 15 -│   │   │   ├── e27d45db014bdec89825b929d031e9d91a3b1d -│   │   │   └── fcb5ebd3d0ee0ea42e718be3b56ae3421de2fe -│   │   ├── 16 -│   │   │   ├── 19830119b2a35fbac54b8ef7ec8f4dd941e13e -│   │   │   └── d551777454bc1ac613a4986f790e4c1d0155d7 -│   │   ├── 17 -│   │   │   ├── 025f68b009800b3cdba8c633b3295507683d80 -│   │   │   ├── bb1b6c5bac6f62ea80ca61b708cf141759ec03 -│   │   │   └── ffce683287d1581571a916b46b11cfc0d7bfb3 -│   │   ├── 18 -│   │   │   ├── 06f02c87be6f23fbf94e67f9a9dc25bfcb257a -│   │   │   ├── d2170df440927a4d13702dee02466eb092978a -│   │   │   └── ea6cb19364c8819939ec39362213906056789c -│   │   ├── 19 -│   │   │   ├── 67fefd262679e184f02cfe8be19f803d45aa7a -│   │   │   └── 905fd157a2dd90d9b29ac75372d4499318aeba -│   │   ├── 1a -│   │   │   └── 1b57d8c51e84288f50689325549a17a90b5a6e -│   │   ├── 1b -│   │   │   ├── 1e79b1990e6bf70f1cc4fd8c9511d537611e9a -│   │   │   ├── 332a9f7385bdd26914567dbb9afbe9fbfa4b54 -│   │   │   ├── 74f2a050f9ec5a05fda1bebd75e17e000b5ec6 -│   │   │   └── cdeeec601845ee073ef95c3b825e0f2f5a6289 -│   │   ├── 1d -│   │   │   ├── 0d6e7a3da8edc367a7a035717f71a4f8750169 -│   │   │   ├── 594fb81533bfc9e495a5f852122dbb8a9ca45a -│   │   │   └── cb1ee8a03a82ce70e842b174e817fcc124ecf2 -│   │   ├── 1e -│   │   │   └── dca836f4188883a40fb690fe8f3e2a80bbb162 -│   │   ├── 1f -│   │   │   ├── 0973f233b4200a89b2af1ced657f3491136499 -│   │   │   ├── 3e07337e0447c81356fe44fda6ad508b5d91f2 -│   │   │   └── 5b5f6ccb5d41f001b8cd7caff309accdc6bc53 -│   │   ├── 21 -│   │   │   ├── b3964688468aedbef65fc0d09acbd9565cb535 -│   │   │   ├── d9b31534eb4b3f7f18717a381bf8726f6e9d48 -│   │   │   └── eaeccc33cb7af9629752ca344b34d6c5fa7696 -│   │   ├── 22 -│   │   │   └── d65bbf7739eb10c3293a8c8460d1b769829d53 -│   │   ├── 23 -│   │   │   ├── 2ddcc1a58aef483f75092badce42ea738e3d9a -│   │   │   ├── 30e846017fc3c28e091ca3635fb92546eda079 -│   │   │   └── b7bfcfdf88ebc8b1851d8e7804f2b63aca8b06 -│   │   ├── 24 -│   │   │   ├── 49345e64627b2217f3cd4c885328ddcc24691c -│   │   │   └── 4d2b180e612410dcb53c311d5b58ee70a7f32f -│   │   ├── 25 -│   │   │   ├── 249fbda5607f801e9b5b9d910a21e12b5ce326 -│   │   │   └── 52c9b6fa3ab3fc03b945045260df88d03658c8 -│   │   ├── 26 -│   │   │   ├── 0d6da978dfc2c0c2f3b40038bb174c057f4d64 -│   │   │   ├── 21d06920010d6f12873aee26056faad914d6eb -│   │   │   └── 364674c4d8ba4826c3f4d10f917b3a5462df35 -│   │   ├── 28 -│   │   │   ├── 3d2549fd284e85f36efd1dc120b3a2f9d0bf08 -│   │   │   ├── 44569d18833f2aefb4ad404944b2dbf74598f7 -│   │   │   └── cc302653c4041995c69a4b34d9ad3b9f9a18e5 -│   │   ├── 29 -│   │   │   ├── 14e9228365478100dd8dd9cca4580aa3308787 -│   │   │   ├── 31b6af4f3e297463f8578708006625717042cd -│   │   │   ├── 63c7ecfb0c6dc81eee9041d91224bcf5375cb6 -│   │   │   └── f2766364ebd089f7262a8e7c53102122c6b4c7 -│   │   ├── 2a -│   │   │   ├── a5cba8bc4859597314576d11e5e5bf192fa693 -│   │   │   ├── d4593a1c01dde13f6ff7279904676cea6b8729 -│   │   │   └── fa0552d5779ea7dbe3723833f6150175365e8c -│   │   ├── 2b -│   │   │   └── fb90656c86e04d53de57f7cae78d267ba84d8a -│   │   ├── 2c -│   │   │   └── e9925c5bf5b08717916594cd66d31622e503ee -│   │   ├── 2d -│   │   │   ├── 142a08254f813eb9c28d2e6dafac808f7d7801 -│   │   │   └── d3bc55e2a9f4b899023f6a0169444b2d6f66a0 -│   │   ├── 2e -│   │   │   ├── 1fa2d52e12f690ce40ac89f77912faba0f993d -│   │   │   ├── 730764ae38af60c979e5196301b2d5a6fefa1b -│   │   │   ├── a393679d2e5aad07e6f6a6c2b581ff2b5e69db -│   │   │   └── aee62aeba71d4fbe3dfbbc1dc42926c572765f -│   │   ├── 2f -│   │   │   ├── 9f38593e2416cccd6f0011df73968f1ed43258 -│   │   │   ├── a66e226afae89c3c530d4b4ea4d9f34d124073 -│   │   │   └── fb627faf3a901e3318121c62e19ab4b4a9feeb -│   │   ├── 30 -│   │   │   └── 1ce798ded1515f361c14b887f4771236ff86ab -│   │   ├── 31 -│   │   │   ├── 03ea2eb0fce24269a6e512f0b5483e0a001690 -│   │   │   ├── 80418ef3c1cdb4b73a311a95ff890136670210 -│   │   │   └── e199dbd25b310367006065fbba734ab18e00db -│   │   ├── 32 -│   │   │   ├── 0ecfb164e9136085db1d1f9cb4d1347dcc53df -│   │   │   ├── 6c4b9818416737cc4d3664fe638f14a767df29 -│   │   │   └── 6da9ca11ffea5c54d82b265900e332b0e97143 -│   │   ├── 33 -│   │   │   ├── 2d5c2bbaeedb7b46fbad35fbee5075942d0499 -│   │   │   ├── c5c0e7334685e9bb40bb72b20597d147759dac -│   │   │   └── e0cc393c037bf37936f32932116b210554c51e -│   │   ├── 34 -│   │   │   ├── 0de303f8476bb33b278a3644a6cf5f1bb53561 -│   │   │   ├── 717aeafb92f8ebf50f3a1006e062c279e547b3 -│   │   │   ├── a2e5e51f098fef033262ea1d2e5db733d24d6f -│   │   │   ├── c20894f495bb2860c4449d4cda8965d0e97283 -│   │   │   ├── c77e960fdaaea01a08c55ca9e37cf60fcb5f1b -│   │   │   ├── c8d322744fcaea50ec32c1795b45218e3ecdb7 -│   │   │   └── d408158d13e7ec2f2024b8070bad2192646cd0 -│   │   ├── 35 -│   │   │   └── b4da70bc8a1a091aaa4681c1359ae249387878 -│   │   ├── 36 -│   │   │   ├── 66658541cea427b27a67faf8cb7a952be9fd9a -│   │   │   ├── a38848a900bd901a0e7e2221615d1dcfcfc6a0 -│   │   │   └── d92339da380ebcde0495819e85a1068196b30f -│   │   ├── 37 -│   │   │   ├── f145e1ed32c01086bf149480dd9335cd1435ed -│   │   │   └── f4a87457e5c6dc97309f68c9925ecb4d1c9b55 -│   │   ├── 38 -│   │   │   ├── 2de3a0747c6526aec3ccc84895764bf4aa3e95 -│   │   │   └── 8b6b2dbb07356f5cd82885db96597da879306b -│   │   ├── 39 -│   │   │   ├── 5d28bfdf8278923b7cf2e9a4c65c04908715a1 -│   │   │   ├── 9d029ee6886be8a65a6d5a180bc873b9f6e81d -│   │   │   ├── d4461e6b0214d6d94e92c91a682a7dc2716d77 -│   │   │   └── fed87e7118b202af9d5a7d3388b85bd6b25f14 -│   │   ├── 3c -│   │   │   ├── 2a68917149dcee38cf92ef3ec6c2fb0ae9c500 -│   │   │   ├── d353f9c653c1acebb1b647644decf936527547 -│   │   │   └── d7078702c6c8a23f28abbbd05f07d3b40feec8 -│   │   ├── 3d -│   │   │   ├── 375b73cc90fd5c9adc18eb51b3e22cbb780ec0 -│   │   │   └── caf05a7cbfbe182206e4d34654a34c36727bd6 -│   │   ├── 3e -│   │   │   ├── be34be14697956333c8f2ba8361974b630a25e -│   │   │   ├── e575523b352ab9a28265543ce2b6d89153493a -│   │   │   └── eaf8c0010259aa396a59b64f7e866e06179326 -│   │   ├── 3f -│   │   │   └── 255697ea47b6d59365118b248b5281982f0526 -│   │   ├── 40 -│   │   │   └── 32c6a2ec52b1bc901fef1e9a4e672251a82d5d -│   │   ├── 41 -│   │   │   └── a20d86f1242f5351a79b9583fe95f09ff036b5 -│   │   ├── 42 -│   │   │   ├── 247496917f6a0d2fab2d80d9770b5a4f7e4bcb -│   │   │   ├── 401762e7299c614589d1e466ffd0d03d02667f -│   │   │   └── c58e524aa6ce6d863b8addb7a83b0b735a6708 -│   │   ├── 43 -│   │   │   └── 1c94b2e473f1f048ecdb8e2fb24f50586b30aa -│   │   ├── 44 -│   │   │   └── a3ae2e41f67685f352bb23de697bd1732711f5 -│   │   ├── 45 -│   │   │   └── 6ef436554d19df9ad2b9d10a372f410115fb8c -│   │   ├── 46 -│   │   │   ├── 2bbbba070ac6c6067632d8babd22427a7770fc -│   │   │   └── 4175335e793b914a578b2d8fa8117a093dfccd -│   │   ├── 47 -│   │   │   ├── 8424a0f7b713386b4e20fe9fbf3bc0ec1ca5a8 -│   │   │   ├── 9b60fc3e824223463580a01e6dcd57d8772044 -│   │   │   └── f950e78d538bee0d8df747922c734bb5b8bc76 -│   │   ├── 4a -│   │   │   ├── 36747c8afa2df784bee1a0f21dd363336e2f55 -│   │   │   ├── 41676c579dd58b043c0e4da190058d62aa9301 -│   │   │   ├── 7c3ceec1e546abd1901f44e1570dc0d159b20c -│   │   │   └── b820bab6cecddce549230be580227bcc3595c9 -│   │   ├── 4b -│   │   │   ├── 0984be5067839829257ca827313cb6ebdb4cd2 -│   │   │   ├── 1bffa3061b36c040dcd8102c4e4dee0117472f -│   │   │   └── 75700fcfd1412f180fea58fcb6407a638b1b02 -│   │   ├── 4d -│   │   │   └── aad07124e0aae3bb81723e550bbae15b5cef82 -│   │   ├── 4e -│   │   │   ├── bd46831ff0020e58c030efa8beffe31b8742ff -│   │   │   └── da0cf71ce1696222b21ace6feffe6c9ebbca73 -│   │   ├── 4f -│   │   │   ├── 799fce729eda2257d7b62269747a19199aed83 -│   │   │   ├── a55a0179025d03f510f394d42b6d723800305a -│   │   │   └── d00273b8b462ec9887c519551218cbd323d333 -│   │   ├── 51 -│   │   │   └── ddcda78a2f4cbacae79f310a821c04815eaab2 -│   │   ├── 52 -│   │   │   ├── 56b8dadb5ff70ce6baaecd944961ae78cf0967 -│   │   │   └── 99c604f9c0d10deb96d4d681bc1e5747639c6f -│   │   ├── 53 -│   │   │   └── 8344df18487e465ea1c621017547ba053855a1 -│   │   ├── 54 -│   │   │   └── 9e6dbec9bbf8de7e6a310b8c791268d1db5445 -│   │   ├── 55 -│   │   │   ├── 5749f6e06e91bb7d0acbba7cc6cfc0724ad19f -│   │   │   ├── 70ebe2499581a8b432ccdcdaa1256dbdeed246 -│   │   │   └── f6bc3ccf02e6e6ecfa7d2fb2976b8f7c4df037 -│   │   ├── 56 -│   │   │   └── 23d8ed4e9d3be352bed439cbb2ca447cb4f9b6 -│   │   ├── 57 -│   │   │   ├── 1f41c3101e8be9f8e0ce6662442176efb2328b -│   │   │   └── badde230cf68d000e46794f523adc5bc94bd2f -│   │   ├── 59 -│   │   │   ├── 3b289dcc5234da96b5528bb67deceab7d4bc1d -│   │   │   ├── 442f2995795bd1b685bb09f828d099e21daee1 -│   │   │   └── c1dd1a181b1fa46398d2b17dc456dbf2f09c89 -│   │   ├── 5a -│   │   │   ├── 0d8d6d142f9685c111a2f035ecfdd94a77331a -│   │   │   ├── 2a4cce1eeb077116fa908b1da862310cf4195c -│   │   │   └── 2d66619d3438942ed423d4b413740ae9a7a91f -│   │   ├── 5b -│   │   │   ├── 0879503d181b3578aef48de2694b73aa154dbe -│   │   │   ├── 90c941123f9563dd757525e793bc407691e335 -│   │   │   ├── ca02ac54dc663706b1d5c5fceef5c2b07ec492 -│   │   │   └── cb1f0e95398f9c4d4709d3394f353726476dbe -│   │   ├── 5c -│   │   │   └── 44d71c782d8961e094be6735d67a9bd32dcba8 -│   │   ├── 5d -│   │   │   ├── 978d0e0068ebffd74b5f2bb02362ce0e7ce451 -│   │   │   └── a2433681c165ccdeb614b9d6880bdb303dc162 -│   │   ├── 5e -│   │   │   ├── 50a65370dd4c0b724de6910d29251a746d5192 -│   │   │   └── 5272738b0e38fa115cc79b87e6e419c02c02ad -│   │   ├── 5f -│   │   │   ├── 40d0992ac0c55a2e51f1ee39321069b9a065a8 -│   │   │   └── 6343de66309fc3bf49c5d6a2131d834b2fc2ab -│   │   ├── 60 -│   │   │   ├── 55926484bd860099faeb9b3230d456bcd16de8 -│   │   │   ├── 62076546b3b92ec6480639c1db40804c4be43a -│   │   │   ├── 718a4caffcbc1d31ff8960156c72d814b1b39d -│   │   │   └── f0d304fb44974146092fad4767b0e19380901b -│   │   ├── 61 -│   │   │   ├── 45a47d1e5f3cf9dd8973f03d50a648ca50db1d -│   │   │   └── 561c17b6ab10c14cbf4fb33a3a83cac5a8efc7 -│   │   ├── 62 -│   │   │   ├── 44fa3da7aed647374b09358161ea8ef33953fa -│   │   │   ├── 703b79783f4fa65524b461c71e3415a1a93614 -│   │   │   └── c3c5684f663ac4e2e72417e4043f3bea473bea -│   │   ├── 63 -│   │   │   ├── 2eeb2b3261f38cc48635a9ba720204cce7481d -│   │   │   ├── c71e3ccfa8fee872811456e65127c7df6ea7fd -│   │   │   └── eaa408b0c89a536a543bca23825dfdabf908ec -│   │   ├── 64 -│   │   │   ├── 02fde22f55d5cad7c3e98a33851b96e1a32cee -│   │   │   └── bc321f16b912b0d51beea2a6fe052739c05f1f -│   │   ├── 65 -│   │   │   └── 3d1e9bda33ad7b9adcac86f158f98a157f5e01 -│   │   ├── 66 -│   │   │   └── d29f99ff4bbd4cc8c1fba5f4e485d24fd53026 -│   │   ├── 6a -│   │   │   ├── 0bb67c531d9d0440b0ad9b411dba9cfaae6d84 -│   │   │   ├── 1e1fb10a12ec24ee0bbf63979063e587c2bc54 -│   │   │   ├── 2423b5174159a5366180df179cf8dae1c4f8a9 -│   │   │   ├── 9014d59dc11c760a50fc4ec41b4c09d4ec5b46 -│   │   │   └── 9ac974a8cd0481ca509de412521d0f0cda32e9 -│   │   ├── 6b -│   │   │   ├── 12bb2c759d2760621de9881a86361b343410ba -│   │   │   ├── 8fd913379a0305966e92abe85cfb376eaaf872 -│   │   │   └── f9008f30c2b5496f8aef8acb4b549c1b21dc92 -│   │   ├── 6c -│   │   │   ├── 45b366a551ecdb1427edb720effafd8c5ab980 -│   │   │   └── 5ff134caacfa7affc823b7ab44c60984d3d5bb -│   │   ├── 6d -│   │   │   ├── 4cc4a23ac3bc9a1616dd15750cab1c05f85fd4 -│   │   │   ├── aa46e7878de94a8382fd306dc454487abd0325 -│   │   │   └── f05066247acab6a06c61d2fc617672a3cf8850 -│   │   ├── 6e -│   │   │   └── 1825dd9532f862e30b136d803cd781f41bdd6a -│   │   ├── 6f -│   │   │   ├── 33d1e1a67f1443daf9b9a9fe550c7cd8451f35 -│   │   │   └── 40ee351a81c5481a08b14b092779336a4f22e6 -│   │   ├── 70 -│   │   │   └── 57c81fc158e6ae1165004367a660a9aeb863e9 -│   │   ├── 71 -│   │   │   └── 9cd2031307a64ba8244f3fdf29bbe6885dae35 -│   │   ├── 72 -│   │   │   ├── 157b253769646fff912a08ffbaae7233fb7391 -│   │   │   ├── 66f8f486f1491ec60c201626ed08c76cb3dd38 -│   │   │   ├── 74a8da5107fa50bc40be46a4f166aa25527520 -│   │   │   └── f3d61c57e495ace746199e0cb70a8547e5b05f -│   │   ├── 73 -│   │   │   ├── 0a4c445beef7a42265864f3bcfa3fb97e37748 -│   │   │   └── 148f32710f86b23dbd7213d4be80666db004fa -│   │   ├── 75 -│   │   │   ├── 1308baa2619ebf3878e22c603cc1e809071e50 -│   │   │   └── 8a521ee46ebddeda4be97d7a62073b24f5922a -│   │   ├── 76 -│   │   │   ├── 07bf6aa1a44d834ec76cc49d52500651f64699 -│   │   │   ├── 95d2515493296b0321365d3d0f19aff32f0c0b -│   │   │   ├── 96be05be44cc171a3f5bd3997c253bc6b3d954 -│   │   │   ├── c4346ea10bafe09741f638930f796258763cb8 -│   │   │   └── e2d33c44a4d69da1e7b3ab7e6d9f83346e0c6e -│   │   ├── 77 -│   │   │   └── ca2f4135f2235e74caf173e6bbd38629fb0064 -│   │   ├── 78 -│   │   │   ├── 44b58d0b0c49bfc0c1e3e926f8b4f2d0d375d1 -│   │   │   ├── 76dd55e1b6277455f7c8164fe2c773bc9bcaab -│   │   │   └── 947c69c5a8b540add16eddd457e48b3242e18c -│   │   ├── 79 -│   │   │   ├── 4edf159e331684c45c62b143e91a896e01069e -│   │   │   ├── 5409a55763bc12be50ecee250b0abfe86d6c3e -│   │   │   └── 92adf82ec23383795088b408361d37cdf70a9d -│   │   ├── 7a -│   │   │   ├── 1b5b23154be6879265e8bf95f8f1e3e0b09ca1 -│   │   │   └── 811c9a5517603341cac8cda47492af1b862468 -│   │   ├── 7b -│   │   │   ├── 2ec856b2d1281b5887bee7d0aef0302f7c6f00 -│   │   │   ├── 82626539b96845e9b432af4cb9a0e50337bc29 -│   │   │   └── ddab1cb2493e02a29ecf8b864e02616ebfa73e -│   │   ├── 7c -│   │   │   ├── abecd8637efbc5716d56ecc9ca77df2d3d9979 -│   │   │   ├── b4205c9c2f0642691452e270cdce4ef1055ac6 -│   │   │   └── cefe572877d26b7bd394ef4a6221e3f3a2d18a -│   │   ├── 7d -│   │   │   ├── 481dcea1510ca64dab9419c16037b35ceb5367 -│   │   │   ├── 61fbaae24896e860b77df24cacd03524c634ba -│   │   │   ├── 741b76055f1f9881d13141188208d8a844279b -│   │   │   └── 779a2820b48ac1f58ce9d03f4726f6ac3fa565 -│   │   ├── 7e -│   │   │   ├── 0cb2654273e8129e3e440a6f6e89fef46860c0 -│   │   │   └── ed75a91b7559761fb56a4d1404b257c6340efd -│   │   ├── 7f -│   │   │   ├── 1bcd17da83c646b88e622f4cc6835f00284488 -│   │   │   ├── 4392bc18c72e95957879b593ff5fd7084a27fb -│   │   │   ├── c88f6554e2df0d43e7d10d2bfef62d916773e6 -│   │   │   ├── d4f68a3074c4e5a3d67c2b4389676bccf5a5a3 -│   │   │   └── dace27dde60a075003f157fb62e5488ae91b36 -│   │   ├── 80 -│   │   │   ├── 79ee0d403b656fc35cb748a517afc5d88b9203 -│   │   │   ├── 8f81efc34e9e0f03eebbd52c2a5bb6c0af6816 -│   │   │   ├── e4221b1a01faaf06f8eb602efc8be75c8ebddc -│   │   │   └── f01af7f4207911aabad328abefa29d8a75b209 -│   │   ├── 81 -│   │   │   ├── 0996ca30fb1d4bd8cb64761c16d2dfd01a779c -│   │   │   ├── 2244d22fca37a3584d66bd5d1d4756d31da15f -│   │   │   └── 3d52f64cc143fb033e7f13ec57f6b82d9c1a54 -│   │   ├── 82 -│   │   │   ├── 1b21f78f6bac05bdbc4ef5ee5cb5d8df436986 -│   │   │   └── bb37f51fff826c9ee751a2758f812a6c19222d -│   │   ├── 83 -│   │   │   ├── 1c90ce5e7f589ca3578e5e990919aeee34a341 -│   │   │   ├── 49ad8e396aebd9e229258747b253f47ba1bcac -│   │   │   ├── 99f06045b1fa79af8f1108e96210ae52f330e7 -│   │   │   └── a10fb1b190120212f27da5fcb6f4858647fe39 -│   │   ├── 84 -│   │   │   └── a8e644945ab4d0673d44f2f49a5813d01d1ba0 -│   │   ├── 85 -│   │   │   ├── 072710f37213c62ec882691b5eacac168efdb3 -│   │   │   ├── 9d8add220b3ce47c25abdf3e14dbf3ae21d703 -│   │   │   ├── b73a9bd53e58b6a33609ab0289c78dfdbce186 -│   │   │   ├── cc979c0937729e99ec1b06b57c3da14462e658 -│   │   │   └── ed42cf937c2ea544d05b70b529dcaac892e843 -│   │   ├── 86 -│   │   │   └── 22e97318ad1637ce8ca9f53df9b37950d94aa9 -│   │   ├── 87 -│   │   │   ├── 7799b0ee5140c77e5ef68f209116fc378b3d93 -│   │   │   └── edb7f1b3b86769d9a09732c584e66c28568ea5 -│   │   ├── 8a -│   │   │   └── 3dce0a2dc18ed3900ce416182caad8be910ebd -│   │   ├── 8c -│   │   │   └── 1b221519aaf05f660e7196bbcfa7f2453da429 -│   │   ├── 8d -│   │   │   ├── 41b6480120c348708d68ed34fefc17e8fda4ff -│   │   │   └── 84c38f838e9c5f980daf718c331aa073feb57f -│   │   ├── 8e -│   │   │   ├── 2b9b35ac292cd1bbdc5ce48ed58a87bacb6b0a -│   │   │   ├── 98288b4d745dab30d14dca42bbeea9ad4bb105 -│   │   │   ├── bc8660367c356e521a46f4837cb8a30d80e107 -│   │   │   ├── ce9435829e4667676d3e1e5a3396b54d742e8c -│   │   │   └── e0bb66e82539d2ff04bee98052d1f9657ded2d -│   │   ├── 8f -│   │   │   └── eff1fe0dda8838c1944b76d05035cd44fc35d8 -│   │   ├── 90 -│   │   │   ├── 0a7fa93f8ae018a466cdc922baf18522911750 -│   │   │   ├── 5b981c1e4c238ca5437a431bf6f64d9e4e2c93 -│   │   │   └── 94e836a7078044df874b70ab1f9ff90fa5018b -│   │   ├── 91 -│   │   │   ├── 286e0dcf9ee01a4c0a9e7ad5311a2ddad17308 -│   │   │   ├── 3799ae73f1afe583bad3a060edd097165d3f57 -│   │   │   ├── 9c42e7c70900d63143092390a55070fb0b72a1 -│   │   │   ├── ac4a498b6be54f5ce300b094b1e0ceecdbd606 -│   │   │   └── e5d598271b84e66c162b78c678ccd147773fd8 -│   │   ├── 92 -│   │   │   ├── 9793dc51d1515d7417cae1fcdda58b855ef55f -│   │   │   └── d69867a6f8caf8bf3db135c69c350b71e37994 -│   │   ├── 94 -│   │   │   ├── 548af5beba7825284af746324c8dc5b2f1ea31 -│   │   │   └── e437388d29170f8ccb912dcb770070e874b35f -│   │   ├── 95 -│   │   │   ├── 7898482cc56abceb3abbc4dcfacf1d45fedac7 -│   │   │   ├── 9b59796cb52edc2f33366299647152dbaed24b -│   │   │   ├── a1366decc132301e3c0977b97d6c71ca78cb08 -│   │   │   └── e26ffe26288791ba8582fae26cbc850ecd639b -│   │   ├── 96 -│   │   │   └── 9f5989d50bee5f91f49f629eb32d0b5eac5a3c -│   │   ├── 98 -│   │   │   ├── 08125099e4c319a1a12ec37b82a6b1ee6c6747 -│   │   │   └── 5189df59b3f254a8a6e4fd745f48a777141cef -│   │   ├── 99 -│   │   │   ├── 00c017bf3edbe0b50ef113409e44bc6f9750b4 -│   │   │   ├── 1e79b6eba415a94a2ebecc2a66907290f1ad4e -│   │   │   ├── 27bef0c33f22e494516d8db92b911889536e9a -│   │   │   ├── 615b1a7f7c331cba6b0cc7bf713739a0b85a85 -│   │   │   └── da9616c06426627a5ef0ab468b538d97a9d6b0 -│   │   ├── 9a -│   │   │   ├── 795a57cf035276ae381132c18139530a76b64f -│   │   │   └── f49b13f6da539007e71d61383dbc1c46801e55 -│   │   ├── 9b -│   │   │   └── 5d543d83233c7e27a08856c9b10c3b1c130570 -│   │   ├── 9c -│   │   │   └── c51ebf606b78432d0bfac6225eb5b0ccf0a954 -│   │   ├── 9d -│   │   │   ├── 6e3b52d9a2c20170ca536ae75af5bb9f1e216e -│   │   │   └── 7f539e95cf5c07f2236e683eeb5dcfb73b5130 -│   │   ├── 9e -│   │   │   ├── 4ebfa99088b833b5ee376dc2bb1561cdc40307 -│   │   │   └── e9dae5ea38e0749d3c9c78e850e223ba8d0bae -│   │   ├── 9f -│   │   │   ├── 4636e41665101e4840d6d742d31a78cdc33bfc -│   │   │   ├── 65010b8c3f8f8eeb11e7b6b71aae01fae3692f -│   │   │   ├── 847d436c244ee1c2f37cbe7a17d66dcf41ccc7 -│   │   │   └── 9c1f7f88a536f9a4bb5541dc5230b68fe7c834 -│   │   ├── a1 -│   │   │   ├── 1413f14a84957b1391190c504cf513fc75d26a -│   │   │   ├── 26d1532aebc5a09af39d1b08bb01784171a684 -│   │   │   ├── 34423c9fd2964192d240a08746a5ac8d8f5b35 -│   │   │   └── dd6a018cdf171a4ff1cfe78c81a6fa5d702656 -│   │   ├── a2 -│   │   │   ├── 076f062c1110ae44465d2039b7fbf832658cc7 -│   │   │   ├── 77ae466bb66cf5160ff4447eb712a9be7c9ef8 -│   │   │   ├── 7c86ffe9609ae9b9b0d915598b0c8032b90f1d -│   │   │   └── 971879f0ed643f23241e2cc9bdf8c9317b069d -│   │   ├── a3 -│   │   │   ├── 3c78a446b6870852a8a1f647e65d027b28630c -│   │   │   └── e6436c0a99107f367ca6a7cc2bb49fbe1b3dce -│   │   ├── a4 -│   │   │   ├── 811ebd504fa504aef70028484b625a7689917c -│   │   │   ├── b0308f9c90b300dabefb632a034cfc039eef19 -│   │   │   └── d0832d8f846d0645a25a4819f72499f8693dde -│   │   ├── a5 -│   │   │   ├── 4d0cdd88a355d83c5b948c602e3f1c8b52922a -│   │   │   ├── d57866b1229be583d51e77cf85785895f85d93 -│   │   │   └── f843e8ce0e89b9da59648a49ac817a7e4e67fa -│   │   ├── a6 -│   │   │   ├── 3d439c50fafcc2fd82ecd4b35322cbf4f30eea -│   │   │   └── f41f4a5a25f6d761143f5d94a0f1db65eff71d -│   │   ├── a7 -│   │   │   ├── 66a9cdb635e2d6d42b90eeb4cff4ec77824761 -│   │   │   └── ec06f18cee64998b97f80abd84631d3d7cc48a -│   │   ├── a8 -│   │   │   ├── 8b3ae4fb735f5b9ba828b9b61a3dcd41c61292 -│   │   │   └── beb5e8836d72ef95e8057f848d6a5d37b46aea -│   │   ├── a9 -│   │   │   ├── 0e15bc2c72552be47d85bf8664fb67a73c1d5d -│   │   │   ├── 5d8cf33257be2a759af2fc43cb13c64573ea64 -│   │   │   └── cfd1a7640019ed6ef7cb42d860153cbb6c84c8 -│   │   ├── aa -│   │   │   ├── 0cc30ac683f3e67f147a8d1091396e29b0a7ef -│   │   │   ├── 4905ef4b9d7b8e7056c8f861e31d68a59dcce8 -│   │   │   ├── b01f85bd4ed5d53bebcb2cca767bb36329b8b4 -│   │   │   └── ea5cd31080c2e5e7dc31ee1e8edca717455c21 -│   │   ├── ab -│   │   │   ├── 0077f11a6f8fe6a6f6a1d833c9c69f5cd3163c -│   │   │   └── a7126521c9613752abfa2192dbdbb69ddc1f39 -│   │   ├── ac -│   │   │   ├── 7cd7431a501516612907dc18d2447dfbec782f -│   │   │   ├── a7460593ad193e0a9c815ec5d643ddb2fc1741 -│   │   │   └── e730e54e3786e3828fe97488aa7793d88d4a20 -│   │   ├── ad -│   │   │   ├── 27079dd5b4dc830e5708551951eed074e5df24 -│   │   │   ├── 6988afcc379449a38c28b8bcb300141ac57e49 -│   │   │   └── 9903f36477c8127958d9daacda138b31aac678 -│   │   ├── ae -│   │   │   ├── b777c21731dd51338f2a07cb11501f531f1d4b -│   │   │   └── c2a179096aba84562511eada400fb5e1da2ea8 -│   │   ├── af -│   │   │   ├── 0a5c6c5ffa64ebf2e1e17f43667574d00a418e -│   │   │   └── c6ef19bebb0c3798f662399dfbb23b6ded1133 -│   │   ├── b0 -│   │   │   └── 1de38c12ece789cc847166efdb238f677b3304 -│   │   ├── b1 -│   │   │   └── 680a956f099c23bebe4653b0c92f4d56e60c57 -│   │   ├── b2 -│   │   │   ├── 7674ea271e6b43bf46cb55a77808231b291652 -│   │   │   └── c9d9077418fc4a2eb5b034dca3929a953b2b34 -│   │   ├── b4 -│   │   │   └── b70208d83eddb3c6ea5ca5ca2ea7236573bc29 -│   │   ├── b5 -│   │   │   ├── e84a8c87a25cce05547b43df936db28172e98c -│   │   │   ├── ed21ea8e3b7fe5d98a12a467505bf6f925a3d0 -│   │   │   └── f004e7b8f2a246002715901e848cbe21baf306 -│   │   ├── b6 -│   │   │   ├── 61ddb8f3965ff4c682f95c9c343c5fd534740e -│   │   │   └── f0c94225c3c22ae29b1b29b4b6d2ad1fb85bc5 -│   │   ├── b7 -│   │   │   ├── 1cd3fcbb4878b8c8caa28d723fdb159d18c0bd -│   │   │   ├── 6f43e09d5da458dcf6823b2ff6276b634ba40e -│   │   │   ├── 8a86278c95887366bbde4e2c26352d2ecf4b9f -│   │   │   └── cd768082b3573c269a9a087dd63fd41e0da7cb -│   │   ├── b9 -│   │   │   └── 6741faf719d1c5bb46aae0bac4c25a98696dbd -│   │   ├── ba -│   │   │   ├── 1435a865140f0d219c5b5cc84563d9f64197ff -│   │   │   ├── 32533affe3b0f6e1bfa8a3dd120d958a59400e -│   │   │   ├── d0989f4c6a79e22018d39d3ec6b42158c8574c -│   │   │   └── d25602a75f5e947d4e5a19498b92a8cd97346e -│   │   ├── bb -│   │   │   ├── 0eaab85f14fc2d316b096bebed9cf4d1815421 -│   │   │   └── 6af0f05ae1148b511ae0b7e6c316d4de0ebeb4 -│   │   ├── bd -│   │   │   ├── 3a34aa524cb57d95e1c506cfd56b6c95c0ea45 -│   │   │   ├── a4ac522bbeabac5ee4bf91310663460122dc64 -│   │   │   ├── b26ee56eb4221dca032eb762e7d8038e8de49e -│   │   │   └── d3ea6dccf17bb86cc41abcf46b84a57f090ad9 -│   │   ├── be -│   │   │   ├── 03f8fc54ddbfdaf7070975e524561c051c737c -│   │   │   └── ca2570cf9cee845017b311e928ff028e3eaa30 -│   │   ├── bf -│   │   │   ├── 22e80409d76f481b5f347531d19262db7f290c -│   │   │   ├── 51b705ecb24d9082609bb18f10abfe02f0386c -│   │   │   └── c6b22f76ba56ff709f0f5c788b0f92448a08e1 -│   │   ├── c0 -│   │   │   ├── 277207c296ff22b200a318bae434aacb4db4d5 -│   │   │   ├── 69a6fe2b12a6e28c5632b760ef1953d3b04317 -│   │   │   ├── 7a53686c1fb4355accf9923b83d2dbdc494168 -│   │   │   ├── ae0a5e7a91c05da98cce898727205325bf84f7 -│   │   │   └── f71b7d6978ac79d4e84e82dd73314ad0b3da7e -│   │   ├── c1 -│   │   │   ├── 0c6d39a1ad4ac1ec64fd2109f14b9539c2504e -│   │   │   ├── 1e197ed19ba6e30d2367518137eb684bf46bb2 -│   │   │   └── 5e2c96489831c031f1df707a9b3c62efe637fc -│   │   ├── c2 -│   │   │   └── 3f1fdf660db2d06b08d3da2bf83f9f03085b1c -│   │   ├── c3 -│   │   │   ├── 2576b2e8cdfd3426dd53b8cd2a65ddba947702 -│   │   │   └── 85d36b92db980d688f3e08de6918bc3952ba9b -│   │   ├── c5 -│   │   │   └── dd9210567d787a009dc69ea55067a5414879f7 -│   │   ├── c6 -│   │   │   ├── 8049aae26053f144754ffe08de31b3ea71ea4b -│   │   │   └── 961c94526e1e073b45e00d53dda716f3712cda -│   │   ├── c7 -│   │   │   ├── 1be2276757118d0244d92f38da29c5fc3b60ca -│   │   │   ├── d02afa631b946b824fe3de57d51b82489282e5 -│   │   │   ├── d59983d51f6c82086610cb075db1ac36c831b4 -│   │   │   └── f58e6aef1dcb504bc72e435162b9e1e895aed3 -│   │   ├── c8 -│   │   │   ├── 6df58673ed3332438c1bc7f41db69790640f29 -│   │   │   └── 9e10952a97849c8db34897cab3a9b3498fa7f7 -│   │   ├── c9 -│   │   │   └── 9fac9d944325dbb6691f4b135eeb94d6ee1deb -│   │   ├── ca -│   │   │   ├── 14868a2793ff2f98eac57e59ebedb2de0f00ee -│   │   │   ├── f02213db4fe871313d07f39ed9d1dbfd08f78b -│   │   │   └── f87f759909d26ff319bc524a55a40faef783da -│   │   ├── cb -│   │   │   ├── 78770cbae6ddcffba6fab8d12066bb6ee20e43 -│   │   │   └── 90c8299e73060c2d43dfc1591e30c6a3fd796a -│   │   ├── cc -│   │   │   ├── 3bb738029f549ef106b13877ec41b8db37e9e1 -│   │   │   └── 76317df77d3592c9f83fe9b897f757535f92bc -│   │   ├── cd -│   │   │   ├── 40424f809e76e4d7bdd1a5adc41d49504ff3ac -│   │   │   ├── 5b94f92f86e6616b68d03aeea4a352d360a2cc -│   │   │   ├── 7307506870e952346a598e85c9e057385fa45e -│   │   │   ├── 7a263aff218bd4deba8c13098273d62ffc9a7c -│   │   │   ├── 8c913b1edee9e4357280e1949303dd87e4ef9e -│   │   │   └── e96d0163eec3f814e3cd278afaba4d43acc5fb -│   │   ├── cf -│   │   │   └── 7951010bf287a5a4a4e07cc4ef9a95cbda8eb6 -│   │   ├── d0 -│   │   │   ├── 605dcfbbc9d5a8341c970040e5c52132d17c4b -│   │   │   ├── 6c0558a95111d00b765977cd355b6075997c00 -│   │   │   └── eba6b8874162e9ec21cbf12530aed4d1bb8d92 -│   │   ├── d1 -│   │   │   └── 846725cc8c45106b6cbd96599cb4971424166a -│   │   ├── d2 -│   │   │   ├── 710456fb756f38229920eeb8100a2a1a32df2a -│   │   │   └── d389ea9155c287e39d52411598064b27d7d78d -│   │   ├── d3 -│   │   │   ├── 3340171f63679cfa6c11ee5534265754d63143 -│   │   │   ├── a27b9e1e15bd3bdc33d3b741fecf8009b087bf -│   │   │   └── eb694e4c815e8389171321c0b32a9fa2b7aa42 -│   │   ├── d5 -│   │   │   ├── 34f12f7412d036eb157e1463952b9e2b826b2d -│   │   │   ├── 3ae29427a8c8ab8209b1ab2eb34829eb8fd18e -│   │   │   ├── b6cb22cdaa55bc94bc32263ea6fa66c9acf7e0 -│   │   │   └── efe2392b7591d8db048884da111b6cd631f3de -│   │   ├── d6 -│   │   │   └── c4451719f4293b8b2e1032e5b9936a20e880c0 -│   │   ├── d7 -│   │   │   └── ece7aab26abdc6e73a3387c2899ec161782cb0 -│   │   ├── d8 -│   │   │   ├── 6fa86c3685a8279255fd91a596cb596a01db91 -│   │   │   ├── a6f1340be0fda3a0b6a52befa75f8dc73ddc44 -│   │   │   ├── b4157aef93a9ac611766077b969c9f87628941 -│   │   │   └── c63857af9fce855ddf06b6a9dc5e453efdcab2 -│   │   ├── d9 -│   │   │   └── 14d2cb0bf8ef7559433fa77dc74318814b4e28 -│   │   ├── da -│   │   │   ├── 2b2037e734bc9462ea175531d71992bc5d1c6b -│   │   │   ├── b8b2457172c15bbd9e11eb0ab58cfc48a0ea13 -│   │   │   └── caaff74bb51f1e25e515db3cfe162093688d0f -│   │   ├── db -│   │   │   ├── 1f6660330540dd944ee8b033e3bfb4611351c9 -│   │   │   ├── 21e93fd568877d551680d8d8c8558bacea2df1 -│   │   │   ├── 661906b38b76d99cbb76bd21191117ff6e651c -│   │   │   └── cfc8867c2f50e8775c8aabc1ecbbc95b8bdb1b -│   │   ├── dc -│   │   │   ├── 26f5befa4806659c2e94fc9c6d359e7673ce75 -│   │   │   ├── 8d7a30e62ee91fe2ec987ea736657180eef687 -│   │   │   ├── a4f1cda9dd270d23874fd9b5fa501bb038c107 -│   │   │   └── a566404efbe089d7037d128f3d03567648e378 -│   │   ├── dd -│   │   │   ├── 3247b11dfeedc0ee11fd963ea53ee4b91de423 -│   │   │   ├── 9b59165093fe754212afbb3e366f52ceddf602 -│   │   │   ├── b2bf06973e7e41a08e33c52fd5a2086171c883 -│   │   │   ├── cc62b35ec63a8053b4a5aa8368c850b57ab788 -│   │   │   └── f14bcbed7fea21c0ecb1bc677b10214e2df657 -│   │   ├── de -│   │   │   └── 34ea62972bdd7cfee48a60d140c04dae8db6de -│   │   ├── df -│   │   │   └── dd23c2a442cbe857eccd185d7f0ce539b30dcb -│   │   ├── e0 -│   │   │   ├── 4f87e101b1164d605c38663193e31c9b9fc236 -│   │   │   └── 81058537a2f8c4145ba4b2ca57ca86dc6dee5b -│   │   ├── e1 -│   │   │   ├── 1fc347fe0904f65b757bebba4d4ef9b04fae87 -│   │   │   ├── 2a0da5b1ce71ce6c09c4af18c9f4e47628c870 -│   │   │   └── ec411a4b64ae4dad52c0077d73654baa27d027 -│   │   ├── e3 -│   │   │   ├── 20ebd247104828e27f3769fb9d200612905584 -│   │   │   └── ca0e26158ef8868cad4093cac3d5b890808ecd -│   │   ├── e4 -│   │   │   └── 4ac28e5a399a7d52abd8011865d5eb0e805434 -│   │   ├── e5 -│   │   │   ├── a7d90ea53ba355d3cd94bbdfd3bbcd8a448d81 -│   │   │   └── fba20bba48ae8a8c31ad14401998ac2e231448 -│   │   ├── e6 -│   │   │   ├── 16ff025825be4339eb051be03fff9489b99c27 -│   │   │   ├── 445b1c49963745c6ef34ec72230d2a2dbb2d50 -│   │   │   ├── 9de29bb2d1d6434b8b29ae775ad8c2e48c5391 -│   │   │   └── c347bb0b19a6adce33fed71a507051c27254c3 -│   │   ├── e9 -│   │   │   ├── 0134484f53da9fed7986ca16f9e2e6f7a08d9f -│   │   │   └── 99b6cfd75061b62f554d39504291c99fd1bcd0 -│   │   ├── ea -│   │   │   ├── 666662132dc3b76a90b4895ac72bc7f7fccada -│   │   │   ├── c11ef898f19d7f871d09a2dd351891e1f8dbd9 -│   │   │   └── c5bb16bc9009077d6987f66d8e76a3da59b0e5 -│   │   ├── eb -│   │   │   └── 4af422e90589f8ec72c63b3bef7c399c2e2a7b -│   │   ├── ec -│   │   │   ├── 0ae64852ab5ab56348e499558848cb5ac6f01c -│   │   │   ├── 2bce68451a79eefd69dab1e99acab1f4401168 -│   │   │   └── 65b67a60837e602a8c0e2a647bbdb20780b048 -│   │   ├── ed -│   │   │   ├── 16807990caed841f82c23d6cc052b0b33ba4cd -│   │   │   ├── 73823aaec97420a3176e3edbb6bf1beb3d71ce -│   │   │   ├── a50b426aafe41ca97d0e4847034e5c390df1d8 -│   │   │   └── b91dd8eeea3a38ef0019cc1442c9394e2a76ca -│   │   ├── ee -│   │   │   ├── 0d80a234d142db1292aae1dfa7800dbef48f58 -│   │   │   └── b17cdc97f6849b628778c80f50119e923fbfff -│   │   ├── ef -│   │   │   └── 73ce7ec9d0da88155c15d6048459be4a32c89c -│   │   ├── f0 -│   │   │   ├── ba637c62ed02809ea84499582ba4a50492d6e9 -│   │   │   ├── cb6a79442ff873008947096058448b231db3a2 -│   │   │   └── e07735dfc9a7e33dea63b8c16cd7589baec38b -│   │   ├── f1 -│   │   │   ├── a38b11d91cd118696f87792d6c33175ce58a6f -│   │   │   ├── d8209a60ff5accc749d90a665af63ee47261f8 -│   │   │   └── f4acf2005ecc97dbbb251588743c3b98200725 -│   │   ├── f2 -│   │   │   └── 39ce8f9dcc0f114a08074cc171a6d44d548766 -│   │   ├── f3 -│   │   │   └── 9073d20fcea6823ebd9cae026338db410fb2ac -│   │   ├── f4 -│   │   │   ├── 4787f0177d3e9149e98fbe69ba371895a14132 -│   │   │   ├── 55992127d2dd29bd21f5f4cf74ac95917612b9 -│   │   │   └── 675fb89791d70649e00b69020a326adfe69bba -│   │   ├── f5 -│   │   │   ├── 2b75d7c714145646f65483e3bac2e83ed79916 -│   │   │   └── c8ad971a13de4f944888a80b115e6f14de4abf -│   │   ├── f6 -│   │   │   ├── 027c941a9622a49df7ccde673b27a2ce55ca17 -│   │   │   └── 98e32d3031ec9789b24c163b0c6e5f08d787aa -│   │   ├── f7 -│   │   │   ├── 5a1a657fbc4448fc499f1ddee43062a101a648 -│   │   │   └── c0e7a78217f82b332079ee74169426d6c360ce -│   │   ├── f8 -│   │   │   └── c330d78de1e847ca770f263982d0506e2d7575 -│   │   ├── f9 -│   │   │   └── 6d7871f62acdd217e85c4daa474e88e2ad9348 -│   │   ├── fa -│   │   │   ├── 40dd5de307b27e7a4298592ed03d8029f13f9a -│   │   │   ├── 479873928ba949ee35c8c6034d69eb5cf7b726 -│   │   │   ├── 52b9d3858961b15e8aabc1601d129efeeb05ea -│   │   │   └── 8920807779c2a7ef89068dd575376a2e77058e -│   │   ├── fb -│   │   │   └── a8e6133daec3febcf7c71dddc37d32d2281500 -│   │   ├── fc -│   │   │   ├── b116e485bc3b63e8ace3585b0e6d833a0fa2d6 -│   │   │   └── d8e5630265e2961e0767596cdeda45c4a11243 -│   │   ├── fd -│   │   │   ├── 3b3e6eb7aa4b9dadd2511b78f833a86015b959 -│   │   │   └── 5e5d2a6d0fbebd8b5c2e24037822c2a28e6880 -│   │   ├── fe -│   │   │   ├── 50313e30a08d85a1bdeef4d84968b5f3097c0c -│   │   │   └── 686a54c43c5e6c13321061d6a4fd840b710570 -│   │   ├── ff -│   │   │   ├── b87509acfce0292c9bfb307d290d60d14184f1 -│   │   │   ├── cdaf034a42abc2c196bf553f55bb79522036cc -│   │   │   └── ee3f0964051c766df3b57037569430b0e1eb73 -│   │   ├── info -│   │   └── pack -│   ├── ORIG_HEAD -│   └── refs -│   ├── heads -│   │   └── main -│   ├── remotes -│   │   └── origin -│   └── tags -├── .gitignore -├── last-ip.ini -├── monitoring -│   ├── data -│   │   └── remote_digest_cache.json -│   ├── docker-exporter -│   │   ├── data -│   │   │   ├── remote_digest_cache.json -│   │   │   └── touch.test -│   │   ├── Dockerfile -│   │   ├── Dockerfile.old -│   │   ├── exporter.py -│   │   └── exporter.py.old -│   ├── gotify -│   │   ├── data -│   │   │   ├── bin -│   │   │   ├── certs -│   │   │   ├── chisel -│   │   │   ├── compose -│   │   │   ├── docker_config -│   │   │   ├── docker-tls -│   │   │   ├── gotify.db -│   │   │   ├── images -│   │   │   ├── plugins -│   │   │   ├── portainer.db -│   │   │   ├── portainer.key -│   │   │   ├── portainer.pub -│   │   │   ├── screenshots -│   │   │   ├── tls -│   │   │   └── upload -│   │   ├── docker-compose.yml -│   │   ├── docker-health-alert -│   │   │   └── last_unhealthy.txt -│   │   └── docker-health-to-gotify.sh -│   ├── grafana -│   │   ├── data -│   │   │   ├── csv -│   │   │   ├── grafana.db -│   │   │   ├── pdf -│   │   │   ├── plugins -│   │   │   ├── png -│   │   │   └── unified-search -│   │   ├── docker-compose.yml -│   │   └── output.txt -│   ├── influxdb -│   │   ├── engine -│   │   │   ├── data -│   │   │   ├── replicationq -│   │   │   └── wal -│   │   ├── influxd.bolt -│   │   └── influxd.sqlite -│   ├── node-red -│   │   ├── data -│   │   │   ├── .config.nodes.json -│   │   │   ├── .config.runtime.json -│   │   │   ├── .config.runtime.json.backup -│   │   │   ├── .config.users.json -│   │   │   ├── .config.users.json.backup -│   │   │   ├── flows_cred.json -│   │   │   ├── flows.json -│   │   │   ├── .flows.json.backup -│   │   │   ├── lib -│   │   │   ├── node_modules -│   │   │   ├── package.json -│   │   │   ├── settings.js -│   │   │   ├── test-container.sh -│   │   │   ├── test-container.sh.old -│   │   │   └── webhook.json -│   │   ├── docker-compose.yml -│   │   └── Dockerfile -│   ├── portainer -│   │   ├── data -│   │   │   ├── backups -│   │   │   ├── bin -│   │   │   ├── certs -│   │   │   ├── chisel -│   │   │   ├── compose -│   │   │   ├── docker_config -│   │   │   ├── portainer.db -│   │   │   ├── portainer.key -│   │   │   ├── portainer.pub -│   │   │   └── tls -│   │   └── docker-compose.yml -│   ├── prometheus -│   │   ├── data -│   │   │   ├── 01KKX2MM2GJP2EMER8E1QX4Y1Q -│   │   │   ├── 01KKZ0E4YQT8RJBV08442XVMWQ -│   │   │   ├── 01KM0Y7PGWQPG5ZBW6RWN43SCD -│   │   │   ├── 01KM2W17G5EB9Q4JP13JCWX9M5 -│   │   │   ├── 01KM4STS1SKXK1J2A34Z56BP13 -│   │   │   ├── 01KM6QM9XF50R0Z6CV3AXH32ES -│   │   │   ├── 01KM8NDVJNS9TNEQ3C52A9M1E2 -│   │   │   ├── 01KMAK7CFYHZRT8R08AGSVGHMT -│   │   │   ├── 01KMCH0YDVB4W0V9A28XFXRJNW -│   │   │   ├── 01KMEETFR1288ZNER530T502NA -│   │   │   ├── 01KMGCM0BT8KZYECCD9Y23665P -│   │   │   ├── 01KMJADGQNFQB33XZRDVWC9X1R -│   │   │   ├── 01KMM871VSYAYGSFW6XK6P1J0J -│   │   │   ├── 01KMP60KVFBZ4JY3E8B0J6R9SM -│   │   │   ├── 01KMR3T4SWYCWW93P6VVAH4C37 -│   │   │   ├── 01KMT1KS682A3SC0E4A4M2VB4A -│   │   │   ├── 01KMVZD77GRGVE834GSZEZ7J67 -│   │   │   ├── 01KMY11YNSRXCHK5V5DWWR0HAV -│   │   │   ├── 01KMZM4K7HES0NBQ6D3EHRV36V -│   │   │   ├── 01KN1RSW901EA3RPT0ZX32D1BF -│   │   │   ├── 01KN26H9AD7KWGNP3MXHHK5ZEN -│   │   │   ├── 01KN2DD0J8M6A0H2442F0TECNM -│   │   │   ├── 01KN2DD0SGYKVX763XENNCGWJW -│   │   │   ├── 01KN2M8QT7PVPWHXCJNQ4PQDME -│   │   │   ├── 01KN2V4F27YHSS9HFRHW6GW57F -│   │   │   ├── chunks_head -│   │   │   ├── healthy -│   │   │   ├── lock -│   │   │   ├── queries.active -│   │   │   └── wal -│   │   ├── docker-compose.yml -│   │   ├── prometheus.yml -│   │   ├── prometheus.yml.old -│   │   └── rules -│   │   └── alerts.yml -│   ├── telegraf -│   │   └── telegraf.conf -│   └── uptime-kuma -│   ├── data -│   │   ├── db-config.json -│   │   ├── docker.sock -│   │   ├── docker-tls -│   │   ├── kuma.db -│   │   ├── kuma.db-shm -│   │   ├── kuma.db-wal -│   │   ├── screenshots -│   │   └── upload -│   ├── docker-compose.yml -│   ├── result -> /nix/var/nix/profiles/system-50-link -│   └── uptime-kuma -│   ├── docker-tls -│   ├── kuma.db -│   ├── screenshots -│   └── upload -├── services-up.sh -├── tree.out -├── update-containers.log -├── update-containers.py -├── update-containers.sh -├── update-firewall.log -├── update-firewall.sh -└── venv - ├── bin - │   ├── activate - │   ├── activate.csh - │   ├── activate.fish - │   ├── Activate.ps1 - │   ├── chardetect - │   ├── docker-compose - │   ├── jsonschema - │   ├── normalizer - │   ├── pip - │   ├── pip3 - │   ├── pip3.12 - │   ├── __pycache__ - │   │   └── wsdump.cpython-312.pyc - │   ├── python -> python3 - │   ├── python3 -> /nix/store/8w718rm43x7z73xhw9d6vh8s4snrq67h-python3-3.12.10/bin/python3 - │   └── python3.12 -> python3 - ├── include - │   └── python3.12 - ├── lib - │   └── python3.12 - │   └── site-packages - ├── lib64 -> lib - └── pyvenv.cfg - -519 directories, 1374 files