modified: default-environment.env

modified: monitoring/node-red/data/.flows.json.backup modified: monitoring/node-red/data/update-events.ndjson
Merge pull request #39 from beatz174-bit/codex/refactor-and-extend-mtls-bridge-for-dynamic-endpoints
2026-04-14 12:33:18 +10:00 · 2026-04-14 12:08:25 +10:00 · 2026-04-14 12:07:17 +10:00
6 changed files with 137 additions and 126 deletions
@@ -65,9 +65,10 @@ DOCKER_SOCKET_PROXY_LOG_LEVEL=info
 NODE_COMPOSE_ROOT=/compose

 # mTLS bridge
-MTLS_BRIDGE_LOG_LEVEL=INFO
+MTLS_BRIDGE_LOG_LEVEL=DEBUG
 MTLS_BRIDGE_TIMEOUT=5
 MTLS_BRIDGE_CLIENT_KEY=/certs/clients/office-pc/office-pc.key
 MTLS_BRIDGE_CLIENT_CERT=/certs/clients/office-pc/office-pc.crt
-MTLS_BRIDGE_TARGET_URL=http://node-red:1880/docker-update-lockouts/clear
+MTLS_BRIDGE_TARGET_URL=http://node-red:1880
+MTLS_BRIDGE_ALLOWED_PATHS_FILE=
 MTLS_BRIDGE_CORS_ALLOW_ORIGIN=https://grafana.lan.ddnsgeek.com
@@ -5,24 +5,33 @@ Internal HTTP-to-mTLS bridge for services that cannot present client certificate
 ## How it works

 1. Accepts plain HTTP requests inside the Docker network.
-2. Forwards requests to an HTTPS upstream.
-3. Presents a client certificate/key pair for mTLS authentication.
+2. Forwards requests to an upstream base URL.
+3. Preserves the incoming request path/method/body/query string.
+4. Presents a client certificate/key pair for mTLS authentication.

 ## Environment variables

- `TARGET_URL` (required): HTTPS upstream base URL.
+- `TARGET_URL` (required): upstream base URL (for example `http://node-red:1880`).
 - `CLIENT_CERT` (default `/certs/client.crt`): client certificate path.
 - `CLIENT_KEY` (default `/certs/client.key`): client private key path.
- `CA_CERT` (default `/certs/ca.crt`): CA certificate bundle used to verify upstream TLS.
+- `UPSTREAM_CA_CERT` (optional, alias: `CA_CERT`): CA bundle path to verify upstream TLS. Use `false`/`0`/`no` to disable verification.
 - `TIMEOUT` (default `5`): request timeout in seconds.
 - `LOG_LEVEL` (default `INFO`): Python logging level.
+- `HEALTH_ENDPOINT` (default `/_mtls_bridge/health`): local container health endpoint path.
+- `ALLOWED_PATHS_FILE` (optional): file path containing one allowed endpoint path per line (for example `/health`). Blank lines and `#` comments are ignored. If unset, all paths are allowed.
 - `MTLS_BRIDGE_BASIC_AUTH_USERS` (required for Traefik auth): value for `traefik.http.middlewares.*.basicauth.users` (e.g. `user:$$apr1$$...`).
 - `MTLS_BRIDGE_CORS_ALLOW_ORIGIN` (default `https://grafana.lan.ddnsgeek.com`): origin allowed for browser-based panel actions.

 ## Endpoints

- `GET /health` returns `200 OK` for container health checks.
- `/*` proxies requests to `${TARGET_URL}` with method/body/headers preserved.
+- `GET /_mtls_bridge/health` returns `200 OK` for container health checks.
+- `/*` proxies requests to `${TARGET_URL}/*` with method/body/headers/query string preserved (subject to optional allow-list checks).
+
+Examples with `TARGET_URL=http://node-red:1880`:
+
+- `https://mtls-bridge.../docker-update-lockouts/clear` -> `http://node-red:1880/docker-update-lockouts/clear`
+- `https://mtls-bridge.../health` -> `http://node-red:1880/health`
+- `https://mtls-bridge.../uptime-kuma` -> `http://node-red:1880/uptime-kuma`

 ## Compose integration

@@ -35,6 +44,17 @@ This repository includes `monitoring/mtls-bridge/docker-compose.yml`:
 ## Example test

 ```bash
-curl http://mtls-bridge:8080/health
-curl -X POST http://mtls-bridge:8080
+curl http://mtls-bridge:8080/_mtls_bridge/health
+curl -X POST http://mtls-bridge:8080/docker-update-lockouts/clear
 ```
+
+## Allow-list file example
+
+```text
+# one path per line
+/docker-update-lockouts/clear
+/health
+/uptime-kuma
+```
+
+When `ALLOWED_PATHS_FILE` is set, any path not listed returns `403 Endpoint not allowed`.
@@ -1,6 +1,7 @@
 import logging
 import os
 import time
+from urllib.parse import urljoin

 import requests
 from flask import Flask, Response, g, request
@@ -15,23 +16,39 @@ logger = logging.getLogger("mtls-bridge")
 logging.getLogger("werkzeug").setLevel(logging.WARNING)

 # Config via env
-TARGET_URL = os.environ.get("TARGET_URL")
+TARGET_URL = (os.environ.get("TARGET_URL") or "").strip()
 CLIENT_CERT = os.environ.get("CLIENT_CERT", "/certs/client.crt")
 CLIENT_KEY = os.environ.get("CLIENT_KEY", "/certs/client.key")
 UPSTREAM_CA_CERT = os.environ.get("UPSTREAM_CA_CERT", os.environ.get("CA_CERT", "")).strip()
-# Backward-compat alias: keep CA_CERT defined so legacy code paths/log statements don't crash.
-CA_CERT = UPSTREAM_CA_CERT
 TIMEOUT = int(os.environ.get("TIMEOUT", "5"))
+HEALTH_ENDPOINT = os.environ.get("HEALTH_ENDPOINT", "/_mtls_bridge/health")
+ALLOWED_PATHS_FILE = (os.environ.get("ALLOWED_PATHS_FILE") or "").strip()

-logger.info(
-    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s ca=%s log_level=%s",
-    TARGET_URL,
-    TIMEOUT,
-    CLIENT_CERT,
-    CLIENT_KEY,
-    CA_CERT,
-    os.environ.get("LOG_LEVEL", "INFO"),
-)
+
+def normalize_path(path: str) -> str:
+    if not path or path == "/":
+        return "/"
+    return f"/{path.lstrip('/')}"
+
+
+def load_allowed_paths() -> set[str]:
+    if not ALLOWED_PATHS_FILE:
+        return set()
+
+    if not os.path.exists(ALLOWED_PATHS_FILE):
+        logger.warning("ALLOWED_PATHS_FILE does not exist: %s (allow-list disabled)", ALLOWED_PATHS_FILE)
+        return set()
+
+    allowed_paths = set()
+    with open(ALLOWED_PATHS_FILE, encoding="utf-8") as f:
+        for line in f:
+            entry = line.strip()
+            if not entry or entry.startswith("#"):
+                continue
+            allowed_paths.add(normalize_path(entry))
+
+    logger.info("loaded %s allowed path(s) from %s", len(allowed_paths), ALLOWED_PATHS_FILE)
+    return allowed_paths


 def get_verify_setting():
@@ -54,95 +71,47 @@ def get_verify_setting():


 VERIFY_SETTING = get_verify_setting()
-
-logger.info(
-    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s log_level=%s",
-    TARGET_URL,
-    TIMEOUT,
-    CLIENT_CERT,
-    CLIENT_KEY,
-    VERIFY_SETTING,
-    os.environ.get("LOG_LEVEL", "INFO"),
-)
-
-
-def get_verify_setting():
-    if not UPSTREAM_CA_CERT:
-        return True
-
-    lowered = UPSTREAM_CA_CERT.lower()
-    if lowered in {"false", "0", "no"}:
-        logger.warning("TLS verification for upstream is disabled via UPSTREAM_CA_CERT=%s", UPSTREAM_CA_CERT)
-        return False
-
-    if not os.path.exists(UPSTREAM_CA_CERT):
-        logger.warning(
-            "Configured UPSTREAM_CA_CERT path does not exist: %s (falling back to system CA bundle)",
-            UPSTREAM_CA_CERT,
-        )
-        return True
-
-    return UPSTREAM_CA_CERT
-
-
-VERIFY_SETTING = get_verify_setting()
-
-logger.info(
-    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s log_level=%s",
-    TARGET_URL,
-    TIMEOUT,
-    CLIENT_CERT,
-    CLIENT_KEY,
-    VERIFY_SETTING,
-    os.environ.get("LOG_LEVEL", "INFO"),
-)
+ALLOWED_PATHS = load_allowed_paths()

 if TARGET_URL and TARGET_URL.lower().startswith("http://"):
-    logger.warning(
-        "TARGET_URL uses http://; upstream may redirect to https:// and change request behavior: %s",
-        TARGET_URL,
-    )
-
-
-def get_verify_setting():
-    if not UPSTREAM_CA_CERT:
-        return True
-
-    lowered = UPSTREAM_CA_CERT.lower()
-    if lowered in {"false", "0", "no"}:
-        logger.warning("TLS verification for upstream is disabled via UPSTREAM_CA_CERT=%s", UPSTREAM_CA_CERT)
-        return False
-
-    if not os.path.exists(UPSTREAM_CA_CERT):
-        logger.warning(
-            "Configured UPSTREAM_CA_CERT path does not exist: %s (falling back to system CA bundle)",
-            UPSTREAM_CA_CERT,
-        )
-        return True
-
-    return UPSTREAM_CA_CERT
-
-
-VERIFY_SETTING = get_verify_setting()
+    logger.warning("TARGET_URL uses http:// (plaintext): %s", TARGET_URL)

 logger.info(
-    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s log_level=%s",
+    "mtls-bridge starting target_url=%s timeout=%ss cert=%s key=%s verify=%s health_endpoint=%s allow_list_file=%s allow_list_entries=%s log_level=%s",
    TARGET_URL,
    TIMEOUT,
    CLIENT_CERT,
    CLIENT_KEY,
    VERIFY_SETTING,
+    HEALTH_ENDPOINT,
+    ALLOWED_PATHS_FILE,
+    len(ALLOWED_PATHS),
    os.environ.get("LOG_LEVEL", "INFO"),
 )

-if TARGET_URL and TARGET_URL.lower().startswith("http://"):
-    logger.warning(
-        "TARGET_URL uses http://; upstream may redirect to https:// and change request behavior: %s",
-        TARGET_URL,
-    )
+
+def build_upstream_url(path: str) -> str:
+    """Map incoming path directly onto TARGET_URL origin/base path."""
+    if not TARGET_URL:
+        raise ValueError("TARGET_URL is not set")
+
+    normalized_target = TARGET_URL.rstrip("/") + "/"
+    normalized_path = path.lstrip("/")
+    upstream_url = urljoin(normalized_target, normalized_path)
+
+    if request.query_string:
+        upstream_url = f"{upstream_url}?{request.query_string.decode('utf-8', 'ignore')}"
+
+    return upstream_url


-@app.route("/health", methods=["GET"])
+def is_path_allowed(request_path: str) -> bool:
+    if not ALLOWED_PATHS:
+        return True
+    return request_path in ALLOWED_PATHS
+
+
+@app.route(HEALTH_ENDPOINT, methods=["GET"])
 def health():
    logger.debug("healthcheck request from %s", request.remote_addr)
    return "OK", 200
@@ -156,7 +125,7 @@ def before_request():
@app.after_request
 def after_request(response):
    elapsed_ms = int((time.time() - g.request_start) * 1000)
-    if request.path != "/health":
+    if request.path != HEALTH_ENDPOINT:
        logger.info(
            "request complete method=%s path=%s status=%s elapsed_ms=%s",
            request.method,
@@ -179,7 +148,7 @@ def after_request(response):
    provide_automatic_options=False,
 )
 def proxy(path):
-    request_path = f"/{path}" if path else "/"
+    request_path = normalize_path(path)
    request_size = len(request.get_data(cache=True))
    logger.info(
        "incoming request method=%s path=%s query=%s remote=%s bytes=%s",
@@ -190,50 +159,49 @@ def proxy(path):
        request_size,
    )

-    if not TARGET_URL:
-        logger.error("TARGET_URL is not set; cannot proxy request")
-        return Response("TARGET_URL is not set", status=500)
+    if not is_path_allowed(request_path):
+        logger.warning("request blocked by allow-list path=%s", request_path)
+        return Response("Endpoint not allowed", status=403)

    try:
-        url = f"{TARGET_URL.rstrip('/')}/{path}".rstrip("/")
-        start_time = time.time()
+        upstream_url = build_upstream_url(path)
+
        headers = {k: v for k, v in request.headers if k.lower() != "host"}
        headers["X-Forwarded-By"] = "mtls-bridge"

-        logger.debug("forwarding request to upstream url=%s headers=%s", url, headers)
-
+        start_time = time.time()
        resp = requests.request(
            method=request.method,
-            url=url,
+            url=upstream_url,
            headers=headers,
            data=request.get_data(cache=True),
            cookies=request.cookies,
            cert=(CLIENT_CERT, CLIENT_KEY),
            verify=VERIFY_SETTING,
            timeout=TIMEOUT,
+            allow_redirects=False,
        )

        elapsed_ms = int((time.time() - start_time) * 1000)
        logger.info(
            "upstream response status=%s url=%s elapsed_ms=%s response_bytes=%s",
            resp.status_code,
-            url,
+            upstream_url,
            elapsed_ms,
            len(resp.content),
        )

-        excluded_headers = ["content-encoding", "content-length", "transfer-encoding", "connection"]
-        response_headers = [
-            (k, v)
-            for k, v in resp.raw.headers.items()
-            if k.lower() not in excluded_headers
-        ]
+        excluded_headers = {"content-encoding", "content-length", "transfer-encoding", "connection"}
+        response_headers = [(k, v) for k, v in resp.headers.items() if k.lower() not in excluded_headers]

        return Response(resp.content, resp.status_code, response_headers)

-    except Exception as e:
+    except ValueError as exc:
+        logger.error("proxy request failed: %s", exc)
+        return Response(str(exc), status=500)
+    except Exception as exc:  # noqa: BLE001
        logger.exception("proxy request failed")
-        return Response(str(e), status=500)
+        return Response(str(exc), status=500)


 if __name__ == "__main__":
@@ -13,6 +13,7 @@ services:
      - TIMEOUT=${MTLS_BRIDGE_TIMEOUT}
      - LOG_LEVEL=${MTLS_BRIDGE_LOG_LEVEL:-INFO}
      - UPSTREAM_CA_CERT=${MTLS_BRIDGE_UPSTREAM_CA_CERT:-}
+      - ALLOWED_PATHS_FILE=${MTLS_BRIDGE_ALLOWED_PATHS_FILE:-}
    volumes:
      - ${PROJECT_ROOT}/core/traefik/certs:/certs:ro
    labels:
@@ -38,7 +39,7 @@ services:
       - "traefik.http.services.mtls-bridge.loadbalancer.server.port=8080"
       - "traefik.docker.network=core_traefik"
    healthcheck:
-      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health', timeout=3).read()"]
+      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/_mtls_bridge/health', timeout=3).read()"]
      interval: 30s
      timeout: 5s
      retries: 3
@@ -771,7 +771,8 @@
        "wires": [
            [
                "46affc0319a44fa9",
-                "ab7d36db8e9316f8"
+                "ab7d36db8e9316f8",
+                "058ef296282aa774"
            ]
        ]
    },
@@ -819,6 +820,23 @@
            ]
        ]
    },
+    {
+        "id": "058ef296282aa774",
+        "type": "debug",
+        "z": "d0f45d2b6f117365",
+        "name": "debug 1",
+        "active": true,
+        "tosidebar": true,
+        "console": false,
+        "tostatus": false,
+        "complete": "payload",
+        "targetType": "msg",
+        "statusVal": "",
+        "statusType": "auto",
+        "x": 560,
+        "y": 280,
+        "wires": []
+    },
    {
        "id": "0301a3cebd1eb23f",
        "type": "http request",
@@ -876,7 +894,7 @@
        "initialize": "",
        "finalize": "",
        "libs": [],
-        "x": 500,
+        "x": 480,
        "y": 100,
        "wires": [
            [
@@ -916,7 +934,7 @@
        "initialize": "",
        "finalize": "",
        "libs": [],
-        "x": 490,
+        "x": 710,
        "y": 200,
        "wires": [
            [
@@ -952,7 +970,7 @@
        "initialize": "",
        "finalize": "",
        "libs": [],
-        "x": 490,
+        "x": 710,
        "y": 300,
        "wires": [
            [
@@ -988,7 +1006,7 @@
        "initialize": "",
        "finalize": "",
        "libs": [],
-        "x": 490,
+        "x": 790,
        "y": 380,
        "wires": [
            [
@@ -1024,7 +1042,7 @@
        "initialize": "",
        "finalize": "",
        "libs": [],
-        "x": 520,
+        "x": 800,
        "y": 460,
        "wires": [
            [
@@ -1060,7 +1078,7 @@
        "initialize": "",
        "finalize": "",
        "libs": [],
-        "x": 490,
+        "x": 750,
        "y": 520,
        "wires": [
            [
@@ -1111,7 +1129,7 @@
        "initialize": "",
        "finalize": "",
        "libs": [],
-        "x": 480,
+        "x": 740,
        "y": 580,
        "wires": [
            [
@@ -1273,7 +1291,7 @@
        "initialize": "",
        "finalize": "",
        "libs": [],
-        "x": 470,
+        "x": 460,
        "y": 580,
        "wires": [
            [
@@ -134,3 +134,6 @@
 {"ts":"2026-04-14T00:20:04.308Z","flow":"docker-updates","event":"completed","container":"telegraf","project":"unknown","host":"docker","status":"locked","success":0,"failed":1,"duration_ms":0,"code":0,"error":""}
 {"ts":"2026-04-14T01:24:35.295Z","flow":"docker-updates","event":"completed","container":"update-test","project":"unknown","host":"docker","status":"success","success":1,"failed":0,"duration_ms":0,"code":0,"error":""}
 {"ts":"2026-04-14T01:25:29.131Z","flow":"docker-updates","event":"completed","container":"telegraf","project":"unknown","host":"docker","status":"failed","success":0,"failed":1,"duration_ms":0,"code":0,"error":""}
+{"ts":"2026-04-14T02:30:22.079Z","flow":"docker-updates","event":"completed","container":"update-test","project":"unknown","host":"docker","status":"success","success":1,"failed":0,"duration_ms":0,"code":0,"error":""}
+{"ts":"2026-04-14T02:31:15.896Z","flow":"docker-updates","event":"completed","container":"telegraf","project":"unknown","host":"docker","status":"failed","success":0,"failed":1,"duration_ms":0,"code":0,"error":""}
+{"ts":"2026-04-14T02:31:15.896Z","flow":"docker-updates","event":"completed","container":"telegraf","project":"unknown","host":"docker","status":"locked","success":0,"failed":1,"duration_ms":0,"code":0,"error":""}
Author	SHA1	Message	Date
git	624d7be17c	modified: default-environment.env modified: monitoring/node-red/data/.flows.json.backup modified: monitoring/node-red/data/update-events.ndjson	2026-04-14 12:33:18 +10:00
beatz174-bit	b17adee7cc	Merge pull request #39 from beatz174-bit/codex/refactor-and-extend-mtls-bridge-for-dynamic-endpoints mtls-bridge: preserve request paths, add allow-list and configurable health endpoint	2026-04-14 12:08:25 +10:00
beatz174-bit	27c5c3f631	Add optional allow-list support for mtls-bridge paths	2026-04-14 12:07:17 +10:00