archived old monitoring system

2026-07-13 14:43:16 +10:00
parent f17a8ee22c
commit dccff1d61c
25 changed files with 2396 additions and 1897 deletions
@@ -0,0 +1,56 @@
+services:
+  prometheus:
+    profiles: ["monitoring","all","prometheus"]
+    image: prom/prometheus:latest
+#    env_file:
+#      - ${PROJECT_ROOT}/secrets/stack-secrets.env
+    container_name: prometheus
+    depends_on:
+#      - alertmanager
+      - telegraf
+      - influxdb
+      - node-exporter
+      - docker-update-exporter
+      - pihole-exporter
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--storage.tsdb.retention.time=15d"
+#    build:
+#      context: ${PROJECT_ROOT}/monitoring/prometheus
+    volumes:
+      - ${PROJECT_ROOT}/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+#      - ${PROJECT_ROOT}/monitoring/prometheus/data:/prometheus
+#      - ${PROJECT_ROOT}/monitoring/prometheus/rules:/etc/prometheus/rules:ro
+      - ${DOCKER_VOLUMES}/prometheus/data:/prometheus
+      - ${DOCKER_VOLUMES}/prometheus/rules:/etc/prometheus/rules:ro
+      - ${PROJECT_ROOT}/secrets/prometheus_kuma_basic_auth_password.txt:/run/secrets/prometheus_kuma_basic_auth_password:ro
+
+    restart: unless-stopped
+    labels:
+       - "traefik.http.routers.prometheus.rule=Host(`prometheus.lan.ddnsgeek.com`)"
+       - "traefik.enable=true"
+       - "traefik.http.routers.prometheus.entrypoints=websecure"
+       - "traefik.http.routers.prometheus.tls.certresolver=myresolver"
+       - "traefik.http.routers.prometheus.tls.options=mtls-private-admin@file"
+       - "io.portainer.accesscontrol.public"
+       - "traefik.http.services.prometheus.loadbalancer.server.port=9090"
+       - "traefik.http.routers.prometheus.middlewares=authelia"
+       - "traefik.docker.network=core_traefik"
+    networks:
+#      - edge
+#      - traefik_reverse_proxy
+      - traefik
+      - monitor
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
+
+#volumes:
+#  prometheus-data:
+#    external: true
+#  prometheus-rules:
+#    external: true
@@ -0,0 +1,166 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+#alerting:
+#  alertmanagers:
+#    - static_configs:
+#        - targets:
+#            - alertmanager:9093
+
+scrape_configs:
+
+  # Prometheus itself
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["prometheus:9090"]
+        labels:
+          role: prometheus
+  # =========================
+  # Node Exporters (ALL hosts)
+  # =========================
+  - job_name: "node"
+    static_configs:
+      - targets:
+          - node-exporter:9100
+        labels:
+          role: docker
+
+      - targets:
+          - raspberrypi.tail13f623.ts.net:9100
+        labels:
+          role: raspberrypi
+
+      - targets:
+          - pve.sweet.home:9100
+        labels:
+          role: proxmox
+
+      - targets:
+          - pbs.sweet.home:9100
+        labels:
+          role: backup
+
+      - targets:
+          - pihole:9100
+        labels:
+          role: pihole
+
+      - targets:
+          - server:9100
+        labels:
+          role: server
+
+      - targets:
+          - nix-cache:9100
+        labels:
+          role: cache
+
+  # =========================
+  # Telegraf (Docker metrics)
+  # =========================
+  - job_name: "telegraf"
+    static_configs:
+      - targets:
+          - telegraf:9273
+          - raspberrypi.tail13f623.ts.net:9273
+        labels:
+          role: docker
+
+  # =========================
+  # Traefik (all instances)
+  # =========================
+  - job_name: "traefik"
+    static_configs:
+      - targets:
+          - traefik.lan.ddnsgeek.com:8080
+        labels:
+          role: docker
+
+      - targets:
+          - raspberrypi.tail13f623.ts.net:8080
+        labels:
+          role: raspberrypi
+
+    metric_relabel_configs:
+      - source_labels: [service]
+        regex: '(.+)@.+'
+        target_label: service
+        replacement: '$1'
+
+  # =========================
+  # Uptime Kuma (separate due to auth)
+  # =========================
+  - job_name: "kuma"
+    metrics_path: /metrics
+    scrape_interval: 30s
+
+    basic_auth:
+      username: wayne.bennett@live.com
+      password_file: /run/secrets/prometheus_kuma_basic_auth_password
+#      password: '4vjCco?[%{=+,t`):C'
+    static_configs:
+      - targets:
+          - monitor-kuma:3001
+        labels:
+          role: docker
+
+      - targets:
+          - kuma.lan.ddnsgeek.com
+        labels:
+          role: raspberrypi
+    tls_config:
+#      ca_file: /prometheus/clients-ca.crt
+      cert_file: /prometheus/office-pc.crt
+      key_file: /prometheus/office-pc.key
+#      server_name: kuma.lan.ddnsgeek.com
+
+  # =========================
+  # Proxmox Storage Exporters
+  # =========================
+  - job_name: "proxmox-storage"
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - pve.sweet.home:9101
+        labels:
+          role: proxmox
+          storage: lvm
+
+#      - targets:
+#          - pbs.sweet.home:9102
+#        labels:
+#          role: backup
+#          storage: datastore
+
+  # =========================
+  # Docker Updates Exporter
+  # =========================
+
+
+  - job_name: "container-updates"
+    static_configs:
+      - targets:
+        - docker-update-exporter:9105
+        labels:
+          role: docker
+      - targets:
+          - raspberrypi.tail13f623.ts.net:9105
+        labels:
+          role: raspberrypi
+
+  # =========================
+  # pihole Exporter
+  # =========================
+
+
+  - job_name: "pihole"
+    static_configs:
+      - targets:
+        - pihole-exporter:9617
+        labels:
+          role: pihole
+
+
+#rule_files:
+#  - /etc/prometheus/rules/*.yml
@@ -0,0 +1,192 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
+scrape_configs:
+  - job_name: "prometheus"
+    static_configs:
+      - targets: ["prometheus:9090"]
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+  - job_name: "docker-node"
+    static_configs:
+      - targets: ["node-exporter:9100"]
+        labels:
+          type: "virtual"
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+#  - job_name: "cadvisor"
+#    static_configs:
+#      - targets: ["cadvisor:8080"]
+#        labels:
+#          type: "container"
+#    relabel_configs:
+#      - source_labels: [__address__]
+#        regex: '([^:]+):.*'
+#        target_label: instance
+  - job_name: "raspberrypi-node"
+    static_configs:
+      - targets:
+          - "raspberrypi.tail13f623.ts.net:9100"
+        labels:
+          type: "physical"
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+  - job_name: "proxmox-node"
+    static_configs:
+      - targets:
+          - "pve.sweet.home:9100"
+        labels:
+          type: "physical"
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+  - job_name: "proxmox-backup-server"
+    static_configs:
+      - targets:
+          - "pbs.sweet.home:9100"
+        labels:
+          type: "virtual"
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+  - job_name: "pihole"
+    static_configs:
+      - targets:
+          - "pihole:9100"
+        labels:
+          type: "virtual"
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+  - job_name: "server"
+    static_configs:
+      - targets:
+          - "server:9100"
+        labels:
+          type: "virtual"
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+  - job_name: "nix-cache"
+    static_configs:
+      - targets:
+          - "nix-cache:9100"
+        labels:
+          type: "virtual"
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+  - job_name: docker
+    static_configs:
+      - targets: ['telegraf:9273']
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+
+  - job_name: 'traefik'
+    static_configs:
+      - targets: ['traefik.lan.ddnsgeek.com:8080']  # replace with your Traefik host:port
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+    metric_relabel_configs:
+      - source_labels: [service]
+        regex: '(.+)@.+'
+        target_label: service
+        replacement: '$1'
+
+  - job_name: 'raspi-traefik'
+    static_configs:
+      - targets: ['raspberrypi.tail13f623.ts.net:8080']  # replace with your Traefik host:port
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+    metric_relabel_configs:
+      - source_labels: [service]
+        regex: '(.+)@.+'
+        target_label: service
+        replacement: '$1'
+
+  - job_name: 'raspi-kuma'
+    metrics_path: /metrics
+    scrape_interval: 30s
+
+    basic_auth:
+      username: wayne.bennett@live.com
+      password: '4vjCco?[%{=+,t`):C'
+
+    static_configs:
+      - targets:
+          - kuma.lan.ddnsgeek.com
+
+  - job_name: 'docker-kuma'
+    metrics_path: /metrics
+    scrape_interval: 30s
+
+    basic_auth:
+      username: wayne.bennett@live.com
+      password: '4vjCco?[%{=+,t`):C'
+
+    static_configs:
+      - targets:
+          - uptime-kuma:3001
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+
+  - job_name: 'proxmox LVM storage'
+    static_configs:
+      - targets: ['pve.sweet.home:9101']
+    metrics_path: /metrics
+    scheme: http
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+  - job_name: 'proxmox backup storage'
+    static_configs:
+      - targets: ['pbs.sweet.home:9102']
+    metrics_path: /metrics
+    scheme: http
+    relabel_configs:
+      - source_labels: [__address__]
+        regex: '([^:]+):.*'
+        target_label: instance
+
+
+rule_files:
+  - /etc/prometheus/rules/*.yml
+