updated response codes on error pages and pve hostname in traefik config

.gitignore

@@ -11,7 +11,7 @@ apps/nextcloud/config/
 core/crowdsec/config/
 **/.env
 !monitoring/node-red/data/
-apps/stockfill/
+apps/stockfill
 apps/shift-recorder/
 
 !apps/searxng/

apps/gitea/docker-compose.yml

@@ -11,7 +11,8 @@ services:
       - GITEA__server__ROOT_URL=${GITEA_ROOT_URL}
       - GITEA__actions__ENABLED=true
     volumes:
-      - ${PROJECT_ROOT}/apps/gitea/data:/data
+#      - ${PROJECT_ROOT}/apps/gitea/data:/data
+      - ${DOCKER_VOLUMES}/gitea/data:/data
     networks:
       - traefik
 
@@ -24,6 +25,7 @@ services:
       - "traefik.http.services.gitea.loadbalancer.server.port=3000"
       - "io.portainer.accesscontrol.public"
       - "traefik.docker.network=core_traefik"
+      - "wud.watch=true"
 
     healthcheck:
       test: ["CMD-SHELL", "curl -fsS http://localhost:3000/api/healthz >/dev/null"]
@@ -47,9 +49,11 @@ services:
       - GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}
       - DOCKER_HOST=${DOCKER_SOCKET_PROXY_HOST}
     volumes:
-      - ${PROJECT_ROOT}/apps/gitea/runner-data:/data
+#      - ${PROJECT_ROOT}/apps/gitea/runner-data:/data
+      - ${DOCKER_VOLUMES}/gitea/runner:/data
     networks:
       - traefik
+      - docker-socket-proxy
 
 #volumes:
 #  gitea_data:

apps/gramps/docker-compose.yml

@@ -26,14 +26,24 @@ services:
       GRAMPSWEB_CELERY_CONFIG__result_backend: redis://gramps-redis:6379/0
       GRAMPSWEB_RATELIMIT_STORAGE_URI: redis://gramps-redis:6379/1
     volumes:
-      - ${PROJECT_ROOT}/apps/gramps/data/users:/app/users
+#      - ${PROJECT_ROOT}/apps/gramps/data/users:/app/users
-      - ${PROJECT_ROOT}/apps/gramps/data/index:/app/indexdir
+#      - ${PROJECT_ROOT}/apps/gramps/data/index:/app/indexdir
-      - ${PROJECT_ROOT}/apps/gramps/data/thumbnail_cache:/app/thumbnail_cache
+#      - ${PROJECT_ROOT}/apps/gramps/data/thumbnail_cache:/app/thumbnail_cache
-      - ${PROJECT_ROOT}/apps/gramps/data/cache:/app/cache
+#      - ${PROJECT_ROOT}/apps/gramps/data/cache:/app/cache
-      - ${PROJECT_ROOT}/apps/gramps/data/secret:/app/secret
+#      - ${PROJECT_ROOT}/apps/gramps/data/secret:/app/secret
-      - ${PROJECT_ROOT}/apps/gramps/data/db:/root/.gramps/grampsdb
+#      - ${PROJECT_ROOT}/apps/gramps/data/db:/root/.gramps/grampsdb
-      - ${PROJECT_ROOT}/apps/gramps/data/media:/app/media
+#      - ${PROJECT_ROOT}/apps/gramps/data/media:/app/media
-      - ${PROJECT_ROOT}/apps/gramps/data/tmp:/tmp
+#      - ${PROJECT_ROOT}/apps/gramps/data/tmp:/tmp
+
+      - ${DOCKER_VOLUMES}/gramps-data/users:/app/users
+      - ${DOCKER_VOLUMES}/gramps-data/index:/app/indexdir
+      - ${DOCKER_VOLUMES}/gramps-data/thumbnail_cache:/app/thumbnail_cache
+      - ${DOCKER_VOLUMES}/gramps-data/cache:/app/cache
+      - ${DOCKER_VOLUMES}/gramps-data/secret:/app/secret
+      - ${DOCKER_VOLUMES}/gramps-data/db:/root/.gramps/grampsdb
+      - ${DOCKER_VOLUMES}/gramps-data/media:/app/media
+      - ${DOCKER_VOLUMES}/gramps-data/tmp:/tmp
+
     labels:
       - "traefik.http.routers.gramps.rule=Host(`familytree.lan.ddnsgeek.com`)"
       - "traefik.enable=true"
@@ -42,6 +52,7 @@ services:
       - "io.portainer.accesscontrol.public"
       - "traefik.http.services.gramps.loadbalancer.server.port=5000"
       - "traefik.docker.network=core_traefik"
+      - "wud.watch=true"
     networks:
       - traefik
       - gramps
@@ -81,14 +92,26 @@ services:
       GRAMPSWEB_CELERY_CONFIG__result_backend: redis://gramps-redis:6379/0
       GRAMPSWEB_RATELIMIT_STORAGE_URI: redis://gramps-redis:6379/1
     volumes:
-      - ${PROJECT_ROOT}/apps/gramps/data/users:/app/users
+#      - ${PROJECT_ROOT}/apps/gramps/data/users:/app/users
-      - ${PROJECT_ROOT}/apps/gramps/data/index:/app/indexdir
+#      - ${PROJECT_ROOT}/apps/gramps/data/index:/app/indexdir
-      - ${PROJECT_ROOT}/apps/gramps/data/thumbnail_cache:/app/thumbnail_cache
+#      - ${PROJECT_ROOT}/apps/gramps/data/thumbnail_cache:/app/thumbnail_cache
-      - ${PROJECT_ROOT}/apps/gramps/data/cache:/app/cache
+#      - ${PROJECT_ROOT}/apps/gramps/data/cache:/app/cache
-      - ${PROJECT_ROOT}/apps/gramps/data/secret:/app/secret
+#      - ${PROJECT_ROOT}/apps/gramps/data/secret:/app/secret
-      - ${PROJECT_ROOT}/apps/gramps/data/db:/root/.gramps/grampsdb
+#      - ${PROJECT_ROOT}/apps/gramps/data/db:/root/.gramps/grampsdb
-      - ${PROJECT_ROOT}/apps/gramps/data/media:/app/media
+#      - ${PROJECT_ROOT}/apps/gramps/data/media:/app/media
-      - ${PROJECT_ROOT}/apps/gramps/data/tmp:/tmp
+#      - ${PROJECT_ROOT}/apps/gramps/data/tmp:/tmp
+
+      - ${DOCKER_VOLUMES}/gramps-data/users:/app/users
+      - ${DOCKER_VOLUMES}/gramps-data/index:/app/indexdir
+      - ${DOCKER_VOLUMES}/gramps-data/thumbnail_cache:/app/thumbnail_cache
+      - ${DOCKER_VOLUMES}/gramps-data/cache:/app/cache
+      - ${DOCKER_VOLUMES}/gramps-data/secret:/app/secret
+      - ${DOCKER_VOLUMES}/gramps-data/db:/root/.gramps/grampsdb
+      - ${DOCKER_VOLUMES}/gramps-data/media:/app/media
+      - ${DOCKER_VOLUMES}/gramps-data/tmp:/tmp
+    labels:
+      - "wud.watch=true"
+
     networks:
       - gramps
     healthcheck:
@@ -102,7 +125,7 @@ services:
 
   gramps-redis:
     profiles: ["apps","all","gramps"]
-    image: valkey/valkey:8-alpine
+    image: valkey/valkey:9-trixie
     container_name: gramps-redis
     restart: always
     networks:
@@ -115,6 +138,8 @@ services:
       timeout: 5s
       retries: 6
       start_period: 10s
+    labels:
+      - "wud.watch=true"
 
 networks:
   gramps:

apps/nextcloud/docker-compose.yml

@@ -1,6 +1,7 @@
 services:
   nextcloud-webapp:
     profiles: ["apps","all","nextcloud"]
+    image: nextcloud:production
     build:
       context: ${PROJECT_ROOT}/apps/nextcloud
     container_name: nextcloud-webapp
@@ -9,10 +10,15 @@ services:
 #    env_file:
 #      - ${SECRETS_ENV_FILE}
     volumes:
-      - ${PROJECT_ROOT}/apps/nextcloud/data:/var/www/html/data:rw
+#      - ${PROJECT_ROOT}/apps/nextcloud/data:/var/www/html/data:rw
-      - ${PROJECT_ROOT}/apps/nextcloud/config:/var/www/html/config:rw
+      - ${NEXTCLOUD_DATA}:/var/www/html/data:rw
+#      - ${PROJECT_ROOT}/apps/nextcloud/config:/var/www/html/config:rw
+      - ${DOCKER_VOLUMES}/nextcloud/config:/var/www/html/config:rw
       - type: tmpfs
         target: /tmp:exec
+#      - ${PROJECT_ROOT}/apps/nextcloud/entrypoint.sh:/entrypoint.sh
+#      - ${PROJECT_ROOT}/apps/nextcloud/custom_apps:/var/www/html/custom_apps
+      - ${DOCKER_VOLUMES}/nextcloud/custom_apps:/var/www/html/custom_apps
     depends_on:
       - nextcloud-db
       - nextcloud-redis
@@ -57,6 +63,7 @@ services:
       - "traefik.http.middlewares.nextcloud-webfinger.redirectregex.regex=https://(.*)/.well-known/webfinger"
       - "traefik.http.middlewares.nextcloud-webfinger.redirectregex.replacement=https://$${1}/nextcloud/index.php/.well-known/webfinger"
       - "traefik.docker.network=core_traefik"
+      - "wud.watch=true"
     healthcheck:
       test:
         - CMD-SHELL
@@ -71,7 +78,7 @@ services:
       start_period: 180s
 
   nextcloud-db:
-    image: mariadb:11.4
+    image: mariadb:${NEXTCLOUD_DB_VERSION}
     restart: always
     profiles: ["apps","all","nextcloud"]
     container_name: nextcloud-db
@@ -80,7 +87,8 @@ services:
 #    env_file:
 #      - ${PROJECT_ROOT}/secrets/stack-secrets.env
     volumes:
-      - ${PROJECT_ROOT}/apps/nextcloud/database:/var/lib/mysql:rw
+#      - ${PROJECT_ROOT}/apps/nextcloud/database:/var/lib/mysql:rw
+      - ${DOCKER_DATABASES}/nextcloud/mysql:/var/lib/mysql:rw
     environment:
       - MYSQL_ROOT_PASSWORD_FILE=/run/secrets/nextcloud_db_root_password
       - MYSQL_PASSWORD_FILE=/run/secrets/nextcloud_db_password
@@ -97,12 +105,16 @@ services:
       - nextcloud
     labels:
       - "io.portainer.accesscontrol.public"
+      - "wud.watch=true"
+      - 'wud.tag.include=^11\.4(?:\.\d+)?$'
+#      - "wud.tag.exclude=${NEXTCLOUD_DB_VERSION}-*"
     healthcheck:
       test: ["CMD-SHELL", "mariadb-admin ping -u $$MYSQL_USER --password=$$(cat /run/secrets/nextcloud_db_password) --silent"]
       interval: 10s
       timeout: 5s
       retries: 12
       start_period: 60s
+      
 
   nextcloud-redis:
     image: "redis"
@@ -114,12 +126,14 @@ services:
 #    secrets:
 #      - nextcloud_redis_password
     volumes:
-      - ${PROJECT_ROOT}/apps/nextcloud/data/redis:/data:rw
+#      - ${PROJECT_ROOT}/apps/nextcloud/data/redis:/data:rw
+      - ${DOCKER_DATABASES}/nextcloud/redis:/data:rw
     restart: always
     networks:
       - nextcloud
     labels:
       - "io.portainer.accesscontrol.public"
+      - "wud.watch=true"
     healthcheck:
 #      test: ["CMD-SHELL", "redis-cli -a \"$$(cat /run/secrets/nextcloud_redis_password)\" PING | grep -q PONG"]
       test: ["CMD-SHELL", "redis-cli  PING | grep -q PONG"]

apps/nextcloud/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+OCC="/var/www/html/occ"
+
+php $OCC app:install memories || true
+php $OCC app:enable memories || true
+
+php $OCC app:install recognize || true
+php $OCC app:enable recognize || true
+
+
+exec apache2-foreground

apps/passbolt/docker-compose.yml

@@ -2,7 +2,7 @@ services:
   passbolt-db:
     profiles: ["apps","all","passbolt"]
     container_name: passbolt-db
-    image: mariadb:12
+    image: mariadb:latest
     restart: always
 #    env_file:
 #      - ${PROJECT_ROOT}/secrets/stack-secrets.env
@@ -14,7 +14,8 @@ services:
     secrets:
       - passbolt_db_password
     volumes:
-      - ${PROJECT_ROOT}/apps/passbolt/data/database:/var/lib/mysql
+#      - ${PROJECT_ROOT}/apps/passbolt/data/database:/var/lib/mysql
+      - ${DOCKER_DATABASES}/passbolt:/var/lib/mysql
     networks:
       - passbolt
     healthcheck:
@@ -24,7 +25,8 @@ services:
       retries: 12
       start_period: 60s
     labels:
-       - "io.portainer.accesscontrol.public"
+      - "io.portainer.accesscontrol.public"
+      - "wud.watch=true"
 
   passbolt-webapp:
     image: passbolt/passbolt:latest-ce
@@ -45,8 +47,12 @@ services:
     secrets:
       - passbolt_db_password
     volumes:
-      - ${PROJECT_ROOT}/apps/passbolt/data/gpg:/etc/passbolt/gpg
+#      - ${PROJECT_ROOT}/apps/passbolt/data/gpg:/etc/passbolt/gpg
-      - ${PROJECT_ROOT}/apps/passbolt/data/jwt:/etc/passbolt/jwt
+#      - ${PROJECT_ROOT}/apps/passbolt/data/jwt:/etc/passbolt/jwt
+
+      - ${DOCKER_VOLUMES}/passbolt-data/gpg:/etc/passbolt/gpg
+      - ${DOCKER_VOLUMES}/passbolt-data/jwt:/etc/passbolt/jwt
+
     command:
       [
         "/usr/bin/wait-for.sh",
@@ -66,6 +72,7 @@ services:
       - "traefik.http.routers.passbolt.tls.certresolver=myresolver"
       - "io.portainer.accesscontrol.public"
       - "traefik.docker.network=core_traefik"
+      - "wud.watch=true"
     healthcheck:
       test: ["CMD-SHELL", "curl -fsS http://localhost/healthcheck/status | grep -qx OK"]
       interval: 30s

apps/searxng/docker-compose.yml

@@ -13,13 +13,14 @@ services:
     networks:
       - traefik
     labels:
-       - "traefik.http.routers.searxng.rule=Host(`searxng.lan.ddnsgeek.com`)"
+      - "traefik.http.routers.searxng.rule=Host(`searxng.lan.ddnsgeek.com`)"
-       - "traefik.enable=true"
+      - "traefik.enable=true"
-       - "traefik.http.routers.searxng.entrypoints=websecure"
+      - "traefik.http.routers.searxng.entrypoints=websecure"
-       - "traefik.http.routers.searxng.tls.certresolver=myresolver"
+      - "traefik.http.routers.searxng.tls.certresolver=myresolver"
-       - "io.portainer.accesscontrol.public"
+      - "io.portainer.accesscontrol.public"
 #       - "traefik.http.routers.searxng.middlewares=crowdsec@file,secHeaders@file,error-pages-middleware"
-       - "traefik.http.services.searxng.loadbalancer.server.port=8080"
+      - "traefik.http.services.searxng.loadbalancer.server.port=8080"
+      - "wud.watch=true"
     healthcheck:
       test: ["CMD-SHELL", "python3 -c \"import urllib.request,sys; r=urllib.request.urlopen('http://127.0.0.1:8080/', timeout=3); sys.exit(0 if 200<=r.status<400 else 1)\""]
       interval: 20s

archive/dynamic-system/monitoring/docker-exporter/docker-compose.yml

@@ -16,7 +16,8 @@ services:
 
     volumes:
       - ~/.docker/config.json:/root/.docker/config.json:ro
-      - ${PROJECT_ROOT}/monitoring/docker-exporter/data:/data:rw
+#      - ${PROJECT_ROOT}/monitoring/docker-exporter/data:/data:rw
+      - ${DOCKER_VOLUMES}/docker-update-exporter-data:/data:rw
       - ${PROJECT_ROOT}:/compose:ro
 #      - ${PROJECT_ROOT}/default-environment.env:/compose/default-environment.env:ro
 #      - ${PROJECT_ROOT}/default-network.yml:/compose/default-network.yml:ro
@@ -53,3 +54,6 @@ services:
       timeout: 5s
       retries: 3
       start_period: 10s
+#volumes:
+#  docker-update-exporter-data:
+#    external: true

archive/dynamic-system/monitoring/grafana/docker-compose.yml

@@ -7,7 +7,8 @@ services:
     environment:
       - GF_SERVER_ROOT_URL=${GRAFANA_ROOT_URL}
     volumes:
-      - ${PROJECT_ROOT}/monitoring/grafana/data:/var/lib/grafana
+#      - ${PROJECT_ROOT}/monitoring/grafana/data:/var/lib/grafana
+      - ${DOCKER_VOLUMES}/grafana-data:/var/lib/grafana
     networks:
 #      - traefik_reverse_proxy
 #      - prometheus_edge
@@ -48,3 +49,7 @@ services:
 #    external: true
 #  prometheus_edge:
 #    external: true
+
+#volumes:
+#  grafana-data:
+#    external: true

archive/dynamic-system/monitoring/influxdb/docker-compose.yml

@@ -1,13 +1,14 @@
 services:
   influxdb:
     profiles: ["monitoring","all","influxdb", "prometheus"]
-    image: influxdb:2.7
+    image: influxdb:2
     container_name: influxdb
     restart: unless-stopped
 #    env_file:
 #      - ${PROJECT_ROOT}/secrets/stack-secrets.env
     volumes:
-      - ${PROJECT_ROOT}/monitoring/influxdb:/var/lib/influxdb2
+#      - ${PROJECT_ROOT}/monitoring/influxdb:/var/lib/influxdb2
+      - ${DOCKER_DATABASES}/influxdb-data:/var/lib/influxdb2
     environment:
       DOCKER_INFLUXDB_INIT_MODE: ${INFLUXDB_INIT_MODE}
       DOCKER_INFLUXDB_INIT_USERNAME: ${INFLUXDB_INIT_USERNAME}
@@ -43,3 +44,7 @@ services:
 secrets:
   influxdb_init_password:
     file: ${PROJECT_ROOT}/secrets/influxdb_init_password.txt
+
+#volumes:
+#  influxdb-data:
+#    external: true

archive/dynamic-system/monitoring/node-red/docker-compose.yml

@@ -19,7 +19,8 @@ services:
 #    ports:
 #      - "1880:1880"
     volumes:
-      - ${PROJECT_ROOT}/monitoring/node-red/data:/data
+#      - ${PROJECT_ROOT}/monitoring/node-red/data:/data
+      - ${DOCKER_VOLUMES}/node-red-data:/data
       - ${PROJECT_ROOT}:/compose/docker:ro
       - /home/nixos/raspi:/compose/raspi:ro
 #      - ${PROJECT_ROOT}:/usr/src/node-red:ro
@@ -61,3 +62,6 @@ services:
       - "io.portainer.accesscontrol.public"
       - "traefik.docker.network=core_traefik"
       - "traefik.http.services.node-red.loadbalancer.server.port=1880"
+#volumes:
+#  node-red-data:
+#    external: true

archive/dynamic-system/monitoring/prometheus/docker-compose.yml

@@ -20,8 +20,10 @@ services:
 #      context: ${PROJECT_ROOT}/monitoring/prometheus
     volumes:
       - ${PROJECT_ROOT}/monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - ${PROJECT_ROOT}/monitoring/prometheus/data:/prometheus
+#      - ${PROJECT_ROOT}/monitoring/prometheus/data:/prometheus
-      - ${PROJECT_ROOT}/monitoring/prometheus/rules:/etc/prometheus/rules:ro
+#      - ${PROJECT_ROOT}/monitoring/prometheus/rules:/etc/prometheus/rules:ro
+      - ${DOCKER_VOLUMES}/prometheus/data:/prometheus
+      - ${DOCKER_VOLUMES}/prometheus/rules:/etc/prometheus/rules:ro
       - ${PROJECT_ROOT}/secrets/prometheus_kuma_basic_auth_password.txt:/run/secrets/prometheus_kuma_basic_auth_password:ro
 
     restart: unless-stopped
@@ -46,3 +48,9 @@ services:
       timeout: 10s
       retries: 3
       start_period: 30s
+
+#volumes:
+#  prometheus-data:
+#    external: true
+#  prometheus-rules:
+#    external: true

archive/dynamic-system/monitoring/prometheus/prometheus.yml

@@ -46,10 +46,10 @@ scrape_configs:
         labels:
           role: pihole
 
-#      - targets:
+      - targets:
-#          - server:9100
+          - server:9100
-#        labels:
+        labels:
-#          role: server
+          role: server
 
       - targets:
           - nix-cache:9100

archive/dynamic-system/monitoring/telegraf/docker-compose.yml

@@ -12,7 +12,9 @@ services:
       - no-new-privileges:true
     volumes:
       - ${PROJECT_ROOT}/monitoring/telegraf/telegraf.conf:/etc/telegraf/telegraf.conf:ro
-      - ${PROJECT_ROOT}/monitoring/node-red/data:/var/log/node-red:ro
+#      - ${PROJECT_ROOT}/monitoring/node-red/data:/var/log/node-red:ro
+
+      - ${DOCKER_VOLUMES}/node-red-data:/var/log/node-red:ro
     networks:
 #      - edge
       - monitor

core/authelia/docker-compose.yml

@@ -12,7 +12,10 @@ services:
 #      - AUTHELIA_SESSION_SECRET=${AUTHELIA_SESSION_SECRET}
 #      - AUTHELIA_STORAGE_ENCRYPTION_KEY=${AUTHELIA_STORAGE_ENCRYPTION_KEY}
     volumes:
-      - ${PROJECT_ROOT}/core/authelia:/config
+#      - ${PROJECT_ROOT}/core/authelia:/config
+      - ${DOCKER_VOLUMES}/authelia-data:/config/data
+      - ${PROJECT_ROOT}/core/authelia/configuration.yml:/config/configuration.yml:ro
+      - ${PROJECT_ROOT}/core/authelia/users_database.yml:/config/users_database.yml:ro
     networks:
 #      - reverse_proxy
       - traefik
@@ -28,3 +31,4 @@ services:
       - traefik.http.middlewares.authelia.forwardauth.trustForwardHeader=true
       - traefik.http.middlewares.authelia.forwardauth.authResponseHeaders=Remote-User,Remote-Groups
       - traefik.http.middlewares.authelia.forwardauth.maxResponseBodySize=2097152
+      - "wud.watch=true"

core/crowdsec/Dockerfile

@@ -1,6 +1,6 @@
 FROM crowdsecurity/crowdsec:latest
 
-COPY config/ /etc/crowdsec
+#COPY config/ /etc/crowdsec
 # Install required components at build time
-RUN cscli hub update && \
+#RUN cscli hub update && \
-    cscli collections install crowdsecurity/traefik --force 
+#    cscli collections install crowdsecurity/traefik --force 

core/crowdsec/docker-compose.yml

@@ -1,17 +1,23 @@
 services:
   crowdsec:
     profiles: ["core","all","crowdsec", "traefik"]
-#    image: crowdsecurity/crowdsec:latest
+    image: crowdsecurity/crowdsec:latest
-    build: ${PROJECT_ROOT}/core/crowdsec
+#    build: ${PROJECT_ROOT}/core/crowdsec
     container_name: crowdsec
     restart: always
     environment:
       - COLLECTIONS=crowdsecurity/traefik
 #      - CROWDSEC_LAPI_KEY=${CROWDSEC_LAPI_KEY}
     volumes:
-      - ${PROJECT_ROOT}/core/crowdsec/logs:/logs:ro
+#      - ${PROJECT_ROOT}/core/crowdsec/logs:/logs:ro
-      - ${PROJECT_ROOT}/core/crowdsec/data:/var/lib/crowdsec/data
+#      - ${PROJECT_ROOT}/core/crowdsec/data:/var/lib/crowdsec/data
-      - ${PROJECT_ROOT}/core/crowdsec/config:/etc/crowdsec
+#      - ${PROJECT_ROOT}/core/crowdsec/config:/etc/crowdsec
+      - ${DOCKER_VOLUMES}/crowdsec/logs:/logs:ro
+      - ${DOCKER_VOLUMES}/crowdsec/data:/var/lib/crowdsec/data
+#      - ${DOCKER_VOLUMES}/crowdsec/config.yaml:/etc/crowdsec/config.yaml
+#      - ${DOCKER_VOLUMES}/crowdsec/config:/etc/crowdsec
+      - ${PROJECT_ROOT}/core/crowdsec/acquis.d:/etc/crowdsec/acquis.d
+      - ${PROJECT_ROOT}/core/crowdsec/profiles.yaml:/etc/crowdsec/profiles.yaml
     networks:
 #      - reverse_proxy
       - traefik
@@ -21,3 +27,5 @@ services:
       timeout: 10s
       retries: 3
       start_period: 15s
+    labels:
+      - "wud.watch=true"

core/crowdsec/profiles.yaml

@@ -0,0 +1,29 @@
+name: default_ip_remediation
+#debug: true
+filters:
+ - Alert.Remediation == true && Alert.GetScope() == "Ip"
+decisions:
+ - type: ban
+   duration: 4h
+#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4)
+# notifications:
+#   - slack_default  # Set the webhook in /etc/crowdsec/notifications/slack.yaml before enabling this.
+#   - splunk_default # Set the splunk url and token in /etc/crowdsec/notifications/splunk.yaml before enabling this.
+#   - http_default   # Set the required http parameters in /etc/crowdsec/notifications/http.yaml before enabling this.
+#   - email_default  # Set the required email parameters in /etc/crowdsec/notifications/email.yaml before enabling this.
+on_success: break
+---
+name: default_range_remediation
+#debug: true
+filters:
+ - Alert.Remediation == true && Alert.GetScope() == "Range"
+decisions:
+ - type: ban
+   duration: 4h
+#duration_expr: Sprintf('%dh', (GetDecisionsCount(Alert.GetValue()) + 1) * 4)
+# notifications:
+#   - slack_default  # Set the webhook in /etc/crowdsec/notifications/slack.yaml before enabling this.
+#   - splunk_default # Set the splunk url and token in /etc/crowdsec/notifications/splunk.yaml before enabling this.
+#   - http_default   # Set the required http parameters in /etc/crowdsec/notifications/http.yaml before enabling this.
+#   - email_default  # Set the required email parameters in /etc/crowdsec/notifications/email.yaml before enabling this.
+on_success: break

core/docker-socket-proxy/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   docker-socket-proxy:
-    profiles: ["monitoring","all","docker-socket-proxy", "core", "traefik", "prometheus"]
+    profiles: ["monitoring","all","docker-socket-proxy", "core", "traefik", "uptime-kuma", "beszel", "wud", "apps"]
     image: tecnativa/docker-socket-proxy:latest
     container_name: docker-socket-proxy
     hostname: docker-socket-proxy
@@ -20,7 +20,7 @@ services:
       SYSTEM: 1
       SERVICES: 1
       SWARM: 1
-      NODES: 1
+      NODES: 1  
       SECRETS: 1
       TASKS: 1
       VERSION: 1
@@ -43,5 +43,10 @@ services:
     security_opt:
       - no-new-privileges:true
     networks:
-      - monitor
+#      - monitor
       - traefik
+      - docker-socket-proxy
+    ports:
+      - 2375:2375
+    labels:
+      - "wud.watch=true"

core/error-pages/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   error-pages:
     profiles: ["core","all","error-pages", "traefik"]
-    image: tarampampam/error-pages:3
+    image: tarampampam/error-pages:latest
     restart: always
     container_name: error-pages
     read_only: true
@@ -19,9 +19,10 @@ services:
       - "traefik.http.routers.error-pages-router.entrypoints=web"
       - "traefik.http.routers.error-pages-router.middlewares=error-pages-middleware"
       # "errors" middleware settings
-      - "traefik.http.middlewares.error-pages-middleware.errors.status=400-599"
+      - "traefik.http.middlewares.error-pages-middleware.errors.status=400,402-502,504-599"
       - "traefik.http.middlewares.error-pages-middleware.errors.service=error-pages-service"
       - "traefik.http.middlewares.error-pages-middleware.errors.query=/{status}.html"
       # define service properties
       - "traefik.http.services.error-pages-service.loadbalancer.server.port=8080"
       - "io.portainer.accesscontrol.public"
+      - "wud.watch=true"

core/traefik/docker-compose.yml

@@ -1,7 +1,7 @@
 services:
   traefik:
     profiles: ["core","all","traefik"]
-    image: traefik:3
+    image: traefik:latest
     container_name: traefik
     restart: always
     read_only: true
@@ -22,13 +22,18 @@ services:
  #     - ${PROJECT_ROOT}/secrets/stack-secrets.env
 
     volumes:
-      - ${PROJECT_ROOT}/core/traefik/data/letsencrypt:/letsencrypt
+#      - ${PROJECT_ROOT}/core/traefik/data/letsencrypt:/letsencrypt
-      - ${PROJECT_ROOT}/core/traefik/data/logs:/logs
+#      - ${PROJECT_ROOT}/core/traefik/data/logs:/logs
       - ${PROJECT_ROOT}/core/traefik/certs:/etc/traefik/certs:ro
       - ${PROJECT_ROOT}/core/traefik/dynamic.yml:/etc/traefik/dynamic.yml:ro
       - ${PROJECT_ROOT}/core/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
-      - ${PROJECT_ROOT}/core/traefik/data/plugins:/plugins-storage
+#      - ${PROJECT_ROOT}/core/traefik/data/plugins:/plugins-storage
-
+      - ${DOCKER_VOLUMES}/traefik-data/letsencrypt:/letsencrypt
+      - ${DOCKER_VOLUMES}/traefik-data/logs:/logs
+#      - ${PROJECT_ROOT}/core/traefik/certs:/etc/traefik/certs:ro
+#      - ${PROJECT_ROOT}/core/traefik/dynamic.yml:/etc/traefik/dynamic.yml:ro
+#      - ${PROJECT_ROOT}/core/traefik/traefik.yml:/etc/traefik/traefik.yml:ro
+      - ${DOCKER_VOLUMES}/traefik-data/plugins:/plugins-storage
     healthcheck:
       test: traefik healthcheck --ping
 
@@ -43,8 +48,10 @@ services:
       - "io.portainer.accesscontrol.public"
       - "traefik.docker.network=core_traefik"
       - "traefik.http.routers.traefik.observability.tracing=true"
-
+      - "wud.watch=true"
+      
     networks:
 #      - reverse_proxy
 #      - prometheus_edge
       - traefik
+      - docker-socket-proxy

core/traefik/dynamic.yml

@@ -32,6 +32,63 @@ http:
 #          - tracing-middleware@file
           - error-pages-middleware@docker
 
+  serversTransports:
+    insecureTransport:
+      insecureSkipVerify: true
+
+  routers:
+    pve1:
+      rule: "Host(`pve1.lan.ddnsgeek.com`)"
+      entryPoints:
+        - websecure
+      service: pve1
+      middlewares:
+        - authelia@docker
+      tls:
+        certResolver: myresolver
+        options: mtls-private-admin@file
+
+    pbs:
+      rule: "Host(`pbs.lan.ddnsgeek.com`)"
+      entryPoints:
+        - websecure
+      service: pbs
+      middlewares:
+        - authelia@docker
+      tls:
+        certResolver: myresolver
+        options: mtls-private-admin@file
+
+    pdm:
+      rule: "Host(`pdm.lan.ddnsgeek.com`)"
+      entryPoints:
+        - websecure
+      service: pdm
+      middlewares:
+        - authelia@docker
+      tls:
+        certResolver: myresolver
+        options: mtls-private-admin@file
+
+  services:
+    pve1:
+      loadBalancer:
+        serversTransport: insecureTransport
+        servers:
+          - url: "https://pve1.sweet.home:8006"
+
+    pbs:
+      loadBalancer:
+        serversTransport: insecureTransport
+        servers:
+          - url: "https://pbs.sweet.home:8007"
+
+    pdm:
+      loadBalancer:
+        serversTransport: insecureTransport
+        servers:
+          - url: "https://pdm.sweet.home:8443"
+
 tls:
   options:
     mtls-private-admin:

core/traefik/traefik.yml

@@ -1,5 +1,5 @@
 log:
-  level: INFO
+  level: DEBUG
 
 accessLog:
   filePath: /logs/access.log

default-environment.env

@@ -1,18 +1,25 @@
-PROJECT_ROOT=/home/nixos/docker
+DATA_ROOT=/mnt/docker
+PROJECT_ROOT=${DATA_ROOT}/config
 DOMAIN=lan.ddnsgeek.com
 TZ=Australia/Brisbane
 EMAIL=wayne.bennett@live.com
 SECRETS_ENV_FILE=${PROJECT_ROOT}/secrets/stack-secrets.env
 
+# Data locations 
+
+DOCKER_VOLUMES=${DATA_ROOT}/volumes
+DOCKER_DATABASES=${DATA_ROOT}/databases
+
 # Core
 CROWDSEC_COLLECTIONS=crowdsecurity/traefik
 ERROR_PAGES_TEMPLATE_NAME=app-down
 
+
 # Gitea
 GITEA_USER_UID=1000
 GITEA_USER_GID=1000
 GITEA_DB_TYPE=sqlite3
-GITEA_ROOT_URL=https://gitea.lan.ddnsgeek.com/
+GITEA_ROOT_URL=https://gitea.${DOMAIN}/
 # Generate a token in Gitea: Site Administration → Actions → Runners
 # or Repo → Settings → Actions → Runners
 GITEA_RUNNER_REGISTRATION_TOKEN=vYDNxzMvayREkXoaAR3x3UREkxQB2PU4eORzmkZ9
@@ -20,12 +27,13 @@ GITEA_RUNNER_NAME=docker-runner-01
 GITEA_RUNNER_LABELS=ubuntu-latest:docker://node:20-bookworm,ubuntu-22.04:docker://node:20-bookworm,linux:docker://node:20-bookworm,docker:docker://docker:cli
 
 # Grafana
-GRAFANA_ROOT_URL=https://grafana.lan.ddnsgeek.com/
+GRAFANA_ROOT_URL=https://grafana.${DOMAIN}/
 
 # Nextcloud
+NEXTCLOUD_DB_VERSION=11.4
 NEXTCLOUD_MYSQL_DATABASE=nextcloud
 NEXTCLOUD_MYSQL_HOST=nextcloud_db:3306
-NEXTCLOUD_TRUSTED_DOMAINS=nextcloud.lan.ddnsgeek.com
+NEXTCLOUD_TRUSTED_DOMAINS=nextcloud.${DOMAIN}
 NEXTCLOUD_OVERWRITEPROTOCOL=https
 NEXTCLOUD_OVERWRITECLIURL=https://${NEXTCLOUD_TRUSTED_DOMAINS}
 NEXTCLOUD_SMTP_HOST=smtp.gmail.com
@@ -35,17 +43,18 @@ NEXTCLOUD_SMTP_AUTHTYPE=login
 NEXTCLOUD_REDIS_HOST=redis
 NEXTCLOUD_REDIS_HOST_PORT=6379
 NEXTCLOUD_MARIADB_AUTO_UPGRADE=1
+NEXTCLOUD_DATA=/mnt/nextcloud-data
 
 # Passbolt
 PASSBOLT_MYSQL_RANDOM_ROOT_PASSWORD=true
 PASSBOLT_MYSQL_DATABASE=passbolt
 PASSBOLT_MYSQL_USER=passbolt
-PASSBOLT_APP_FULL_BASE_URL=https://passbolt.lan.ddnsgeek.com
+PASSBOLT_APP_FULL_BASE_URL=https://passbolt.${DOMAIN}
 PASSBOLT_DATASOURCES_DEFAULT_HOST=passbolt-db
 
 # Gramps
 GRAMPSWEB_TREE=main
-GRAMPSWEB_BASE_URL=https://familytree.lan.ddnsgeek.com
+GRAMPSWEB_BASE_URL=https://familytree.${DOMAIN}
 GRAMPSWEB_REGISTRATION_DISABLED=true
 GRAMPSWEB_EMAIL_HOST=smtp.gmail.com
 GRAMPSWEB_EMAIL_PORT=587
@@ -81,4 +90,14 @@ MTLS_BRIDGE_CLIENT_KEY=/certs/clients/office-pc/office-pc.key
 MTLS_BRIDGE_CLIENT_CERT=/certs/clients/office-pc/office-pc.crt
 MTLS_BRIDGE_TARGET_URL=http://node-red:1880
 MTLS_BRIDGE_ALLOWED_PATHS_FILE=
-MTLS_BRIDGE_CORS_ALLOW_ORIGIN=https://grafana.lan.ddnsgeek.com
+MTLS_BRIDGE_CORS_ALLOW_ORIGIN=https://grafana.${DOMAIN}
+
+# Whats up Docker
+WUD_WATCHER_LOCAL_CRON="0 */6 * * *"
+WUD_WATCHER_LOCAL_HOST=docker-socket-proxy
+WUD_WATCHER_LOCAL_WATCHBYDEFAULT=false
+WUD_WATCHER_RASPI_HOST=raspberrypi.tail13f623.ts.net
+WUD_TRIGGER_GOTIFY_URL=http://gotify
+WUD_TRIGGER_GOTIFY_TOKEN=ARPi3hyrAdEUSu6
+WUD_TRIGGER_GOTIFY_PRIORITY=5
+

default-network.yml

@@ -4,4 +4,5 @@ networks:
     ipam:
       config:
         - subnet: 172.21.0.0/16
-  monitor:
+#  monitor:
+  docker-socket-proxy:

monitoring/beszel/docker-compose.yml

@@ -0,0 +1,63 @@
+services:
+  beszel:
+    profiles: ["monitoring","all","beszel"]
+    image: henrygd/beszel:latest
+    container_name: beszel
+    restart: always
+    environment:
+      APP_URL: https://beszel.lan.ddnsgeek.com
+    ports:
+      - 8090:8090
+    volumes:
+      - ${DOCKER_VOLUMES}/beszel/data:/beszel_data
+      - ${DOCKER_VOLUMES}/beszel/socket:/beszel_socket
+    healthcheck:
+      test: ['CMD', '/beszel', 'health', '--url', 'http://localhost:8090']
+      interval: 120s
+      start_period: 10s
+      timeout: 5s
+    networks:
+      - traefik
+ #     - beszel
+ #     - monitor
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.beszel.rule=Host(`beszel.lan.ddnsgeek.com`)"
+      - "traefik.http.routers.beszel.entrypoints=websecure"
+      - "traefik.http.routers.beszel.tls=true"
+      - "traefik.http.routers.beszel.tls.certresolver=myresolver"
+      - "traefik.http.services.beszel.loadbalancer.server.port=8090"
+      - "io.portainer.accesscontrol.public"
+      - "traefik.docker.network=core_traefik"
+      - "traefik.http.routers.beszel.tls.options=mtls-private-admin@file"
+      - "wud.watch=true"
+
+  beszel-agent:
+    profiles: ["monitoring","all","beszel"]
+    image: henrygd/beszel-agent:latest
+    container_name: beszel-agent
+    restart: always
+    network_mode: host
+    volumes:
+      - ${DOCKER_VOLUMES}/beszel/agent:/var/lib/beszel-agent
+      - ${DOCKER_VOLUMES}/beszel/socket:/beszel_socket
+#      - /var/run/docker.sock:/var/run/docker.sock:ro
+#      - /mnt/docker/volumes/.beszel:/extra-filesystems/docker-volumes:ro
+    environment:
+      LISTEN: /beszel_socket/beszel.sock
+      HUB_URL: http://localhost:8090
+      KEY: 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFPR9kwtC4TAeTRu46A7+opZsYpxqkRJ+x/ZyB2GWCeG'
+      TOKEN: ee2f2df3-d79b-4695-8840-bc936dd7b121
+      DOCKER_HOST: tcp://localhost:2375
+#      EXTRA_FILESYSTEMS: /mnt/docker/volumes
+    healthcheck:
+      test: ['CMD', '/agent', 'health']
+      interval: 120s
+#    networks:
+#      - beszel
+#      - monitor
+    labels:
+      - "wud.watch=true"
+
+#networks:
+#  beszel:

monitoring/gotify/docker-compose.yml

@@ -7,7 +7,9 @@ services:
 #    env_file:
 #      - ${PROJECT_ROOT}/secrets/stack-secrets.env
     volumes:
-      - ${PROJECT_ROOT}/monitoring/gotify/data:/app/data
+#      - ${PROJECT_ROOT}/monitoring/gotify/data:/app/data
+      - ${DOCKER_VOLUMES}/gotify-data:/app/data
+#      - gotify-data:/app/data
     environment:
       - TZ=${TZ}
       - GOTIFY_DEFAULTUSER_NAME=${GOTIFY_DEFAULTUSER_NAME}
@@ -25,5 +27,9 @@ services:
       - "traefik.http.routers.gotify.tls.certresolver=myresolver"
       - "traefik.http.routers.gotify.tls.options=mtls-private-admin@file"
       - "traefik.http.services.gotify.loadbalancer.server.port=80"
+      - "wud.watch=true"
     ports:
       - 8080:80
+#volumes:
+#  gotify-data:
+#    external: true

monitoring/grafana/output.txt

@@ -1,630 +0,0 @@
-tempo  | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
-tempo  |   line 24: field compactor not found in type app.Config
-tempo  | level=info ts=2026-02-28T03:38:16.504252277Z caller=main.go:106 msg="Starting Tempo" version="(version=v2.10.1, branch=main, revision=ad20e59db)" target=all
-tempo  | level=info ts=2026-02-28T03:38:16.506107792Z caller=server.go:284 msg="server listening on addresses" http=[::]:3200 grpc=[::]:9095
-tempo  | level=info ts=2026-02-28T03:38:16.510179623Z caller=cache.go:55 msg="caches available to storage backend" parquet-footer=false bloom=false parquet-offset-idx=false parquet-column-idx=false trace-id-index=false parquet-page=false
-tempo  | level=info ts=2026-02-28T03:38:16.51381011Z caller=cache.go:55 msg="caches available to storage backend" parquet-footer=false bloom=false parquet-offset-idx=false parquet-column-idx=false trace-id-index=false parquet-page=false
-tempo  | level=warn ts=2026-02-28T03:38:16.515946735Z caller=modules.go:682 msg="Scheduler address is empty in single binary mode. Attempting automatic worker configuration." address=127.0.0.1:9095
-tempo  | level=info ts=2026-02-28T03:38:16.515962445Z caller=backendworker.go:80 msg="backend worker starting" worker_id=4777f4a6f30e
-tempo  | level=info ts=2026-02-28T03:38:16.518963163Z caller=frontend.go:88 msg="creating middleware in query frontend"
-tempo  | level=info ts=2026-02-28T03:38:16.521703999Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:38:16.521732753Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:38:16.521738063Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:38:16.52174193Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:38:16.521750647Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:38:16.521756559Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | ts=2026-02-28T03:38:16Z level=info msg="OTel Shim Logger Initialized" component=tempo
-tempo  | level=warn ts=2026-02-28T03:38:16.527971627Z caller=modules.go:397 msg="Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly." address=127.0.0.1:9095
-tempo  | level=info ts=2026-02-28T03:38:16.528090421Z caller=worker.go:108 msg="Starting querier worker connected to query-frontend" frontend=127.0.0.1:9095
-tempo  | level=warn ts=2026-02-28T03:38:16.528643676Z caller=modules.go:295 msg="metrics-generator is not configured." err="no metrics_generator.storage.path configured, metrics generator will be disabled"
-tempo  | level=info ts=2026-02-28T03:38:16.529205548Z caller=module_service.go:82 msg=starting module=cache-provider
-tempo  | level=info ts=2026-02-28T03:38:16.529235324Z caller=module_service.go:82 msg=starting module=optional-store
-tempo  | level=info ts=2026-02-28T03:38:16.529246304Z caller=module_service.go:82 msg=starting module=store
-tempo  | level=info ts=2026-02-28T03:38:16.529310496Z caller=module_service.go:82 msg=starting module=internal-server
-tempo  | level=info ts=2026-02-28T03:38:16.529363827Z caller=module_service.go:82 msg=starting module=server
-tempo  | level=info ts=2026-02-28T03:38:16.529446112Z caller=module_service.go:82 msg=starting module=overrides
-tempo  | level=info ts=2026-02-28T03:38:16.529477541Z caller=module_service.go:82 msg=starting module=memberlist-kv
-tempo  | level=info ts=2026-02-28T03:38:16.529508179Z caller=module_service.go:82 msg=starting module=overrides-api
-tempo  | level=info ts=2026-02-28T03:38:16.529517306Z caller=module_service.go:82 msg=starting module=live-store-ring
-tempo  | level=info ts=2026-02-28T03:38:16.529975612Z caller=module_service.go:82 msg=starting module=metrics-generator-ring
-tempo  | level=info ts=2026-02-28T03:38:16.530025126Z caller=ring.go:372 msg="ring doesn't exist in KV store yet"
-tempo  | level=info ts=2026-02-28T03:38:16.529669724Z caller=ring.go:372 msg="ring doesn't exist in KV store yet"
-tempo  | level=info ts=2026-02-28T03:38:16.530662269Z caller=module_service.go:82 msg=starting module=usage-report
-tempo  | level=info ts=2026-02-28T03:38:16.530661898Z caller=module_service.go:82 msg=starting module=partition-ring
-tempo  | level=info ts=2026-02-28T03:38:16.530845895Z caller=partition_ring_watcher.go:73 msg="partition ring doesn't exist in KV store yet"
-tempo  | level=info ts=2026-02-28T03:38:16.530989036Z caller=module_service.go:82 msg=starting module=live-store
-tempo  | level=info ts=2026-02-28T03:38:16.530999045Z caller=module_service.go:82 msg=starting module=query-frontend
-tempo  | level=info ts=2026-02-28T03:38:16.530994115Z caller=module_service.go:82 msg=starting module=querier
-tempo  | level=info ts=2026-02-28T03:38:16.531011028Z caller=module_service.go:82 msg=starting module=block-builder
-tempo  | level=info ts=2026-02-28T03:38:16.531029903Z caller=module_service.go:82 msg=starting module=backend-worker
-tempo  | level=info ts=2026-02-28T03:38:16.531033931Z caller=module_service.go:82 msg=starting module=distributor
-tempo  | level=info ts=2026-02-28T03:38:16.531047757Z caller=tempodb.go:633 msg="polling enabled" interval=5m0s blocklist_concurrency=50
-tempo  | level=info ts=2026-02-28T03:38:16.531594119Z caller=module_service.go:82 msg=starting module=metrics-generator
-tempo  | level=info ts=2026-02-28T03:38:16.531614337Z caller=module_service.go:82 msg=starting module=backend-scheduler
-tempo  | level=info ts=2026-02-28T03:38:16.531618996Z caller=backendscheduler.go:109 msg="backend scheduler starting"
-tempo  | level=info ts=2026-02-28T03:38:16.534070025Z caller=poller.go:249 msg="blocklist poll complete" seconds=0.002997762
-tempo  | level=info ts=2026-02-28T03:38:16.534101805Z caller=backendworker.go:183 msg="backend worker running"
-tempo  | ts=2026-02-28T03:38:16Z level=info msg="Starting GRPC server" component=tempo endpoint=[::]:4317
-tempo  | level=info ts=2026-02-28T03:38:16.535267337Z caller=worker.go:184 msg="adding connection" addr=127.0.0.1:9095
-tempo  | level=info ts=2026-02-28T03:38:16.536553646Z caller=worker.go:250 msg="total worker concurrency updated" totalConcurrency=20
-tempo  | ts=2026-02-28T03:38:16Z level=info msg="Starting HTTP server" component=tempo endpoint=[::]:4318
-tempo  | level=info ts=2026-02-28T03:38:16.538250531Z caller=backendscheduler.go:162 msg="backend scheduler running"
-tempo  | level=info ts=2026-02-28T03:38:16.538293953Z caller=app.go:214 msg="Tempo started"
-tempo  | level=info ts=2026-02-28T03:38:16.538315674Z caller=compaction.go:102 msg="compaction provider started"
-tempo  | level=info ts=2026-02-28T03:38:16.538325803Z caller=compaction.go:126 msg="compaction provider waiting for poll notification"
-tempo  | level=info ts=2026-02-28T03:38:16.538336152Z caller=retention.go:45 msg="retention provider started"
-tempo  | level=warn ts=2026-02-28T03:38:31.539059217Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.001238779s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:38:31.539333085Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=137.845499ms
-tempo  | level=warn ts=2026-02-28T03:38:47.706949743Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000246082s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:38:47.707402278Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=463.092807ms
-tempo  | level=warn ts=2026-02-28T03:39:03.90316531Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000234022s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:39:03.90345658Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=2.456866248s
-tempo  | level=info ts=2026-02-28T03:39:04.022018905Z caller=app.go:242 msg="=== received SIGINT/SIGTERM ===\n*** exiting"
-tempo  | level=info ts=2026-02-28T03:39:04.022341133Z caller=module_service.go:120 msg="module stopped" module=metrics-generator
-tempo  | level=info ts=2026-02-28T03:39:04.022392981Z caller=compaction.go:131 msg="compaction provider stopping"
-tempo  | level=info ts=2026-02-28T03:39:04.022421094Z caller=module_service.go:120 msg="module stopped" module=live-store
-tempo  | level=info ts=2026-02-28T03:39:04.022436252Z caller=backendscheduler.go:136 msg="stopping provider" provider=1
-tempo  | level=info ts=2026-02-28T03:39:04.022477952Z caller=retention.go:50 msg="retention provider stopping"
-tempo  | level=info ts=2026-02-28T03:39:04.022483221Z caller=backendscheduler.go:136 msg="stopping provider" provider=0
-tempo  | level=info ts=2026-02-28T03:39:04.022499622Z caller=backendscheduler.go:154 msg="all providers stopped"
-tempo  | level=info ts=2026-02-28T03:39:04.022514801Z caller=compaction.go:202 msg="compaction provider measure ticker stopping"
-tempo  | level=info ts=2026-02-28T03:39:04.023035023Z caller=module_service.go:120 msg="module stopped" module=distributor
-tempo  | level=info ts=2026-02-28T03:39:04.023143178Z caller=module_service.go:120 msg="module stopped" module=block-builder
-tempo  | level=warn ts=2026-02-28T03:39:04.023319751Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.481348854s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023397699Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485115377s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023419059Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485051917s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023433387Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485095891s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023447353Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485568464s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.02345667Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485115568s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.02346698Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485072817s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023476648Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485110168s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023486636Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485125947s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023501636Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485212231s msg=gRPC err="queue is stopped"
-tempo  | level=info ts=2026-02-28T03:39:04.023502868Z caller=module_service.go:120 msg="module stopped" module=optional-store
-tempo  | level=warn ts=2026-02-28T03:39:04.023508539Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485663543s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023539146Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485854102s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023551659Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485305717s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.02356266Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485318241s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.02357358Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485326275s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023587017Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.48529687s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023585083Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485268317s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023596484Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485287864s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T03:39:04.023606022Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485293264s msg=gRPC err="queue is stopped"
-tempo  | level=info ts=2026-02-28T03:39:04.023644765Z caller=module_service.go:120 msg="module stopped" module=query-frontend
-tempo  | level=info ts=2026-02-28T03:39:04.023659914Z caller=module_service.go:120 msg="module stopped" module=overrides-api
-tempo  | level=warn ts=2026-02-28T03:39:04.023515852Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.481666904s msg=gRPC err="queue is stopped"
-tempo  | level=info ts=2026-02-28T03:39:04.023804989Z caller=frontend.go:339 msg="received shutdown notification from querier" querier=4777f4a6f30e
-tempo  | level=info ts=2026-02-28T03:39:04.024020695Z caller=module_service.go:120 msg="module stopped" module=querier
-tempo  | level=info ts=2026-02-28T03:39:04.037362837Z caller=module_service.go:120 msg="module stopped" module=partition-ring
-tempo  | level=info ts=2026-02-28T03:39:04.037395037Z caller=module_service.go:120 msg="module stopped" module=metrics-generator-ring
-tempo  | level=info ts=2026-02-28T03:39:04.037503452Z caller=module_service.go:120 msg="module stopped" module=live-store-ring
-tempo  | level=info ts=2026-02-28T03:39:04.436087499Z caller=backendscheduler.go:201 msg="backend scheduler stopping"
-tempo  | level=info ts=2026-02-28T03:39:04.436141891Z caller=module_service.go:120 msg="module stopped" module=backend-scheduler
-tempo  | level=warn ts=2026-02-28T03:39:10.605069202Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=44.344µs msg=gRPC err="rpc error: code = Internal desc = nil job received"
-tempo  | level=error ts=2026-02-28T03:39:10.605239002Z caller=backendworker.go:454 msg="error calling scheduler" err="error getting next job: rpc error: code = Internal desc = nil job received" backoff=8.910321994s
-tempo  | level=info ts=2026-02-28T03:39:14.488350709Z caller=main.go:106 msg="Starting Tempo" version="(version=v2.10.1, branch=main, revision=ad20e59db)" target=all
-tempo  | level=info ts=2026-02-28T03:39:14.488776793Z caller=cache.go:55 msg="caches available to storage backend" parquet-footer=false bloom=false parquet-offset-idx=false parquet-column-idx=false trace-id-index=false parquet-page=false
-tempo  | level=info ts=2026-02-28T03:39:14.48970103Z caller=server.go:284 msg="server listening on addresses" http=[::]:3200 grpc=[::]:9095
-tempo  | level=info ts=2026-02-28T03:39:14.490096587Z caller=cache.go:55 msg="caches available to storage backend" parquet-footer=false bloom=false parquet-offset-idx=false parquet-column-idx=false trace-id-index=false parquet-page=false
-tempo  | level=warn ts=2026-02-28T03:39:14.492106423Z caller=modules.go:682 msg="Scheduler address is empty in single binary mode. Attempting automatic worker configuration." address=127.0.0.1:9095
-tempo  | level=info ts=2026-02-28T03:39:14.492149043Z caller=backendworker.go:80 msg="backend worker starting" worker_id=4777f4a6f30e
-tempo  | level=info ts=2026-02-28T03:39:14.492516217Z caller=frontend.go:88 msg="creating middleware in query frontend"
-tempo  | level=info ts=2026-02-28T03:39:14.492620283Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:39:14.492661331Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:39:14.492681839Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:39:14.492700414Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:39:14.492715484Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=info ts=2026-02-28T03:39:14.492728578Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
-tempo  | level=warn ts=2026-02-28T03:39:14.493177155Z caller=modules.go:397 msg="Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly." address=127.0.0.1:9095
-tempo  | level=info ts=2026-02-28T03:39:14.493339051Z caller=worker.go:108 msg="Starting querier worker connected to query-frontend" frontend=127.0.0.1:9095
-tempo  | ts=2026-02-28T03:39:14Z level=info msg="OTel Shim Logger Initialized" component=tempo
-tempo  | level=warn ts=2026-02-28T03:39:14.495105829Z caller=modules.go:295 msg="metrics-generator is not configured." err="no metrics_generator.storage.path configured, metrics generator will be disabled"
-tempo  | level=info ts=2026-02-28T03:39:14.495667199Z caller=module_service.go:82 msg=starting module=optional-store
-tempo  | level=info ts=2026-02-28T03:39:14.496941125Z caller=module_service.go:82 msg=starting module=internal-server
-tempo  | level=info ts=2026-02-28T03:39:14.497087782Z caller=module_service.go:82 msg=starting module=cache-provider
-tempo  | level=info ts=2026-02-28T03:39:14.497242835Z caller=module_service.go:82 msg=starting module=store
-tempo  | level=info ts=2026-02-28T03:39:14.497359636Z caller=module_service.go:82 msg=starting module=server
-tempo  | level=info ts=2026-02-28T03:39:14.497560034Z caller=module_service.go:82 msg=starting module=overrides
-tempo  | level=info ts=2026-02-28T03:39:14.49825639Z caller=module_service.go:82 msg=starting module=overrides-api
-tempo  | level=info ts=2026-02-28T03:39:14.498445827Z caller=module_service.go:82 msg=starting module=memberlist-kv
-tempo  | level=info ts=2026-02-28T03:39:14.498546206Z caller=module_service.go:82 msg=starting module=metrics-generator-ring
-tempo  | level=info ts=2026-02-28T03:39:14.498747066Z caller=ring.go:372 msg="ring doesn't exist in KV store yet"
-tempo  | level=info ts=2026-02-28T03:39:14.498816477Z caller=module_service.go:82 msg=starting module=live-store-ring
-tempo  | level=info ts=2026-02-28T03:39:14.498829101Z caller=module_service.go:82 msg=starting module=usage-report
-tempo  | level=info ts=2026-02-28T03:39:14.501249322Z caller=module_service.go:82 msg=starting module=query-frontend
-tempo  | level=info ts=2026-02-28T03:39:14.501273138Z caller=module_service.go:82 msg=starting module=backend-scheduler
-tempo  | level=info ts=2026-02-28T03:39:14.501281423Z caller=backendscheduler.go:109 msg="backend scheduler starting"
-tempo  | ts=2026-02-28T03:39:14Z level=info msg="Starting GRPC server" component=tempo endpoint=[::]:4317
-tempo  | ts=2026-02-28T03:39:14Z level=info msg="Starting HTTP server" component=tempo endpoint=[::]:4318
-tempo  | level=info ts=2026-02-28T03:39:14.501985883Z caller=ring.go:372 msg="ring doesn't exist in KV store yet"
-tempo  | level=info ts=2026-02-28T03:39:14.502072537Z caller=module_service.go:82 msg=starting module=partition-ring
-tempo  | level=info ts=2026-02-28T03:39:14.502101092Z caller=module_service.go:82 msg=starting module=backend-worker
-tempo  | level=info ts=2026-02-28T03:39:14.502101442Z caller=partition_ring_watcher.go:73 msg="partition ring doesn't exist in KV store yet"
-tempo  | level=info ts=2026-02-28T03:39:14.502119777Z caller=tempodb.go:633 msg="polling enabled" interval=5m0s blocklist_concurrency=50
-tempo  | level=info ts=2026-02-28T03:39:14.502157537Z caller=module_service.go:82 msg=starting module=metrics-generator
-tempo  | level=info ts=2026-02-28T03:39:14.502189098Z caller=module_service.go:82 msg=starting module=distributor
-tempo  | level=info ts=2026-02-28T03:39:14.502216168Z caller=poller.go:249 msg="blocklist poll complete" seconds=7.9791e-05
-tempo  | level=info ts=2026-02-28T03:39:14.502232729Z caller=backendworker.go:183 msg="backend worker running"
-tempo  | level=info ts=2026-02-28T03:39:14.502586187Z caller=module_service.go:82 msg=starting module=querier
-tempo  | level=info ts=2026-02-28T03:39:14.502630632Z caller=module_service.go:82 msg=starting module=live-store
-tempo  | level=info ts=2026-02-28T03:39:14.502740018Z caller=worker.go:184 msg="adding connection" addr=127.0.0.1:9095
-tempo  | level=info ts=2026-02-28T03:39:14.503119545Z caller=worker.go:250 msg="total worker concurrency updated" totalConcurrency=20
-tempo  | level=info ts=2026-02-28T03:39:14.504169348Z caller=module_service.go:82 msg=starting module=block-builder
-tempo  | level=info ts=2026-02-28T03:39:14.506269895Z caller=backendscheduler.go:162 msg="backend scheduler running"
-tempo  | level=info ts=2026-02-28T03:39:14.506302157Z caller=app.go:214 msg="Tempo started"
-tempo  | level=info ts=2026-02-28T03:39:14.5063145Z caller=compaction.go:102 msg="compaction provider started"
-tempo  | level=info ts=2026-02-28T03:39:14.506320521Z caller=compaction.go:126 msg="compaction provider waiting for poll notification"
-tempo  | level=info ts=2026-02-28T03:39:14.506328516Z caller=retention.go:45 msg="retention provider started"
-tempo  | level=warn ts=2026-02-28T03:39:29.508287056Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000581399s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:39:29.508572244Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=184.86286ms
-tempo  | level=warn ts=2026-02-28T03:39:45.052093517Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000133061s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:39:45.052342347Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=683.918431ms
-tempo  | level=warn ts=2026-02-28T03:40:01.617186207Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000246556s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:40:01.617444254Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=2.230026613s
-tempo  | level=warn ts=2026-02-28T03:40:21.738394906Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000616341s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:40:21.738738376Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=6.928753128s
-tempo  | level=warn ts=2026-02-28T03:41:00.303046594Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000840441s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:41:00.303269665Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=46.690005969s
-tempo  | level=warn ts=2026-02-28T03:42:09.822438562Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.001122933s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:42:09.822714043Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=55.06043985s
-tempo  | level=warn ts=2026-02-28T03:43:23.832379908Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000290546s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:43:23.83267209Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.189518589s
-tempo  | level=info ts=2026-02-28T03:44:14.51080544Z caller=poller.go:249 msg="blocklist poll complete" seconds=0.005466397
-tempo  | level=info ts=2026-02-28T03:44:14.510866645Z caller=compaction.go:142 msg="received empty tenant"
-tempo  | level=warn ts=2026-02-28T03:44:32.167179641Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000322598s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:44:32.167467976Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=59.343922215s
-tempo  | level=warn ts=2026-02-28T03:45:44.442347133Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000682317s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:45:44.442597837Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=55.235447404s
-tempo  | level=warn ts=2026-02-28T03:46:55.621855189Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000935062s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:46:55.622113698Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=57.996223179s
-tempo  | level=warn ts=2026-02-28T03:48:03.715877703Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000435945s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:48:03.716165717Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.637759938s
-tempo  | level=info ts=2026-02-28T03:49:14.505419519Z caller=poller.go:249 msg="blocklist poll complete" seconds=6.9141e-05
-tempo  | level=info ts=2026-02-28T03:49:14.505482128Z caller=compaction.go:142 msg="received empty tenant"
-tempo  | level=warn ts=2026-02-28T03:49:14.842107415Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000257167s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:49:14.842340856Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.010954393s
-tempo  | level=warn ts=2026-02-28T03:50:28.490571505Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000174599s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:50:28.490848819Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=56.688382421s
-tempo  | level=warn ts=2026-02-28T03:51:39.573560321Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000365972s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:51:39.573943555Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=59.697783583s
-tempo  | level=info ts=2026-02-28T03:52:47.227247458Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:47.230208333Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772249867&end=1772250767&scope=&q= (500) 1.277885ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:47.230708187Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772249867&end=1772250767&scope=&q= (500) 37.201µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:47.230898016Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.003652292 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:47.230940145Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772250767&limit=5000&start=1772249867" duration=4.515183ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:47.231008214Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 4.711182ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:55.046107322Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000358106s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:52:55.04635027Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=51.430516974s
-tempo  | level=info ts=2026-02-28T03:52:59.295759743Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.304296915Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 119.124µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.304832928Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 87.075µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.305196605Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0 duration_seconds=0.009433275 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.30531019Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&start=1772249879&tag=resource.service.name" duration=9.561277ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.305348642Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&tag=resource.service.name (500) 9.666326ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.305855119Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
-tempo  | level=info ts=2026-02-28T03:52:59.305855199Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.306587153Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 123.874µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.309044746Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 86.212µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.309177898Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.003320204 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.309206162Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&start=1772249879&tag=name" duration=3.356933ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.309253811Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 3.44016ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.309659288Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.003794991 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.309690206Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&start=1772249879&tag=status" duration=3.832993ms response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:52:59.364894892Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.365471181Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 102.253µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.365832854Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 53.241µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.366012043Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0 duration_seconds=0.001112984 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.366043211Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&start=1772249879&tag=resource.service.name" duration=1.157266ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.366073419Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&tag=resource.service.name (500) 1.236056ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.374586756Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
-tempo  | level=info ts=2026-02-28T03:52:59.374912371Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.375016809Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 103.666µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.375473521Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 57.058µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.375643352Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.001053461 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.375670744Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&start=1772249879&tag=name" duration=1.093838ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.375692655Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 1.189688ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.375839072Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.000923395 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.375865262Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&start=1772249879&tag=status" duration=953.653µs response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:52:59.430639585Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
-tempo  | level=info ts=2026-02-28T03:52:59.430641929Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.431784919Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 54.263µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.431784538Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 92.496µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.432085166Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 48.462µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.432098762Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 43.912µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.43220327Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0 duration_seconds=0.001559207 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.43222465Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.001584123 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.432240359Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&start=1772249879&tag=resource.service.name" duration=1.604001ms response_size=0 status=500
-tempo  | level=info ts=2026-02-28T03:52:59.432241802Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&start=1772249879&tag=name" duration=1.615052ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.4322619Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&tag=resource.service.name (500) 1.679032ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.432262761Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 1.68304ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.437788703Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
-tempo  | level=info ts=2026-02-28T03:52:59.438468568Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.00067752 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.438494086Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&start=1772249879&tag=status" duration=713.858µs response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:52:59.554734737Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.555499843Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 132.17µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.555983276Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 61.475µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.5561281Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001390236 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.556162785Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=resource.service.name" duration=1.445401ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.556196699Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.527045ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.570117374Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
-tempo  | level=info ts=2026-02-28T03:52:59.570117354Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.571119107Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 54.263µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.57120513Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.001085222 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.571256697Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=status" duration=1.154452ms response_size=130 status=200
-tempo  | level=warn ts=2026-02-28T03:52:59.571389749Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 48.662µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.57159701Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001479817 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.571636224Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=name" duration=1.535432ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.571666902Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.607797ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.604904298Z caller=search_handlers.go:225 msg="search request" tenant=single-tenant query={} range_seconds=3600 limit=20 spans_per_spanset=3
-tempo  | level=warn ts=2026-02-28T03:52:59.605700042Z caller=server.go:2286 msg="GET /querier/api/search?start=1772250479&end=1772250779&limit=20&spss=3&q=%7B%7D (500) 129.284µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.606087353Z caller=server.go:2286 msg="GET /querier/api/search?start=1772250479&end=1772250779&limit=20&spss=3&q=%7B%7D (500) 58.489µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.606187743Z caller=server.go:2286 msg="GET /querier/api/search?start=1772249879&end=1772250179&limit=20&spss=3&q=%7B%7D (500) 45.105µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.606188826Z caller=server.go:2286 msg="GET /querier/api/search?start=1772250179&end=1772250479&limit=20&spss=3&q=%7B%7D (500) 48.352µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.606414472Z caller=server.go:2286 msg="GET /querier/api/search?start=1772250179&end=1772250479&limit=20&spss=3&q=%7B%7D (500) 36.369µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.606446532Z caller=server.go:2286 msg="GET /querier/api/search?start=1772249879&end=1772250179&limit=20&spss=3&q=%7B%7D (500) 37.09µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.606580405Z caller=search_handlers.go:184 msg="search response - no resp" tenant=single-tenant duration_seconds=0.001723747 status_code=500 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.606602738Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/search?end=1772250779&limit=20&q=%7B%7D&start=1772247196" duration=1.748845ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.606631933Z caller=server.go:2286 msg="GET /api/search?end=1772250796&limit=20&q=%7B%7D&start=1772247196 (500) 1.819327ms Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.644836935Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
-tempo  | level=info ts=2026-02-28T03:52:59.644835402Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.646233004Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 58.19µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.646354614Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.001516506 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.64637912Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=status" duration=1.555229ms response_size=130 status=200
-tempo  | level=warn ts=2026-02-28T03:52:59.646532249Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 40.095µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.646628131Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001792909 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.646645163Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=name" duration=1.822845ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.646662535Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.877126ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.651077217Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.651670158Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 67.368µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.651989281Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 71.174µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.652129425Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001046398 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.652183919Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=resource.service.name" duration=1.105269ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.652218053Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.183345ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.834908067Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.83583599Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 180.561µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.83622182Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 53.04µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.836628539Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001717005 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.83666094Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=resource.service.name" duration=1.762972ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.836695936Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.845125ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.844675474Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
-tempo  | level=info ts=2026-02-28T03:52:59.844676606Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:52:59.845559664Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 94.889µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:52:59.846061863Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 46.718µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:52:59.846141675Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.001464176 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.846167934Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=status" duration=1.50822ms response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:52:59.846173765Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001499192 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:52:59.846215905Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=name" duration=1.557683ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:52:59.846233909Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.626814ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:54:06.387117532Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000155449s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:54:06.387384306Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.337824347s
-tempo  | level=info ts=2026-02-28T03:54:07.747790047Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:54:07.748598275Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249947&end=1772250847&q=%7B%7D (500) 118.444µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:54:07.74898729Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249947&end=1772250847&q=%7B%7D (500) 90.04µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.749157522Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001365771 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:54:07.749184953Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250847&limit=5000&q=%7B%7D&start=1772249947&tag=resource.service.name" duration=1.405937ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:54:07.749223236Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.496548ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.75446989Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:54:07.754987538Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772249947&end=1772250847&scope=&q= (500) 70.694µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.754546215Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:54:07.755500718Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772249947&end=1772250847&scope=&q= (500) 28.594µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.755823719Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.001353137 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:54:07.755849367Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772250847&limit=5000&start=1772249947" duration=1.403232ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:54:07.755871819Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 1.466611ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.756469869Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.001920669 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:54:07.756492221Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250847&limit=5000&q=%7B%7D&start=1772249947&tag=status" duration=1.96836ms response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:54:07.767792876Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:54:07.768349809Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249947&end=1772250847&q=%7B%7D (500) 87.937µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:54:07.768676455Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249947&end=1772250847&q=%7B%7D (500) 54.313µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.768857398Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001062668 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:54:07.768881633Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250847&limit=5000&q=%7B%7D&start=1772249947&tag=name" duration=1.097895ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:54:07.768904576Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.15937ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.805215311Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:54:07.80596677Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249947&end=1772250847&q=%7Bresource.service.name%3Dtraefik%7D (500) 89.989µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:54:07.806306162Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249947&end=1772250847&q=%7Bresource.service.name%3Dtraefik%7D (500) 61.466µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.806466906Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001251445 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:54:07.806493576Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250847&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772249947&tag=name" duration=1.294836ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:54:07.806545344Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.387872ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:54:07.810069784Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:54:07.8114648Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001388143 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:54:07.811500197Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250847&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772249947&tag=status" duration=1.443909ms response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:54:14.505512462Z caller=poller.go:249 msg="blocklist poll complete" seconds=6.3169e-05
-tempo  | level=info ts=2026-02-28T03:54:14.505573466Z caller=compaction.go:142 msg="received empty tenant"
-tempo  | level=warn ts=2026-02-28T03:55:17.230319246Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.00109829s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:55:17.23056488Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.331397281s
-tempo  | level=warn ts=2026-02-28T03:56:27.212672373Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000757225s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:56:27.212924711Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=54.995665223s
-tempo  | level=info ts=2026-02-28T03:57:35.447493981Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:57:35.448262373Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 82.466µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.448835105Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 57.639µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.448972244Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001477442 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.448997662Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251055&limit=5000&q=%7B%7D&start=1772250155&tag=resource.service.name" duration=1.516175ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:57:35.44902242Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.596868ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=error ts=2026-02-28T03:57:35.449625368Z caller=search_handlers.go:115 msg="search: could not create combiner" err="invalid TraceQL query: parse error at line 1, col 24: unknown identifier: traefik"
-tempo  | level=info ts=2026-02-28T03:57:35.449658121Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/search?end=1772251072&limit=20&q=%7Bresource.service.name%3Dtraefik%7D&start=1772247472" duration=2.110898ms response_size=0 status=400
-tempo  | level=info ts=2026-02-28T03:57:35.452827149Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:57:35.453724735Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 79.12µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.454072382Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 73.098µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.454754481Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001925969 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.454809205Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251055&limit=5000&q=%7B%7D&start=1772250155&tag=resource.service.name" duration=1.993728ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:57:35.454834763Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 2.066896ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.464569659Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
-tempo  | level=info ts=2026-02-28T03:57:35.464647065Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
-tempo  | level=info ts=2026-02-28T03:57:35.465734159Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:57:35.466270002Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250155&end=1772251055&scope=&q= (500) 51.658µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.466468006Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q= (500) 65.935µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.466467926Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q= (500) 68.098µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.466578975Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250155&end=1772251055&scope=&q= (500) 23.484µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.466683073Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.000949535 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.466703331Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772251055&limit=5000&start=1772250155" duration=1.013645ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:57:35.4667241Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 1.062919ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.466757072Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q= (500) 46.428µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.467000492Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
-tempo  | level=info ts=2026-02-28T03:57:35.467747053Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.003097213 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.467810633Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251055&limit=5000&start=1772250155&tag=name" duration=3.183225ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:57:35.467847663Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 3.248088ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.46786673Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q= (500) 42.22µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.468069162Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.003499453 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.468109167Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251055&limit=5000&start=1772250155&tag=name" duration=3.552883ms response_size=0 status=500
-tempo  | level=info ts=2026-02-28T03:57:35.468133974Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.001130987 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=warn ts=2026-02-28T03:57:35.468138273Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 3.643706ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.468153011Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251055&limit=5000&start=1772250155&tag=status" duration=1.158009ms response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:57:35.48429799Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:57:35.484956764Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q=%7Bresource.service.name%3Dtraefik%7D (500) 104.417µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.485242635Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q=%7Bresource.service.name%3Dtraefik%7D (500) 50.867µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.485397968Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001097994 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.485429208Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251055&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250155&tag=name" duration=1.142479ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:57:35.485458763Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.215597ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.4890928Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:57:35.489884336Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.00078816 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.489910275Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251055&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250155&tag=status" duration=825.11µs response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:57:35.533791838Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
-tempo  | level=info ts=2026-02-28T03:57:35.534540833Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.000745989 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.534565951Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251055&limit=5000&start=1772250155&tag=status" duration=783.882µs response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:57:35.563874004Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:57:35.564416319Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 78.509µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:35.564750501Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 53.641µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.564892479Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001016812 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.564919931Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251055&limit=5000&q=%7B%7D&start=1772250155&tag=resource.service.name" duration=1.055645ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:57:35.564948625Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.131119ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.573722094Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:57:35.573743404Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:57:35.574654356Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q=%7Bresource.service.name%3Dtraefik%7D (500) 72.246µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.574864683Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001118294 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.574890291Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251055&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250155&tag=status" duration=1.149151ms response_size=130 status=200
-tempo  | level=warn ts=2026-02-28T03:57:35.575274137Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q=%7Bresource.service.name%3Dtraefik%7D (500) 91.864µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:57:35.575494063Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.00177241 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:57:35.575515052Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251055&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250155&tag=name" duration=1.803948ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:57:35.575546121Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.873881ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:57:36.820760121Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000451663s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:57:36.821000996Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=58.35515471s
-tempo  | level=info ts=2026-02-28T03:58:32.220909942Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:32.222185944Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250212&end=1772251112&q=%7B%7D (500) 113.223µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:32.222648478Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250212&end=1772251112&q=%7B%7D (500) 83.147µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:32.222821986Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001911723 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:32.22293528Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251112&limit=5000&q=%7B%7D&start=1772250212&tag=resource.service.name" duration=2.02115ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:32.223024188Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 2.182405ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:32.22766645Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:32.229001193Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250212&end=1772251112&q=%7Bresource.service.name%3Dtraefik%7D (500) 95.671µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:32.22933287Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250212&end=1772251112&q=%7Bresource.service.name%3Dtraefik%7D (500) 68.77µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:32.229489626Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001819599 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:32.229537877Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251112&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250212&tag=name" duration=1.893367ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:32.22980944Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 2.228621ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:32.230137901Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:58:32.230151897Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:32.231243921Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250212&end=1772251112&scope=&q= (500) 42.661µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:32.231768121Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250212&end=1772251112&scope=&q= (500) 34.455µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:32.231879883Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.001727085 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:32.231927694Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772251112&limit=5000&start=1772250212" duration=1.798167ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:32.231954043Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 1.85241ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:32.232134333Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001961206 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:32.232167276Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251112&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250212&tag=status" duration=2.034485ms response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:58:32.285507543Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
-tempo  | level=info ts=2026-02-28T03:58:32.286381996Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.000871818 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:32.286407925Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251112&limit=5000&q=%7B%7D&start=1772250212&tag=status" duration=916.472µs response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:58:32.294280212Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:32.29475576Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250212&end=1772251112&q=%7B%7D (500) 84.451µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:32.295220318Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250212&end=1772251112&q=%7B%7D (500) 54.984µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:32.295346977Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001065514 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:32.295370862Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251112&limit=5000&q=%7B%7D&start=1772250212&tag=name" duration=1.109325ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:32.295423582Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.210067ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:45.987186027Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:45.987781302Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250225&end=1772251125&q=%7B%7D (500) 89.659µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:45.988143777Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250225&end=1772251125&q=%7B%7D (500) 45.104µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:45.988296276Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001109726 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:45.988326693Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251125&limit=5000&q=%7B%7D&start=1772250225&tag=resource.service.name" duration=1.153651ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:45.988367049Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.244051ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:45.993858696Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:45.994205922Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250225&end=1772251125&scope=&q= (500) 48.631µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:45.994567956Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250225&end=1772251125&scope=&q= (500) 25.388µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:45.994712711Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.00085203 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:45.994736776Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772251125&limit=5000&start=1772250225" duration=901.995µs response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:45.994761041Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 974.271µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:45.998993379Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:45.999499025Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250225&end=1772251125&q=%7B%7D (500) 96.371µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:45.999960998Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250225&end=1772251125&q=%7B%7D (500) 58.551µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:46.000109869Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001115388 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:46.000191904Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251125&limit=5000&q=%7B%7D&start=1772250225&tag=name" duration=1.204145ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:46.000245305Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.295628ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:46.005858884Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
-tempo  | level=info ts=2026-02-28T03:58:46.006440202Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.000578262 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:46.00646579Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251126&limit=5000&q=%7B%7D&start=1772250226&tag=status" duration=617.327µs response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:58:46.038874293Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:46.039289979Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250226&end=1772251126&q=%7Bresource.service.name%3Dtraefik%7D (500) 80.222µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:46.039656171Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250226&end=1772251126&q=%7Bresource.service.name%3Dtraefik%7D (500) 69.872µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:46.039798289Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.00092052 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:46.039821953Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251126&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250226&tag=name" duration=957.809µs response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:46.039844978Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.035867ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:46.049178986Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:58:46.050280237Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001097534 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:46.050326674Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251126&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250226&tag=status" duration=1.151936ms response_size=130 status=200
-tempo  | level=info ts=2026-02-28T03:58:48.785841897Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:48.786673308Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250228&end=1772251128&q=%7B%7D (500) 100.36µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:48.786993352Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250228&end=1772251128&q=%7B%7D (500) 56.375µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:48.787118058Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.00127548 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:48.787142945Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251128&limit=5000&q=%7B%7D&start=1772250228&tag=resource.service.name" duration=1.314524ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:48.787165358Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.390929ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:48.794066079Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:58:48.794090214Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:58:48.795203578Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001109156 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=warn ts=2026-02-28T03:58:48.795226541Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250228&end=1772251128&q=%7Bresource.service.name%3Dtraefik%7D (500) 62.438µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:48.795230719Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251128&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250228&tag=status" duration=1.145034ms response_size=130 status=200
-tempo  | level=warn ts=2026-02-28T03:58:48.795561594Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250228&end=1772251128&q=%7Bresource.service.name%3Dtraefik%7D (500) 70.834µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:48.795744581Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001676498 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:48.795765059Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251128&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250228&tag=name" duration=1.710554ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:48.79579207Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.790703ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=error ts=2026-02-28T03:58:48.83299873Z caller=search_handlers.go:115 msg="search: could not create combiner" err="invalid TraceQL query: parse error at line 1, col 24: unknown identifier: traefik"
-tempo  | level=info ts=2026-02-28T03:58:48.83304106Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/search?end=1772251146&limit=20&q=%7Bresource.service.name%3Dtraefik%7D&start=1772247546" duration=90.15µs response_size=0 status=400
-tempo  | level=info ts=2026-02-28T03:58:48.874206161Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:48.87493086Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250228&end=1772251128&q=%7B%7D (500) 80.572µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:48.875247888Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250228&end=1772251128&q=%7B%7D (500) 64.743µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:48.875412961Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001204506 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:48.875441434Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251128&limit=5000&q=%7B%7D&start=1772250228&tag=resource.service.name" duration=1.249171ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:48.875475068Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.333641ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:48.883983787Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:58:48.883976754Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:48.88471521Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250228&end=1772251128&q=%7Bresource.service.name%3Dtraefik%7D (500) 98.757µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:48.8851653Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001185801 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:48.885194765Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251128&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250228&tag=status" duration=1.232518ms response_size=130 status=200
-tempo  | level=warn ts=2026-02-28T03:58:48.88522882Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250228&end=1772251128&q=%7Bresource.service.name%3Dtraefik%7D (500) 64.813µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:48.885360759Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.00137542 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:48.885386357Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251128&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250228&tag=name" duration=1.404595ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:48.885410463Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.467232ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:49.014415446Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:49.015154923Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250229&end=1772251129&q=%7B%7D (500) 70.854µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:49.015511007Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250229&end=1772251129&q=%7B%7D (500) 65.524µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:49.015653155Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001236126 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:49.015676499Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251129&limit=5000&q=%7B%7D&start=1772250229&tag=resource.service.name" duration=1.27538ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:49.015698921Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.347867ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:49.024025686Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T03:58:49.024063037Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T03:58:49.024960494Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250229&end=1772251129&q=%7Bresource.service.name%3Dtraefik%7D (500) 119.997µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:49.025149951Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001083918 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:49.02517548Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251129&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250229&tag=status" duration=1.115689ms response_size=130 status=200
-tempo  | level=warn ts=2026-02-28T03:58:49.02544081Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250229&end=1772251129&q=%7Bresource.service.name%3Dtraefik%7D (500) 98.416µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T03:58:49.025664895Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001639639 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T03:58:49.025713907Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251129&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250229&tag=name" duration=1.700363ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T03:58:49.025744665Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.778792ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T03:58:49.467994461Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000736018s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T03:58:49.468248942Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=51.485834687s
-tempo  | level=info ts=2026-02-28T03:59:14.505516299Z caller=poller.go:249 msg="blocklist poll complete" seconds=9.005e-05
-tempo  | level=info ts=2026-02-28T03:59:14.505593114Z caller=compaction.go:142 msg="received empty tenant"
-tempo  | level=warn ts=2026-02-28T04:00:00.62141836Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000885343s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T04:00:00.62175716Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.674604584s
-tempo  | level=warn ts=2026-02-28T04:01:09.533862097Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.00112131s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T04:01:09.534125144Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=59.264069146s
-tempo  | level=info ts=2026-02-28T04:01:29.941563869Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=info ts=2026-02-28T04:01:29.941563849Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query="{status=ok}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T04:01:29.943318265Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250389&end=1772251289&q=%7Bstatus%3Dok%7D (500) 110.207µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:01:29.943868216Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250389&end=1772251289&q=%7Bstatus%3Dok%7D (500) 61.116µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:29.944076008Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query="{status=ok}" range_seconds=0 duration_seconds=0.002508052 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:29.944123869Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251289&limit=5000&q=%7Bstatus%3Dok%7D&start=1772250389&tag=resource.service.name" duration=2.576181ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T04:01:29.944211835Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7Bstatus%3Dok%7D&tag=resource.service.name (500) 2.752084ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:29.944825355Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.003260233 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:29.944849992Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251289&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250389&tag=status" duration=3.301083ms response_size=130 status=200
-tempo  | level=info ts=2026-02-28T04:01:29.94544159Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik && status=ok}" range_seconds=0
-tempo  | level=info ts=2026-02-28T04:01:29.94544145Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T04:01:29.946054078Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250389&end=1772251289&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D (500) 79.67µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:01:29.946098603Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250389&end=1772251289&q=%7Bresource.service.name%3Dtraefik%7D (500) 49.093µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:01:29.946857016Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250389&end=1772251289&q=%7Bresource.service.name%3Dtraefik%7D (500) 70.944µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:01:29.946905057Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250389&end=1772251289&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D (500) 61.215µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:29.947220684Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik && status=ok}" range_seconds=0 duration_seconds=0.001777269 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:29.947235412Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001791837 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:29.947247615Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251289&limit=5000&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D&start=1772250389&tag=name" duration=1.812708ms response_size=0 status=500
-tempo  | level=info ts=2026-02-28T04:01:29.947258776Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251289&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250389&tag=name" duration=1.826764ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T04:01:29.947270758Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%20%26%26%20status%3Dok%7D&tag=name (500) 1.87809ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:01:29.947276138Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.88339ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:29.960342984Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
-tempo  | level=warn ts=2026-02-28T04:01:29.960838761Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250389&end=1772251289&scope=&q= (500) 39.715µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:01:29.961101268Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250389&end=1772251289&scope=&q= (500) 32.621µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:29.961265038Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.000922725 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:29.96129317Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772251289&limit=5000&start=1772250389" duration=973.351µs response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T04:01:29.961326062Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 1.049643ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:33.604410488Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query="{status=ok}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T04:01:33.604981117Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250393&end=1772251293&q=%7Bstatus%3Dok%7D (500) 84.439µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:01:33.605337871Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250393&end=1772251293&q=%7Bstatus%3Dok%7D (500) 52.799µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:33.605460594Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query="{status=ok}" range_seconds=0 duration_seconds=0.001049545 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:33.605484029Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251293&limit=5000&q=%7Bstatus%3Dok%7D&start=1772250393&tag=resource.service.name" duration=1.086664ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T04:01:33.605510328Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7Bstatus%3Dok%7D&tag=resource.service.name (500) 1.170984ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:33.613938413Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik && status=ok}" range_seconds=0
-tempo  | level=info ts=2026-02-28T04:01:33.614308312Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
-tempo  | level=warn ts=2026-02-28T04:01:33.615192815Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250393&end=1772251293&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D (500) 106.932µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:33.615309065Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.000996795 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:33.615334522Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251293&limit=5000&q=%7B%7D&start=1772250393&tag=status" duration=1.027463ms response_size=130 status=200
-tempo  | level=warn ts=2026-02-28T04:01:33.615558405Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250393&end=1772251293&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D (500) 66.005µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:33.615953593Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik && status=ok}" range_seconds=0 duration_seconds=0.002014959 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:33.615976316Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251293&limit=5000&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D&start=1772250393&tag=name" duration=2.050226ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T04:01:33.615996604Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%20%26%26%20status%3Dok%7D&tag=name (500) 2.11554ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:33.684220173Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{status=ok}" range_seconds=0
-tempo  | level=warn ts=2026-02-28T04:01:33.684911631Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250393&end=1772251293&q=%7Bstatus%3Dok%7D (500) 74.18µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:01:33.685479776Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250393&end=1772251293&q=%7Bstatus%3Dok%7D (500) 116.079µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=info ts=2026-02-28T04:01:33.685639858Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{status=ok}" range_seconds=0 duration_seconds=0.001419274 inspected_bytes=0 request_throughput=0 error=null
-tempo  | level=info ts=2026-02-28T04:01:33.685697567Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251293&limit=5000&q=%7Bstatus%3Dok%7D&start=1772250393&tag=name" duration=1.488815ms response_size=0 status=500
-tempo  | level=warn ts=2026-02-28T04:01:33.685734456Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bstatus%3Dok%7D&tag=name (500) 1.576119ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
-tempo  | level=warn ts=2026-02-28T04:02:21.401664876Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000979208s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T04:02:21.401988659Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=58.963135457s
-tempo  | level=info ts=2026-02-28T04:02:54.506996729Z caller=reporter.go:272 msg="reporting cluster stats" date=2026-02-28T04:02:54.506995536Z
-tempo  | level=warn ts=2026-02-28T04:03:30.924861907Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000152254s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T04:03:30.925110638Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=52.941417365s
-tempo  | level=info ts=2026-02-28T04:04:14.505701243Z caller=poller.go:249 msg="blocklist poll complete" seconds=6.7217e-05
-tempo  | level=info ts=2026-02-28T04:04:14.505766966Z caller=compaction.go:142 msg="received empty tenant"
-tempo  | level=warn ts=2026-02-28T04:04:38.96769366Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000596524s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T04:04:38.967991853Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.427260392s
-tempo  | level=warn ts=2026-02-28T04:05:54.649236109Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000145477s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T04:05:54.649500619Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=59.111777322s
-tempo  | level=warn ts=2026-02-28T04:07:09.031108125Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000783057s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T04:07:09.031392622Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.643106953s
-tempo  | level=warn ts=2026-02-28T04:08:16.800283365Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.00088407s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
-tempo  | level=error ts=2026-02-28T04:08:16.800508742Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=55.946244881s
-tempo  | level=info ts=2026-02-28T04:08:55.544817027Z caller=app.go:242 msg="=== received SIGINT/SIGTERM ===\n*** exiting"
-tempo  | level=info ts=2026-02-28T04:08:55.545170515Z caller=module_service.go:120 msg="module stopped" module=metrics-generator
-tempo  | level=info ts=2026-02-28T04:08:55.545143745Z caller=backendscheduler.go:136 msg="stopping provider" provider=0
-tempo  | level=info ts=2026-02-28T04:08:55.545422351Z caller=module_service.go:120 msg="module stopped" module=live-store
-tempo  | level=info ts=2026-02-28T04:08:55.545509286Z caller=module_service.go:120 msg="module stopped" module=optional-store
-tempo  | level=info ts=2026-02-28T04:08:55.54555359Z caller=module_service.go:120 msg="module stopped" module=block-builder
-tempo  | level=warn ts=2026-02-28T04:08:55.545588966Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.03861475s msg=gRPC err="queue is stopped"
-tempo  | level=info ts=2026-02-28T04:08:55.545861381Z caller=compaction.go:142 msg="received empty tenant"
-tempo  | level=info ts=2026-02-28T04:08:55.545877252Z caller=compaction.go:131 msg="compaction provider stopping"
-tempo  | level=info ts=2026-02-28T04:08:55.545888232Z caller=compaction.go:202 msg="compaction provider measure ticker stopping"
-tempo  | level=warn ts=2026-02-28T04:08:55.546132804Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039274788s msg=gRPC err="queue is stopped"
-tempo  | level=info ts=2026-02-28T04:08:55.546235459Z caller=module_service.go:120 msg="module stopped" module=query-frontend
-tempo  | level=info ts=2026-02-28T04:08:55.546254915Z caller=module_service.go:120 msg="module stopped" module=overrides-api
-tempo  | level=info ts=2026-02-28T04:08:55.546299981Z caller=backendscheduler.go:136 msg="stopping provider" provider=1
-tempo  | level=info ts=2026-02-28T04:08:55.546316602Z caller=backendscheduler.go:154 msg="all providers stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546425368Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039448657s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546452799Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039388243s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546466765Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039674203s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546477736Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.03989506s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546492986Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039421806s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546505519Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039507939s msg=gRPC err="queue is stopped"
-tempo  | level=info ts=2026-02-28T04:08:55.546601099Z caller=module_service.go:120 msg="module stopped" module=distributor
-tempo  | level=info ts=2026-02-28T04:08:55.546634613Z caller=frontend.go:339 msg="received shutdown notification from querier" querier=4777f4a6f30e
-tempo  | level=warn ts=2026-02-28T04:08:55.546518493Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039596125s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.54669618Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039697237s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546723351Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039625551s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546745622Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039909457s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546760261Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039945426s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546772995Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039820781s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546783324Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039696415s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546810495Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039784952s msg=gRPC err="queue is stopped"
-tempo  | level=info ts=2026-02-28T04:08:55.546674528Z caller=retention.go:50 msg="retention provider stopping"
-tempo  | level=warn ts=2026-02-28T04:08:55.546819051Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039939374s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546827608Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.040065051s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546845331Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.03982558s msg=gRPC err="queue is stopped"
-tempo  | level=warn ts=2026-02-28T04:08:55.546880507Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039980712s msg=gRPC err="queue is stopped"
-tempo  | level=info ts=2026-02-28T04:08:55.547572095Z caller=module_service.go:120 msg="module stopped" module=querier
-tempo  | level=info ts=2026-02-28T04:08:55.561796142Z caller=module_service.go:120 msg="module stopped" module=partition-ring
-tempo  | level=info ts=2026-02-28T04:08:55.561812693Z caller=module_service.go:120 msg="module stopped" module=metrics-generator-ring
-tempo  | level=info ts=2026-02-28T04:08:55.662029127Z caller=module_service.go:120 msg="module stopped" module=live-store-ring
-tempo  | level=info ts=2026-02-28T04:08:55.979001962Z caller=backendscheduler.go:201 msg="backend scheduler stopping"
-tempo  | level=info ts=2026-02-28T04:08:55.979054602Z caller=module_service.go:120 msg="module stopped" module=backend-scheduler
-tempo  | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
-tempo  |   line 36: field enabled not found in type frontend.SearchConfig
-tempo  | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
-tempo  |   line 36: field enabled not found in type frontend.SearchConfig
-tempo  |   line 37: field search_enabled not found in type app.Config
-tempo  | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
-tempo  |   line 27: field compactor not found in type app.Config
-tempo  |   line 36: field enabled not found in type frontend.SearchConfig
-tempo  | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
-tempo  |   line 27: field compactor not found in type app.Config
-tempo  |   line 36: field enabled not found in type frontend.SearchConfig

monitoring/node-red/data/.config.nodes.json

@@ -1,430 +0,0 @@
-{
-    "node-red": {
-        "name": "node-red",
-        "version": "5.0.0",
-        "local": false,
-        "user": false,
-        "nodes": {
-            "junction": {
-                "name": "junction",
-                "types": [
-                    "junction"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/05-junction.js"
-            },
-            "inject": {
-                "name": "inject",
-                "types": [
-                    "inject"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/20-inject.js"
-            },
-            "debug": {
-                "name": "debug",
-                "types": [
-                    "debug"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/21-debug.js"
-            },
-            "complete": {
-                "name": "complete",
-                "types": [
-                    "complete"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/24-complete.js"
-            },
-            "catch": {
-                "name": "catch",
-                "types": [
-                    "catch"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/25-catch.js"
-            },
-            "status": {
-                "name": "status",
-                "types": [
-                    "status"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/25-status.js"
-            },
-            "link": {
-                "name": "link",
-                "types": [
-                    "link in",
-                    "link out",
-                    "link call"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/60-link.js"
-            },
-            "comment": {
-                "name": "comment",
-                "types": [
-                    "comment"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/90-comment.js"
-            },
-            "global-config": {
-                "name": "global-config",
-                "types": [
-                    "global-config"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/91-global-config.js"
-            },
-            "unknown": {
-                "name": "unknown",
-                "types": [
-                    "unknown"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/98-unknown.js"
-            },
-            "function": {
-                "name": "function",
-                "types": [
-                    "function"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/10-function.js"
-            },
-            "switch": {
-                "name": "switch",
-                "types": [
-                    "switch"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/10-switch.js"
-            },
-            "change": {
-                "name": "change",
-                "types": [
-                    "change"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/15-change.js"
-            },
-            "range": {
-                "name": "range",
-                "types": [
-                    "range"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/16-range.js"
-            },
-            "template": {
-                "name": "template",
-                "types": [
-                    "template"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/80-template.js"
-            },
-            "delay": {
-                "name": "delay",
-                "types": [
-                    "delay"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/89-delay.js"
-            },
-            "trigger": {
-                "name": "trigger",
-                "types": [
-                    "trigger"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/89-trigger.js"
-            },
-            "exec": {
-                "name": "exec",
-                "types": [
-                    "exec"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/90-exec.js"
-            },
-            "rbe": {
-                "name": "rbe",
-                "types": [
-                    "rbe"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/rbe.js"
-            },
-            "tls": {
-                "name": "tls",
-                "types": [
-                    "tls-config"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/05-tls.js"
-            },
-            "httpproxy": {
-                "name": "httpproxy",
-                "types": [
-                    "http proxy"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/06-httpproxy.js"
-            },
-            "mqtt": {
-                "name": "mqtt",
-                "types": [
-                    "mqtt in",
-                    "mqtt out",
-                    "mqtt-broker"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/10-mqtt.js"
-            },
-            "httpin": {
-                "name": "httpin",
-                "types": [
-                    "http in",
-                    "http response"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/21-httpin.js"
-            },
-            "httprequest": {
-                "name": "httprequest",
-                "types": [
-                    "http request"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/21-httprequest.js"
-            },
-            "websocket": {
-                "name": "websocket",
-                "types": [
-                    "websocket in",
-                    "websocket out",
-                    "websocket-listener",
-                    "websocket-client"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/22-websocket.js"
-            },
-            "tcpin": {
-                "name": "tcpin",
-                "types": [
-                    "tcp in",
-                    "tcp out",
-                    "tcp request"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/31-tcpin.js"
-            },
-            "udp": {
-                "name": "udp",
-                "types": [
-                    "udp in",
-                    "udp out"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/32-udp.js"
-            },
-            "CSV": {
-                "name": "CSV",
-                "types": [
-                    "csv"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-CSV.js"
-            },
-            "HTML": {
-                "name": "HTML",
-                "types": [
-                    "html"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-HTML.js"
-            },
-            "JSON": {
-                "name": "JSON",
-                "types": [
-                    "json"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-JSON.js"
-            },
-            "XML": {
-                "name": "XML",
-                "types": [
-                    "xml"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-XML.js"
-            },
-            "YAML": {
-                "name": "YAML",
-                "types": [
-                    "yaml"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-YAML.js"
-            },
-            "split": {
-                "name": "split",
-                "types": [
-                    "split",
-                    "join"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/17-split.js"
-            },
-            "sort": {
-                "name": "sort",
-                "types": [
-                    "sort"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/18-sort.js"
-            },
-            "batch": {
-                "name": "batch",
-                "types": [
-                    "batch"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/19-batch.js"
-            },
-            "file": {
-                "name": "file",
-                "types": [
-                    "file",
-                    "file in"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/storage/10-file.js"
-            },
-            "watch": {
-                "name": "watch",
-                "types": [
-                    "watch"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/storage/23-watch.js"
-            }
-        }
-    }
-}

monitoring/node-red/data/.config.nodes.json.backup

@@ -1,430 +0,0 @@
-{
-    "node-red": {
-        "name": "node-red",
-        "version": "4.1.11",
-        "local": false,
-        "user": false,
-        "nodes": {
-            "junction": {
-                "name": "junction",
-                "types": [
-                    "junction"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/05-junction.js"
-            },
-            "inject": {
-                "name": "inject",
-                "types": [
-                    "inject"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/20-inject.js"
-            },
-            "debug": {
-                "name": "debug",
-                "types": [
-                    "debug"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/21-debug.js"
-            },
-            "complete": {
-                "name": "complete",
-                "types": [
-                    "complete"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/24-complete.js"
-            },
-            "catch": {
-                "name": "catch",
-                "types": [
-                    "catch"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/25-catch.js"
-            },
-            "status": {
-                "name": "status",
-                "types": [
-                    "status"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/25-status.js"
-            },
-            "link": {
-                "name": "link",
-                "types": [
-                    "link in",
-                    "link out",
-                    "link call"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/60-link.js"
-            },
-            "comment": {
-                "name": "comment",
-                "types": [
-                    "comment"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/90-comment.js"
-            },
-            "global-config": {
-                "name": "global-config",
-                "types": [
-                    "global-config"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/91-global-config.js"
-            },
-            "unknown": {
-                "name": "unknown",
-                "types": [
-                    "unknown"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/common/98-unknown.js"
-            },
-            "function": {
-                "name": "function",
-                "types": [
-                    "function"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/10-function.js"
-            },
-            "switch": {
-                "name": "switch",
-                "types": [
-                    "switch"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/10-switch.js"
-            },
-            "change": {
-                "name": "change",
-                "types": [
-                    "change"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/15-change.js"
-            },
-            "range": {
-                "name": "range",
-                "types": [
-                    "range"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/16-range.js"
-            },
-            "template": {
-                "name": "template",
-                "types": [
-                    "template"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/80-template.js"
-            },
-            "delay": {
-                "name": "delay",
-                "types": [
-                    "delay"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/89-delay.js"
-            },
-            "trigger": {
-                "name": "trigger",
-                "types": [
-                    "trigger"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/89-trigger.js"
-            },
-            "exec": {
-                "name": "exec",
-                "types": [
-                    "exec"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/90-exec.js"
-            },
-            "rbe": {
-                "name": "rbe",
-                "types": [
-                    "rbe"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/function/rbe.js"
-            },
-            "tls": {
-                "name": "tls",
-                "types": [
-                    "tls-config"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/05-tls.js"
-            },
-            "httpproxy": {
-                "name": "httpproxy",
-                "types": [
-                    "http proxy"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/06-httpproxy.js"
-            },
-            "mqtt": {
-                "name": "mqtt",
-                "types": [
-                    "mqtt in",
-                    "mqtt out",
-                    "mqtt-broker"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/10-mqtt.js"
-            },
-            "httpin": {
-                "name": "httpin",
-                "types": [
-                    "http in",
-                    "http response"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/21-httpin.js"
-            },
-            "httprequest": {
-                "name": "httprequest",
-                "types": [
-                    "http request"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/21-httprequest.js"
-            },
-            "websocket": {
-                "name": "websocket",
-                "types": [
-                    "websocket in",
-                    "websocket out",
-                    "websocket-listener",
-                    "websocket-client"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/22-websocket.js"
-            },
-            "tcpin": {
-                "name": "tcpin",
-                "types": [
-                    "tcp in",
-                    "tcp out",
-                    "tcp request"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/31-tcpin.js"
-            },
-            "udp": {
-                "name": "udp",
-                "types": [
-                    "udp in",
-                    "udp out"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/network/32-udp.js"
-            },
-            "CSV": {
-                "name": "CSV",
-                "types": [
-                    "csv"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-CSV.js"
-            },
-            "HTML": {
-                "name": "HTML",
-                "types": [
-                    "html"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-HTML.js"
-            },
-            "JSON": {
-                "name": "JSON",
-                "types": [
-                    "json"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-JSON.js"
-            },
-            "XML": {
-                "name": "XML",
-                "types": [
-                    "xml"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-XML.js"
-            },
-            "YAML": {
-                "name": "YAML",
-                "types": [
-                    "yaml"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/parsers/70-YAML.js"
-            },
-            "split": {
-                "name": "split",
-                "types": [
-                    "split",
-                    "join"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/17-split.js"
-            },
-            "sort": {
-                "name": "sort",
-                "types": [
-                    "sort"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/18-sort.js"
-            },
-            "batch": {
-                "name": "batch",
-                "types": [
-                    "batch"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/sequence/19-batch.js"
-            },
-            "file": {
-                "name": "file",
-                "types": [
-                    "file",
-                    "file in"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/storage/10-file.js"
-            },
-            "watch": {
-                "name": "watch",
-                "types": [
-                    "watch"
-                ],
-                "enabled": true,
-                "local": false,
-                "user": false,
-                "module": "node-red",
-                "file": "/usr/src/node-red/node_modules/@node-red/nodes/core/storage/23-watch.js"
-            }
-        }
-    }
-}

monitoring/node-red/data/.config.runtime.json

@@ -1,5 +0,0 @@
-{
-    "instanceId": "22480a0b20f99f0c",
-    "_credentialSecret": "c128ee08a3b994abec9602b578cc23c8a395fd1259896c0c21609b46861a093e",
-    "telemetryEnabled": true
-}

monitoring/node-red/data/.config.runtime.json.backup

@@ -1,4 +0,0 @@
-{
-    "instanceId": "22480a0b20f99f0c",
-    "_credentialSecret": "c128ee08a3b994abec9602b578cc23c8a395fd1259896c0c21609b46861a093e"
-}

monitoring/node-red/data/.config.users.json

@@ -1,29 +0,0 @@
-{
-    "_": {
-        "editor": {
-            "view": {
-                "view-store-zoom": false,
-                "view-store-position": false,
-                "view-show-grid": true,
-                "view-snap-grid": true,
-                "view-grid-size": 20,
-                "view-node-status": true,
-                "view-node-info-icon": true,
-                "view-node-show-label": true,
-                "view-show-tips": true,
-                "view-show-welcome-tours": true
-            },
-            "tours": {
-                "welcome": "4.1.8"
-            },
-            "dialog": {
-                "export": {
-                    "pretty": true,
-                    "json-view": true
-                }
-            }
-        },
-        "menu-menu-item-sidebar": true,
-        "menu-menu-item-palette": true
-    }
-}

monitoring/node-red/data/.config.users.json.backup

@@ -1,29 +0,0 @@
-{
-    "_": {
-        "editor": {
-            "view": {
-                "view-store-zoom": false,
-                "view-store-position": false,
-                "view-show-grid": true,
-                "view-snap-grid": true,
-                "view-grid-size": 20,
-                "view-node-status": true,
-                "view-node-info-icon": true,
-                "view-node-show-label": true,
-                "view-show-tips": true,
-                "view-show-welcome-tours": true
-            },
-            "tours": {
-                "welcome": "4.1.8"
-            },
-            "dialog": {
-                "export": {
-                    "pretty": true,
-                    "json-view": true
-                }
-            }
-        },
-        "menu-menu-item-sidebar": false,
-        "menu-menu-item-palette": true
-    }
-}

monitoring/node-red/data/.flows_cred.json.backup

@@ -1,3 +0,0 @@
-{
-    "$": "0875f76c323e5c597b616b8d0802ec74qpYy2kDXF8kqK4a8NneEiGhCLy5Lp+vI1FkwjR0umSjQBv2nQKyxpVC2eFxg"
-}

monitoring/node-red/data/.npm/_cacache/content-v2/sha512/c2/c2/b64870ea5c5a42b5772106f51123cf684d3c8381de10ccc07d01168d111d0a1ab79ee26fda320b3027c76cfc856119f7b440845a83c9f22d7d731643e62f

@@ -1 +0,0 @@
-{"_id":"node-red-debugger","_rev":"4-77842405de49ac8fcd21dc06e98f929a","name":"node-red-debugger","dist-tags":{"latest":"1.1.1"},"versions":{"1.0.0":{"name":"node-red-debugger","version":"1.0.0","description":"A flow debugger for Node-RED 2.x","repository":{"type":"git","url":"git+https://github.com/node-red/node-red-debugger.git"},"scripts":{"build":"(tsc || exit 1) && npm run tslint && npm run copyAssets","tslint":"tslint -c tslint.json -p tsconfig.json","copyAssets":"node scripts/copy-static-assets.js","dev":"nodemon --exec 'npm run build'  -i dist -e 'ts html'","test":"npm run build"},"keywords":["node-red","debugger"],"author":{"name":"Nick O'Leary","email":"nick.oleary@gmail.com"},"license":"Apache-2","node-red":{"version":">=2.0.0","plugins":{"flow-debugger":"dist/flow-debugger.js"}},"devDependencies":{"@types/node":"^15.12.2","fs-extra":"^10.0.0","html-minifier":"^4.0.0","nodemon":"^2.0.7","tslint":"^6.1.3","typescript":"^4.3.2"},"gitHead":"3c27f6d64e6dbd2313e2f859f8ffb1bf9772ca72","bugs":{"url":"https://github.com/node-red/node-red-debugger/issues"},"homepage":"https://github.com/node-red/node-red-debugger#readme","_id":"node-red-debugger@1.0.0","_nodeVersion":"14.17.0","_npmVersion":"6.14.13","dist":{"integrity":"sha512-XaWptBinVBjXZYWmURwbTDTCD1XxmRzOidYyput3Pqnb+H7g+Q2wtdBVX/ROEL1pWAqL1rzIK4/wXPYF6A6WFA==","shasum":"75dfa1ed1c2197e69700769c18e7cd56c6a0c409","tarball":"https://registry.npmjs.org/node-red-debugger/-/node-red-debugger-1.0.0.tgz","fileCount":17,"unpackedSize":73748,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgw4TGCRA9TVsSAnZWagAAi1gP/jTTNax+Tw2C83Udjf/1\nqxYSS1goc55dtQ3F8wTLPuZwOx3eo2ylp6oc9YicXU4nOfRmz4mVsUlLE0fe\nlJYqbCITTBvfx82ZLnx9Xfy3F6rNbJWRlDKqjuIuyFbFXaZfsZYictHa7eh6\nhFIrLDaG8CedEQbPRgUKCNVpMVaCcMuNOUMccSv6gGkssOlNctRFHrmvPU2M\nwwwwMZ3XIoPdt5hJ2MN1XxsCYl4ehtXavq3TmNGrzm4IdX0iQRhkQ6ji9odi\nuJX5hkblHedIhJseA6pHeBkh6bdTBeJKPiYDxx+io4qw5xU8ehyo4MTeqp89\njzaiTkMQmr7+SUL1wFj+3crY54WBr5hDQDeKd/xYfGneYWgqPho+AW/eQOJl\nKmuhOqJNy/hF63AaoFXuXBwVoyYeYAf9M63EWmxPcaYGxOwJws4Je2LK6e+d\niuggG+rlAoHz5HSVj+JUxw/iE0gKiXQNcsS6a4RpDaV1PiiNqQaEvQfDNtH/\neoK104u4nk2MvDmPhJeCcWFR//Jmwgh/IQIWdJO2UZ0gvBWgXWe/aV91Ru/A\n3C5YKWdpWB7Y64QLsTBqHXkPZTP6mFJfKWZm+lGaWqWcx42wc7xYEHceHJBu\nQNcI3gqztCPVz0h5J4l451Wr4qtwehPDmfYKXT/O7K/RdAEc3qZoShDktufX\nF4C0\r\n=JLZ9\r\n-----END PGP SIGNATURE-----\r\n","signatures":[{"keyid":"SHA256:jl3bwswu80PjjokCgh0o2w5c2U4LhQAE57gj9cz1kzA","sig":"MEUCIQCFGkrByzKFV6QjHlfZa2JB176kwRJelihrKhT5+RfkTwIgPlLJhsoaRH6y2L140UGqgd+L4IW6epxKq9s0RxNQgrk="}]},"_npmUser":{"name":"knolleary","email":"nick.oleary@gmail.com"},"directories":{},"maintainers":[{"name":"knolleary","email":"nick.oleary@gmail.com"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/node-red-debugger_1.0.0_1623426246605_0.7855937877363888"},"_hasShrinkwrap":false},"1.0.1":{"name":"node-red-debugger","version":"1.0.1","description":"A flow debugger for Node-RED 2.x","repository":{"type":"git","url":"git+https://github.com/node-red/node-red-debugger.git"},"scripts":{"build":"(tsc || exit 1) && npm run tslint && npm run copyAssets","tslint":"tslint -c tslint.json -p tsconfig.json","copyAssets":"node scripts/copy-static-assets.js","dev":"nodemon --exec 'npm run build'  -i dist -e 'ts html'","test":"npm run build"},"keywords":["node-red","debugger"],"author":{"name":"Nick O'Leary","email":"nick.oleary@gmail.com"},"license":"Apache-2","node-red":{"version":">=2.0.0","plugins":{"flow-debugger":"dist/flow-debugger.js"}},"devDependencies":{"@types/node":"^15.12.2","fs-extra":"^10.0.0","html-minifier":"^4.0.0","nodemon":"^2.0.7","tslint":"^6.1.3","typescript":"^4.3.2"},"gitHead":"de2973e309e8b59b8d11f4e67dc4fe062142b10d","bugs":{"url":"https://github.com/node-red/node-red-debugger/issues"},"homepage":"https://github.com/node-red/node-red-debugger#readme","_id":"node-red-debugger@1.0.1","_nodeVersion":"14.17.0","_npmVersion":"6.14.13","dist":{"integrity":"sha512-7zOaX/vCUgwpiHrhLbk6P9d3kZY2P9G4t/0x9P69KlKRT1NxekGYtYKRKjFA1GRX0pna//nfJMXwAX4eyzhRZg==","shasum":"42c902dacc3a02993553e62c96a230b7cdf53df3","tarball":"https://registry.npmjs.org/node-red-debugger/-/node-red-debugger-1.0.1.tgz","fileCount":17,"unpackedSize":73748,"npm-signature":"-----BEGIN PGP SIGNATURE-----\r\nVersion: OpenPGP.js v3.0.13\r\nComment: https://openpgpjs.org\r\n\r\nwsFcBAEBCAAQBQJgw5S2CRA9TVsSAnZWagAAlDsP/1WhoDXGylQd7dBfg6oA\n+JRNKrQcqkkguKCIMKr+hHQU/6JiyswmwRTVZvT35pPSsk9Cw66H7SPWNqIN\ncZBDt/k6Jb03rJl5GFv32+vmxSrOIfvaVjiosIqVWmynPNqSsClZknufZpqb\niSD3RhQ6fWmv2FoENTw+MapZ1QV6mZyFnZG62Qe9nnUETBcuLW5/QHjENd16\nI5iHf4GK02qMDGHc34RSjZx5DGLvEqqgKBlAXKlc1yjTorxMl6D+CWiCg0fn\nRZj8wCPwSZRs5oyKjTLJ3Z3DFihRH8ecV+n57U4ncAQ3g1xpBFxBO/wbb9ct\nNyQ0vvmLb/CiwQv5UgPvxng7phzpaVqMNFo1xeGLckei2c7tOIcBktD8WPHS\nGNIcPZb1Fgiis9zqtT5u7pusBs8ftFd81xTitFioHSqTCO0UALhtb7a5eXuK\ngDhlVbqAhKaRHjzQ23rKi/xc+zrmgyUvIJgxMOtMv4nSwKJ1/4I9dkmcGbcQ\n4g+zvtRSwpN+kq60f4rweTX6Vle3kLine truncated

monitoring/node-red/data/.npm/_cacache/index-v5/15/a4/2638498d877ec2c8c3d88cb9c08d7867c52d3fceb6fc64cc5abde73b01a9

@@ -1,2 +0,0 @@
-
-910aac156dabb4fcf662d29b6bd825588e1e1844	{"key":"make-fetch-happen:request-cache:https://registry.npmjs.org/node-red-debugger/-/node-red-debugger-1.1.1.tgz","integrity":"sha512-sEfBRYZk+ptqCOkDURC1IxJ6lr1yhdGUctxwL13EmLknQSsOzTJzcI+/nWF1RSBZmsCw4R8+TE1Kx4TnjX2X/g==","time":1775352975936,"size":22123,"metadata":{"time":1775352975918,"url":"https://registry.npmjs.org/node-red-debugger/-/node-red-debugger-1.1.1.tgz","reqHeaders":{},"resHeaders":{"cache-control":"public, immutable, max-age=31557600","content-type":"application/octet-stream","date":"Sun, 05 Apr 2026 01:36:15 GMT","etag":"\"8d40a313fd218e46c3e4fe78121b9267\"","last-modified":"Wed, 21 Jul 2021 11:15:11 GMT","vary":"Accept-Encoding"},"options":{"compress":true}}}

monitoring/node-red/data/.npm/_cacache/index-v5/48/03/b8903b717bbc1ad41b3f37148db48f54e0828c1aef870973f6672895d689

@@ -1,2 +0,0 @@
-
-aaa2f94f98f0490d343865ece4ec971006a4f1c3	{"key":"make-fetch-happen:request-cache:https://registry.npmjs.org/node-red-debugger","integrity":"sha512-wsK2SHDqXFpCtXchBvURI89oTTyDgd4QzMB9ARaNER0KGree4m/aMgswJ8ds/IVhGfe0QIRag8nyLX1zFkPmLw==","time":1775352975866,"size":15640,"metadata":{"time":1775352975857,"url":"https://registry.npmjs.org/node-red-debugger","reqHeaders":{"accept":"application/json"},"resHeaders":{"cache-control":"public, max-age=300","content-encoding":"gzip","content-type":"application/json","date":"Sun, 05 Apr 2026 01:36:15 GMT","etag":"W/\"907dbbf9d3078f923fc58d814f174384\"","last-modified":"Wed, 11 May 2022 04:22:14 GMT","vary":"accept-encoding, accept"},"options":{"compress":true}}}