modified: .gitignore
new file: .gitignore.gpt new file: .gitignore.old deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/add-environment-setup-in-conftest.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/add-logging-to-geocode.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/add-logging-to-route_metrics.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/add-logging-to-tracking-simulator.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/extend-sqlite-tuning-in-database.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/fix-route-handling-in-routing.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/handle-api-response-errors-in-routing.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/refactor-database-path-handling-in-database.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/update-fcm-message-construction-in-notifications.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/update-role-check-in-ws.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/update-user-seed-in-database.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/add-environment-setup-in-conftest.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/add-logging-to-geocode.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/add-logging-to-route_metrics.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/add-logging-to-tracking-simulator.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/extend-sqlite-tuning-in-database.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/fix-route-handling-in-routing.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/handle-api-response-errors-in-routing.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/refactor-database-path-handling-in-database.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/update-fcm-message-construction-in-notifications.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/update-role-check-in-ws.py deleted: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/update-user-seed-in-database.py deleted: apps/searxng/dockerfiles/docker-entrypoint.sh deleted: apps/searxng/docs/conf.py deleted: apps/searxng/docs/user/.gitignore deleted: apps/searxng/examples/basic_engine.py deleted: apps/searxng/searx/__init__.py deleted: apps/searxng/searx/answerers/__init__.py deleted: apps/searxng/searx/answerers/random/answerer.py deleted: apps/searxng/searx/answerers/statistics/answerer.py deleted: apps/searxng/searx/autocomplete.py deleted: apps/searxng/searx/babel_extract.py deleted: apps/searxng/searx/botdetection/__init__.py deleted: apps/searxng/searx/botdetection/_helpers.py deleted: apps/searxng/searx/botdetection/http_accept.py deleted: apps/searxng/searx/botdetection/http_accept_encoding.py deleted: apps/searxng/searx/botdetection/http_accept_language.py deleted: apps/searxng/searx/botdetection/http_connection.py deleted: apps/searxng/searx/botdetection/http_user_agent.py deleted: apps/searxng/searx/botdetection/ip_limit.py deleted: apps/searxng/searx/botdetection/ip_lists.py deleted: apps/searxng/searx/botdetection/limiter.py deleted: apps/searxng/searx/botdetection/link_token.py deleted: apps/searxng/searx/compat.py deleted: apps/searxng/searx/data/__init__.py deleted: apps/searxng/searx/enginelib/__init__.py deleted: apps/searxng/searx/enginelib/traits.py deleted: apps/searxng/searx/engines/1337x.py deleted: apps/searxng/searx/engines/9gag.py deleted: apps/searxng/searx/engines/__init__.py deleted: apps/searxng/searx/engines/ahmia.py deleted: apps/searxng/searx/engines/annas_archive.py deleted: apps/searxng/searx/engines/apkmirror.py deleted: apps/searxng/searx/engines/apple_app_store.py deleted: apps/searxng/searx/engines/apple_maps.py deleted: apps/searxng/searx/engines/archlinux.py deleted: apps/searxng/searx/engines/artic.py deleted: apps/searxng/searx/engines/arxiv.py deleted: apps/searxng/searx/engines/bandcamp.py deleted: apps/searxng/searx/engines/base.py deleted: apps/searxng/searx/engines/bing.py deleted: apps/searxng/searx/engines/bing_images.py deleted: apps/searxng/searx/engines/bing_news.py deleted: apps/searxng/searx/engines/bing_videos.py deleted: apps/searxng/searx/engines/brave.py deleted: apps/searxng/searx/engines/bt4g.py deleted: apps/searxng/searx/engines/btdigg.py deleted: apps/searxng/searx/engines/command.py deleted: apps/searxng/searx/engines/core.py deleted: apps/searxng/searx/engines/crossref.py deleted: apps/searxng/searx/engines/currency_convert.py deleted: apps/searxng/searx/engines/dailymotion.py deleted: apps/searxng/searx/engines/deepl.py deleted: apps/searxng/searx/engines/deezer.py deleted: apps/searxng/searx/engines/demo_offline.py deleted: apps/searxng/searx/engines/demo_online.py deleted: apps/searxng/searx/engines/deviantart.py deleted: apps/searxng/searx/engines/dictzone.py deleted: apps/searxng/searx/engines/digbt.py deleted: apps/searxng/searx/engines/docker_hub.py deleted: apps/searxng/searx/engines/doku.py deleted: apps/searxng/searx/engines/duckduckgo.py deleted: apps/searxng/searx/engines/duckduckgo_definitions.py deleted: apps/searxng/searx/engines/duckduckgo_images.py deleted: apps/searxng/searx/engines/duckduckgo_weather.py deleted: apps/searxng/searx/engines/duden.py deleted: apps/searxng/searx/engines/dummy-offline.py deleted: apps/searxng/searx/engines/dummy.py deleted: apps/searxng/searx/engines/ebay.py deleted: apps/searxng/searx/engines/elasticsearch.py deleted: apps/searxng/searx/engines/emojipedia.py deleted: apps/searxng/searx/engines/fdroid.py deleted: apps/searxng/searx/engines/flickr.py deleted: apps/searxng/searx/engines/flickr_noapi.py deleted: apps/searxng/searx/engines/framalibre.py deleted: apps/searxng/searx/engines/freesound.py deleted: apps/searxng/searx/engines/frinkiac.py deleted: apps/searxng/searx/engines/genius.py deleted: apps/searxng/searx/engines/gentoo.py deleted: apps/searxng/searx/engines/github.py deleted: apps/searxng/searx/engines/google.py deleted: apps/searxng/searx/engines/google_images.py deleted: apps/searxng/searx/engines/google_news.py deleted: apps/searxng/searx/engines/google_play.py deleted: apps/searxng/searx/engines/google_scholar.py deleted: apps/searxng/searx/engines/google_videos.py deleted: apps/searxng/searx/engines/imdb.py deleted: apps/searxng/searx/engines/ina.py deleted: apps/searxng/searx/engines/invidious.py deleted: apps/searxng/searx/engines/jisho.py deleted: apps/searxng/searx/engines/json_engine.py deleted: apps/searxng/searx/engines/kickass.py deleted: apps/searxng/searx/engines/lemmy.py deleted: apps/searxng/searx/engines/lingva.py deleted: apps/searxng/searx/engines/loc.py deleted: apps/searxng/searx/engines/mediathekviewweb.py deleted: apps/searxng/searx/engines/mediawiki.py deleted: apps/searxng/searx/engines/meilisearch.py deleted: apps/searxng/searx/engines/metacpan.py deleted: apps/searxng/searx/engines/mixcloud.py deleted: apps/searxng/searx/engines/mongodb.py deleted: apps/searxng/searx/engines/mysql_server.py deleted: apps/searxng/searx/engines/nyaa.py deleted: apps/searxng/searx/engines/opensemantic.py deleted: apps/searxng/searx/engines/openstreetmap.py deleted: apps/searxng/searx/engines/openverse.py deleted: apps/searxng/searx/engines/pdbe.py deleted: apps/searxng/searx/engines/peertube.py deleted: apps/searxng/searx/engines/photon.py deleted: apps/searxng/searx/engines/piped.py deleted: apps/searxng/searx/engines/piratebay.py deleted: apps/searxng/searx/engines/postgresql.py deleted: apps/searxng/searx/engines/pubmed.py deleted: apps/searxng/searx/engines/qwant.py deleted: apps/searxng/searx/engines/recoll.py deleted: apps/searxng/searx/engines/reddit.py deleted: apps/searxng/searx/engines/redis_server.py deleted: apps/searxng/searx/engines/rumble.py deleted: apps/searxng/searx/engines/scanr_structures.py deleted: apps/searxng/searx/engines/searchcode_code.py deleted: apps/searxng/searx/engines/searx_engine.py deleted: apps/searxng/searx/engines/semantic_scholar.py deleted: apps/searxng/searx/engines/sepiasearch.py deleted: apps/searxng/searx/engines/seznam.py deleted: apps/searxng/searx/engines/sjp.py deleted: apps/searxng/searx/engines/solidtorrents.py deleted: apps/searxng/searx/engines/solr.py deleted: apps/searxng/searx/engines/soundcloud.py deleted: apps/searxng/searx/engines/spotify.py deleted: apps/searxng/searx/engines/springer.py deleted: apps/searxng/searx/engines/sqlite.py deleted: apps/searxng/searx/engines/stackexchange.py deleted: apps/searxng/searx/engines/startpage.py deleted: apps/searxng/searx/engines/tagesschau.py deleted: apps/searxng/searx/engines/tineye.py deleted: apps/searxng/searx/engines/tokyotoshokan.py deleted: apps/searxng/searx/engines/torznab.py deleted: apps/searxng/searx/engines/translated.py deleted: apps/searxng/searx/engines/twitter.py deleted: apps/searxng/searx/engines/unsplash.py deleted: apps/searxng/searx/engines/vimeo.py deleted: apps/searxng/searx/engines/wikidata.py deleted: apps/searxng/searx/engines/wikipedia.py deleted: apps/searxng/searx/engines/wolframalpha_api.py deleted: apps/searxng/searx/engines/wolframalpha_noapi.py deleted: apps/searxng/searx/engines/wordnik.py deleted: apps/searxng/searx/engines/wttr.py deleted: apps/searxng/searx/engines/www1x.py deleted: apps/searxng/searx/engines/xpath.py deleted: apps/searxng/searx/engines/yacy.py deleted: apps/searxng/searx/engines/yahoo.py deleted: apps/searxng/searx/engines/yahoo_news.py deleted: apps/searxng/searx/engines/youtube_api.py deleted: apps/searxng/searx/engines/youtube_noapi.py deleted: apps/searxng/searx/engines/zlibrary.py deleted: apps/searxng/searx/exceptions.py deleted: apps/searxng/searx/external_bang.py deleted: apps/searxng/searx/external_urls.py deleted: apps/searxng/searx/flaskfix.py deleted: apps/searxng/searx/infopage/__init__.py deleted: apps/searxng/searx/locales.py deleted: apps/searxng/searx/metrics/__init__.py deleted: apps/searxng/searx/metrics/error_recorder.py deleted: apps/searxng/searx/metrics/models.py deleted: apps/searxng/searx/network/__init__.py deleted: apps/searxng/searx/network/client.py deleted: apps/searxng/searx/network/network.py deleted: apps/searxng/searx/network/raise_for_httperror.py deleted: apps/searxng/searx/plugins/__init__.py deleted: apps/searxng/searx/plugins/ahmia_filter.py deleted: apps/searxng/searx/plugins/hash_plugin.py deleted: apps/searxng/searx/plugins/hostname_replace.py deleted: apps/searxng/searx/plugins/limiter.py deleted: apps/searxng/searx/plugins/oa_doi_rewrite.py deleted: apps/searxng/searx/plugins/search_on_category_select.py deleted: apps/searxng/searx/plugins/self_info.py deleted: apps/searxng/searx/plugins/tor_check.py deleted: apps/searxng/searx/plugins/tracker_url_remover.py deleted: apps/searxng/searx/plugins/vim_hotkeys.py deleted: apps/searxng/searx/preferences.py deleted: apps/searxng/searx/query.py deleted: apps/searxng/searx/redisdb.py deleted: apps/searxng/searx/redislib.py deleted: apps/searxng/searx/results.py deleted: apps/searxng/searx/search/__init__.py deleted: apps/searxng/searx/search/checker/__init__.py deleted: apps/searxng/searx/search/checker/__main__.py deleted: apps/searxng/searx/search/checker/background.py deleted: apps/searxng/searx/search/checker/impl.py deleted: apps/searxng/searx/search/checker/scheduler.py deleted: apps/searxng/searx/search/models.py deleted: apps/searxng/searx/search/processors/__init__.py deleted: apps/searxng/searx/search/processors/abstract.py deleted: apps/searxng/searx/search/processors/offline.py deleted: apps/searxng/searx/search/processors/online.py deleted: apps/searxng/searx/search/processors/online_currency.py deleted: apps/searxng/searx/search/processors/online_dictionary.py deleted: apps/searxng/searx/search/processors/online_url_search.py deleted: apps/searxng/searx/settings.yml deleted: apps/searxng/searx/settings_defaults.py deleted: apps/searxng/searx/settings_loader.py deleted: apps/searxng/searx/static/plugins/external_plugins/.gitignore deleted: apps/searxng/searx/static/themes/simple/.gitattributes deleted: apps/searxng/searx/static/themes/simple/.gitignore deleted: apps/searxng/searx/sxng_locales.py deleted: apps/searxng/searx/tools/__init__.py deleted: apps/searxng/searx/tools/config.py deleted: apps/searxng/searx/unixthreadname.py deleted: apps/searxng/searx/utils.py deleted: apps/searxng/searx/version.py deleted: apps/searxng/searx/webadapter.py deleted: apps/searxng/searx/webapp.py deleted: apps/searxng/searx/webutils.py deleted: apps/searxng/searxng_extra/standalone_searx.py deleted: apps/searxng/searxng_extra/update/__init__.py deleted: apps/searxng/searxng_extra/update/update_ahmia_blacklist.py deleted: apps/searxng/searxng_extra/update/update_currencies.py deleted: apps/searxng/searxng_extra/update/update_engine_descriptions.py deleted: apps/searxng/searxng_extra/update/update_engine_traits.py deleted: apps/searxng/searxng_extra/update/update_external_bangs.py deleted: apps/searxng/searxng_extra/update/update_firefox_version.py deleted: apps/searxng/searxng_extra/update/update_osm_keys_tags.py deleted: apps/searxng/searxng_extra/update/update_pygments.py deleted: apps/searxng/searxng_extra/update/update_wikidata_units.py deleted: apps/searxng/setup.py deleted: apps/searxng/tests/__init__.py deleted: apps/searxng/tests/robot/__init__.py deleted: apps/searxng/tests/robot/__main__.py deleted: apps/searxng/tests/robot/settings_robot.yml deleted: apps/searxng/tests/robot/test_webapp.py deleted: apps/searxng/tests/unit/__init__.py deleted: apps/searxng/tests/unit/engines/test_command.py deleted: apps/searxng/tests/unit/engines/test_xpath.py deleted: apps/searxng/tests/unit/network/__init__.py deleted: apps/searxng/tests/unit/network/test_network.py deleted: apps/searxng/tests/unit/settings/empty_settings.yml deleted: apps/searxng/tests/unit/settings/syntaxerror_settings.yml deleted: apps/searxng/tests/unit/settings/test_settings.yml deleted: apps/searxng/tests/unit/settings/user_settings.yml deleted: apps/searxng/tests/unit/settings/user_settings_keep_only.yml deleted: apps/searxng/tests/unit/settings/user_settings_remove.yml deleted: apps/searxng/tests/unit/settings/user_settings_remove2.yml deleted: apps/searxng/tests/unit/settings/user_settings_simple.yml deleted: apps/searxng/tests/unit/test_answerers.py deleted: apps/searxng/tests/unit/test_engines_init.py deleted: apps/searxng/tests/unit/test_exceptions.py deleted: apps/searxng/tests/unit/test_external_bangs.py deleted: apps/searxng/tests/unit/test_locales.py deleted: apps/searxng/tests/unit/test_plugins.py deleted: apps/searxng/tests/unit/test_preferences.py deleted: apps/searxng/tests/unit/test_query.py deleted: apps/searxng/tests/unit/test_results.py deleted: apps/searxng/tests/unit/test_search.py deleted: apps/searxng/tests/unit/test_settings_loader.py deleted: apps/searxng/tests/unit/test_utils.py deleted: apps/searxng/tests/unit/test_webadapter.py deleted: apps/searxng/tests/unit/test_webapp.py deleted: apps/searxng/tests/unit/test_webutils.py deleted: apps/searxng/utils/build_env.py deleted: apps/searxng/utils/filtron.sh deleted: apps/searxng/utils/lib.sh deleted: apps/searxng/utils/lib_go.sh deleted: apps/searxng/utils/lib_nvm.sh deleted: apps/searxng/utils/lib_redis.sh deleted: apps/searxng/utils/lib_sxng_data.sh deleted: apps/searxng/utils/lib_sxng_node.sh deleted: apps/searxng/utils/lib_sxng_static.sh deleted: apps/searxng/utils/lib_sxng_test.sh deleted: apps/searxng/utils/lib_sxng_themes.sh deleted: apps/searxng/utils/lib_sxng_weblate.sh deleted: apps/searxng/utils/lxc.sh deleted: apps/searxng/utils/morty.sh deleted: apps/searxng/utils/searx.sh deleted: apps/searxng/utils/searxng.sh deleted: apps/searxng/utils/searxng_check.py deleted: apps/searxng/utils/templates/etc/searxng/settings.yml deleted: apps/shift-recorder deleted: apps/stockfill deleted: archive/esphome/data/.gitignore deleted: archive/esphome/data/esphome-garage.yaml deleted: archive/esphome/data/esphome-waynes-room.yaml deleted: core/crowdsec/data/detect.yaml deleted: core/traefik/data/dynamic.yaml deleted: core/traefik/data/plugins.yaml new file: default-environment.env new file: docker new file: last-ip.ini new file: monitoring/docker-exporter/Dockerfile.old new file: monitoring/docker-exporter/exporter.py.old renamed: apps/searxng/searxng_extra/__init__.py -> monitoring/gotify/docker-health-alert/last_unhealthy.txt new file: monitoring/grafana/output.txt new file: monitoring/influxdb/influxd.bolt new file: monitoring/influxdb/influxd.sqlite deleted: monitoring/node-red/data/test-container.sh new file: monitoring/prometheus/prometheus.yml.old new file: monitoring/telegraf/telegraf.conf new file: monitoring/uptime-kuma/result new file: monitoring/uptime-kuma/uptime-kuma/kuma.db new file: tree.out new file: update-containers.log new file: update-firewall.log
This commit is contained in:
git
@@ -0,0 +1,8 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY exporter.py .
|
||||
|
||||
RUN pip install docker prometheus_client requests pyyaml
|
||||
|
||||
CMD ["python", "exporter.py"]
|
||||
@@ -0,0 +1,514 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
import docker
|
||||
import requests
|
||||
import yaml
|
||||
from prometheus_client import Gauge, start_http_server
|
||||
|
||||
# --- Logging ---
|
||||
LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
|
||||
|
||||
logging.basicConfig(
|
||||
level=getattr(logging, LOG_LEVEL, logging.INFO),
|
||||
format="%(asctime)s [%(levelname)s] %(message)s"
|
||||
)
|
||||
|
||||
logger = logging.getLogger("docker-update-exporter")
|
||||
|
||||
# --- Config ---
|
||||
EXPORTER_PORT = 9105
|
||||
CHECK_INTERVAL = 60
|
||||
CACHE_TTL = 6 * 3600
|
||||
SERVICES_UP_SCRIPT = "/compose/services-up.sh"
|
||||
CACHE_FILE = "/data/remote_digest_cache.json"
|
||||
|
||||
client = docker.from_env()
|
||||
|
||||
# --- Metrics ---
|
||||
CONTAINER_UPDATE = Gauge(
|
||||
"docker_container_update_available",
|
||||
"1 if container image is out of date (compose drift or registry), 0 otherwise",
|
||||
["container", "compose_image", "running_image", "com_docker_compose_project"]
|
||||
)
|
||||
|
||||
LAST_CHECK = Gauge(
|
||||
"docker_image_update_last_check_timestamp",
|
||||
"Last time the update check ran (unix timestamp)"
|
||||
)
|
||||
|
||||
# --- Persistent Cache ---
|
||||
|
||||
def load_cache():
|
||||
if not os.path.exists(CACHE_FILE):
|
||||
logger.info(f"Cache file does not exist yet: {CACHE_FILE}")
|
||||
return {}
|
||||
|
||||
try:
|
||||
with open(CACHE_FILE, "r") as f:
|
||||
cache = json.load(f)
|
||||
logger.info(f"Loaded {len(cache)} cached remote digests")
|
||||
logger.debug(f"Cache contents: {cache}")
|
||||
return cache
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to load cache from {CACHE_FILE}: {e}")
|
||||
return {}
|
||||
|
||||
def save_cache():
|
||||
try:
|
||||
os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True)
|
||||
with open(CACHE_FILE, "w") as f:
|
||||
json.dump(REMOTE_DIGEST_CACHE, f)
|
||||
|
||||
logger.debug(
|
||||
f"Saved {len(REMOTE_DIGEST_CACHE)} entries to cache file {CACHE_FILE}"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save cache to {CACHE_FILE}: {e}")
|
||||
|
||||
REMOTE_DIGEST_CACHE = load_cache()
|
||||
|
||||
# --- Helpers ---
|
||||
|
||||
def get_project_prefix_from_script(script_path):
|
||||
project_prefix = "core-" # fallback
|
||||
|
||||
if not os.path.exists(script_path):
|
||||
logger.warning(
|
||||
f"services-up script not found at {script_path}, using fallback project prefix {project_prefix}"
|
||||
)
|
||||
return project_prefix
|
||||
|
||||
try:
|
||||
with open(script_path, "r") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
m = re.match(r'PROJECT\s*=\s*["\']?([^"\']+)["\']?', line)
|
||||
if m:
|
||||
project_prefix = m.group(1) + "-"
|
||||
logger.debug(
|
||||
f"Detected compose project prefix from script: {project_prefix}"
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Failed reading project prefix from {script_path}: {e}")
|
||||
|
||||
return project_prefix
|
||||
|
||||
def get_local_digest(image_name):
|
||||
try:
|
||||
img = client.images.get(image_name)
|
||||
digests = img.attrs.get("RepoDigests", [])
|
||||
|
||||
logger.debug(f"Local RepoDigests for {image_name}: {digests}")
|
||||
|
||||
if digests:
|
||||
digest = digests[0].split("@")[1]
|
||||
logger.debug(f"Local digest for {image_name}: {digest}")
|
||||
return digest
|
||||
|
||||
logger.info(f"No local digest found for image {image_name}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to retrieve local digest for {image_name}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def get_remote_digest(image_name):
|
||||
now = time.time()
|
||||
original = image_name
|
||||
|
||||
# Cache hit
|
||||
if original in REMOTE_DIGEST_CACHE:
|
||||
digest, ts = REMOTE_DIGEST_CACHE[original]
|
||||
age = now - ts
|
||||
|
||||
if age < CACHE_TTL:
|
||||
logger.debug(
|
||||
f"Using cached remote digest for {original} "
|
||||
f"(age={int(age)}s, ttl={CACHE_TTL}s): {digest}"
|
||||
)
|
||||
return digest
|
||||
|
||||
logger.info(
|
||||
f"Cache entry expired for {original} "
|
||||
f"(age={int(age)}s > ttl={CACHE_TTL}s)"
|
||||
)
|
||||
|
||||
try:
|
||||
if "/" not in image_name:
|
||||
registry = "docker.io"
|
||||
repo = "library/" + image_name
|
||||
else:
|
||||
parts = image_name.split("/")
|
||||
if "." in parts[0] or ":" in parts[0]:
|
||||
registry = parts[0]
|
||||
repo = "/".join(parts[1:])
|
||||
else:
|
||||
registry = "docker.io"
|
||||
repo = image_name
|
||||
|
||||
if ":" in repo:
|
||||
repo, tag = repo.rsplit(":", 1)
|
||||
else:
|
||||
tag = "latest"
|
||||
|
||||
logger.debug(
|
||||
f"Resolving remote digest for {original}: "
|
||||
f"registry={registry}, repo={repo}, tag={tag}"
|
||||
)
|
||||
|
||||
token = None
|
||||
manifest_url = None
|
||||
|
||||
if registry in ["docker.io", "registry-1.docker.io"]:
|
||||
logger.debug(f"Requesting Docker Hub token for {repo}")
|
||||
|
||||
token_res = requests.get(
|
||||
"https://auth.docker.io/token",
|
||||
params={
|
||||
"service": "registry.docker.io",
|
||||
"scope": f"repository:{repo}:pull"
|
||||
},
|
||||
timeout=10
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Docker Hub token response for {repo}: "
|
||||
f"status={token_res.status_code}"
|
||||
)
|
||||
|
||||
token = token_res.json().get("token")
|
||||
manifest_url = (
|
||||
f"https://registry-1.docker.io/v2/{repo}/manifests/{tag}"
|
||||
)
|
||||
|
||||
elif registry == "ghcr.io":
|
||||
logger.debug(f"Requesting GHCR token for {repo}")
|
||||
|
||||
token_res = requests.get(
|
||||
"https://ghcr.io/token",
|
||||
params={
|
||||
"service": "ghcr.io",
|
||||
"scope": f"repository:{repo}:pull"
|
||||
},
|
||||
timeout=10
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"GHCR token response for {repo}: "
|
||||
f"status={token_res.status_code}"
|
||||
)
|
||||
|
||||
token = token_res.json().get("token")
|
||||
manifest_url = f"https://ghcr.io/v2/{repo}/manifests/{tag}"
|
||||
|
||||
else:
|
||||
logger.warning(
|
||||
f"Unsupported registry '{registry}' for image {original}"
|
||||
)
|
||||
return None
|
||||
|
||||
if not token:
|
||||
logger.warning(
|
||||
f"No authentication token returned for {original}"
|
||||
)
|
||||
return None
|
||||
|
||||
logger.debug(f"Requesting manifest for {original}: {manifest_url}")
|
||||
|
||||
res = requests.get(
|
||||
manifest_url,
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.docker.distribution.manifest.v2+json"
|
||||
},
|
||||
timeout=10
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Manifest response for {original}: "
|
||||
f"status={res.status_code}"
|
||||
)
|
||||
|
||||
if res.status_code == 200:
|
||||
digest = res.headers.get("Docker-Content-Digest")
|
||||
|
||||
logger.info(
|
||||
f"Fetched remote digest for {original}: {digest}"
|
||||
)
|
||||
|
||||
REMOTE_DIGEST_CACHE[original] = (digest, now)
|
||||
save_cache()
|
||||
|
||||
logger.debug(
|
||||
f"Cached remote digest for {original}: {digest}"
|
||||
)
|
||||
|
||||
return digest
|
||||
|
||||
if res.status_code == 429:
|
||||
logger.warning(
|
||||
f"Registry rate limit hit while fetching {original}"
|
||||
)
|
||||
elif res.status_code in [401, 403]:
|
||||
logger.warning(
|
||||
f"Authentication failed while fetching {original}: "
|
||||
f"status={res.status_code}"
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
f"Unexpected manifest response for {original}: "
|
||||
f"status={res.status_code}, body={res.text[:250]}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to fetch remote digest for {original}: {e}")
|
||||
|
||||
return None
|
||||
|
||||
def get_compose_files_from_script(script_path):
|
||||
files = []
|
||||
|
||||
if not os.path.exists(script_path):
|
||||
logger.error(f"services-up script not found: {script_path}")
|
||||
return files
|
||||
|
||||
base_dir = os.path.dirname(script_path)
|
||||
|
||||
try:
|
||||
with open(script_path, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
match = re.search(r'FILES\s*=\s*\((.*?)\)', content, re.DOTALL)
|
||||
|
||||
if not match:
|
||||
logger.warning(
|
||||
f"No FILES=(...) block found in {script_path}"
|
||||
)
|
||||
return files
|
||||
|
||||
lines = match.group(1).splitlines()
|
||||
|
||||
for line in lines:
|
||||
line = line.strip()
|
||||
|
||||
if line.startswith("-f"):
|
||||
rel_path = line[2:].strip()
|
||||
|
||||
if rel_path:
|
||||
full_path = os.path.normpath(
|
||||
os.path.join(base_dir, rel_path)
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"Resolved compose file: {rel_path} -> {full_path}"
|
||||
)
|
||||
|
||||
files.append(full_path)
|
||||
|
||||
logger.info(f"Found {len(files)} compose files")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed parsing compose files from {script_path}: {e}")
|
||||
|
||||
return files
|
||||
|
||||
def parse_compose_files(compose_files):
|
||||
service_to_image = {}
|
||||
|
||||
for f in compose_files:
|
||||
if not os.path.exists(f):
|
||||
logger.warning(f"Compose file missing: {f}")
|
||||
continue
|
||||
|
||||
try:
|
||||
with open(f, "r") as stream:
|
||||
data = yaml.safe_load(stream) or {}
|
||||
services = data.get("services", {})
|
||||
|
||||
logger.debug(
|
||||
f"Parsing {len(services)} services from compose file {f}"
|
||||
)
|
||||
|
||||
for service_name, service_def in services.items():
|
||||
image = service_def.get("image")
|
||||
is_built = False
|
||||
|
||||
if not image:
|
||||
is_built = True
|
||||
build_ctx = service_def.get("build")
|
||||
|
||||
logger.debug(
|
||||
f"Service {service_name} is build-based, build config={build_ctx}"
|
||||
)
|
||||
|
||||
if isinstance(build_ctx, dict):
|
||||
context_path = build_ctx.get("context", ".")
|
||||
dockerfile_path = os.path.join(
|
||||
context_path,
|
||||
build_ctx.get("dockerfile", "Dockerfile")
|
||||
)
|
||||
elif isinstance(build_ctx, str):
|
||||
context_path = build_ctx
|
||||
dockerfile_path = os.path.join(
|
||||
context_path, "Dockerfile"
|
||||
)
|
||||
else:
|
||||
dockerfile_path = None
|
||||
|
||||
if dockerfile_path and os.path.exists(dockerfile_path):
|
||||
try:
|
||||
with open(dockerfile_path, "r") as df:
|
||||
for line in df:
|
||||
line = line.strip()
|
||||
if (
|
||||
line.upper().startswith("LABEL")
|
||||
and "image=" in line
|
||||
):
|
||||
m = re.search(
|
||||
r'image=["\']?([^"\']+)["\']?',
|
||||
line
|
||||
)
|
||||
if m:
|
||||
image = m.group(1)
|
||||
logger.debug(
|
||||
f"Found upstream image label for {service_name}: {image}"
|
||||
)
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
f"Failed reading Dockerfile {dockerfile_path}: {e}"
|
||||
)
|
||||
|
||||
if not image:
|
||||
image = f"{service_name}:latest"
|
||||
logger.info(
|
||||
f"No image label found for build service {service_name}, "
|
||||
f"defaulting to {image}"
|
||||
)
|
||||
|
||||
service_to_image[service_name] = (image, is_built)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed parsing compose file {f}: {e}")
|
||||
|
||||
logger.info(f"Mapped {len(service_to_image)} compose services to images")
|
||||
logger.debug(f"Service/image mapping: {service_to_image}")
|
||||
|
||||
return service_to_image
|
||||
|
||||
def check_containers():
|
||||
logger.info("Starting container update check")
|
||||
|
||||
CONTAINER_UPDATE.clear()
|
||||
|
||||
project_prefix = get_project_prefix_from_script(SERVICES_UP_SCRIPT)
|
||||
compose_files = get_compose_files_from_script(SERVICES_UP_SCRIPT)
|
||||
service_to_image = parse_compose_files(compose_files)
|
||||
|
||||
containers = client.containers.list()
|
||||
logger.info(f"Checking {len(containers)} running containers")
|
||||
|
||||
for container in containers:
|
||||
project_label = container.labels.get("com.docker.compose.project")
|
||||
|
||||
if not project_label:
|
||||
logger.debug(
|
||||
f"Skipping non-compose container {container.name}"
|
||||
)
|
||||
continue
|
||||
|
||||
service_label = container.labels.get("com.docker.compose.service")
|
||||
running_image = container.attrs["Config"]["Image"]
|
||||
|
||||
logger.debug(
|
||||
f"Evaluating container={container.name}, "
|
||||
f"service={service_label}, project={project_label}, "
|
||||
f"running_image={running_image}"
|
||||
)
|
||||
|
||||
compose_image = None
|
||||
is_built = False
|
||||
|
||||
if service_label and service_label in service_to_image:
|
||||
compose_image, is_built = service_to_image[service_label]
|
||||
|
||||
if is_built:
|
||||
compose_image_name, _, _ = compose_image.partition(":")
|
||||
compose_image = f"{project_prefix}{compose_image_name}"
|
||||
|
||||
update_flag = 0
|
||||
|
||||
if is_built:
|
||||
if running_image != compose_image:
|
||||
logger.info(
|
||||
f"Update detected for build-based container {container.name}: "
|
||||
f"running image {running_image} != expected {compose_image}"
|
||||
)
|
||||
update_flag = 1
|
||||
else:
|
||||
local_digest = get_local_digest(running_image)
|
||||
remote_digest = get_remote_digest(
|
||||
service_to_image[service_label][0]
|
||||
)
|
||||
|
||||
if local_digest and remote_digest and local_digest != remote_digest:
|
||||
logger.info(
|
||||
f"Remote image update available for {container.name}: "
|
||||
f"{local_digest} != {remote_digest}"
|
||||
)
|
||||
update_flag = 1
|
||||
else:
|
||||
if running_image != compose_image:
|
||||
logger.info(
|
||||
f"Compose drift detected for {container.name}: "
|
||||
f"running image {running_image} != compose image {compose_image}"
|
||||
)
|
||||
update_flag = 1
|
||||
else:
|
||||
local_digest = get_local_digest(running_image)
|
||||
remote_digest = get_remote_digest(running_image)
|
||||
|
||||
if local_digest and remote_digest and local_digest != remote_digest:
|
||||
logger.info(
|
||||
f"Registry update available for {container.name}: "
|
||||
f"{local_digest} != {remote_digest}"
|
||||
)
|
||||
update_flag = 1
|
||||
|
||||
CONTAINER_UPDATE.labels(
|
||||
container=container.name,
|
||||
compose_image=compose_image if compose_image else "unknown",
|
||||
running_image=running_image,
|
||||
com_docker_compose_project=project_label
|
||||
).set(update_flag)
|
||||
|
||||
logger.info(
|
||||
f"Container {container.name}: "
|
||||
f"running={running_image}, "
|
||||
f"compose={compose_image}, "
|
||||
f"update_available={update_flag}"
|
||||
)
|
||||
|
||||
LAST_CHECK.set(time.time())
|
||||
logger.info("Container update check complete")
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger.info(
|
||||
f"Starting Docker update exporter on port {EXPORTER_PORT} "
|
||||
f"with LOG_LEVEL={LOG_LEVEL}"
|
||||
)
|
||||
|
||||
start_http_server(EXPORTER_PORT)
|
||||
|
||||
while True:
|
||||
try:
|
||||
check_containers()
|
||||
except Exception as e:
|
||||
logger.exception(f"Unhandled error during update check: {e}")
|
||||
|
||||
time.sleep(CHECK_INTERVAL)
|
||||
@@ -0,0 +1,630 @@
|
||||
tempo | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
|
||||
tempo | line 24: field compactor not found in type app.Config
|
||||
tempo | level=info ts=2026-02-28T03:38:16.504252277Z caller=main.go:106 msg="Starting Tempo" version="(version=v2.10.1, branch=main, revision=ad20e59db)" target=all
|
||||
tempo | level=info ts=2026-02-28T03:38:16.506107792Z caller=server.go:284 msg="server listening on addresses" http=[::]:3200 grpc=[::]:9095
|
||||
tempo | level=info ts=2026-02-28T03:38:16.510179623Z caller=cache.go:55 msg="caches available to storage backend" parquet-footer=false bloom=false parquet-offset-idx=false parquet-column-idx=false trace-id-index=false parquet-page=false
|
||||
tempo | level=info ts=2026-02-28T03:38:16.51381011Z caller=cache.go:55 msg="caches available to storage backend" parquet-footer=false bloom=false parquet-offset-idx=false parquet-column-idx=false trace-id-index=false parquet-page=false
|
||||
tempo | level=warn ts=2026-02-28T03:38:16.515946735Z caller=modules.go:682 msg="Scheduler address is empty in single binary mode. Attempting automatic worker configuration." address=127.0.0.1:9095
|
||||
tempo | level=info ts=2026-02-28T03:38:16.515962445Z caller=backendworker.go:80 msg="backend worker starting" worker_id=4777f4a6f30e
|
||||
tempo | level=info ts=2026-02-28T03:38:16.518963163Z caller=frontend.go:88 msg="creating middleware in query frontend"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.521703999Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:38:16.521732753Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:38:16.521738063Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:38:16.52174193Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:38:16.521750647Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:38:16.521756559Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | ts=2026-02-28T03:38:16Z level=info msg="OTel Shim Logger Initialized" component=tempo
|
||||
tempo | level=warn ts=2026-02-28T03:38:16.527971627Z caller=modules.go:397 msg="Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly." address=127.0.0.1:9095
|
||||
tempo | level=info ts=2026-02-28T03:38:16.528090421Z caller=worker.go:108 msg="Starting querier worker connected to query-frontend" frontend=127.0.0.1:9095
|
||||
tempo | level=warn ts=2026-02-28T03:38:16.528643676Z caller=modules.go:295 msg="metrics-generator is not configured." err="no metrics_generator.storage.path configured, metrics generator will be disabled"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529205548Z caller=module_service.go:82 msg=starting module=cache-provider
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529235324Z caller=module_service.go:82 msg=starting module=optional-store
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529246304Z caller=module_service.go:82 msg=starting module=store
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529310496Z caller=module_service.go:82 msg=starting module=internal-server
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529363827Z caller=module_service.go:82 msg=starting module=server
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529446112Z caller=module_service.go:82 msg=starting module=overrides
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529477541Z caller=module_service.go:82 msg=starting module=memberlist-kv
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529508179Z caller=module_service.go:82 msg=starting module=overrides-api
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529517306Z caller=module_service.go:82 msg=starting module=live-store-ring
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529975612Z caller=module_service.go:82 msg=starting module=metrics-generator-ring
|
||||
tempo | level=info ts=2026-02-28T03:38:16.530025126Z caller=ring.go:372 msg="ring doesn't exist in KV store yet"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.529669724Z caller=ring.go:372 msg="ring doesn't exist in KV store yet"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.530662269Z caller=module_service.go:82 msg=starting module=usage-report
|
||||
tempo | level=info ts=2026-02-28T03:38:16.530661898Z caller=module_service.go:82 msg=starting module=partition-ring
|
||||
tempo | level=info ts=2026-02-28T03:38:16.530845895Z caller=partition_ring_watcher.go:73 msg="partition ring doesn't exist in KV store yet"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.530989036Z caller=module_service.go:82 msg=starting module=live-store
|
||||
tempo | level=info ts=2026-02-28T03:38:16.530999045Z caller=module_service.go:82 msg=starting module=query-frontend
|
||||
tempo | level=info ts=2026-02-28T03:38:16.530994115Z caller=module_service.go:82 msg=starting module=querier
|
||||
tempo | level=info ts=2026-02-28T03:38:16.531011028Z caller=module_service.go:82 msg=starting module=block-builder
|
||||
tempo | level=info ts=2026-02-28T03:38:16.531029903Z caller=module_service.go:82 msg=starting module=backend-worker
|
||||
tempo | level=info ts=2026-02-28T03:38:16.531033931Z caller=module_service.go:82 msg=starting module=distributor
|
||||
tempo | level=info ts=2026-02-28T03:38:16.531047757Z caller=tempodb.go:633 msg="polling enabled" interval=5m0s blocklist_concurrency=50
|
||||
tempo | level=info ts=2026-02-28T03:38:16.531594119Z caller=module_service.go:82 msg=starting module=metrics-generator
|
||||
tempo | level=info ts=2026-02-28T03:38:16.531614337Z caller=module_service.go:82 msg=starting module=backend-scheduler
|
||||
tempo | level=info ts=2026-02-28T03:38:16.531618996Z caller=backendscheduler.go:109 msg="backend scheduler starting"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.534070025Z caller=poller.go:249 msg="blocklist poll complete" seconds=0.002997762
|
||||
tempo | level=info ts=2026-02-28T03:38:16.534101805Z caller=backendworker.go:183 msg="backend worker running"
|
||||
tempo | ts=2026-02-28T03:38:16Z level=info msg="Starting GRPC server" component=tempo endpoint=[::]:4317
|
||||
tempo | level=info ts=2026-02-28T03:38:16.535267337Z caller=worker.go:184 msg="adding connection" addr=127.0.0.1:9095
|
||||
tempo | level=info ts=2026-02-28T03:38:16.536553646Z caller=worker.go:250 msg="total worker concurrency updated" totalConcurrency=20
|
||||
tempo | ts=2026-02-28T03:38:16Z level=info msg="Starting HTTP server" component=tempo endpoint=[::]:4318
|
||||
tempo | level=info ts=2026-02-28T03:38:16.538250531Z caller=backendscheduler.go:162 msg="backend scheduler running"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.538293953Z caller=app.go:214 msg="Tempo started"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.538315674Z caller=compaction.go:102 msg="compaction provider started"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.538325803Z caller=compaction.go:126 msg="compaction provider waiting for poll notification"
|
||||
tempo | level=info ts=2026-02-28T03:38:16.538336152Z caller=retention.go:45 msg="retention provider started"
|
||||
tempo | level=warn ts=2026-02-28T03:38:31.539059217Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.001238779s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:38:31.539333085Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=137.845499ms
|
||||
tempo | level=warn ts=2026-02-28T03:38:47.706949743Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000246082s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:38:47.707402278Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=463.092807ms
|
||||
tempo | level=warn ts=2026-02-28T03:39:03.90316531Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000234022s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:39:03.90345658Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=2.456866248s
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022018905Z caller=app.go:242 msg="=== received SIGINT/SIGTERM ===\n*** exiting"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022341133Z caller=module_service.go:120 msg="module stopped" module=metrics-generator
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022392981Z caller=compaction.go:131 msg="compaction provider stopping"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022421094Z caller=module_service.go:120 msg="module stopped" module=live-store
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022436252Z caller=backendscheduler.go:136 msg="stopping provider" provider=1
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022477952Z caller=retention.go:50 msg="retention provider stopping"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022483221Z caller=backendscheduler.go:136 msg="stopping provider" provider=0
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022499622Z caller=backendscheduler.go:154 msg="all providers stopped"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.022514801Z caller=compaction.go:202 msg="compaction provider measure ticker stopping"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.023035023Z caller=module_service.go:120 msg="module stopped" module=distributor
|
||||
tempo | level=info ts=2026-02-28T03:39:04.023143178Z caller=module_service.go:120 msg="module stopped" module=block-builder
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023319751Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.481348854s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023397699Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485115377s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023419059Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485051917s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023433387Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485095891s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023447353Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485568464s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.02345667Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485115568s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.02346698Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485072817s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023476648Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485110168s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023486636Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485125947s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023501636Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485212231s msg=gRPC err="queue is stopped"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.023502868Z caller=module_service.go:120 msg="module stopped" module=optional-store
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023508539Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485663543s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023539146Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485854102s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023551659Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485305717s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.02356266Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485318241s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.02357358Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485326275s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023587017Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.48529687s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023585083Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485268317s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023596484Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485287864s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023606022Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.485293264s msg=gRPC err="queue is stopped"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.023644765Z caller=module_service.go:120 msg="module stopped" module=query-frontend
|
||||
tempo | level=info ts=2026-02-28T03:39:04.023659914Z caller=module_service.go:120 msg="module stopped" module=overrides-api
|
||||
tempo | level=warn ts=2026-02-28T03:39:04.023515852Z caller=server.go:1560 method=/frontend.Frontend/Process duration=47.481666904s msg=gRPC err="queue is stopped"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.023804989Z caller=frontend.go:339 msg="received shutdown notification from querier" querier=4777f4a6f30e
|
||||
tempo | level=info ts=2026-02-28T03:39:04.024020695Z caller=module_service.go:120 msg="module stopped" module=querier
|
||||
tempo | level=info ts=2026-02-28T03:39:04.037362837Z caller=module_service.go:120 msg="module stopped" module=partition-ring
|
||||
tempo | level=info ts=2026-02-28T03:39:04.037395037Z caller=module_service.go:120 msg="module stopped" module=metrics-generator-ring
|
||||
tempo | level=info ts=2026-02-28T03:39:04.037503452Z caller=module_service.go:120 msg="module stopped" module=live-store-ring
|
||||
tempo | level=info ts=2026-02-28T03:39:04.436087499Z caller=backendscheduler.go:201 msg="backend scheduler stopping"
|
||||
tempo | level=info ts=2026-02-28T03:39:04.436141891Z caller=module_service.go:120 msg="module stopped" module=backend-scheduler
|
||||
tempo | level=warn ts=2026-02-28T03:39:10.605069202Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=44.344µs msg=gRPC err="rpc error: code = Internal desc = nil job received"
|
||||
tempo | level=error ts=2026-02-28T03:39:10.605239002Z caller=backendworker.go:454 msg="error calling scheduler" err="error getting next job: rpc error: code = Internal desc = nil job received" backoff=8.910321994s
|
||||
tempo | level=info ts=2026-02-28T03:39:14.488350709Z caller=main.go:106 msg="Starting Tempo" version="(version=v2.10.1, branch=main, revision=ad20e59db)" target=all
|
||||
tempo | level=info ts=2026-02-28T03:39:14.488776793Z caller=cache.go:55 msg="caches available to storage backend" parquet-footer=false bloom=false parquet-offset-idx=false parquet-column-idx=false trace-id-index=false parquet-page=false
|
||||
tempo | level=info ts=2026-02-28T03:39:14.48970103Z caller=server.go:284 msg="server listening on addresses" http=[::]:3200 grpc=[::]:9095
|
||||
tempo | level=info ts=2026-02-28T03:39:14.490096587Z caller=cache.go:55 msg="caches available to storage backend" parquet-footer=false bloom=false parquet-offset-idx=false parquet-column-idx=false trace-id-index=false parquet-page=false
|
||||
tempo | level=warn ts=2026-02-28T03:39:14.492106423Z caller=modules.go:682 msg="Scheduler address is empty in single binary mode. Attempting automatic worker configuration." address=127.0.0.1:9095
|
||||
tempo | level=info ts=2026-02-28T03:39:14.492149043Z caller=backendworker.go:80 msg="backend worker starting" worker_id=4777f4a6f30e
|
||||
tempo | level=info ts=2026-02-28T03:39:14.492516217Z caller=frontend.go:88 msg="creating middleware in query frontend"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.492620283Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:39:14.492661331Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:39:14.492681839Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:39:14.492700414Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:39:14.492715484Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=info ts=2026-02-28T03:39:14.492728578Z caller=sync_handler_cache.go:131 msg="init frontend cache" role=frontend-search enabled=false
|
||||
tempo | level=warn ts=2026-02-28T03:39:14.493177155Z caller=modules.go:397 msg="Worker address is empty in single binary mode. Attempting automatic worker configuration. If queries are unresponsive consider configuring the worker explicitly." address=127.0.0.1:9095
|
||||
tempo | level=info ts=2026-02-28T03:39:14.493339051Z caller=worker.go:108 msg="Starting querier worker connected to query-frontend" frontend=127.0.0.1:9095
|
||||
tempo | ts=2026-02-28T03:39:14Z level=info msg="OTel Shim Logger Initialized" component=tempo
|
||||
tempo | level=warn ts=2026-02-28T03:39:14.495105829Z caller=modules.go:295 msg="metrics-generator is not configured." err="no metrics_generator.storage.path configured, metrics generator will be disabled"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.495667199Z caller=module_service.go:82 msg=starting module=optional-store
|
||||
tempo | level=info ts=2026-02-28T03:39:14.496941125Z caller=module_service.go:82 msg=starting module=internal-server
|
||||
tempo | level=info ts=2026-02-28T03:39:14.497087782Z caller=module_service.go:82 msg=starting module=cache-provider
|
||||
tempo | level=info ts=2026-02-28T03:39:14.497242835Z caller=module_service.go:82 msg=starting module=store
|
||||
tempo | level=info ts=2026-02-28T03:39:14.497359636Z caller=module_service.go:82 msg=starting module=server
|
||||
tempo | level=info ts=2026-02-28T03:39:14.497560034Z caller=module_service.go:82 msg=starting module=overrides
|
||||
tempo | level=info ts=2026-02-28T03:39:14.49825639Z caller=module_service.go:82 msg=starting module=overrides-api
|
||||
tempo | level=info ts=2026-02-28T03:39:14.498445827Z caller=module_service.go:82 msg=starting module=memberlist-kv
|
||||
tempo | level=info ts=2026-02-28T03:39:14.498546206Z caller=module_service.go:82 msg=starting module=metrics-generator-ring
|
||||
tempo | level=info ts=2026-02-28T03:39:14.498747066Z caller=ring.go:372 msg="ring doesn't exist in KV store yet"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.498816477Z caller=module_service.go:82 msg=starting module=live-store-ring
|
||||
tempo | level=info ts=2026-02-28T03:39:14.498829101Z caller=module_service.go:82 msg=starting module=usage-report
|
||||
tempo | level=info ts=2026-02-28T03:39:14.501249322Z caller=module_service.go:82 msg=starting module=query-frontend
|
||||
tempo | level=info ts=2026-02-28T03:39:14.501273138Z caller=module_service.go:82 msg=starting module=backend-scheduler
|
||||
tempo | level=info ts=2026-02-28T03:39:14.501281423Z caller=backendscheduler.go:109 msg="backend scheduler starting"
|
||||
tempo | ts=2026-02-28T03:39:14Z level=info msg="Starting GRPC server" component=tempo endpoint=[::]:4317
|
||||
tempo | ts=2026-02-28T03:39:14Z level=info msg="Starting HTTP server" component=tempo endpoint=[::]:4318
|
||||
tempo | level=info ts=2026-02-28T03:39:14.501985883Z caller=ring.go:372 msg="ring doesn't exist in KV store yet"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502072537Z caller=module_service.go:82 msg=starting module=partition-ring
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502101092Z caller=module_service.go:82 msg=starting module=backend-worker
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502101442Z caller=partition_ring_watcher.go:73 msg="partition ring doesn't exist in KV store yet"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502119777Z caller=tempodb.go:633 msg="polling enabled" interval=5m0s blocklist_concurrency=50
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502157537Z caller=module_service.go:82 msg=starting module=metrics-generator
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502189098Z caller=module_service.go:82 msg=starting module=distributor
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502216168Z caller=poller.go:249 msg="blocklist poll complete" seconds=7.9791e-05
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502232729Z caller=backendworker.go:183 msg="backend worker running"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502586187Z caller=module_service.go:82 msg=starting module=querier
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502630632Z caller=module_service.go:82 msg=starting module=live-store
|
||||
tempo | level=info ts=2026-02-28T03:39:14.502740018Z caller=worker.go:184 msg="adding connection" addr=127.0.0.1:9095
|
||||
tempo | level=info ts=2026-02-28T03:39:14.503119545Z caller=worker.go:250 msg="total worker concurrency updated" totalConcurrency=20
|
||||
tempo | level=info ts=2026-02-28T03:39:14.504169348Z caller=module_service.go:82 msg=starting module=block-builder
|
||||
tempo | level=info ts=2026-02-28T03:39:14.506269895Z caller=backendscheduler.go:162 msg="backend scheduler running"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.506302157Z caller=app.go:214 msg="Tempo started"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.5063145Z caller=compaction.go:102 msg="compaction provider started"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.506320521Z caller=compaction.go:126 msg="compaction provider waiting for poll notification"
|
||||
tempo | level=info ts=2026-02-28T03:39:14.506328516Z caller=retention.go:45 msg="retention provider started"
|
||||
tempo | level=warn ts=2026-02-28T03:39:29.508287056Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000581399s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:39:29.508572244Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=184.86286ms
|
||||
tempo | level=warn ts=2026-02-28T03:39:45.052093517Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000133061s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:39:45.052342347Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=683.918431ms
|
||||
tempo | level=warn ts=2026-02-28T03:40:01.617186207Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000246556s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:40:01.617444254Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=2.230026613s
|
||||
tempo | level=warn ts=2026-02-28T03:40:21.738394906Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000616341s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:40:21.738738376Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=6.928753128s
|
||||
tempo | level=warn ts=2026-02-28T03:41:00.303046594Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000840441s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:41:00.303269665Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=46.690005969s
|
||||
tempo | level=warn ts=2026-02-28T03:42:09.822438562Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.001122933s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:42:09.822714043Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=55.06043985s
|
||||
tempo | level=warn ts=2026-02-28T03:43:23.832379908Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000290546s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:43:23.83267209Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.189518589s
|
||||
tempo | level=info ts=2026-02-28T03:44:14.51080544Z caller=poller.go:249 msg="blocklist poll complete" seconds=0.005466397
|
||||
tempo | level=info ts=2026-02-28T03:44:14.510866645Z caller=compaction.go:142 msg="received empty tenant"
|
||||
tempo | level=warn ts=2026-02-28T03:44:32.167179641Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000322598s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:44:32.167467976Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=59.343922215s
|
||||
tempo | level=warn ts=2026-02-28T03:45:44.442347133Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000682317s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:45:44.442597837Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=55.235447404s
|
||||
tempo | level=warn ts=2026-02-28T03:46:55.621855189Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000935062s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:46:55.622113698Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=57.996223179s
|
||||
tempo | level=warn ts=2026-02-28T03:48:03.715877703Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000435945s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:48:03.716165717Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.637759938s
|
||||
tempo | level=info ts=2026-02-28T03:49:14.505419519Z caller=poller.go:249 msg="blocklist poll complete" seconds=6.9141e-05
|
||||
tempo | level=info ts=2026-02-28T03:49:14.505482128Z caller=compaction.go:142 msg="received empty tenant"
|
||||
tempo | level=warn ts=2026-02-28T03:49:14.842107415Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000257167s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:49:14.842340856Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.010954393s
|
||||
tempo | level=warn ts=2026-02-28T03:50:28.490571505Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000174599s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:50:28.490848819Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=56.688382421s
|
||||
tempo | level=warn ts=2026-02-28T03:51:39.573560321Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000365972s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:51:39.573943555Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=59.697783583s
|
||||
tempo | level=info ts=2026-02-28T03:52:47.227247458Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:47.230208333Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772249867&end=1772250767&scope=&q= (500) 1.277885ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:47.230708187Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772249867&end=1772250767&scope=&q= (500) 37.201µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:47.230898016Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.003652292 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:47.230940145Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772250767&limit=5000&start=1772249867" duration=4.515183ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:47.231008214Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 4.711182ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:55.046107322Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000358106s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:52:55.04635027Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=51.430516974s
|
||||
tempo | level=info ts=2026-02-28T03:52:59.295759743Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.304296915Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 119.124µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.304832928Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 87.075µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.305196605Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0 duration_seconds=0.009433275 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.30531019Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&start=1772249879&tag=resource.service.name" duration=9.561277ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.305348642Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&tag=resource.service.name (500) 9.666326ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.305855119Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:52:59.305855199Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.306587153Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 123.874µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.309044746Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 86.212µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.309177898Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.003320204 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.309206162Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&start=1772249879&tag=name" duration=3.356933ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.309253811Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 3.44016ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.309659288Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.003794991 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.309690206Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&start=1772249879&tag=status" duration=3.832993ms response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:52:59.364894892Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.365471181Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 102.253µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.365832854Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 53.241µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.366012043Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0 duration_seconds=0.001112984 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.366043211Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&start=1772249879&tag=resource.service.name" duration=1.157266ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.366073419Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&tag=resource.service.name (500) 1.236056ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.374586756Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:52:59.374912371Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.375016809Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 103.666µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.375473521Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 57.058µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.375643352Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.001053461 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.375670744Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&start=1772249879&tag=name" duration=1.093838ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.375692655Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 1.189688ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.375839072Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.000923395 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.375865262Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&start=1772249879&tag=status" duration=953.653µs response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:52:59.430639585Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:52:59.430641929Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.431784919Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 54.263µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.431784538Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 92.496µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.432085166Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q= (500) 48.462µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.432098762Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q= (500) 43.912µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.43220327Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query= range_seconds=0 duration_seconds=0.001559207 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.43222465Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.001584123 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.432240359Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&start=1772249879&tag=resource.service.name" duration=1.604001ms response_size=0 status=500
|
||||
tempo | level=info ts=2026-02-28T03:52:59.432241802Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&start=1772249879&tag=name" duration=1.615052ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.4322619Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&tag=resource.service.name (500) 1.679032ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.432262761Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 1.68304ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.437788703Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:52:59.438468568Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.00067752 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.438494086Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&start=1772249879&tag=status" duration=713.858µs response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:52:59.554734737Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.555499843Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 132.17µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.555983276Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 61.475µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.5561281Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001390236 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.556162785Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=resource.service.name" duration=1.445401ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.556196699Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.527045ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.570117374Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:52:59.570117354Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.571119107Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 54.263µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.57120513Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.001085222 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.571256697Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=status" duration=1.154452ms response_size=130 status=200
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.571389749Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 48.662µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.57159701Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001479817 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.571636224Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=name" duration=1.535432ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.571666902Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.607797ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.604904298Z caller=search_handlers.go:225 msg="search request" tenant=single-tenant query={} range_seconds=3600 limit=20 spans_per_spanset=3
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.605700042Z caller=server.go:2286 msg="GET /querier/api/search?start=1772250479&end=1772250779&limit=20&spss=3&q=%7B%7D (500) 129.284µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.606087353Z caller=server.go:2286 msg="GET /querier/api/search?start=1772250479&end=1772250779&limit=20&spss=3&q=%7B%7D (500) 58.489µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.606187743Z caller=server.go:2286 msg="GET /querier/api/search?start=1772249879&end=1772250179&limit=20&spss=3&q=%7B%7D (500) 45.105µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.606188826Z caller=server.go:2286 msg="GET /querier/api/search?start=1772250179&end=1772250479&limit=20&spss=3&q=%7B%7D (500) 48.352µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.606414472Z caller=server.go:2286 msg="GET /querier/api/search?start=1772250179&end=1772250479&limit=20&spss=3&q=%7B%7D (500) 36.369µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.606446532Z caller=server.go:2286 msg="GET /querier/api/search?start=1772249879&end=1772250179&limit=20&spss=3&q=%7B%7D (500) 37.09µs Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.606580405Z caller=search_handlers.go:184 msg="search response - no resp" tenant=single-tenant duration_seconds=0.001723747 status_code=500 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.606602738Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/search?end=1772250779&limit=20&q=%7B%7D&start=1772247196" duration=1.748845ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.606631933Z caller=server.go:2286 msg="GET /api/search?end=1772250796&limit=20&q=%7B%7D&start=1772247196 (500) 1.819327ms Response: \"error querying live-stores in Querier.SearchRecent: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.644836935Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:52:59.644835402Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.646233004Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 58.19µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.646354614Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.001516506 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.64637912Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=status" duration=1.555229ms response_size=130 status=200
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.646532249Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 40.095µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.646628131Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001792909 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.646645163Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=name" duration=1.822845ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.646662535Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.877126ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.651077217Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.651670158Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 67.368µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.651989281Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 71.174µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.652129425Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001046398 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.652183919Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=resource.service.name" duration=1.105269ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.652218053Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.183345ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.834908067Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.83583599Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 180.561µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.83622182Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 53.04µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.836628539Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001717005 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.83666094Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=resource.service.name" duration=1.762972ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.836695936Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.845125ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.844675474Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:52:59.844676606Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.845559664Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 94.889µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.846061863Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249879&end=1772250779&q=%7B%7D (500) 46.718µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:52:59.846141675Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.001464176 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.846167934Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=status" duration=1.50822ms response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:52:59.846173765Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001499192 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:52:59.846215905Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250779&limit=5000&q=%7B%7D&start=1772249879&tag=name" duration=1.557683ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:52:59.846233909Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.626814ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:54:06.387117532Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000155449s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:54:06.387384306Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.337824347s
|
||||
tempo | level=info ts=2026-02-28T03:54:07.747790047Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.748598275Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249947&end=1772250847&q=%7B%7D (500) 118.444µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.74898729Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772249947&end=1772250847&q=%7B%7D (500) 90.04µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.749157522Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001365771 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:54:07.749184953Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772250847&limit=5000&q=%7B%7D&start=1772249947&tag=resource.service.name" duration=1.405937ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.749223236Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.496548ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.75446989Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.754987538Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772249947&end=1772250847&scope=&q= (500) 70.694µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.754546215Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.755500718Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772249947&end=1772250847&scope=&q= (500) 28.594µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.755823719Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.001353137 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:54:07.755849367Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772250847&limit=5000&start=1772249947" duration=1.403232ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.755871819Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 1.466611ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.756469869Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.001920669 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:54:07.756492221Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250847&limit=5000&q=%7B%7D&start=1772249947&tag=status" duration=1.96836ms response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:54:07.767792876Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.768349809Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249947&end=1772250847&q=%7B%7D (500) 87.937µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.768676455Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249947&end=1772250847&q=%7B%7D (500) 54.313µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.768857398Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001062668 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:54:07.768881633Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250847&limit=5000&q=%7B%7D&start=1772249947&tag=name" duration=1.097895ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.768904576Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.15937ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.805215311Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.80596677Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249947&end=1772250847&q=%7Bresource.service.name%3Dtraefik%7D (500) 89.989µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.806306162Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772249947&end=1772250847&q=%7Bresource.service.name%3Dtraefik%7D (500) 61.466µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.806466906Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001251445 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:54:07.806493576Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772250847&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772249947&tag=name" duration=1.294836ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:54:07.806545344Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.387872ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:54:07.810069784Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:54:07.8114648Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001388143 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:54:07.811500197Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772250847&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772249947&tag=status" duration=1.443909ms response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:54:14.505512462Z caller=poller.go:249 msg="blocklist poll complete" seconds=6.3169e-05
|
||||
tempo | level=info ts=2026-02-28T03:54:14.505573466Z caller=compaction.go:142 msg="received empty tenant"
|
||||
tempo | level=warn ts=2026-02-28T03:55:17.230319246Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.00109829s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:55:17.23056488Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.331397281s
|
||||
tempo | level=warn ts=2026-02-28T03:56:27.212672373Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000757225s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:56:27.212924711Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=54.995665223s
|
||||
tempo | level=info ts=2026-02-28T03:57:35.447493981Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.448262373Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 82.466µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.448835105Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 57.639µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.448972244Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001477442 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.448997662Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251055&limit=5000&q=%7B%7D&start=1772250155&tag=resource.service.name" duration=1.516175ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.44902242Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.596868ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=error ts=2026-02-28T03:57:35.449625368Z caller=search_handlers.go:115 msg="search: could not create combiner" err="invalid TraceQL query: parse error at line 1, col 24: unknown identifier: traefik"
|
||||
tempo | level=info ts=2026-02-28T03:57:35.449658121Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/search?end=1772251072&limit=20&q=%7Bresource.service.name%3Dtraefik%7D&start=1772247472" duration=2.110898ms response_size=0 status=400
|
||||
tempo | level=info ts=2026-02-28T03:57:35.452827149Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.453724735Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 79.12µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.454072382Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 73.098µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.454754481Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001925969 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.454809205Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251055&limit=5000&q=%7B%7D&start=1772250155&tag=resource.service.name" duration=1.993728ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.454834763Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 2.066896ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.464569659Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:57:35.464647065Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:57:35.465734159Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.466270002Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250155&end=1772251055&scope=&q= (500) 51.658µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.466468006Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q= (500) 65.935µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.466467926Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q= (500) 68.098µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.466578975Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250155&end=1772251055&scope=&q= (500) 23.484µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.466683073Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.000949535 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.466703331Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772251055&limit=5000&start=1772250155" duration=1.013645ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.4667241Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 1.062919ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.466757072Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q= (500) 46.428µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.467000492Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:57:35.467747053Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.003097213 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.467810633Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251055&limit=5000&start=1772250155&tag=name" duration=3.183225ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.467847663Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 3.248088ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.46786673Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q= (500) 42.22µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.468069162Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query= range_seconds=0 duration_seconds=0.003499453 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.468109167Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251055&limit=5000&start=1772250155&tag=name" duration=3.552883ms response_size=0 status=500
|
||||
tempo | level=info ts=2026-02-28T03:57:35.468133974Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.001130987 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.468138273Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&tag=name (500) 3.643706ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.468153011Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251055&limit=5000&start=1772250155&tag=status" duration=1.158009ms response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:57:35.48429799Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.484956764Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q=%7Bresource.service.name%3Dtraefik%7D (500) 104.417µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.485242635Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q=%7Bresource.service.name%3Dtraefik%7D (500) 50.867µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.485397968Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001097994 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.485429208Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251055&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250155&tag=name" duration=1.142479ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.485458763Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.215597ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.4890928Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:57:35.489884336Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.00078816 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.489910275Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251055&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250155&tag=status" duration=825.11µs response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:57:35.533791838Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:57:35.534540833Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query= range_seconds=0 duration_seconds=0.000745989 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.534565951Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251055&limit=5000&start=1772250155&tag=status" duration=783.882µs response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:57:35.563874004Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.564416319Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 78.509µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.564750501Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250155&end=1772251055&q=%7B%7D (500) 53.641µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.564892479Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001016812 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.564919931Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251055&limit=5000&q=%7B%7D&start=1772250155&tag=resource.service.name" duration=1.055645ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.564948625Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.131119ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.573722094Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:57:35.573743404Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.574654356Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q=%7Bresource.service.name%3Dtraefik%7D (500) 72.246µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.574864683Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001118294 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.574890291Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251055&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250155&tag=status" duration=1.149151ms response_size=130 status=200
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.575274137Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250155&end=1772251055&q=%7Bresource.service.name%3Dtraefik%7D (500) 91.864µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:57:35.575494063Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.00177241 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:57:35.575515052Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251055&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250155&tag=name" duration=1.803948ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:57:35.575546121Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.873881ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:57:36.820760121Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000451663s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:57:36.821000996Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=58.35515471s
|
||||
tempo | level=info ts=2026-02-28T03:58:32.220909942Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.222185944Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250212&end=1772251112&q=%7B%7D (500) 113.223µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.222648478Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250212&end=1772251112&q=%7B%7D (500) 83.147µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:32.222821986Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001911723 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:32.22293528Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251112&limit=5000&q=%7B%7D&start=1772250212&tag=resource.service.name" duration=2.02115ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.223024188Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 2.182405ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:32.22766645Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.229001193Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250212&end=1772251112&q=%7Bresource.service.name%3Dtraefik%7D (500) 95.671µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.22933287Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250212&end=1772251112&q=%7Bresource.service.name%3Dtraefik%7D (500) 68.77µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:32.229489626Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001819599 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:32.229537877Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251112&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250212&tag=name" duration=1.893367ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.22980944Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 2.228621ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:32.230137901Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:58:32.230151897Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.231243921Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250212&end=1772251112&scope=&q= (500) 42.661µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.231768121Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250212&end=1772251112&scope=&q= (500) 34.455µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:32.231879883Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.001727085 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:32.231927694Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772251112&limit=5000&start=1772250212" duration=1.798167ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.231954043Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 1.85241ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:32.232134333Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001961206 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:32.232167276Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251112&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250212&tag=status" duration=2.034485ms response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:58:32.285507543Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:58:32.286381996Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.000871818 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:32.286407925Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251112&limit=5000&q=%7B%7D&start=1772250212&tag=status" duration=916.472µs response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:58:32.294280212Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.29475576Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250212&end=1772251112&q=%7B%7D (500) 84.451µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.295220318Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250212&end=1772251112&q=%7B%7D (500) 54.984µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:32.295346977Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001065514 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:32.295370862Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251112&limit=5000&q=%7B%7D&start=1772250212&tag=name" duration=1.109325ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:32.295423582Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.210067ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:45.987186027Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:45.987781302Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250225&end=1772251125&q=%7B%7D (500) 89.659µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:45.988143777Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250225&end=1772251125&q=%7B%7D (500) 45.104µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:45.988296276Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001109726 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:45.988326693Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251125&limit=5000&q=%7B%7D&start=1772250225&tag=resource.service.name" duration=1.153651ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:45.988367049Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.244051ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:45.993858696Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:45.994205922Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250225&end=1772251125&scope=&q= (500) 48.631µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:45.994567956Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250225&end=1772251125&scope=&q= (500) 25.388µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:45.994712711Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.00085203 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:45.994736776Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772251125&limit=5000&start=1772250225" duration=901.995µs response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:45.994761041Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 974.271µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:45.998993379Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:45.999499025Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250225&end=1772251125&q=%7B%7D (500) 96.371µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:45.999960998Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250225&end=1772251125&q=%7B%7D (500) 58.551µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:46.000109869Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query={} range_seconds=0 duration_seconds=0.001115388 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:46.000191904Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251125&limit=5000&q=%7B%7D&start=1772250225&tag=name" duration=1.204145ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:46.000245305Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7B%7D&tag=name (500) 1.295628ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:46.005858884Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:58:46.006440202Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.000578262 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:46.00646579Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251126&limit=5000&q=%7B%7D&start=1772250226&tag=status" duration=617.327µs response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:58:46.038874293Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:46.039289979Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250226&end=1772251126&q=%7Bresource.service.name%3Dtraefik%7D (500) 80.222µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:46.039656171Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250226&end=1772251126&q=%7Bresource.service.name%3Dtraefik%7D (500) 69.872µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:46.039798289Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.00092052 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:46.039821953Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251126&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250226&tag=name" duration=957.809µs response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:46.039844978Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.035867ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:46.049178986Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:58:46.050280237Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001097534 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:46.050326674Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251126&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250226&tag=status" duration=1.151936ms response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T03:58:48.785841897Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.786673308Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250228&end=1772251128&q=%7B%7D (500) 100.36µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.786993352Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250228&end=1772251128&q=%7B%7D (500) 56.375µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:48.787118058Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.00127548 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:48.787142945Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251128&limit=5000&q=%7B%7D&start=1772250228&tag=resource.service.name" duration=1.314524ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.787165358Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.390929ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:48.794066079Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:58:48.794090214Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:58:48.795203578Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001109156 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.795226541Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250228&end=1772251128&q=%7Bresource.service.name%3Dtraefik%7D (500) 62.438µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:48.795230719Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251128&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250228&tag=status" duration=1.145034ms response_size=130 status=200
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.795561594Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250228&end=1772251128&q=%7Bresource.service.name%3Dtraefik%7D (500) 70.834µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:48.795744581Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001676498 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:48.795765059Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251128&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250228&tag=name" duration=1.710554ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.79579207Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.790703ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=error ts=2026-02-28T03:58:48.83299873Z caller=search_handlers.go:115 msg="search: could not create combiner" err="invalid TraceQL query: parse error at line 1, col 24: unknown identifier: traefik"
|
||||
tempo | level=info ts=2026-02-28T03:58:48.83304106Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/search?end=1772251146&limit=20&q=%7Bresource.service.name%3Dtraefik%7D&start=1772247546" duration=90.15µs response_size=0 status=400
|
||||
tempo | level=info ts=2026-02-28T03:58:48.874206161Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.87493086Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250228&end=1772251128&q=%7B%7D (500) 80.572µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.875247888Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250228&end=1772251128&q=%7B%7D (500) 64.743µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:48.875412961Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001204506 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:48.875441434Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251128&limit=5000&q=%7B%7D&start=1772250228&tag=resource.service.name" duration=1.249171ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.875475068Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.333641ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:48.883983787Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:58:48.883976754Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.88471521Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250228&end=1772251128&q=%7Bresource.service.name%3Dtraefik%7D (500) 98.757µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:48.8851653Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001185801 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:48.885194765Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251128&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250228&tag=status" duration=1.232518ms response_size=130 status=200
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.88522882Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250228&end=1772251128&q=%7Bresource.service.name%3Dtraefik%7D (500) 64.813µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:48.885360759Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.00137542 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:48.885386357Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251128&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250228&tag=name" duration=1.404595ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:48.885410463Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.467232ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:49.014415446Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:49.015154923Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250229&end=1772251129&q=%7B%7D (500) 70.854µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:49.015511007Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250229&end=1772251129&q=%7B%7D (500) 65.524µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:49.015653155Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query={} range_seconds=0 duration_seconds=0.001236126 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:49.015676499Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251129&limit=5000&q=%7B%7D&start=1772250229&tag=resource.service.name" duration=1.27538ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:49.015698921Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7B%7D&tag=resource.service.name (500) 1.347867ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:49.024025686Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T03:58:49.024063037Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T03:58:49.024960494Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250229&end=1772251129&q=%7Bresource.service.name%3Dtraefik%7D (500) 119.997µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:49.025149951Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001083918 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:49.02517548Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251129&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250229&tag=status" duration=1.115689ms response_size=130 status=200
|
||||
tempo | level=warn ts=2026-02-28T03:58:49.02544081Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250229&end=1772251129&q=%7Bresource.service.name%3Dtraefik%7D (500) 98.416µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T03:58:49.025664895Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001639639 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T03:58:49.025713907Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251129&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250229&tag=name" duration=1.700363ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T03:58:49.025744665Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.778792ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T03:58:49.467994461Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000736018s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T03:58:49.468248942Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=51.485834687s
|
||||
tempo | level=info ts=2026-02-28T03:59:14.505516299Z caller=poller.go:249 msg="blocklist poll complete" seconds=9.005e-05
|
||||
tempo | level=info ts=2026-02-28T03:59:14.505593114Z caller=compaction.go:142 msg="received empty tenant"
|
||||
tempo | level=warn ts=2026-02-28T04:00:00.62141836Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000885343s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T04:00:00.62175716Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.674604584s
|
||||
tempo | level=warn ts=2026-02-28T04:01:09.533862097Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.00112131s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T04:01:09.534125144Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=59.264069146s
|
||||
tempo | level=info ts=2026-02-28T04:01:29.941563869Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T04:01:29.941563849Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query="{status=ok}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.943318265Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250389&end=1772251289&q=%7Bstatus%3Dok%7D (500) 110.207µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.943868216Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250389&end=1772251289&q=%7Bstatus%3Dok%7D (500) 61.116µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:29.944076008Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query="{status=ok}" range_seconds=0 duration_seconds=0.002508052 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:29.944123869Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251289&limit=5000&q=%7Bstatus%3Dok%7D&start=1772250389&tag=resource.service.name" duration=2.576181ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.944211835Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7Bstatus%3Dok%7D&tag=resource.service.name (500) 2.752084ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:29.944825355Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.003260233 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:29.944849992Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251289&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250389&tag=status" duration=3.301083ms response_size=130 status=200
|
||||
tempo | level=info ts=2026-02-28T04:01:29.94544159Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik && status=ok}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T04:01:29.94544145Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.946054078Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250389&end=1772251289&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D (500) 79.67µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.946098603Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250389&end=1772251289&q=%7Bresource.service.name%3Dtraefik%7D (500) 49.093µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.946857016Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250389&end=1772251289&q=%7Bresource.service.name%3Dtraefik%7D (500) 70.944µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.946905057Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250389&end=1772251289&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D (500) 61.215µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:29.947220684Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik && status=ok}" range_seconds=0 duration_seconds=0.001777269 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:29.947235412Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik}" range_seconds=0 duration_seconds=0.001791837 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:29.947247615Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251289&limit=5000&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D&start=1772250389&tag=name" duration=1.812708ms response_size=0 status=500
|
||||
tempo | level=info ts=2026-02-28T04:01:29.947258776Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251289&limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&start=1772250389&tag=name" duration=1.826764ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.947270758Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%20%26%26%20status%3Dok%7D&tag=name (500) 1.87809ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.947276138Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%7D&tag=name (500) 1.88339ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:29.960342984Z caller=tag_handlers.go:518 msg="search tag request" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.960838761Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250389&end=1772251289&scope=&q= (500) 39.715µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.961101268Z caller=server.go:2286 msg="GET /querier/api/v2/search/tags?start=1772250389&end=1772251289&scope=&q= (500) 32.621µs Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:29.961265038Z caller=tag_handlers.go:527 msg="search tag response" tenant=single-tenant handler=SearchTagsV2 scope= range_seconds=0 duration_seconds=0.000922725 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:29.96129317Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tags?end=1772251289&limit=5000&start=1772250389" duration=973.351µs response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T04:01:29.961326062Z caller=server.go:2286 msg="GET /api/v2/search/tags?limit=5000 (500) 1.049643ms Response: \"error querying live-stores in Querier.SearchTags: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:33.604410488Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query="{status=ok}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.604981117Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250393&end=1772251293&q=%7Bstatus%3Dok%7D (500) 84.439µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.605337871Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/resource.service.name/values?start=1772250393&end=1772251293&q=%7Bstatus%3Dok%7D (500) 52.799µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:33.605460594Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=resource.service.name query="{status=ok}" range_seconds=0 duration_seconds=0.001049545 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:33.605484029Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/resource.service.name/values?end=1772251293&limit=5000&q=%7Bstatus%3Dok%7D&start=1772250393&tag=resource.service.name" duration=1.086664ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.605510328Z caller=server.go:2286 msg="GET /api/v2/search/tag/resource.service.name/values?limit=5000&q=%7Bstatus%3Dok%7D&tag=resource.service.name (500) 1.170984ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:33.613938413Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik && status=ok}" range_seconds=0
|
||||
tempo | level=info ts=2026-02-28T04:01:33.614308312Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.615192815Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250393&end=1772251293&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D (500) 106.932µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:33.615309065Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=status query={} range_seconds=0 duration_seconds=0.000996795 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:33.615334522Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/status/values?end=1772251293&limit=5000&q=%7B%7D&start=1772250393&tag=status" duration=1.027463ms response_size=130 status=200
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.615558405Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250393&end=1772251293&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D (500) 66.005µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:33.615953593Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{resource.service.name=traefik && status=ok}" range_seconds=0 duration_seconds=0.002014959 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:33.615976316Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251293&limit=5000&q=%7Bresource.service.name%3Dtraefik+%26%26+status%3Dok%7D&start=1772250393&tag=name" duration=2.050226ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.615996604Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bresource.service.name%3Dtraefik%20%26%26%20status%3Dok%7D&tag=name (500) 2.11554ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:33.684220173Z caller=tag_handlers.go:540 msg="search tag values request" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{status=ok}" range_seconds=0
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.684911631Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250393&end=1772251293&q=%7Bstatus%3Dok%7D (500) 74.18µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.685479776Z caller=server.go:2286 msg="GET /querier/api/v2/search/tag/name/values?start=1772250393&end=1772251293&q=%7Bstatus%3Dok%7D (500) 116.079µs Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=info ts=2026-02-28T04:01:33.685639858Z caller=tag_handlers.go:550 msg="search tag values response" tenant=single-tenant handler=SearchTagValuesV2 tag=name query="{status=ok}" range_seconds=0 duration_seconds=0.001419274 inspected_bytes=0 request_throughput=0 error=null
|
||||
tempo | level=info ts=2026-02-28T04:01:33.685697567Z caller=handler.go:135 msg="query stats" tenant=single-tenant method=GET traceID= url="/api/v2/search/tag/name/values?end=1772251293&limit=5000&q=%7Bstatus%3Dok%7D&start=1772250393&tag=name" duration=1.488815ms response_size=0 status=500
|
||||
tempo | level=warn ts=2026-02-28T04:01:33.685734456Z caller=server.go:2286 msg="GET /api/v2/search/tag/name/values?limit=5000&q=%7Bstatus%3Dok%7D&tag=name (500) 1.576119ms Response: \"error querying live-stores in Querier.SearchTagValues: error finding partition ring replicas: empty ring\\n\""
|
||||
tempo | level=warn ts=2026-02-28T04:02:21.401664876Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000979208s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T04:02:21.401988659Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=58.963135457s
|
||||
tempo | level=info ts=2026-02-28T04:02:54.506996729Z caller=reporter.go:272 msg="reporting cluster stats" date=2026-02-28T04:02:54.506995536Z
|
||||
tempo | level=warn ts=2026-02-28T04:03:30.924861907Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000152254s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T04:03:30.925110638Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=52.941417365s
|
||||
tempo | level=info ts=2026-02-28T04:04:14.505701243Z caller=poller.go:249 msg="blocklist poll complete" seconds=6.7217e-05
|
||||
tempo | level=info ts=2026-02-28T04:04:14.505766966Z caller=compaction.go:142 msg="received empty tenant"
|
||||
tempo | level=warn ts=2026-02-28T04:04:38.96769366Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000596524s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T04:04:38.967991853Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.427260392s
|
||||
tempo | level=warn ts=2026-02-28T04:05:54.649236109Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000145477s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T04:05:54.649500619Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=59.111777322s
|
||||
tempo | level=warn ts=2026-02-28T04:07:09.031108125Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.000783057s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T04:07:09.031392622Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=53.643106953s
|
||||
tempo | level=warn ts=2026-02-28T04:08:16.800283365Z caller=server.go:1230 method=/tempopb.BackendScheduler/Next duration=15.00088407s msg=gRPC err="rpc error: code = NotFound desc = no jobs found"
|
||||
tempo | level=error ts=2026-02-28T04:08:16.800508742Z caller=backendworker.go:454 msg="error calling scheduler" err="rpc error: code = NotFound desc = no jobs found" backoff=55.946244881s
|
||||
tempo | level=info ts=2026-02-28T04:08:55.544817027Z caller=app.go:242 msg="=== received SIGINT/SIGTERM ===\n*** exiting"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.545170515Z caller=module_service.go:120 msg="module stopped" module=metrics-generator
|
||||
tempo | level=info ts=2026-02-28T04:08:55.545143745Z caller=backendscheduler.go:136 msg="stopping provider" provider=0
|
||||
tempo | level=info ts=2026-02-28T04:08:55.545422351Z caller=module_service.go:120 msg="module stopped" module=live-store
|
||||
tempo | level=info ts=2026-02-28T04:08:55.545509286Z caller=module_service.go:120 msg="module stopped" module=optional-store
|
||||
tempo | level=info ts=2026-02-28T04:08:55.54555359Z caller=module_service.go:120 msg="module stopped" module=block-builder
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.545588966Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.03861475s msg=gRPC err="queue is stopped"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.545861381Z caller=compaction.go:142 msg="received empty tenant"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.545877252Z caller=compaction.go:131 msg="compaction provider stopping"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.545888232Z caller=compaction.go:202 msg="compaction provider measure ticker stopping"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546132804Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039274788s msg=gRPC err="queue is stopped"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.546235459Z caller=module_service.go:120 msg="module stopped" module=query-frontend
|
||||
tempo | level=info ts=2026-02-28T04:08:55.546254915Z caller=module_service.go:120 msg="module stopped" module=overrides-api
|
||||
tempo | level=info ts=2026-02-28T04:08:55.546299981Z caller=backendscheduler.go:136 msg="stopping provider" provider=1
|
||||
tempo | level=info ts=2026-02-28T04:08:55.546316602Z caller=backendscheduler.go:154 msg="all providers stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546425368Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039448657s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546452799Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039388243s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546466765Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039674203s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546477736Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.03989506s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546492986Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039421806s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546505519Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039507939s msg=gRPC err="queue is stopped"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.546601099Z caller=module_service.go:120 msg="module stopped" module=distributor
|
||||
tempo | level=info ts=2026-02-28T04:08:55.546634613Z caller=frontend.go:339 msg="received shutdown notification from querier" querier=4777f4a6f30e
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546518493Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039596125s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.54669618Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039697237s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546723351Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039625551s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546745622Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039909457s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546760261Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039945426s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546772995Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039820781s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546783324Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039696415s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546810495Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039784952s msg=gRPC err="queue is stopped"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.546674528Z caller=retention.go:50 msg="retention provider stopping"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546819051Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039939374s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546827608Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.040065051s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546845331Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.03982558s msg=gRPC err="queue is stopped"
|
||||
tempo | level=warn ts=2026-02-28T04:08:55.546880507Z caller=server.go:1560 method=/frontend.Frontend/Process duration=29m41.039980712s msg=gRPC err="queue is stopped"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.547572095Z caller=module_service.go:120 msg="module stopped" module=querier
|
||||
tempo | level=info ts=2026-02-28T04:08:55.561796142Z caller=module_service.go:120 msg="module stopped" module=partition-ring
|
||||
tempo | level=info ts=2026-02-28T04:08:55.561812693Z caller=module_service.go:120 msg="module stopped" module=metrics-generator-ring
|
||||
tempo | level=info ts=2026-02-28T04:08:55.662029127Z caller=module_service.go:120 msg="module stopped" module=live-store-ring
|
||||
tempo | level=info ts=2026-02-28T04:08:55.979001962Z caller=backendscheduler.go:201 msg="backend scheduler stopping"
|
||||
tempo | level=info ts=2026-02-28T04:08:55.979054602Z caller=module_service.go:120 msg="module stopped" module=backend-scheduler
|
||||
tempo | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
|
||||
tempo | line 36: field enabled not found in type frontend.SearchConfig
|
||||
tempo | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
|
||||
tempo | line 36: field enabled not found in type frontend.SearchConfig
|
||||
tempo | line 37: field search_enabled not found in type app.Config
|
||||
tempo | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
|
||||
tempo | line 27: field compactor not found in type app.Config
|
||||
tempo | line 36: field enabled not found in type frontend.SearchConfig
|
||||
tempo | failed parsing config: failed to parse configFile /etc/tempo/config.yaml: yaml: unmarshal errors:
|
||||
tempo | line 27: field compactor not found in type app.Config
|
||||
tempo | line 36: field enabled not found in type frontend.SearchConfig
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,45 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-container.sh
|
||||
# Usage: ./test-container.sh container_name
|
||||
|
||||
container="$1"
|
||||
test_name="testing-${container}"
|
||||
compose_script="/compose/services-up.sh"
|
||||
|
||||
# Run container in detached mode
|
||||
$compose_script --profile all run -d --name "$test_name" --build "$container" >/dev/null 2>&1
|
||||
|
||||
# Poll health status
|
||||
timeout=60 # seconds
|
||||
interval=2 # seconds
|
||||
elapsed=0
|
||||
result=1 # default to failure
|
||||
|
||||
while [ $elapsed -lt $timeout ]; do
|
||||
status=$(docker inspect --format='{{.State.Health.Status}}' "$test_name" 2>/dev/null)
|
||||
if [ "$status" == "healthy" ]; then
|
||||
# echo "healthy"
|
||||
result=0 # success
|
||||
break
|
||||
elif [ "$status" == "unhealthy" ]; then
|
||||
# echo "unhealthy"
|
||||
result=1 # failure
|
||||
break
|
||||
fi
|
||||
sleep $interval
|
||||
elapsed=$((elapsed + interval))
|
||||
done
|
||||
|
||||
# Timeout case
|
||||
if [ $elapsed -ge $timeout ]; then
|
||||
# echo "timeout"
|
||||
result=1
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
docker rm "$test_name" --force >/dev/null 2>&1
|
||||
|
||||
#echo "Exiting with $result" >&2
|
||||
|
||||
echo $result
|
||||
exit $result
|
||||
@@ -0,0 +1,192 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
alerting:
|
||||
alertmanagers:
|
||||
- static_configs:
|
||||
- targets:
|
||||
- alertmanager:9093
|
||||
|
||||
scrape_configs:
|
||||
- job_name: "prometheus"
|
||||
static_configs:
|
||||
- targets: ["prometheus:9090"]
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
- job_name: "docker-node"
|
||||
static_configs:
|
||||
- targets: ["node-exporter:9100"]
|
||||
labels:
|
||||
type: "virtual"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
# - job_name: "cadvisor"
|
||||
# static_configs:
|
||||
# - targets: ["cadvisor:8080"]
|
||||
# labels:
|
||||
# type: "container"
|
||||
# relabel_configs:
|
||||
# - source_labels: [__address__]
|
||||
# regex: '([^:]+):.*'
|
||||
# target_label: instance
|
||||
- job_name: "raspberrypi-node"
|
||||
static_configs:
|
||||
- targets:
|
||||
- "raspberrypi.tail13f623.ts.net:9100"
|
||||
labels:
|
||||
type: "physical"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
- job_name: "proxmox-node"
|
||||
static_configs:
|
||||
- targets:
|
||||
- "pve.sweet.home:9100"
|
||||
labels:
|
||||
type: "physical"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
- job_name: "proxmox-backup-server"
|
||||
static_configs:
|
||||
- targets:
|
||||
- "pbs.sweet.home:9100"
|
||||
labels:
|
||||
type: "virtual"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
- job_name: "pihole"
|
||||
static_configs:
|
||||
- targets:
|
||||
- "pihole:9100"
|
||||
labels:
|
||||
type: "virtual"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
- job_name: "server"
|
||||
static_configs:
|
||||
- targets:
|
||||
- "server:9100"
|
||||
labels:
|
||||
type: "virtual"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
- job_name: "nix-cache"
|
||||
static_configs:
|
||||
- targets:
|
||||
- "nix-cache:9100"
|
||||
labels:
|
||||
type: "virtual"
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
- job_name: docker
|
||||
static_configs:
|
||||
- targets: ['telegraf:9273']
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
|
||||
- job_name: 'traefik'
|
||||
static_configs:
|
||||
- targets: ['traefik.lan.ddnsgeek.com:8080'] # replace with your Traefik host:port
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
metric_relabel_configs:
|
||||
- source_labels: [service]
|
||||
regex: '(.+)@.+'
|
||||
target_label: service
|
||||
replacement: '$1'
|
||||
|
||||
- job_name: 'raspi-traefik'
|
||||
static_configs:
|
||||
- targets: ['raspberrypi.tail13f623.ts.net:8080'] # replace with your Traefik host:port
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
metric_relabel_configs:
|
||||
- source_labels: [service]
|
||||
regex: '(.+)@.+'
|
||||
target_label: service
|
||||
replacement: '$1'
|
||||
|
||||
- job_name: 'raspi-kuma'
|
||||
metrics_path: /metrics
|
||||
scrape_interval: 30s
|
||||
|
||||
basic_auth:
|
||||
username: wayne.bennett@live.com
|
||||
password: '4vjCco?[%{=+,t`):C'
|
||||
|
||||
static_configs:
|
||||
- targets:
|
||||
- kuma.lan.ddnsgeek.com
|
||||
|
||||
- job_name: 'docker-kuma'
|
||||
metrics_path: /metrics
|
||||
scrape_interval: 30s
|
||||
|
||||
basic_auth:
|
||||
username: wayne.bennett@live.com
|
||||
password: '4vjCco?[%{=+,t`):C'
|
||||
|
||||
static_configs:
|
||||
- targets:
|
||||
- uptime-kuma:3001
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
|
||||
- job_name: 'proxmox LVM storage'
|
||||
static_configs:
|
||||
- targets: ['pve.sweet.home:9101']
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
- job_name: 'proxmox backup storage'
|
||||
static_configs:
|
||||
- targets: ['pbs.sweet.home:9102']
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
relabel_configs:
|
||||
- source_labels: [__address__]
|
||||
regex: '([^:]+):.*'
|
||||
target_label: instance
|
||||
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
[agent]
|
||||
interval = "10s"
|
||||
|
||||
[[inputs.docker]]
|
||||
endpoint = "unix:///var/run/docker.sock"
|
||||
gather_services = false
|
||||
|
||||
[[outputs.prometheus_client]]
|
||||
listen = ":9273"
|
||||
Symlink
+1
@@ -0,0 +1 @@
|
||||
/nix/var/nix/profiles/system-50-link
|
||||
Executable
BIN
Binary file not shown.
Reference in New Issue
Block a user