modified: .gitignore
new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/add-environment-setup-in-conftest.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/add-logging-to-geocode.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/add-logging-to-route_metrics.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/add-logging-to-tracking-simulator.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/extend-sqlite-tuning-in-database.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/fix-route-handling-in-routing.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/handle-api-response-errors-in-routing.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/refactor-database-path-handling-in-database.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/update-fcm-message-construction-in-notifications.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/update-role-check-in-ws.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/logs/refs/heads/codex/update-user-seed-in-database.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/add-environment-setup-in-conftest.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/add-logging-to-geocode.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/add-logging-to-route_metrics.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/add-logging-to-tracking-simulator.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/extend-sqlite-tuning-in-database.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/fix-route-handling-in-routing.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/handle-api-response-errors-in-routing.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/refactor-database-path-handling-in-database.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/update-fcm-message-construction-in-notifications.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/update-role-check-in-ws.py new file: apps/gitea/data/git/repositories/beatzaplenty/limo-booking-app.git/refs/heads/codex/update-user-seed-in-database.py renamed: gitea/docker-compose.yml -> apps/gitea/docker-compose.yml new file: apps/gramps/docker-compose.yml renamed: nextcloud/Dockerfile -> apps/nextcloud/Dockerfile new file: apps/nextcloud/docker-compose.yml renamed: passbolt/Dockerfile -> apps/passbolt/Dockerfile renamed: passbolt/docker-compose.yml -> apps/passbolt/docker-compose.yml renamed: searxng/Dockerfile -> apps/searxng/Dockerfile renamed: searxng/docker-compose.yml -> apps/searxng/docker-compose.yml renamed: searxng/dockerfiles/docker-entrypoint.sh -> apps/searxng/dockerfiles/docker-entrypoint.sh renamed: searxng/docs/conf.py -> apps/searxng/docs/conf.py renamed: searxng/docs/user/.gitignore -> apps/searxng/docs/user/.gitignore renamed: searxng/examples/basic_engine.py -> apps/searxng/examples/basic_engine.py renamed: searxng/searx/__init__.py -> apps/searxng/searx/__init__.py renamed: searxng/searx/answerers/__init__.py -> apps/searxng/searx/answerers/__init__.py renamed: searxng/searx/answerers/random/answerer.py -> apps/searxng/searx/answerers/random/answerer.py renamed: searxng/searx/answerers/statistics/answerer.py -> apps/searxng/searx/answerers/statistics/answerer.py renamed: searxng/searx/autocomplete.py -> apps/searxng/searx/autocomplete.py renamed: searxng/searx/babel_extract.py -> apps/searxng/searx/babel_extract.py renamed: searxng/searx/botdetection/__init__.py -> apps/searxng/searx/botdetection/__init__.py renamed: searxng/searx/botdetection/_helpers.py -> apps/searxng/searx/botdetection/_helpers.py renamed: searxng/searx/botdetection/http_accept.py -> apps/searxng/searx/botdetection/http_accept.py renamed: searxng/searx/botdetection/http_accept_encoding.py -> apps/searxng/searx/botdetection/http_accept_encoding.py renamed: searxng/searx/botdetection/http_accept_language.py -> apps/searxng/searx/botdetection/http_accept_language.py renamed: searxng/searx/botdetection/http_connection.py -> apps/searxng/searx/botdetection/http_connection.py renamed: searxng/searx/botdetection/http_user_agent.py -> apps/searxng/searx/botdetection/http_user_agent.py renamed: searxng/searx/botdetection/ip_limit.py -> apps/searxng/searx/botdetection/ip_limit.py renamed: searxng/searx/botdetection/ip_lists.py -> apps/searxng/searx/botdetection/ip_lists.py renamed: searxng/searx/botdetection/limiter.py -> apps/searxng/searx/botdetection/limiter.py renamed: searxng/searx/botdetection/link_token.py -> apps/searxng/searx/botdetection/link_token.py renamed: searxng/searx/compat.py -> apps/searxng/searx/compat.py renamed: searxng/searx/data/__init__.py -> apps/searxng/searx/data/__init__.py renamed: searxng/searx/enginelib/__init__.py -> apps/searxng/searx/enginelib/__init__.py renamed: searxng/searx/enginelib/traits.py -> apps/searxng/searx/enginelib/traits.py renamed: searxng/searx/engines/1337x.py -> apps/searxng/searx/engines/1337x.py renamed: searxng/searx/engines/9gag.py -> apps/searxng/searx/engines/9gag.py renamed: searxng/searx/engines/__init__.py -> apps/searxng/searx/engines/__init__.py renamed: searxng/searx/engines/ahmia.py -> apps/searxng/searx/engines/ahmia.py renamed: searxng/searx/engines/annas_archive.py -> apps/searxng/searx/engines/annas_archive.py renamed: searxng/searx/engines/apkmirror.py -> apps/searxng/searx/engines/apkmirror.py renamed: searxng/searx/engines/apple_app_store.py -> apps/searxng/searx/engines/apple_app_store.py renamed: searxng/searx/engines/apple_maps.py -> apps/searxng/searx/engines/apple_maps.py renamed: searxng/searx/engines/archlinux.py -> apps/searxng/searx/engines/archlinux.py renamed: searxng/searx/engines/artic.py -> apps/searxng/searx/engines/artic.py renamed: searxng/searx/engines/arxiv.py -> apps/searxng/searx/engines/arxiv.py renamed: searxng/searx/engines/bandcamp.py -> apps/searxng/searx/engines/bandcamp.py renamed: searxng/searx/engines/base.py -> apps/searxng/searx/engines/base.py renamed: searxng/searx/engines/bing.py -> apps/searxng/searx/engines/bing.py renamed: searxng/searx/engines/bing_images.py -> apps/searxng/searx/engines/bing_images.py renamed: searxng/searx/engines/bing_news.py -> apps/searxng/searx/engines/bing_news.py renamed: searxng/searx/engines/bing_videos.py -> apps/searxng/searx/engines/bing_videos.py renamed: searxng/searx/engines/brave.py -> apps/searxng/searx/engines/brave.py renamed: searxng/searx/engines/bt4g.py -> apps/searxng/searx/engines/bt4g.py renamed: searxng/searx/engines/btdigg.py -> apps/searxng/searx/engines/btdigg.py renamed: searxng/searx/engines/command.py -> apps/searxng/searx/engines/command.py renamed: searxng/searx/engines/core.py -> apps/searxng/searx/engines/core.py renamed: searxng/searx/engines/crossref.py -> apps/searxng/searx/engines/crossref.py renamed: searxng/searx/engines/currency_convert.py -> apps/searxng/searx/engines/currency_convert.py renamed: searxng/searx/engines/dailymotion.py -> apps/searxng/searx/engines/dailymotion.py renamed: searxng/searx/engines/deepl.py -> apps/searxng/searx/engines/deepl.py renamed: searxng/searx/engines/deezer.py -> apps/searxng/searx/engines/deezer.py renamed: searxng/searx/engines/demo_offline.py -> apps/searxng/searx/engines/demo_offline.py renamed: searxng/searx/engines/demo_online.py -> apps/searxng/searx/engines/demo_online.py renamed: searxng/searx/engines/deviantart.py -> apps/searxng/searx/engines/deviantart.py renamed: searxng/searx/engines/dictzone.py -> apps/searxng/searx/engines/dictzone.py renamed: searxng/searx/engines/digbt.py -> apps/searxng/searx/engines/digbt.py renamed: searxng/searx/engines/docker_hub.py -> apps/searxng/searx/engines/docker_hub.py renamed: searxng/searx/engines/doku.py -> apps/searxng/searx/engines/doku.py renamed: searxng/searx/engines/duckduckgo.py -> apps/searxng/searx/engines/duckduckgo.py renamed: searxng/searx/engines/duckduckgo_definitions.py -> apps/searxng/searx/engines/duckduckgo_definitions.py renamed: searxng/searx/engines/duckduckgo_images.py -> apps/searxng/searx/engines/duckduckgo_images.py renamed: searxng/searx/engines/duckduckgo_weather.py -> apps/searxng/searx/engines/duckduckgo_weather.py renamed: searxng/searx/engines/duden.py -> apps/searxng/searx/engines/duden.py renamed: searxng/searx/engines/dummy-offline.py -> apps/searxng/searx/engines/dummy-offline.py renamed: searxng/searx/engines/dummy.py -> apps/searxng/searx/engines/dummy.py renamed: searxng/searx/engines/ebay.py -> apps/searxng/searx/engines/ebay.py renamed: searxng/searx/engines/elasticsearch.py -> apps/searxng/searx/engines/elasticsearch.py renamed: searxng/searx/engines/emojipedia.py -> apps/searxng/searx/engines/emojipedia.py renamed: searxng/searx/engines/fdroid.py -> apps/searxng/searx/engines/fdroid.py renamed: searxng/searx/engines/flickr.py -> apps/searxng/searx/engines/flickr.py renamed: searxng/searx/engines/flickr_noapi.py -> apps/searxng/searx/engines/flickr_noapi.py renamed: searxng/searx/engines/framalibre.py -> apps/searxng/searx/engines/framalibre.py renamed: searxng/searx/engines/freesound.py -> apps/searxng/searx/engines/freesound.py renamed: searxng/searx/engines/frinkiac.py -> apps/searxng/searx/engines/frinkiac.py renamed: searxng/searx/engines/genius.py -> apps/searxng/searx/engines/genius.py renamed: searxng/searx/engines/gentoo.py -> apps/searxng/searx/engines/gentoo.py renamed: searxng/searx/engines/github.py -> apps/searxng/searx/engines/github.py renamed: searxng/searx/engines/google.py -> apps/searxng/searx/engines/google.py renamed: searxng/searx/engines/google_images.py -> apps/searxng/searx/engines/google_images.py renamed: searxng/searx/engines/google_news.py -> apps/searxng/searx/engines/google_news.py renamed: searxng/searx/engines/google_play.py -> apps/searxng/searx/engines/google_play.py renamed: searxng/searx/engines/google_scholar.py -> apps/searxng/searx/engines/google_scholar.py renamed: searxng/searx/engines/google_videos.py -> apps/searxng/searx/engines/google_videos.py renamed: searxng/searx/engines/imdb.py -> apps/searxng/searx/engines/imdb.py renamed: searxng/searx/engines/ina.py -> apps/searxng/searx/engines/ina.py renamed: searxng/searx/engines/invidious.py -> apps/searxng/searx/engines/invidious.py renamed: searxng/searx/engines/jisho.py -> apps/searxng/searx/engines/jisho.py renamed: searxng/searx/engines/json_engine.py -> apps/searxng/searx/engines/json_engine.py renamed: searxng/searx/engines/kickass.py -> apps/searxng/searx/engines/kickass.py renamed: searxng/searx/engines/lemmy.py -> apps/searxng/searx/engines/lemmy.py renamed: searxng/searx/engines/lingva.py -> apps/searxng/searx/engines/lingva.py renamed: searxng/searx/engines/loc.py -> apps/searxng/searx/engines/loc.py renamed: searxng/searx/engines/mediathekviewweb.py -> apps/searxng/searx/engines/mediathekviewweb.py renamed: searxng/searx/engines/mediawiki.py -> apps/searxng/searx/engines/mediawiki.py renamed: searxng/searx/engines/meilisearch.py -> apps/searxng/searx/engines/meilisearch.py renamed: searxng/searx/engines/metacpan.py -> apps/searxng/searx/engines/metacpan.py renamed: searxng/searx/engines/mixcloud.py -> apps/searxng/searx/engines/mixcloud.py renamed: searxng/searx/engines/mongodb.py -> apps/searxng/searx/engines/mongodb.py renamed: searxng/searx/engines/mysql_server.py -> apps/searxng/searx/engines/mysql_server.py renamed: searxng/searx/engines/nyaa.py -> apps/searxng/searx/engines/nyaa.py renamed: searxng/searx/engines/opensemantic.py -> apps/searxng/searx/engines/opensemantic.py renamed: searxng/searx/engines/openstreetmap.py -> apps/searxng/searx/engines/openstreetmap.py renamed: searxng/searx/engines/openverse.py -> apps/searxng/searx/engines/openverse.py renamed: searxng/searx/engines/pdbe.py -> apps/searxng/searx/engines/pdbe.py renamed: searxng/searx/engines/peertube.py -> apps/searxng/searx/engines/peertube.py renamed: searxng/searx/engines/photon.py -> apps/searxng/searx/engines/photon.py renamed: searxng/searx/engines/piped.py -> apps/searxng/searx/engines/piped.py renamed: searxng/searx/engines/piratebay.py -> apps/searxng/searx/engines/piratebay.py renamed: searxng/searx/engines/postgresql.py -> apps/searxng/searx/engines/postgresql.py renamed: searxng/searx/engines/pubmed.py -> apps/searxng/searx/engines/pubmed.py renamed: searxng/searx/engines/qwant.py -> apps/searxng/searx/engines/qwant.py renamed: searxng/searx/engines/recoll.py -> apps/searxng/searx/engines/recoll.py renamed: searxng/searx/engines/reddit.py -> apps/searxng/searx/engines/reddit.py renamed: searxng/searx/engines/redis_server.py -> apps/searxng/searx/engines/redis_server.py renamed: searxng/searx/engines/rumble.py -> apps/searxng/searx/engines/rumble.py renamed: searxng/searx/engines/scanr_structures.py -> apps/searxng/searx/engines/scanr_structures.py renamed: searxng/searx/engines/searchcode_code.py -> apps/searxng/searx/engines/searchcode_code.py renamed: searxng/searx/engines/searx_engine.py -> apps/searxng/searx/engines/searx_engine.py renamed: searxng/searx/engines/semantic_scholar.py -> apps/searxng/searx/engines/semantic_scholar.py renamed: searxng/searx/engines/sepiasearch.py -> apps/searxng/searx/engines/sepiasearch.py renamed: searxng/searx/engines/seznam.py -> apps/searxng/searx/engines/seznam.py renamed: searxng/searx/engines/sjp.py -> apps/searxng/searx/engines/sjp.py renamed: searxng/searx/engines/solidtorrents.py -> apps/searxng/searx/engines/solidtorrents.py renamed: searxng/searx/engines/solr.py -> apps/searxng/searx/engines/solr.py renamed: searxng/searx/engines/soundcloud.py -> apps/searxng/searx/engines/soundcloud.py renamed: searxng/searx/engines/spotify.py -> apps/searxng/searx/engines/spotify.py renamed: searxng/searx/engines/springer.py -> apps/searxng/searx/engines/springer.py renamed: searxng/searx/engines/sqlite.py -> apps/searxng/searx/engines/sqlite.py renamed: searxng/searx/engines/stackexchange.py -> apps/searxng/searx/engines/stackexchange.py renamed: searxng/searx/engines/startpage.py -> apps/searxng/searx/engines/startpage.py renamed: searxng/searx/engines/tagesschau.py -> apps/searxng/searx/engines/tagesschau.py renamed: searxng/searx/engines/tineye.py -> apps/searxng/searx/engines/tineye.py renamed: searxng/searx/engines/tokyotoshokan.py -> apps/searxng/searx/engines/tokyotoshokan.py renamed: searxng/searx/engines/torznab.py -> apps/searxng/searx/engines/torznab.py renamed: searxng/searx/engines/translated.py -> apps/searxng/searx/engines/translated.py renamed: searxng/searx/engines/twitter.py -> apps/searxng/searx/engines/twitter.py renamed: searxng/searx/engines/unsplash.py -> apps/searxng/searx/engines/unsplash.py renamed: searxng/searx/engines/vimeo.py -> apps/searxng/searx/engines/vimeo.py renamed: searxng/searx/engines/wikidata.py -> apps/searxng/searx/engines/wikidata.py renamed: searxng/searx/engines/wikipedia.py -> apps/searxng/searx/engines/wikipedia.py renamed: searxng/searx/engines/wolframalpha_api.py -> apps/searxng/searx/engines/wolframalpha_api.py renamed: searxng/searx/engines/wolframalpha_noapi.py -> apps/searxng/searx/engines/wolframalpha_noapi.py renamed: searxng/searx/engines/wordnik.py -> apps/searxng/searx/engines/wordnik.py renamed: searxng/searx/engines/wttr.py -> apps/searxng/searx/engines/wttr.py renamed: searxng/searx/engines/www1x.py -> apps/searxng/searx/engines/www1x.py renamed: searxng/searx/engines/xpath.py -> apps/searxng/searx/engines/xpath.py renamed: searxng/searx/engines/yacy.py -> apps/searxng/searx/engines/yacy.py renamed: searxng/searx/engines/yahoo.py -> apps/searxng/searx/engines/yahoo.py renamed: searxng/searx/engines/yahoo_news.py -> apps/searxng/searx/engines/yahoo_news.py renamed: searxng/searx/engines/youtube_api.py -> apps/searxng/searx/engines/youtube_api.py renamed: searxng/searx/engines/youtube_noapi.py -> apps/searxng/searx/engines/youtube_noapi.py renamed: searxng/searx/engines/zlibrary.py -> apps/searxng/searx/engines/zlibrary.py renamed: searxng/searx/exceptions.py -> apps/searxng/searx/exceptions.py renamed: searxng/searx/external_bang.py -> apps/searxng/searx/external_bang.py renamed: searxng/searx/external_urls.py -> apps/searxng/searx/external_urls.py renamed: searxng/searx/flaskfix.py -> apps/searxng/searx/flaskfix.py renamed: searxng/searx/infopage/__init__.py -> apps/searxng/searx/infopage/__init__.py renamed: searxng/searx/locales.py -> apps/searxng/searx/locales.py renamed: searxng/searx/metrics/__init__.py -> apps/searxng/searx/metrics/__init__.py renamed: searxng/searx/metrics/error_recorder.py -> apps/searxng/searx/metrics/error_recorder.py renamed: searxng/searx/metrics/models.py -> apps/searxng/searx/metrics/models.py renamed: searxng/searx/network/__init__.py -> apps/searxng/searx/network/__init__.py renamed: searxng/searx/network/client.py -> apps/searxng/searx/network/client.py renamed: searxng/searx/network/network.py -> apps/searxng/searx/network/network.py renamed: searxng/searx/network/raise_for_httperror.py -> apps/searxng/searx/network/raise_for_httperror.py renamed: searxng/searx/plugins/__init__.py -> apps/searxng/searx/plugins/__init__.py renamed: searxng/searx/plugins/ahmia_filter.py -> apps/searxng/searx/plugins/ahmia_filter.py renamed: searxng/searx/plugins/hash_plugin.py -> apps/searxng/searx/plugins/hash_plugin.py renamed: searxng/searx/plugins/hostname_replace.py -> apps/searxng/searx/plugins/hostname_replace.py renamed: searxng/searx/plugins/limiter.py -> apps/searxng/searx/plugins/limiter.py renamed: searxng/searx/plugins/oa_doi_rewrite.py -> apps/searxng/searx/plugins/oa_doi_rewrite.py renamed: searxng/searx/plugins/search_on_category_select.py -> apps/searxng/searx/plugins/search_on_category_select.py renamed: searxng/searx/plugins/self_info.py -> apps/searxng/searx/plugins/self_info.py renamed: searxng/searx/plugins/tor_check.py -> apps/searxng/searx/plugins/tor_check.py renamed: searxng/searx/plugins/tracker_url_remover.py -> apps/searxng/searx/plugins/tracker_url_remover.py renamed: searxng/searx/plugins/vim_hotkeys.py -> apps/searxng/searx/plugins/vim_hotkeys.py renamed: searxng/searx/preferences.py -> apps/searxng/searx/preferences.py renamed: searxng/searx/query.py -> apps/searxng/searx/query.py renamed: searxng/searx/redisdb.py -> apps/searxng/searx/redisdb.py renamed: searxng/searx/redislib.py -> apps/searxng/searx/redislib.py renamed: searxng/searx/results.py -> apps/searxng/searx/results.py renamed: searxng/searx/search/__init__.py -> apps/searxng/searx/search/__init__.py renamed: searxng/searx/search/checker/__init__.py -> apps/searxng/searx/search/checker/__init__.py renamed: searxng/searx/search/checker/__main__.py -> apps/searxng/searx/search/checker/__main__.py renamed: searxng/searx/search/checker/background.py -> apps/searxng/searx/search/checker/background.py renamed: searxng/searx/search/checker/impl.py -> apps/searxng/searx/search/checker/impl.py renamed: searxng/searx/search/checker/scheduler.py -> apps/searxng/searx/search/checker/scheduler.py renamed: searxng/searx/search/models.py -> apps/searxng/searx/search/models.py renamed: searxng/searx/search/processors/__init__.py -> apps/searxng/searx/search/processors/__init__.py renamed: searxng/searx/search/processors/abstract.py -> apps/searxng/searx/search/processors/abstract.py renamed: searxng/searx/search/processors/offline.py -> apps/searxng/searx/search/processors/offline.py renamed: searxng/searx/search/processors/online.py -> apps/searxng/searx/search/processors/online.py renamed: searxng/searx/search/processors/online_currency.py -> apps/searxng/searx/search/processors/online_currency.py renamed: searxng/searx/search/processors/online_dictionary.py -> apps/searxng/searx/search/processors/online_dictionary.py renamed: searxng/searx/search/processors/online_url_search.py -> apps/searxng/searx/search/processors/online_url_search.py renamed: searxng/searx/settings.yml -> apps/searxng/searx/settings.yml renamed: searxng/searx/settings_defaults.py -> apps/searxng/searx/settings_defaults.py renamed: searxng/searx/settings_loader.py -> apps/searxng/searx/settings_loader.py renamed: searxng/searx/static/plugins/external_plugins/.gitignore -> apps/searxng/searx/static/plugins/external_plugins/.gitignore renamed: searxng/searx/static/themes/simple/.gitattributes -> apps/searxng/searx/static/themes/simple/.gitattributes renamed: searxng/searx/static/themes/simple/.gitignore -> apps/searxng/searx/static/themes/simple/.gitignore renamed: searxng/searx/sxng_locales.py -> apps/searxng/searx/sxng_locales.py renamed: searxng/searx/tools/__init__.py -> apps/searxng/searx/tools/__init__.py renamed: searxng/searx/tools/config.py -> apps/searxng/searx/tools/config.py renamed: searxng/searx/unixthreadname.py -> apps/searxng/searx/unixthreadname.py renamed: searxng/searx/utils.py -> apps/searxng/searx/utils.py renamed: searxng/searx/version.py -> apps/searxng/searx/version.py renamed: searxng/searx/webadapter.py -> apps/searxng/searx/webadapter.py renamed: searxng/searx/webapp.py -> apps/searxng/searx/webapp.py renamed: searxng/searx/webutils.py -> apps/searxng/searx/webutils.py renamed: searxng/searxng_extra/__init__.py -> apps/searxng/searxng_extra/__init__.py renamed: searxng/searxng_extra/standalone_searx.py -> apps/searxng/searxng_extra/standalone_searx.py renamed: searxng/searxng_extra/update/__init__.py -> apps/searxng/searxng_extra/update/__init__.py renamed: searxng/searxng_extra/update/update_ahmia_blacklist.py -> apps/searxng/searxng_extra/update/update_ahmia_blacklist.py renamed: searxng/searxng_extra/update/update_currencies.py -> apps/searxng/searxng_extra/update/update_currencies.py renamed: searxng/searxng_extra/update/update_engine_descriptions.py -> apps/searxng/searxng_extra/update/update_engine_descriptions.py renamed: searxng/searxng_extra/update/update_engine_traits.py -> apps/searxng/searxng_extra/update/update_engine_traits.py renamed: searxng/searxng_extra/update/update_external_bangs.py -> apps/searxng/searxng_extra/update/update_external_bangs.py renamed: searxng/searxng_extra/update/update_firefox_version.py -> apps/searxng/searxng_extra/update/update_firefox_version.py renamed: searxng/searxng_extra/update/update_osm_keys_tags.py -> apps/searxng/searxng_extra/update/update_osm_keys_tags.py renamed: searxng/searxng_extra/update/update_pygments.py -> apps/searxng/searxng_extra/update/update_pygments.py renamed: searxng/searxng_extra/update/update_wikidata_units.py -> apps/searxng/searxng_extra/update/update_wikidata_units.py renamed: searxng/setup.py -> apps/searxng/setup.py renamed: searxng/tests/__init__.py -> apps/searxng/tests/__init__.py renamed: searxng/tests/robot/__init__.py -> apps/searxng/tests/robot/__init__.py renamed: searxng/tests/robot/__main__.py -> apps/searxng/tests/robot/__main__.py renamed: searxng/tests/robot/settings_robot.yml -> apps/searxng/tests/robot/settings_robot.yml renamed: searxng/tests/robot/test_webapp.py -> apps/searxng/tests/robot/test_webapp.py renamed: searxng/tests/unit/__init__.py -> apps/searxng/tests/unit/__init__.py renamed: searxng/tests/unit/engines/test_command.py -> apps/searxng/tests/unit/engines/test_command.py renamed: searxng/tests/unit/engines/test_xpath.py -> apps/searxng/tests/unit/engines/test_xpath.py renamed: searxng/tests/unit/network/__init__.py -> apps/searxng/tests/unit/network/__init__.py renamed: searxng/tests/unit/network/test_network.py -> apps/searxng/tests/unit/network/test_network.py renamed: searxng/tests/unit/settings/empty_settings.yml -> apps/searxng/tests/unit/settings/empty_settings.yml renamed: searxng/tests/unit/settings/syntaxerror_settings.yml -> apps/searxng/tests/unit/settings/syntaxerror_settings.yml renamed: searxng/tests/unit/settings/test_settings.yml -> apps/searxng/tests/unit/settings/test_settings.yml renamed: searxng/tests/unit/settings/user_settings.yml -> apps/searxng/tests/unit/settings/user_settings.yml renamed: searxng/tests/unit/settings/user_settings_keep_only.yml -> apps/searxng/tests/unit/settings/user_settings_keep_only.yml renamed: searxng/tests/unit/settings/user_settings_remove.yml -> apps/searxng/tests/unit/settings/user_settings_remove.yml renamed: searxng/tests/unit/settings/user_settings_remove2.yml -> apps/searxng/tests/unit/settings/user_settings_remove2.yml renamed: searxng/tests/unit/settings/user_settings_simple.yml -> apps/searxng/tests/unit/settings/user_settings_simple.yml renamed: searxng/tests/unit/test_answerers.py -> apps/searxng/tests/unit/test_answerers.py renamed: searxng/tests/unit/test_engines_init.py -> apps/searxng/tests/unit/test_engines_init.py renamed: searxng/tests/unit/test_exceptions.py -> apps/searxng/tests/unit/test_exceptions.py renamed: searxng/tests/unit/test_external_bangs.py -> apps/searxng/tests/unit/test_external_bangs.py renamed: searxng/tests/unit/test_locales.py -> apps/searxng/tests/unit/test_locales.py renamed: searxng/tests/unit/test_plugins.py -> apps/searxng/tests/unit/test_plugins.py renamed: searxng/tests/unit/test_preferences.py -> apps/searxng/tests/unit/test_preferences.py renamed: searxng/tests/unit/test_query.py -> apps/searxng/tests/unit/test_query.py renamed: searxng/tests/unit/test_results.py -> apps/searxng/tests/unit/test_results.py renamed: searxng/tests/unit/test_search.py -> apps/searxng/tests/unit/test_search.py renamed: searxng/tests/unit/test_settings_loader.py -> apps/searxng/tests/unit/test_settings_loader.py renamed: searxng/tests/unit/test_utils.py -> apps/searxng/tests/unit/test_utils.py renamed: searxng/tests/unit/test_webadapter.py -> apps/searxng/tests/unit/test_webadapter.py renamed: searxng/tests/unit/test_webapp.py -> apps/searxng/tests/unit/test_webapp.py renamed: searxng/tests/unit/test_webutils.py -> apps/searxng/tests/unit/test_webutils.py renamed: searxng/utils/build_env.py -> apps/searxng/utils/build_env.py renamed: searxng/utils/filtron.sh -> apps/searxng/utils/filtron.sh renamed: searxng/utils/lib.sh -> apps/searxng/utils/lib.sh renamed: searxng/utils/lib_go.sh -> apps/searxng/utils/lib_go.sh renamed: searxng/utils/lib_nvm.sh -> apps/searxng/utils/lib_nvm.sh renamed: searxng/utils/lib_redis.sh -> apps/searxng/utils/lib_redis.sh renamed: searxng/utils/lib_sxng_data.sh -> apps/searxng/utils/lib_sxng_data.sh renamed: searxng/utils/lib_sxng_node.sh -> apps/searxng/utils/lib_sxng_node.sh renamed: searxng/utils/lib_sxng_static.sh -> apps/searxng/utils/lib_sxng_static.sh renamed: searxng/utils/lib_sxng_test.sh -> apps/searxng/utils/lib_sxng_test.sh renamed: searxng/utils/lib_sxng_themes.sh -> apps/searxng/utils/lib_sxng_themes.sh renamed: searxng/utils/lib_sxng_weblate.sh -> apps/searxng/utils/lib_sxng_weblate.sh renamed: searxng/utils/lxc.sh -> apps/searxng/utils/lxc.sh renamed: searxng/utils/morty.sh -> apps/searxng/utils/morty.sh renamed: searxng/utils/searx.sh -> apps/searxng/utils/searx.sh renamed: searxng/utils/searxng.sh -> apps/searxng/utils/searxng.sh renamed: searxng/utils/searxng_check.py -> apps/searxng/utils/searxng_check.py renamed: searxng/utils/templates/etc/searxng/settings.yml -> apps/searxng/utils/templates/etc/searxng/settings.yml new file: apps/shift-recorder new file: apps/stockfill new file: core/authelia/configuration.yml new file: core/authelia/users_database.yml new file: core/crowdsec/Dockerfile new file: core/crowdsec/data/detect.yaml new file: core/docker-compose.yml new file: core/test/Dockerfile new file: core/test/docker-compose.yml new file: core/test/exporter.py new file: core/traefik/data/dynamic.yaml renamed: traefik/data/plugins.yaml -> core/traefik/data/plugins.yaml new file: core/traefik/dynamic.yml new file: core/traefik/traefik.yml new file: default-network.yml new file: monitoring/docker-exporter/Dockerfile new file: monitoring/docker-exporter/exporter.py new file: monitoring/gotify/docker-compose.yml new file: monitoring/gotify/docker-health-to-gotify.sh new file: monitoring/grafana/docker-compose.yml new file: monitoring/node-red/Dockerfile new file: monitoring/node-red/data/test-container.sh new file: monitoring/node-red/docker-compose.yml new file: monitoring/portainer/docker-compose.yml new file: monitoring/prometheus/docker-compose.yml new file: monitoring/prometheus/prometheus.yml new file: monitoring/prometheus/rules/alerts.yml new file: monitoring/uptime-kuma/docker-compose.yml deleted: nextcloud/docker-compose.yml new file: services-up.sh deleted: traefik/docker-compose.yml deleted: traefik/traefik.Dockerfile modified: update-containers.py modified: update-containers.sh modified: apps/shift-recorder (modified content) modified: apps/stockfill (modified content)
This commit is contained in:
git
Executable
+27
@@ -0,0 +1,27 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
""".. _botdetection src:
|
||||
|
||||
X-Forwarded-For
|
||||
===============
|
||||
|
||||
.. attention::
|
||||
|
||||
A correct setup of the HTTP request headers ``X-Forwarded-For`` and
|
||||
``X-Real-IP`` is essential to be able to assign a request to an IP correctly:
|
||||
|
||||
- `NGINX RequestHeader`_
|
||||
- `Apache RequestHeader`_
|
||||
|
||||
.. _NGINX RequestHeader:
|
||||
https://docs.searxng.org/admin/installation-nginx.html#nginx-s-searxng-site
|
||||
.. _Apache RequestHeader:
|
||||
https://docs.searxng.org/admin/installation-apache.html#apache-s-searxng-site
|
||||
|
||||
.. autofunction:: searx.botdetection.get_real_ip
|
||||
|
||||
"""
|
||||
|
||||
from ._helpers import dump_request
|
||||
from ._helpers import get_real_ip
|
||||
from ._helpers import too_many_requests
|
||||
Executable
+120
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
# pylint: disable=missing-module-docstring, invalid-name
|
||||
from __future__ import annotations
|
||||
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
IPv4Address,
|
||||
IPv6Address,
|
||||
ip_network,
|
||||
)
|
||||
import flask
|
||||
import werkzeug
|
||||
|
||||
from searx.tools import config
|
||||
from searx import logger
|
||||
|
||||
logger = logger.getChild('botdetection')
|
||||
|
||||
|
||||
def dump_request(request: flask.Request):
|
||||
return (
|
||||
request.path
|
||||
+ " || X-Forwarded-For: %s" % request.headers.get('X-Forwarded-For')
|
||||
+ " || X-Real-IP: %s" % request.headers.get('X-Real-IP')
|
||||
+ " || form: %s" % request.form
|
||||
+ " || Accept: %s" % request.headers.get('Accept')
|
||||
+ " || Accept-Language: %s" % request.headers.get('Accept-Language')
|
||||
+ " || Accept-Encoding: %s" % request.headers.get('Accept-Encoding')
|
||||
+ " || Content-Type: %s" % request.headers.get('Content-Type')
|
||||
+ " || Content-Length: %s" % request.headers.get('Content-Length')
|
||||
+ " || Connection: %s" % request.headers.get('Connection')
|
||||
+ " || User-Agent: %s" % request.headers.get('User-Agent')
|
||||
)
|
||||
|
||||
|
||||
def too_many_requests(network: IPv4Network | IPv6Network, log_msg: str) -> werkzeug.Response | None:
|
||||
"""Returns a HTTP 429 response object and writes a ERROR message to the
|
||||
'botdetection' logger. This function is used in part by the filter methods
|
||||
to return the default ``Too Many Requests`` response.
|
||||
|
||||
"""
|
||||
|
||||
logger.debug("BLOCK %s: %s", network.compressed, log_msg)
|
||||
return flask.make_response(('Too Many Requests', 429))
|
||||
|
||||
|
||||
def get_network(real_ip: IPv4Address | IPv6Address, cfg: config.Config) -> IPv4Network | IPv6Network:
|
||||
"""Returns the (client) network of whether the real_ip is part of."""
|
||||
|
||||
if real_ip.version == 6:
|
||||
prefix = cfg['real_ip.ipv6_prefix']
|
||||
else:
|
||||
prefix = cfg['real_ip.ipv4_prefix']
|
||||
network = ip_network(f"{real_ip}/{prefix}", strict=False)
|
||||
# logger.debug("get_network(): %s", network.compressed)
|
||||
return network
|
||||
|
||||
|
||||
def get_real_ip(request: flask.Request) -> str:
|
||||
"""Returns real IP of the request. Since not all proxies set all the HTTP
|
||||
headers and incoming headers can be faked it may happen that the IP cannot
|
||||
be determined correctly.
|
||||
|
||||
.. sidebar:: :py:obj:`flask.Request.remote_addr`
|
||||
|
||||
SearXNG uses Werkzeug's ProxyFix_ (with it default ``x_for=1``).
|
||||
|
||||
This function tries to get the remote IP in the order listed below,
|
||||
additional some tests are done and if inconsistencies or errors are
|
||||
detected, they are logged.
|
||||
|
||||
The remote IP of the request is taken from (first match):
|
||||
|
||||
- X-Forwarded-For_ header
|
||||
- `X-real-IP header <https://github.com/searxng/searxng/issues/1237#issuecomment-1147564516>`__
|
||||
- :py:obj:`flask.Request.remote_addr`
|
||||
|
||||
.. _ProxyFix:
|
||||
https://werkzeug.palletsprojects.com/middleware/proxy_fix/
|
||||
|
||||
.. _X-Forwarded-For:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
|
||||
|
||||
"""
|
||||
|
||||
forwarded_for = request.headers.get("X-Forwarded-For")
|
||||
real_ip = request.headers.get('X-Real-IP')
|
||||
remote_addr = request.remote_addr
|
||||
# logger.debug(
|
||||
# "X-Forwarded-For: %s || X-Real-IP: %s || request.remote_addr: %s", forwarded_for, real_ip, remote_addr
|
||||
# )
|
||||
|
||||
if not forwarded_for:
|
||||
logger.error("X-Forwarded-For header is not set!")
|
||||
else:
|
||||
from .limiter import get_cfg # pylint: disable=import-outside-toplevel, cyclic-import
|
||||
|
||||
forwarded_for = [x.strip() for x in forwarded_for.split(',')]
|
||||
x_for: int = get_cfg()['real_ip.x_for'] # type: ignore
|
||||
forwarded_for = forwarded_for[-min(len(forwarded_for), x_for)]
|
||||
|
||||
if not real_ip:
|
||||
logger.error("X-Real-IP header is not set!")
|
||||
|
||||
if forwarded_for and real_ip and forwarded_for != real_ip:
|
||||
logger.warning("IP from X-Real-IP (%s) is not equal to IP from X-Forwarded-For (%s)", real_ip, forwarded_for)
|
||||
|
||||
if forwarded_for and remote_addr and forwarded_for != remote_addr:
|
||||
logger.warning(
|
||||
"IP from WSGI environment (%s) is not equal to IP from X-Forwarded-For (%s)", remote_addr, forwarded_for
|
||||
)
|
||||
|
||||
if real_ip and remote_addr and real_ip != remote_addr:
|
||||
logger.warning("IP from WSGI environment (%s) is not equal to IP from X-Real-IP (%s)", remote_addr, real_ip)
|
||||
|
||||
request_ip = forwarded_for or real_ip or remote_addr or '0.0.0.0'
|
||||
# logger.debug("get_real_ip() -> %s", request_ip)
|
||||
return request_ip
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""
|
||||
Method ``http_accept``
|
||||
----------------------
|
||||
|
||||
The ``http_accept`` method evaluates a request as the request of a bot if the
|
||||
Accept_ header ..
|
||||
|
||||
- did not contain ``text/html``
|
||||
|
||||
.. _Accept:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept
|
||||
|
||||
"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
from __future__ import annotations
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
)
|
||||
|
||||
import flask
|
||||
import werkzeug
|
||||
|
||||
from searx.tools import config
|
||||
from ._helpers import too_many_requests
|
||||
|
||||
|
||||
def filter_request(
|
||||
network: IPv4Network | IPv6Network,
|
||||
request: flask.Request,
|
||||
cfg: config.Config,
|
||||
) -> werkzeug.Response | None:
|
||||
|
||||
if 'text/html' not in request.accept_mimetypes:
|
||||
return too_many_requests(network, "HTTP header Accept did not contain text/html")
|
||||
return None
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""
|
||||
Method ``http_accept_encoding``
|
||||
-------------------------------
|
||||
|
||||
The ``http_accept_encoding`` method evaluates a request as the request of a
|
||||
bot if the Accept-Encoding_ header ..
|
||||
|
||||
- did not contain ``gzip`` AND ``deflate`` (if both values are missed)
|
||||
- did not contain ``text/html``
|
||||
|
||||
.. _Accept-Encoding:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Encoding
|
||||
|
||||
"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
from __future__ import annotations
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
)
|
||||
|
||||
import flask
|
||||
import werkzeug
|
||||
|
||||
from searx.tools import config
|
||||
from ._helpers import too_many_requests
|
||||
|
||||
|
||||
def filter_request(
|
||||
network: IPv4Network | IPv6Network,
|
||||
request: flask.Request,
|
||||
cfg: config.Config,
|
||||
) -> werkzeug.Response | None:
|
||||
|
||||
accept_list = [l.strip() for l in request.headers.get('Accept-Encoding', '').split(',')]
|
||||
if not ('gzip' in accept_list or 'deflate' in accept_list):
|
||||
return too_many_requests(network, "HTTP header Accept-Encoding did not contain gzip nor deflate")
|
||||
return None
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""
|
||||
Method ``http_accept_language``
|
||||
-------------------------------
|
||||
|
||||
The ``http_accept_language`` method evaluates a request as the request of a bot
|
||||
if the Accept-Language_ header is unset.
|
||||
|
||||
.. _Accept-Language:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
|
||||
|
||||
"""
|
||||
# pylint: disable=unused-argument
|
||||
from __future__ import annotations
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
)
|
||||
|
||||
import flask
|
||||
import werkzeug
|
||||
|
||||
from searx.tools import config
|
||||
from ._helpers import too_many_requests
|
||||
|
||||
|
||||
def filter_request(
|
||||
network: IPv4Network | IPv6Network,
|
||||
request: flask.Request,
|
||||
cfg: config.Config,
|
||||
) -> werkzeug.Response | None:
|
||||
if request.headers.get('Accept-Language', '').strip() == '':
|
||||
return too_many_requests(network, "missing HTTP header Accept-Language")
|
||||
return None
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""
|
||||
Method ``http_connection``
|
||||
--------------------------
|
||||
|
||||
The ``http_connection`` method evaluates a request as the request of a bot if
|
||||
the Connection_ header is set to ``close``.
|
||||
|
||||
.. _Connection:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection
|
||||
|
||||
"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
from __future__ import annotations
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
)
|
||||
|
||||
import flask
|
||||
import werkzeug
|
||||
|
||||
from searx.tools import config
|
||||
from ._helpers import too_many_requests
|
||||
|
||||
|
||||
def filter_request(
|
||||
network: IPv4Network | IPv6Network,
|
||||
request: flask.Request,
|
||||
cfg: config.Config,
|
||||
) -> werkzeug.Response | None:
|
||||
|
||||
if request.headers.get('Connection', '').strip() == 'close':
|
||||
return too_many_requests(network, "HTTP header 'Connection=close")
|
||||
return None
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""
|
||||
Method ``http_user_agent``
|
||||
--------------------------
|
||||
|
||||
The ``http_user_agent`` method evaluates a request as the request of a bot if
|
||||
the User-Agent_ header is unset or matches the regular expression
|
||||
:py:obj:`USER_AGENT`.
|
||||
|
||||
.. _User-Agent:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent
|
||||
|
||||
"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
from __future__ import annotations
|
||||
import re
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
)
|
||||
|
||||
import flask
|
||||
import werkzeug
|
||||
|
||||
from searx.tools import config
|
||||
from ._helpers import too_many_requests
|
||||
|
||||
|
||||
USER_AGENT = (
|
||||
r'('
|
||||
+ r'unknown'
|
||||
+ r'|[Cc][Uu][Rr][Ll]|[wW]get|Scrapy|splash|JavaFX|FeedFetcher|python-requests|Go-http-client|Java|Jakarta|okhttp'
|
||||
+ r'|HttpClient|Jersey|Python|libwww-perl|Ruby|SynHttpClient|UniversalFeedParser|Googlebot|GoogleImageProxy'
|
||||
+ r'|bingbot|Baiduspider|yacybot|YandexMobileBot|YandexBot|Yahoo! Slurp|MJ12bot|AhrefsBot|archive.org_bot|msnbot'
|
||||
+ r'|MJ12bot|SeznamBot|linkdexbot|Netvibes|SMTBot|zgrab|James BOT|Sogou|Abonti|Pixray|Spinn3r|SemrushBot|Exabot'
|
||||
+ r'|ZmEu|BLEXBot|bitlybot'
|
||||
# unmaintained Farside instances
|
||||
+ r'|'
|
||||
+ re.escape(r'Mozilla/5.0 (compatible; Farside/0.1.0; +https://farside.link)')
|
||||
# other bots and client to block
|
||||
+ '|.*PetalBot.*'
|
||||
+ r')'
|
||||
)
|
||||
"""Regular expression that matches to User-Agent_ from known *bots*"""
|
||||
|
||||
_regexp = None
|
||||
|
||||
|
||||
def regexp_user_agent():
|
||||
global _regexp # pylint: disable=global-statement
|
||||
if not _regexp:
|
||||
_regexp = re.compile(USER_AGENT)
|
||||
return _regexp
|
||||
|
||||
|
||||
def filter_request(
|
||||
network: IPv4Network | IPv6Network,
|
||||
request: flask.Request,
|
||||
cfg: config.Config,
|
||||
) -> werkzeug.Response | None:
|
||||
|
||||
user_agent = request.headers.get('User-Agent', 'unknown')
|
||||
if regexp_user_agent().match(user_agent):
|
||||
return too_many_requests(network, f"bot detected, HTTP header User-Agent: {user_agent}")
|
||||
return None
|
||||
Executable
+148
@@ -0,0 +1,148 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
""".. _botdetection.ip_limit:
|
||||
|
||||
Method ``ip_limit``
|
||||
-------------------
|
||||
|
||||
The ``ip_limit`` method counts request from an IP in *sliding windows*. If
|
||||
there are to many requests in a sliding window, the request is evaluated as a
|
||||
bot request. This method requires a redis DB and needs a HTTP X-Forwarded-For_
|
||||
header. To take privacy only the hash value of an IP is stored in the redis DB
|
||||
and at least for a maximum of 10 minutes.
|
||||
|
||||
The :py:obj:`.link_token` method can be used to investigate whether a request is
|
||||
*suspicious*. To activate the :py:obj:`.link_token` method in the
|
||||
:py:obj:`.ip_limit` method add the following to your
|
||||
``/etc/searxng/limiter.toml``:
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[botdetection.ip_limit]
|
||||
link_token = true
|
||||
|
||||
If the :py:obj:`.link_token` method is activated and a request is *suspicious*
|
||||
the request rates are reduced:
|
||||
|
||||
- :py:obj:`BURST_MAX` -> :py:obj:`BURST_MAX_SUSPICIOUS`
|
||||
- :py:obj:`LONG_MAX` -> :py:obj:`LONG_MAX_SUSPICIOUS`
|
||||
|
||||
To intercept bots that get their IPs from a range of IPs, there is a
|
||||
:py:obj:`SUSPICIOUS_IP_WINDOW`. In this window the suspicious IPs are stored
|
||||
for a longer time. IPs stored in this sliding window have a maximum of
|
||||
:py:obj:`SUSPICIOUS_IP_MAX` accesses before they are blocked. As soon as the IP
|
||||
makes a request that is not suspicious, the sliding window for this IP is
|
||||
droped.
|
||||
|
||||
.. _X-Forwarded-For:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
)
|
||||
|
||||
import flask
|
||||
import werkzeug
|
||||
from searx.tools import config
|
||||
|
||||
from searx import redisdb
|
||||
from searx.redislib import incr_sliding_window, drop_counter
|
||||
|
||||
from . import link_token
|
||||
from ._helpers import (
|
||||
too_many_requests,
|
||||
logger,
|
||||
)
|
||||
|
||||
|
||||
logger = logger.getChild('ip_limit')
|
||||
|
||||
BURST_WINDOW = 20
|
||||
"""Time (sec) before sliding window for *burst* requests expires."""
|
||||
|
||||
BURST_MAX = 15
|
||||
"""Maximum requests from one IP in the :py:obj:`BURST_WINDOW`"""
|
||||
|
||||
BURST_MAX_SUSPICIOUS = 2
|
||||
"""Maximum of suspicious requests from one IP in the :py:obj:`BURST_WINDOW`"""
|
||||
|
||||
LONG_WINDOW = 600
|
||||
"""Time (sec) before the longer sliding window expires."""
|
||||
|
||||
LONG_MAX = 150
|
||||
"""Maximum requests from one IP in the :py:obj:`LONG_WINDOW`"""
|
||||
|
||||
LONG_MAX_SUSPICIOUS = 10
|
||||
"""Maximum suspicious requests from one IP in the :py:obj:`LONG_WINDOW`"""
|
||||
|
||||
API_WONDOW = 3600
|
||||
"""Time (sec) before sliding window for API requests (format != html) expires."""
|
||||
|
||||
API_MAX = 4
|
||||
"""Maximum requests from one IP in the :py:obj:`API_WONDOW`"""
|
||||
|
||||
SUSPICIOUS_IP_WINDOW = 3600 * 24 * 30
|
||||
"""Time (sec) before sliding window for one suspicious IP expires."""
|
||||
|
||||
SUSPICIOUS_IP_MAX = 3
|
||||
"""Maximum requests from one suspicious IP in the :py:obj:`SUSPICIOUS_IP_WINDOW`."""
|
||||
|
||||
|
||||
def filter_request(
|
||||
network: IPv4Network | IPv6Network,
|
||||
request: flask.Request,
|
||||
cfg: config.Config,
|
||||
) -> werkzeug.Response | None:
|
||||
|
||||
# pylint: disable=too-many-return-statements
|
||||
redis_client = redisdb.client()
|
||||
|
||||
if network.is_link_local and not cfg['botdetection.ip_limit.filter_link_local']:
|
||||
logger.debug("network %s is link-local -> not monitored by ip_limit method", network.compressed)
|
||||
return None
|
||||
|
||||
if request.args.get('format', 'html') != 'html':
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.API_WONDOW:' + network.compressed, API_WONDOW)
|
||||
if c > API_MAX:
|
||||
return too_many_requests(network, "too many request in API_WINDOW")
|
||||
|
||||
if cfg['botdetection.ip_limit.link_token']:
|
||||
|
||||
suspicious = link_token.is_suspicious(network, request, True)
|
||||
|
||||
if not suspicious:
|
||||
# this IP is no longer suspicious: release ip again / delete the counter of this IP
|
||||
drop_counter(redis_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed)
|
||||
return None
|
||||
|
||||
# this IP is suspicious: count requests from this IP
|
||||
c = incr_sliding_window(
|
||||
redis_client, 'ip_limit.SUSPICIOUS_IP_WINDOW' + network.compressed, SUSPICIOUS_IP_WINDOW
|
||||
)
|
||||
if c > SUSPICIOUS_IP_MAX:
|
||||
logger.error("BLOCK: too many request from %s in SUSPICIOUS_IP_WINDOW (redirect to /)", network)
|
||||
return flask.redirect(flask.url_for('index'), code=302)
|
||||
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
|
||||
if c > BURST_MAX_SUSPICIOUS:
|
||||
return too_many_requests(network, "too many request in BURST_WINDOW (BURST_MAX_SUSPICIOUS)")
|
||||
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
|
||||
if c > LONG_MAX_SUSPICIOUS:
|
||||
return too_many_requests(network, "too many request in LONG_WINDOW (LONG_MAX_SUSPICIOUS)")
|
||||
|
||||
return None
|
||||
|
||||
# vanilla limiter without extensions counts BURST_MAX and LONG_MAX
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.BURST_WINDOW' + network.compressed, BURST_WINDOW)
|
||||
if c > BURST_MAX:
|
||||
return too_many_requests(network, "too many request in BURST_WINDOW (BURST_MAX)")
|
||||
|
||||
c = incr_sliding_window(redis_client, 'ip_limit.LONG_WINDOW' + network.compressed, LONG_WINDOW)
|
||||
if c > LONG_MAX:
|
||||
return too_many_requests(network, "too many request in LONG_WINDOW (LONG_MAX)")
|
||||
|
||||
return None
|
||||
Executable
+85
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
""".. _botdetection.ip_lists:
|
||||
|
||||
Method ``ip_lists``
|
||||
-------------------
|
||||
|
||||
The ``ip_lists`` method implements IP :py:obj:`block- <block_ip>` and
|
||||
:py:obj:`pass-lists <pass_ip>`.
|
||||
|
||||
.. code:: toml
|
||||
|
||||
[botdetection.ip_lists]
|
||||
|
||||
pass_ip = [
|
||||
'140.238.172.132', # IPv4 of check.searx.space
|
||||
'192.168.0.0/16', # IPv4 private network
|
||||
'fe80::/10' # IPv6 linklocal
|
||||
]
|
||||
block_ip = [
|
||||
'93.184.216.34', # IPv4 of example.org
|
||||
'257.1.1.1', # invalid IP --> will be ignored, logged in ERROR class
|
||||
]
|
||||
|
||||
"""
|
||||
# pylint: disable=unused-argument
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Tuple
|
||||
from ipaddress import (
|
||||
ip_network,
|
||||
IPv4Address,
|
||||
IPv6Address,
|
||||
)
|
||||
|
||||
from searx.tools import config
|
||||
from ._helpers import logger
|
||||
|
||||
logger = logger.getChild('ip_limit')
|
||||
|
||||
SEARXNG_ORG = [
|
||||
# https://github.com/searxng/searxng/pull/2484#issuecomment-1576639195
|
||||
'140.238.172.132', # IPv4 check.searx.space
|
||||
'2603:c022:0:4900::/56', # IPv6 check.searx.space
|
||||
]
|
||||
"""Passlist of IPs from the SearXNG organization, e.g. `check.searx.space`."""
|
||||
|
||||
|
||||
def pass_ip(real_ip: IPv4Address | IPv6Address, cfg: config.Config) -> Tuple[bool, str]:
|
||||
"""Checks if the IP on the subnet is in one of the members of the
|
||||
``botdetection.ip_lists.pass_ip`` list.
|
||||
"""
|
||||
|
||||
if cfg.get('botdetection.ip_lists.pass_searxng_org', default=True):
|
||||
for net in SEARXNG_ORG:
|
||||
net = ip_network(net, strict=False)
|
||||
if real_ip.version == net.version and real_ip in net:
|
||||
return True, f"IP matches {net.compressed} in SEARXNG_ORG list."
|
||||
return ip_is_subnet_of_member_in_list(real_ip, 'botdetection.ip_lists.pass_ip', cfg)
|
||||
|
||||
|
||||
def block_ip(real_ip: IPv4Address | IPv6Address, cfg: config.Config) -> Tuple[bool, str]:
|
||||
"""Checks if the IP on the subnet is in one of the members of the
|
||||
``botdetection.ip_lists.block_ip`` list.
|
||||
"""
|
||||
|
||||
block, msg = ip_is_subnet_of_member_in_list(real_ip, 'botdetection.ip_lists.block_ip', cfg)
|
||||
if block:
|
||||
msg += " To remove IP from list, please contact the maintainer of the service."
|
||||
return block, msg
|
||||
|
||||
|
||||
def ip_is_subnet_of_member_in_list(
|
||||
real_ip: IPv4Address | IPv6Address, list_name: str, cfg: config.Config
|
||||
) -> Tuple[bool, str]:
|
||||
|
||||
for net in cfg.get(list_name, default=[]):
|
||||
try:
|
||||
net = ip_network(net, strict=False)
|
||||
except ValueError:
|
||||
logger.error("invalid IP %s in %s", net, list_name)
|
||||
continue
|
||||
if real_ip.version == net.version and real_ip in net:
|
||||
return True, f"IP matches {net.compressed} in {list_name}."
|
||||
return False, f"IP is not a member of an item in the f{list_name} list"
|
||||
Executable
+147
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
""".. _limiter src:
|
||||
|
||||
Limiter
|
||||
=======
|
||||
|
||||
.. sidebar:: info
|
||||
|
||||
The limiter requires a :ref:`Redis <settings redis>` database.
|
||||
|
||||
Bot protection / IP rate limitation. The intention of rate limitation is to
|
||||
limit suspicious requests from an IP. The motivation behind this is the fact
|
||||
that SearXNG passes through requests from bots and is thus classified as a bot
|
||||
itself. As a result, the SearXNG engine then receives a CAPTCHA or is blocked
|
||||
by the search engine (the origin) in some other way.
|
||||
|
||||
To avoid blocking, the requests from bots to SearXNG must also be blocked, this
|
||||
is the task of the limiter. To perform this task, the limiter uses the methods
|
||||
from the :py:obj:`searx.botdetection`.
|
||||
|
||||
To enable the limiter activate:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
server:
|
||||
...
|
||||
limiter: true # rate limit the number of request on the instance, block some bots
|
||||
|
||||
and set the redis-url connection. Check the value, it depends on your redis DB
|
||||
(see :ref:`settings redis`), by example:
|
||||
|
||||
.. code:: yaml
|
||||
|
||||
redis:
|
||||
url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from ipaddress import ip_address
|
||||
import flask
|
||||
import werkzeug
|
||||
|
||||
from searx.tools import config
|
||||
from searx import logger
|
||||
|
||||
from . import (
|
||||
http_accept,
|
||||
http_accept_encoding,
|
||||
http_accept_language,
|
||||
http_connection,
|
||||
http_user_agent,
|
||||
ip_limit,
|
||||
ip_lists,
|
||||
)
|
||||
|
||||
from ._helpers import (
|
||||
get_network,
|
||||
get_real_ip,
|
||||
dump_request,
|
||||
)
|
||||
|
||||
logger = logger.getChild('botdetection.limiter')
|
||||
|
||||
CFG: config.Config = None # type: ignore
|
||||
|
||||
LIMITER_CFG_SCHEMA = Path(__file__).parent / "limiter.toml"
|
||||
"""Base configuration (schema) of the botdetection."""
|
||||
|
||||
LIMITER_CFG = Path('/etc/searxng/limiter.toml')
|
||||
"""Lokal Limiter configuration."""
|
||||
|
||||
CFG_DEPRECATED = {
|
||||
# "dummy.old.foo": "config 'dummy.old.foo' exists only for tests. Don't use it in your real project config."
|
||||
}
|
||||
|
||||
|
||||
def get_cfg() -> config.Config:
|
||||
global CFG # pylint: disable=global-statement
|
||||
if CFG is None:
|
||||
CFG = config.Config.from_toml(LIMITER_CFG_SCHEMA, LIMITER_CFG, CFG_DEPRECATED)
|
||||
return CFG
|
||||
|
||||
|
||||
def filter_request(request: flask.Request) -> werkzeug.Response | None:
|
||||
# pylint: disable=too-many-return-statements
|
||||
|
||||
cfg = get_cfg()
|
||||
real_ip = ip_address(get_real_ip(request))
|
||||
network = get_network(real_ip, cfg)
|
||||
|
||||
if request.path == '/healthz':
|
||||
return None
|
||||
|
||||
# link-local
|
||||
|
||||
if network.is_link_local:
|
||||
return None
|
||||
|
||||
# block- & pass- lists
|
||||
#
|
||||
# 1. The IP of the request is first checked against the pass-list; if the IP
|
||||
# matches an entry in the list, the request is not blocked.
|
||||
# 2. If no matching entry is found in the pass-list, then a check is made against
|
||||
# the block list; if the IP matches an entry in the list, the request is
|
||||
# blocked.
|
||||
# 3. If the IP is not in either list, the request is not blocked.
|
||||
|
||||
match, msg = ip_lists.pass_ip(real_ip, cfg)
|
||||
if match:
|
||||
logger.warning("PASS %s: matched PASSLIST - %s", network.compressed, msg)
|
||||
return None
|
||||
|
||||
match, msg = ip_lists.block_ip(real_ip, cfg)
|
||||
if match:
|
||||
logger.error("BLOCK %s: matched BLOCKLIST - %s", network.compressed, msg)
|
||||
return flask.make_response(('IP is on BLOCKLIST - %s' % msg, 429))
|
||||
|
||||
# methods applied on /
|
||||
|
||||
for func in [
|
||||
http_user_agent,
|
||||
]:
|
||||
val = func.filter_request(network, request, cfg)
|
||||
if val is not None:
|
||||
return val
|
||||
|
||||
# methods applied on /search
|
||||
|
||||
if request.path == '/search':
|
||||
|
||||
for func in [
|
||||
http_accept,
|
||||
http_accept_encoding,
|
||||
http_accept_language,
|
||||
http_connection,
|
||||
http_user_agent,
|
||||
ip_limit,
|
||||
]:
|
||||
val = func.filter_request(network, request, cfg)
|
||||
if val is not None:
|
||||
return val
|
||||
logger.debug(f"OK {network}: %s", dump_request(flask.request))
|
||||
return None
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
"""
|
||||
Method ``link_token``
|
||||
---------------------
|
||||
|
||||
The ``link_token`` method evaluates a request as :py:obj:`suspicious
|
||||
<is_suspicious>` if the URL ``/client<token>.css`` is not requested by the
|
||||
client. By adding a random component (the token) in the URL, a bot can not send
|
||||
a ping by request a static URL.
|
||||
|
||||
.. note::
|
||||
|
||||
This method requires a redis DB and needs a HTTP X-Forwarded-For_ header.
|
||||
|
||||
To get in use of this method a flask URL route needs to be added:
|
||||
|
||||
.. code:: python
|
||||
|
||||
@app.route('/client<token>.css', methods=['GET', 'POST'])
|
||||
def client_token(token=None):
|
||||
link_token.ping(request, token)
|
||||
return Response('', mimetype='text/css')
|
||||
|
||||
And in the HTML template from flask a stylesheet link is needed (the value of
|
||||
``link_token`` comes from :py:obj:`get_token`):
|
||||
|
||||
.. code:: html
|
||||
|
||||
<link rel="stylesheet"
|
||||
href="{{ url_for('client_token', token=link_token) }}"
|
||||
type="text/css" />
|
||||
|
||||
.. _X-Forwarded-For:
|
||||
https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/X-Forwarded-For
|
||||
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from ipaddress import (
|
||||
IPv4Network,
|
||||
IPv6Network,
|
||||
ip_address,
|
||||
)
|
||||
|
||||
import string
|
||||
import random
|
||||
import flask
|
||||
|
||||
from searx import logger
|
||||
from searx import redisdb
|
||||
from searx.redislib import secret_hash
|
||||
|
||||
from ._helpers import (
|
||||
get_network,
|
||||
get_real_ip,
|
||||
)
|
||||
|
||||
TOKEN_LIVE_TIME = 600
|
||||
"""Livetime (sec) of limiter's CSS token."""
|
||||
|
||||
PING_LIVE_TIME = 3600
|
||||
"""Livetime (sec) of the ping-key from a client (request)"""
|
||||
|
||||
PING_KEY = 'SearXNG_limiter.ping'
|
||||
"""Prefix of all ping-keys generated by :py:obj:`get_ping_key`"""
|
||||
|
||||
TOKEN_KEY = 'SearXNG_limiter.token'
|
||||
"""Key for which the current token is stored in the DB"""
|
||||
|
||||
logger = logger.getChild('botdetection.link_token')
|
||||
|
||||
|
||||
def is_suspicious(network: IPv4Network | IPv6Network, request: flask.Request, renew: bool = False):
|
||||
"""Checks whether a valid ping is exists for this (client) network, if not
|
||||
this request is rated as *suspicious*. If a valid ping exists and argument
|
||||
``renew`` is ``True`` the expire time of this ping is reset to
|
||||
:py:obj:`PING_LIVE_TIME`.
|
||||
|
||||
"""
|
||||
redis_client = redisdb.client()
|
||||
if not redis_client:
|
||||
return False
|
||||
|
||||
ping_key = get_ping_key(network, request)
|
||||
if not redis_client.get(ping_key):
|
||||
logger.warning("missing ping (IP: %s) / request: %s", network.compressed, ping_key)
|
||||
return True
|
||||
|
||||
if renew:
|
||||
redis_client.set(ping_key, 1, ex=PING_LIVE_TIME)
|
||||
|
||||
logger.debug("found ping for (client) network %s -> %s", network.compressed, ping_key)
|
||||
return False
|
||||
|
||||
|
||||
def ping(request: flask.Request, token: str):
|
||||
"""This function is called by a request to URL ``/client<token>.css``. If
|
||||
``token`` is valid a :py:obj:`PING_KEY` for the client is stored in the DB.
|
||||
The expire time of this ping-key is :py:obj:`PING_LIVE_TIME`.
|
||||
|
||||
"""
|
||||
from . import limiter # pylint: disable=import-outside-toplevel, cyclic-import
|
||||
|
||||
redis_client = redisdb.client()
|
||||
if not redis_client:
|
||||
return
|
||||
if not token_is_valid(token):
|
||||
return
|
||||
|
||||
cfg = limiter.get_cfg()
|
||||
real_ip = ip_address(get_real_ip(request))
|
||||
network = get_network(real_ip, cfg)
|
||||
|
||||
ping_key = get_ping_key(network, request)
|
||||
logger.debug("store ping_key for (client) network %s (IP %s) -> %s", network.compressed, real_ip, ping_key)
|
||||
redis_client.set(ping_key, 1, ex=PING_LIVE_TIME)
|
||||
|
||||
|
||||
def get_ping_key(network: IPv4Network | IPv6Network, request: flask.Request) -> str:
|
||||
"""Generates a hashed key that fits (more or less) to a *WEB-browser
|
||||
session* in a network."""
|
||||
return (
|
||||
PING_KEY
|
||||
+ "["
|
||||
+ secret_hash(
|
||||
network.compressed + request.headers.get('Accept-Language', '') + request.headers.get('User-Agent', '')
|
||||
)
|
||||
+ "]"
|
||||
)
|
||||
|
||||
|
||||
def token_is_valid(token) -> bool:
|
||||
valid = token == get_token()
|
||||
logger.debug("token is valid --> %s", valid)
|
||||
return valid
|
||||
|
||||
|
||||
def get_token() -> str:
|
||||
"""Returns current token. If there is no currently active token a new token
|
||||
is generated randomly and stored in the redis DB.
|
||||
|
||||
- :py:obj:`TOKEN_LIVE_TIME`
|
||||
- :py:obj:`TOKEN_KEY`
|
||||
|
||||
"""
|
||||
redis_client = redisdb.client()
|
||||
if not redis_client:
|
||||
# This function is also called when limiter is inactive / no redis DB
|
||||
# (see render function in webapp.py)
|
||||
return '12345678'
|
||||
token = redis_client.get(TOKEN_KEY)
|
||||
if token:
|
||||
token = token.decode('UTF-8')
|
||||
else:
|
||||
token = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(16))
|
||||
redis_client.set(TOKEN_KEY, token, ex=TOKEN_LIVE_TIME)
|
||||
return token
|
||||
Reference in New Issue
Block a user