first commit
This commit is contained in:
git
+37
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python
|
||||
# lint: pylint
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""This script saves `Ahmia's blacklist`_ for onion sites.
|
||||
|
||||
Output file: :origin:`searx/data/ahmia_blacklist.txt` (:origin:`CI Update data
|
||||
... <.github/workflows/data-update.yml>`).
|
||||
|
||||
.. _Ahmia's blacklist: https://ahmia.fi/blacklist/
|
||||
|
||||
"""
|
||||
# pylint: disable=use-dict-literal
|
||||
|
||||
from os.path import join
|
||||
|
||||
import requests
|
||||
from searx import searx_dir
|
||||
|
||||
URL = 'https://ahmia.fi/blacklist/banned/'
|
||||
|
||||
|
||||
def fetch_ahmia_blacklist():
|
||||
resp = requests.get(URL, timeout=3.0)
|
||||
if resp.status_code != 200:
|
||||
# pylint: disable=broad-exception-raised
|
||||
raise Exception("Error fetching Ahmia blacklist, HTTP code " + resp.status_code)
|
||||
return resp.text.split()
|
||||
|
||||
|
||||
def get_ahmia_blacklist_filename():
|
||||
return join(join(searx_dir, "data"), "ahmia_blacklist.txt")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
blacklist = fetch_ahmia_blacklist()
|
||||
with open(get_ahmia_blacklist_filename(), "w", encoding='utf-8') as f:
|
||||
f.write('\n'.join(blacklist))
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python
|
||||
# lint: pylint
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
"""Fetch currencies from :origin:`searx/engines/wikidata.py` engine.
|
||||
|
||||
Output file: :origin:`searx/data/currencies.json` (:origin:`CI Update data ...
|
||||
<.github/workflows/data-update.yml>`).
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
|
||||
import re
|
||||
import unicodedata
|
||||
import json
|
||||
|
||||
# set path
|
||||
from os.path import join
|
||||
|
||||
from searx import searx_dir
|
||||
from searx.locales import LOCALE_NAMES, locales_initialize
|
||||
from searx.engines import wikidata, set_loggers
|
||||
|
||||
set_loggers(wikidata, 'wikidata')
|
||||
locales_initialize()
|
||||
|
||||
# ORDER BY (with all the query fields) is important to keep a deterministic result order
|
||||
# so multiple invocation of this script doesn't change currencies.json
|
||||
SARQL_REQUEST = """
|
||||
SELECT DISTINCT ?iso4217 ?unit ?unicode ?label ?alias WHERE {
|
||||
?item wdt:P498 ?iso4217; rdfs:label ?label.
|
||||
OPTIONAL { ?item skos:altLabel ?alias FILTER (LANG (?alias) = LANG(?label)). }
|
||||
OPTIONAL { ?item wdt:P5061 ?unit. }
|
||||
OPTIONAL { ?item wdt:P489 ?symbol.
|
||||
?symbol wdt:P487 ?unicode. }
|
||||
MINUS { ?item wdt:P582 ?end_data . } # Ignore monney with an end date
|
||||
MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . } # Ignore "former entity" (obsolete currency)
|
||||
FILTER(LANG(?label) IN (%LANGUAGES_SPARQL%)).
|
||||
}
|
||||
ORDER BY ?iso4217 ?unit ?unicode ?label ?alias
|
||||
"""
|
||||
|
||||
# ORDER BY (with all the query fields) is important to keep a deterministic result order
|
||||
# so multiple invocation of this script doesn't change currencies.json
|
||||
SPARQL_WIKIPEDIA_NAMES_REQUEST = """
|
||||
SELECT DISTINCT ?iso4217 ?article_name WHERE {
|
||||
?item wdt:P498 ?iso4217 .
|
||||
?article schema:about ?item ;
|
||||
schema:name ?article_name ;
|
||||
schema:isPartOf [ wikibase:wikiGroup "wikipedia" ]
|
||||
MINUS { ?item wdt:P582 ?end_data . } # Ignore monney with an end date
|
||||
MINUS { ?item wdt:P31/wdt:P279* wd:Q15893266 . } # Ignore "former entity" (obsolete currency)
|
||||
FILTER(LANG(?article_name) IN (%LANGUAGES_SPARQL%)).
|
||||
}
|
||||
ORDER BY ?iso4217 ?article_name
|
||||
"""
|
||||
|
||||
|
||||
LANGUAGES = LOCALE_NAMES.keys()
|
||||
LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))
|
||||
|
||||
|
||||
def remove_accents(name):
|
||||
return unicodedata.normalize('NFKD', name).lower()
|
||||
|
||||
|
||||
def remove_extra(name):
|
||||
for c in ('(', ':'):
|
||||
if c in name:
|
||||
name = name.split(c)[0].strip()
|
||||
return name
|
||||
|
||||
|
||||
def _normalize_name(name):
|
||||
name = re.sub(' +', ' ', remove_accents(name.lower()).replace('-', ' '))
|
||||
name = remove_extra(name)
|
||||
return name
|
||||
|
||||
|
||||
def add_currency_name(db, name, iso4217, normalize_name=True):
|
||||
db_names = db['names']
|
||||
|
||||
if normalize_name:
|
||||
name = _normalize_name(name)
|
||||
|
||||
iso4217_set = db_names.setdefault(name, [])
|
||||
if iso4217 not in iso4217_set:
|
||||
iso4217_set.insert(0, iso4217)
|
||||
|
||||
|
||||
def add_currency_label(db, label, iso4217, language):
|
||||
labels = db['iso4217'].setdefault(iso4217, {})
|
||||
labels[language] = label
|
||||
|
||||
|
||||
def wikidata_request_result_iterator(request):
|
||||
result = wikidata.send_wikidata_query(request.replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
|
||||
if result is not None:
|
||||
for r in result['results']['bindings']:
|
||||
yield r
|
||||
|
||||
|
||||
def fetch_db():
|
||||
db = {
|
||||
'names': {},
|
||||
'iso4217': {},
|
||||
}
|
||||
|
||||
for r in wikidata_request_result_iterator(SPARQL_WIKIPEDIA_NAMES_REQUEST):
|
||||
iso4217 = r['iso4217']['value']
|
||||
article_name = r['article_name']['value']
|
||||
article_lang = r['article_name']['xml:lang']
|
||||
add_currency_name(db, article_name, iso4217)
|
||||
add_currency_label(db, article_name, iso4217, article_lang)
|
||||
|
||||
for r in wikidata_request_result_iterator(SARQL_REQUEST):
|
||||
iso4217 = r['iso4217']['value']
|
||||
if 'label' in r:
|
||||
label = r['label']['value']
|
||||
label_lang = r['label']['xml:lang']
|
||||
add_currency_name(db, label, iso4217)
|
||||
add_currency_label(db, label, iso4217, label_lang)
|
||||
|
||||
if 'alias' in r:
|
||||
add_currency_name(db, r['alias']['value'], iso4217)
|
||||
|
||||
if 'unicode' in r:
|
||||
add_currency_name(db, r['unicode']['value'], iso4217, normalize_name=False)
|
||||
|
||||
if 'unit' in r:
|
||||
add_currency_name(db, r['unit']['value'], iso4217, normalize_name=False)
|
||||
|
||||
return db
|
||||
|
||||
|
||||
def get_filename():
|
||||
return join(join(searx_dir, "data"), "currencies.json")
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
db = fetch_db()
|
||||
|
||||
# static
|
||||
add_currency_name(db, "euro", 'EUR')
|
||||
add_currency_name(db, "euros", 'EUR')
|
||||
add_currency_name(db, "dollar", 'USD')
|
||||
add_currency_name(db, "dollars", 'USD')
|
||||
add_currency_name(db, "peso", 'MXN')
|
||||
add_currency_name(db, "pesos", 'MXN')
|
||||
|
||||
# reduce memory usage:
|
||||
# replace lists with one item by the item. see
|
||||
# searx.search.processors.online_currency.name_to_iso4217
|
||||
for name in db['names']:
|
||||
if len(db['names'][name]) == 1:
|
||||
db['names'][name] = db['names'][name][0]
|
||||
|
||||
with open(get_filename(), 'w', encoding='utf8') as f:
|
||||
json.dump(db, f, ensure_ascii=False, indent=4)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+370
@@ -0,0 +1,370 @@
|
||||
#!/usr/bin/env python
|
||||
# lint: pylint
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
"""Fetch website description from websites and from
|
||||
:origin:`searx/engines/wikidata.py` engine.
|
||||
|
||||
Output file: :origin:`searx/data/engine_descriptions.json`.
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=invalid-name, global-statement
|
||||
|
||||
import json
|
||||
from urllib.parse import urlparse
|
||||
from os.path import join
|
||||
|
||||
from lxml.html import fromstring
|
||||
|
||||
from searx.engines import wikidata, set_loggers
|
||||
from searx.utils import extract_text, searx_useragent
|
||||
from searx.locales import LOCALE_NAMES, locales_initialize, match_locale
|
||||
from searx import searx_dir
|
||||
from searx.utils import gen_useragent, detect_language
|
||||
import searx.search
|
||||
import searx.network
|
||||
|
||||
set_loggers(wikidata, 'wikidata')
|
||||
locales_initialize()
|
||||
|
||||
# you can run the query in https://query.wikidata.org
|
||||
# replace %IDS% by Wikidata entities separated by spaces with the prefix wd:
|
||||
# for example wd:Q182496 wd:Q1540899
|
||||
# replace %LANGUAGES_SPARQL% by languages
|
||||
SPARQL_WIKIPEDIA_ARTICLE = """
|
||||
SELECT DISTINCT ?item ?name ?article ?lang
|
||||
WHERE {
|
||||
hint:Query hint:optimizer "None".
|
||||
VALUES ?item { %IDS% }
|
||||
?article schema:about ?item ;
|
||||
schema:inLanguage ?lang ;
|
||||
schema:name ?name ;
|
||||
schema:isPartOf [ wikibase:wikiGroup "wikipedia" ] .
|
||||
FILTER(?lang in (%LANGUAGES_SPARQL%)) .
|
||||
FILTER (!CONTAINS(?name, ':')) .
|
||||
}
|
||||
ORDER BY ?item ?lang
|
||||
"""
|
||||
|
||||
SPARQL_DESCRIPTION = """
|
||||
SELECT DISTINCT ?item ?itemDescription
|
||||
WHERE {
|
||||
VALUES ?item { %IDS% }
|
||||
?item schema:description ?itemDescription .
|
||||
FILTER (lang(?itemDescription) in (%LANGUAGES_SPARQL%))
|
||||
}
|
||||
ORDER BY ?itemLang
|
||||
"""
|
||||
|
||||
NOT_A_DESCRIPTION = [
|
||||
'web site',
|
||||
'site web',
|
||||
'komputa serĉilo',
|
||||
'interreta serĉilo',
|
||||
'bilaketa motor',
|
||||
'web search engine',
|
||||
'wikimedia täpsustuslehekülg',
|
||||
]
|
||||
|
||||
SKIP_ENGINE_SOURCE = [
|
||||
# fmt: off
|
||||
('gitlab', 'wikidata')
|
||||
# descriptions are about wikipedia disambiguation pages
|
||||
# fmt: on
|
||||
]
|
||||
|
||||
WIKIPEDIA_LANGUAGES = {}
|
||||
LANGUAGES_SPARQL = ''
|
||||
IDS = None
|
||||
WIKIPEDIA_LANGUAGE_VARIANTS = {'zh_Hant': 'zh-tw'}
|
||||
|
||||
|
||||
descriptions = {}
|
||||
wd_to_engine_name = {}
|
||||
|
||||
|
||||
def normalize_description(description):
|
||||
for c in [chr(c) for c in range(0, 31)]:
|
||||
description = description.replace(c, ' ')
|
||||
description = ' '.join(description.strip().split())
|
||||
return description
|
||||
|
||||
|
||||
def update_description(engine_name, lang, description, source, replace=True):
|
||||
if not isinstance(description, str):
|
||||
return
|
||||
description = normalize_description(description)
|
||||
if description.lower() == engine_name.lower():
|
||||
return
|
||||
if description.lower() in NOT_A_DESCRIPTION:
|
||||
return
|
||||
if (engine_name, source) in SKIP_ENGINE_SOURCE:
|
||||
return
|
||||
if ' ' not in description:
|
||||
# skip unique word description (like "website")
|
||||
return
|
||||
if replace or lang not in descriptions[engine_name]:
|
||||
descriptions[engine_name][lang] = [description, source]
|
||||
|
||||
|
||||
def get_wikipedia_summary(wikipedia_url, searxng_locale):
|
||||
# get the REST API URL from the HTML URL
|
||||
|
||||
# Headers
|
||||
headers = {'User-Agent': searx_useragent()}
|
||||
|
||||
if searxng_locale in WIKIPEDIA_LANGUAGE_VARIANTS:
|
||||
headers['Accept-Language'] = WIKIPEDIA_LANGUAGE_VARIANTS.get(searxng_locale)
|
||||
|
||||
# URL path : from HTML URL to REST API URL
|
||||
parsed_url = urlparse(wikipedia_url)
|
||||
# remove the /wiki/ prefix
|
||||
article_name = parsed_url.path.split('/wiki/')[1]
|
||||
# article_name is already encoded but not the / which is required for the REST API call
|
||||
encoded_article_name = article_name.replace('/', '%2F')
|
||||
path = '/api/rest_v1/page/summary/' + encoded_article_name
|
||||
wikipedia_rest_url = parsed_url._replace(path=path).geturl()
|
||||
try:
|
||||
response = searx.network.get(wikipedia_rest_url, headers=headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
print(" ", wikipedia_url, e)
|
||||
return None
|
||||
api_result = json.loads(response.text)
|
||||
return api_result.get('extract')
|
||||
|
||||
|
||||
def get_website_description(url, lang1, lang2=None):
|
||||
headers = {
|
||||
'User-Agent': gen_useragent(),
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||||
'DNT': '1',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Sec-GPC': '1',
|
||||
'Cache-Control': 'max-age=0',
|
||||
}
|
||||
if lang1 is not None:
|
||||
lang_list = [lang1]
|
||||
if lang2 is not None:
|
||||
lang_list.append(lang2)
|
||||
headers['Accept-Language'] = f'{",".join(lang_list)};q=0.8'
|
||||
try:
|
||||
response = searx.network.get(url, headers=headers, timeout=10)
|
||||
response.raise_for_status()
|
||||
except Exception: # pylint: disable=broad-except
|
||||
return (None, None)
|
||||
|
||||
try:
|
||||
html = fromstring(response.text)
|
||||
except ValueError:
|
||||
html = fromstring(response.content)
|
||||
|
||||
description = extract_text(html.xpath('/html/head/meta[@name="description"]/@content'))
|
||||
if not description:
|
||||
description = extract_text(html.xpath('/html/head/meta[@property="og:description"]/@content'))
|
||||
if not description:
|
||||
description = extract_text(html.xpath('/html/head/title'))
|
||||
lang = extract_text(html.xpath('/html/@lang'))
|
||||
if lang is None and len(lang1) > 0:
|
||||
lang = lang1
|
||||
lang = detect_language(description) or lang or 'en'
|
||||
lang = lang.split('_')[0]
|
||||
lang = lang.split('-')[0]
|
||||
return (lang, description)
|
||||
|
||||
|
||||
def initialize():
|
||||
global IDS, LANGUAGES_SPARQL
|
||||
searx.search.initialize()
|
||||
wikipedia_engine = searx.engines.engines['wikipedia']
|
||||
|
||||
locale2lang = {'nl-BE': 'nl'}
|
||||
for sxng_ui_lang in LOCALE_NAMES:
|
||||
|
||||
sxng_ui_alias = locale2lang.get(sxng_ui_lang, sxng_ui_lang)
|
||||
wiki_lang = None
|
||||
|
||||
if sxng_ui_alias in wikipedia_engine.traits.custom['WIKIPEDIA_LANGUAGES']:
|
||||
wiki_lang = sxng_ui_alias
|
||||
if not wiki_lang:
|
||||
wiki_lang = wikipedia_engine.traits.get_language(sxng_ui_alias)
|
||||
if not wiki_lang:
|
||||
print(f"WIKIPEDIA_LANGUAGES missing {sxng_ui_lang}")
|
||||
continue
|
||||
WIKIPEDIA_LANGUAGES[sxng_ui_lang] = wiki_lang
|
||||
|
||||
LANGUAGES_SPARQL = ', '.join(f"'{l}'" for l in set(WIKIPEDIA_LANGUAGES.values()))
|
||||
for engine_name, engine in searx.engines.engines.items():
|
||||
descriptions[engine_name] = {}
|
||||
wikidata_id = getattr(engine, "about", {}).get('wikidata_id')
|
||||
if wikidata_id is not None:
|
||||
wd_to_engine_name.setdefault(wikidata_id, set()).add(engine_name)
|
||||
|
||||
IDS = ' '.join(list(map(lambda wd_id: 'wd:' + wd_id, wd_to_engine_name.keys())))
|
||||
|
||||
|
||||
def fetch_wikidata_descriptions():
|
||||
print('Fetching wikidata descriptions')
|
||||
searx.network.set_timeout_for_thread(60)
|
||||
result = wikidata.send_wikidata_query(
|
||||
SPARQL_DESCRIPTION.replace('%IDS%', IDS).replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL)
|
||||
)
|
||||
if result is not None:
|
||||
for binding in result['results']['bindings']:
|
||||
wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '')
|
||||
wikidata_lang = binding['itemDescription']['xml:lang']
|
||||
desc = binding['itemDescription']['value']
|
||||
for engine_name in wd_to_engine_name[wikidata_id]:
|
||||
for searxng_locale in LOCALE_NAMES:
|
||||
if WIKIPEDIA_LANGUAGES[searxng_locale] != wikidata_lang:
|
||||
continue
|
||||
print(
|
||||
f" engine: {engine_name:20} / wikidata_lang: {wikidata_lang:5}",
|
||||
f"/ len(wikidata_desc): {len(desc)}",
|
||||
)
|
||||
update_description(engine_name, searxng_locale, desc, 'wikidata')
|
||||
|
||||
|
||||
def fetch_wikipedia_descriptions():
|
||||
print('Fetching wikipedia descriptions')
|
||||
result = wikidata.send_wikidata_query(
|
||||
SPARQL_WIKIPEDIA_ARTICLE.replace('%IDS%', IDS).replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL)
|
||||
)
|
||||
if result is not None:
|
||||
for binding in result['results']['bindings']:
|
||||
wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '')
|
||||
wikidata_lang = binding['name']['xml:lang']
|
||||
wikipedia_url = binding['article']['value'] # for example the URL https://de.wikipedia.org/wiki/PubMed
|
||||
for engine_name in wd_to_engine_name[wikidata_id]:
|
||||
for searxng_locale in LOCALE_NAMES:
|
||||
if WIKIPEDIA_LANGUAGES[searxng_locale] != wikidata_lang:
|
||||
continue
|
||||
desc = get_wikipedia_summary(wikipedia_url, searxng_locale)
|
||||
if not desc:
|
||||
continue
|
||||
print(
|
||||
f" engine: {engine_name:20} / wikidata_lang: {wikidata_lang:5}",
|
||||
f"/ len(wikipedia_desc): {len(desc)}",
|
||||
)
|
||||
update_description(engine_name, searxng_locale, desc, 'wikipedia')
|
||||
|
||||
|
||||
def normalize_url(url):
|
||||
url = url.replace('{language}', 'en')
|
||||
url = urlparse(url)._replace(path='/', params='', query='', fragment='').geturl()
|
||||
url = url.replace('https://api.', 'https://')
|
||||
return url
|
||||
|
||||
|
||||
def fetch_website_description(engine_name, website):
|
||||
print(f"- fetch website descr: {engine_name} / {website}")
|
||||
default_lang, default_description = get_website_description(website, None, None)
|
||||
|
||||
if default_lang is None or default_description is None:
|
||||
# the front page can't be fetched: skip this engine
|
||||
return
|
||||
|
||||
# to specify an order in where the most common languages are in front of the
|
||||
# language list ..
|
||||
languages = ['en', 'es', 'pt', 'ru', 'tr', 'fr']
|
||||
languages = languages + [l for l in LOCALE_NAMES if l not in languages]
|
||||
|
||||
previous_matched_lang = None
|
||||
previous_count = 0
|
||||
|
||||
for lang in languages:
|
||||
|
||||
if lang in descriptions[engine_name]:
|
||||
continue
|
||||
|
||||
fetched_lang, desc = get_website_description(website, lang, WIKIPEDIA_LANGUAGES[lang])
|
||||
if fetched_lang is None or desc is None:
|
||||
continue
|
||||
|
||||
# check if desc changed with the different lang values
|
||||
|
||||
if fetched_lang == previous_matched_lang:
|
||||
previous_count += 1
|
||||
if previous_count == 6:
|
||||
# the website has returned the same description for 6 different languages in Accept-Language header
|
||||
# stop now
|
||||
break
|
||||
else:
|
||||
previous_matched_lang = fetched_lang
|
||||
previous_count = 0
|
||||
|
||||
# Don't trust in the value of fetched_lang, some websites return
|
||||
# for some inappropriate values, by example bing-images::
|
||||
#
|
||||
# requested lang: zh-Hans-CN / fetched lang: ceb / desc: 查看根据您的兴趣量身定制的提要
|
||||
#
|
||||
# The lang ceb is "Cebuano" but the description is given in zh-Hans-CN
|
||||
|
||||
print(
|
||||
f" engine: {engine_name:20} / requested lang:{lang:7}"
|
||||
f" / fetched lang: {fetched_lang:7} / len(desc): {len(desc)}"
|
||||
)
|
||||
|
||||
matched_lang = match_locale(fetched_lang, LOCALE_NAMES.keys(), fallback=lang)
|
||||
update_description(engine_name, matched_lang, desc, website, replace=False)
|
||||
|
||||
|
||||
def fetch_website_descriptions():
|
||||
print('Fetching website descriptions')
|
||||
for engine_name, engine in searx.engines.engines.items():
|
||||
website = getattr(engine, "about", {}).get('website')
|
||||
if website is None and hasattr(engine, "search_url"):
|
||||
website = normalize_url(getattr(engine, "search_url"))
|
||||
if website is None and hasattr(engine, "base_url"):
|
||||
website = normalize_url(getattr(engine, "base_url"))
|
||||
if website is not None:
|
||||
fetch_website_description(engine_name, website)
|
||||
|
||||
|
||||
def get_engine_descriptions_filename():
|
||||
return join(join(searx_dir, "data"), "engine_descriptions.json")
|
||||
|
||||
|
||||
def get_output():
|
||||
"""
|
||||
From descriptions[engine][language] = [description, source]
|
||||
To
|
||||
|
||||
* output[language][engine] = description_and_source
|
||||
* description_and_source can be:
|
||||
* [description, source]
|
||||
* description (if source = "wikipedia")
|
||||
* [f"engine:lang", "ref"] (reference to another existing description)
|
||||
"""
|
||||
output = {locale: {} for locale in LOCALE_NAMES}
|
||||
|
||||
seen_descriptions = {}
|
||||
|
||||
for engine_name, lang_descriptions in descriptions.items():
|
||||
for language, description in lang_descriptions.items():
|
||||
if description[0] in seen_descriptions:
|
||||
ref = seen_descriptions[description[0]]
|
||||
description = [f'{ref[0]}:{ref[1]}', 'ref']
|
||||
else:
|
||||
seen_descriptions[description[0]] = (engine_name, language)
|
||||
if description[1] == 'wikipedia':
|
||||
description = description[0]
|
||||
output.setdefault(language, {}).setdefault(engine_name, description)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
initialize()
|
||||
fetch_wikidata_descriptions()
|
||||
fetch_wikipedia_descriptions()
|
||||
fetch_website_descriptions()
|
||||
|
||||
output = get_output()
|
||||
with open(get_engine_descriptions_filename(), 'w', encoding='utf8') as f:
|
||||
f.write(json.dumps(output, indent=1, separators=(',', ':'), ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python
|
||||
# lint: pylint
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Update :py:obj:`searx.enginelib.traits.EngineTraitsMap` and :origin:`searx/languages.py`
|
||||
|
||||
:py:obj:`searx.enginelib.traits.EngineTraitsMap.ENGINE_TRAITS_FILE`:
|
||||
Persistence of engines traits, fetched from the engines.
|
||||
|
||||
:origin:`searx/languages.py`
|
||||
Is generated from intersecting each engine's supported traits.
|
||||
|
||||
The script :origin:`searxng_extra/update/update_engine_traits.py` is called in
|
||||
the :origin:`CI Update data ... <.github/workflows/data-update.yml>`
|
||||
|
||||
"""
|
||||
|
||||
# pylint: disable=invalid-name
|
||||
from unicodedata import lookup
|
||||
from pathlib import Path
|
||||
from pprint import pformat
|
||||
import babel
|
||||
|
||||
from searx import settings, searx_dir
|
||||
from searx import network
|
||||
from searx.engines import load_engines
|
||||
from searx.enginelib.traits import EngineTraitsMap
|
||||
|
||||
# Output files.
|
||||
languages_file = Path(searx_dir) / 'sxng_locales.py'
|
||||
languages_file_header = """\
|
||||
# -*- coding: utf-8 -*-
|
||||
'''List of SearXNG's locale codes.
|
||||
|
||||
This file is generated automatically by::
|
||||
|
||||
./manage pyenv.cmd searxng_extra/update/update_engine_traits.py
|
||||
'''
|
||||
|
||||
sxng_locales = (
|
||||
"""
|
||||
languages_file_footer = """,
|
||||
)
|
||||
'''
|
||||
A list of five-digit tuples:
|
||||
|
||||
0. SearXNG's internal locale tag (a language or region tag)
|
||||
1. Name of the language (:py:obj:`babel.core.Locale.get_language_name`)
|
||||
2. For region tags the name of the region (:py:obj:`babel.core.Locale.get_territory_name`).
|
||||
Empty string for language tags.
|
||||
3. English language name (from :py:obj:`babel.core.Locale.english_name`)
|
||||
4. Unicode flag (emoji) that fits to SearXNG's internal region tag. Languages
|
||||
are represented by a globe (\U0001F310)
|
||||
|
||||
.. code:: python
|
||||
|
||||
('en', 'English', '', 'English', '\U0001f310'),
|
||||
('en-CA', 'English', 'Canada', 'English', '\U0001f1e8\U0001f1e6'),
|
||||
('en-US', 'English', 'United States', 'English', '\U0001f1fa\U0001f1f8'),
|
||||
..
|
||||
('fr', 'Français', '', 'French', '\U0001f310'),
|
||||
('fr-BE', 'Français', 'Belgique', 'French', '\U0001f1e7\U0001f1ea'),
|
||||
('fr-CA', 'Français', 'Canada', 'French', '\U0001f1e8\U0001f1e6'),
|
||||
|
||||
:meta hide-value:
|
||||
'''
|
||||
"""
|
||||
|
||||
|
||||
lang2emoji = {
|
||||
'ha': '\U0001F1F3\U0001F1EA', # Hausa / Niger
|
||||
'bs': '\U0001F1E7\U0001F1E6', # Bosnian / Bosnia & Herzegovina
|
||||
'jp': '\U0001F1EF\U0001F1F5', # Japanese
|
||||
'ua': '\U0001F1FA\U0001F1E6', # Ukrainian
|
||||
'he': '\U0001F1EE\U0001F1F1', # Hebrew
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
load_engines(settings['engines'])
|
||||
# traits_map = EngineTraitsMap.from_data()
|
||||
traits_map = fetch_traits_map()
|
||||
sxng_tag_list = filter_locales(traits_map)
|
||||
write_languages_file(sxng_tag_list)
|
||||
|
||||
|
||||
def fetch_traits_map():
|
||||
"""Fetchs supported languages for each engine and writes json file with those."""
|
||||
network.set_timeout_for_thread(10.0)
|
||||
|
||||
def log(msg):
|
||||
print(msg)
|
||||
|
||||
traits_map = EngineTraitsMap.fetch_traits(log=log)
|
||||
print("fetched properties from %s engines" % len(traits_map))
|
||||
print("write json file: %s" % traits_map.ENGINE_TRAITS_FILE)
|
||||
traits_map.save_data()
|
||||
return traits_map
|
||||
|
||||
|
||||
def filter_locales(traits_map: EngineTraitsMap):
|
||||
"""Filter language & region tags by a threshold."""
|
||||
|
||||
min_eng_per_region = 11
|
||||
min_eng_per_lang = 13
|
||||
|
||||
_ = {}
|
||||
for eng in traits_map.values():
|
||||
for reg in eng.regions.keys():
|
||||
_[reg] = _.get(reg, 0) + 1
|
||||
|
||||
regions = set(k for k, v in _.items() if v >= min_eng_per_region)
|
||||
lang_from_region = set(k.split('-')[0] for k in regions)
|
||||
|
||||
_ = {}
|
||||
for eng in traits_map.values():
|
||||
for lang in eng.languages.keys():
|
||||
# ignore script types like zh_Hant, zh_Hans or sr_Latin, pa_Arab (they
|
||||
# already counted by existence of 'zh' or 'sr', 'pa')
|
||||
if '_' in lang:
|
||||
# print("ignore %s" % lang)
|
||||
continue
|
||||
_[lang] = _.get(lang, 0) + 1
|
||||
|
||||
languages = set(k for k, v in _.items() if v >= min_eng_per_lang)
|
||||
|
||||
sxng_tag_list = set()
|
||||
sxng_tag_list.update(regions)
|
||||
sxng_tag_list.update(lang_from_region)
|
||||
sxng_tag_list.update(languages)
|
||||
|
||||
return sxng_tag_list
|
||||
|
||||
|
||||
def write_languages_file(sxng_tag_list):
|
||||
|
||||
language_codes = []
|
||||
|
||||
for sxng_tag in sorted(sxng_tag_list):
|
||||
sxng_locale: babel.Locale = babel.Locale.parse(sxng_tag, sep='-')
|
||||
|
||||
flag = get_unicode_flag(sxng_locale) or ''
|
||||
|
||||
item = (
|
||||
sxng_tag,
|
||||
sxng_locale.get_language_name().title(),
|
||||
sxng_locale.get_territory_name() or '',
|
||||
sxng_locale.english_name.split(' (')[0],
|
||||
UnicodeEscape(flag),
|
||||
)
|
||||
|
||||
language_codes.append(item)
|
||||
|
||||
language_codes = tuple(language_codes)
|
||||
|
||||
with open(languages_file, 'w', encoding='utf-8') as new_file:
|
||||
file_content = "{header} {language_codes}{footer}".format(
|
||||
header=languages_file_header,
|
||||
language_codes=pformat(language_codes, width=120, indent=4)[1:-1],
|
||||
footer=languages_file_footer,
|
||||
)
|
||||
new_file.write(file_content)
|
||||
new_file.close()
|
||||
|
||||
|
||||
class UnicodeEscape(str):
|
||||
"""Escape unicode string in :py:obj:`pprint.pformat`"""
|
||||
|
||||
def __repr__(self):
|
||||
return "'" + "".join([chr(c) for c in self.encode('unicode-escape')]) + "'"
|
||||
|
||||
|
||||
def get_unicode_flag(locale: babel.Locale):
|
||||
"""Determine a unicode flag (emoji) that fits to the ``locale``"""
|
||||
|
||||
emoji = lang2emoji.get(locale.language)
|
||||
if emoji:
|
||||
return emoji
|
||||
|
||||
if not locale.territory:
|
||||
return '\U0001F310'
|
||||
|
||||
emoji = lang2emoji.get(locale.territory.lower())
|
||||
if emoji:
|
||||
return emoji
|
||||
|
||||
try:
|
||||
c1 = lookup('REGIONAL INDICATOR SYMBOL LETTER ' + locale.territory[0])
|
||||
c2 = lookup('REGIONAL INDICATOR SYMBOL LETTER ' + locale.territory[1])
|
||||
# print("OK : %s --> %s%s" % (locale, c1, c2))
|
||||
except KeyError as exc:
|
||||
print("ERROR: %s --> %s" % (locale, exc))
|
||||
return None
|
||||
|
||||
return c1 + c2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python
|
||||
# lint: pylint
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Update :origin:`searx/data/external_bangs.json` using the duckduckgo bangs
|
||||
(:origin:`CI Update data ... <.github/workflows/data-update.yml>`).
|
||||
|
||||
https://duckduckgo.com/newbang loads:
|
||||
|
||||
* a javascript which provides the bang version ( https://duckduckgo.com/bv1.js )
|
||||
* a JSON file which contains the bangs ( https://duckduckgo.com/bang.v260.js for example )
|
||||
|
||||
This script loads the javascript, then the bangs.
|
||||
|
||||
The javascript URL may change in the future ( for example
|
||||
https://duckduckgo.com/bv2.js ), but most probably it will requires to update
|
||||
RE_BANG_VERSION
|
||||
|
||||
"""
|
||||
# pylint: disable=C0116
|
||||
|
||||
import json
|
||||
import re
|
||||
from os.path import join
|
||||
|
||||
import httpx
|
||||
|
||||
from searx import searx_dir # pylint: disable=E0401 C0413
|
||||
from searx.external_bang import LEAF_KEY
|
||||
|
||||
# from https://duckduckgo.com/newbang
|
||||
URL_BV1 = 'https://duckduckgo.com/bv1.js'
|
||||
RE_BANG_VERSION = re.compile(r'\/bang\.v([0-9]+)\.js')
|
||||
HTTPS_COLON = 'https:'
|
||||
HTTP_COLON = 'http:'
|
||||
|
||||
|
||||
def get_bang_url():
|
||||
response = httpx.get(URL_BV1)
|
||||
response.raise_for_status()
|
||||
|
||||
r = RE_BANG_VERSION.findall(response.text)
|
||||
return f'https://duckduckgo.com/bang.v{r[0]}.js', r[0]
|
||||
|
||||
|
||||
def fetch_ddg_bangs(url):
|
||||
response = httpx.get(url)
|
||||
response.raise_for_status()
|
||||
return json.loads(response.content.decode())
|
||||
|
||||
|
||||
def merge_when_no_leaf(node):
|
||||
"""Minimize the number of nodes
|
||||
|
||||
``A -> B -> C``
|
||||
|
||||
- ``B`` is child of ``A``
|
||||
- ``C`` is child of ``B``
|
||||
|
||||
If there are no ``C`` equals to ``<LEAF_KEY>``, then each ``C`` are merged
|
||||
into ``A``. For example (5 nodes)::
|
||||
|
||||
d -> d -> g -> <LEAF_KEY> (ddg)
|
||||
-> i -> g -> <LEAF_KEY> (dig)
|
||||
|
||||
becomes (3 noodes)::
|
||||
|
||||
d -> dg -> <LEAF_KEY>
|
||||
-> ig -> <LEAF_KEY>
|
||||
|
||||
"""
|
||||
restart = False
|
||||
if not isinstance(node, dict):
|
||||
return
|
||||
|
||||
# create a copy of the keys so node can be modified
|
||||
keys = list(node.keys())
|
||||
|
||||
for key in keys:
|
||||
if key == LEAF_KEY:
|
||||
continue
|
||||
|
||||
value = node[key]
|
||||
value_keys = list(value.keys())
|
||||
if LEAF_KEY not in value_keys:
|
||||
for value_key in value_keys:
|
||||
node[key + value_key] = value[value_key]
|
||||
merge_when_no_leaf(node[key + value_key])
|
||||
del node[key]
|
||||
restart = True
|
||||
else:
|
||||
merge_when_no_leaf(value)
|
||||
|
||||
if restart:
|
||||
merge_when_no_leaf(node)
|
||||
|
||||
|
||||
def optimize_leaf(parent, parent_key, node):
|
||||
if not isinstance(node, dict):
|
||||
return
|
||||
|
||||
if len(node) == 1 and LEAF_KEY in node and parent is not None:
|
||||
parent[parent_key] = node[LEAF_KEY]
|
||||
else:
|
||||
for key, value in node.items():
|
||||
optimize_leaf(node, key, value)
|
||||
|
||||
|
||||
def parse_ddg_bangs(ddg_bangs):
|
||||
bang_trie = {}
|
||||
bang_urls = {}
|
||||
|
||||
for bang_definition in ddg_bangs:
|
||||
# bang_list
|
||||
bang_url = bang_definition['u']
|
||||
if '{{{s}}}' not in bang_url:
|
||||
# ignore invalid bang
|
||||
continue
|
||||
|
||||
bang_url = bang_url.replace('{{{s}}}', chr(2))
|
||||
|
||||
# only for the https protocol: "https://example.com" becomes "//example.com"
|
||||
if bang_url.startswith(HTTPS_COLON + '//'):
|
||||
bang_url = bang_url[len(HTTPS_COLON) :]
|
||||
|
||||
#
|
||||
if bang_url.startswith(HTTP_COLON + '//') and bang_url[len(HTTP_COLON) :] in bang_urls:
|
||||
# if the bang_url uses the http:// protocol, and the same URL exists in https://
|
||||
# then reuse the https:// bang definition. (written //example.com)
|
||||
bang_def_output = bang_urls[bang_url[len(HTTP_COLON) :]]
|
||||
else:
|
||||
# normal use case : new http:// URL or https:// URL (without "https:", see above)
|
||||
bang_rank = str(bang_definition['r'])
|
||||
bang_def_output = bang_url + chr(1) + bang_rank
|
||||
bang_def_output = bang_urls.setdefault(bang_url, bang_def_output)
|
||||
|
||||
bang_urls[bang_url] = bang_def_output
|
||||
|
||||
# bang name
|
||||
bang = bang_definition['t']
|
||||
|
||||
# bang_trie
|
||||
t = bang_trie
|
||||
for bang_letter in bang:
|
||||
t = t.setdefault(bang_letter, {})
|
||||
t = t.setdefault(LEAF_KEY, bang_def_output)
|
||||
|
||||
# optimize the trie
|
||||
merge_when_no_leaf(bang_trie)
|
||||
optimize_leaf(None, None, bang_trie)
|
||||
|
||||
return bang_trie
|
||||
|
||||
|
||||
def get_bangs_filename():
|
||||
return join(join(searx_dir, "data"), "external_bangs.json")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
bangs_url, bangs_version = get_bang_url()
|
||||
print(f'fetch bangs from {bangs_url}')
|
||||
output = {'version': bangs_version, 'trie': parse_ddg_bangs(fetch_ddg_bangs(bangs_url))}
|
||||
with open(get_bangs_filename(), 'w', encoding="utf8") as fp:
|
||||
json.dump(output, fp, ensure_ascii=False, indent=4)
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python
|
||||
# lint: pylint
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Fetch firefox useragent signatures
|
||||
|
||||
Output file: :origin:`searx/data/useragents.json` (:origin:`CI Update data ...
|
||||
<.github/workflows/data-update.yml>`).
|
||||
|
||||
"""
|
||||
# pylint: disable=use-dict-literal
|
||||
|
||||
import json
|
||||
import re
|
||||
from os.path import join
|
||||
from urllib.parse import urlparse, urljoin
|
||||
from packaging.version import parse
|
||||
|
||||
import requests
|
||||
from lxml import html
|
||||
from searx import searx_dir
|
||||
|
||||
URL = 'https://ftp.mozilla.org/pub/firefox/releases/'
|
||||
RELEASE_PATH = '/pub/firefox/releases/'
|
||||
|
||||
NORMAL_REGEX = re.compile(r'^[0-9]+\.[0-9](\.[0-9])?$')
|
||||
# BETA_REGEX = re.compile(r'.*[0-9]b([0-9\-a-z]+)$')
|
||||
# ESR_REGEX = re.compile(r'^[0-9]+\.[0-9](\.[0-9])?esr$')
|
||||
|
||||
#
|
||||
useragents = {
|
||||
# fmt: off
|
||||
"versions": (),
|
||||
"os": ('Windows NT 10.0; Win64; x64',
|
||||
'X11; Linux x86_64'),
|
||||
"ua": "Mozilla/5.0 ({os}; rv:{version}) Gecko/20100101 Firefox/{version}",
|
||||
# fmt: on
|
||||
}
|
||||
|
||||
|
||||
def fetch_firefox_versions():
|
||||
resp = requests.get(URL, timeout=2.0)
|
||||
if resp.status_code != 200:
|
||||
# pylint: disable=broad-exception-raised
|
||||
raise Exception("Error fetching firefox versions, HTTP code " + resp.status_code)
|
||||
dom = html.fromstring(resp.text)
|
||||
versions = []
|
||||
|
||||
for link in dom.xpath('//a/@href'):
|
||||
url = urlparse(urljoin(URL, link))
|
||||
path = url.path
|
||||
if path.startswith(RELEASE_PATH):
|
||||
version = path[len(RELEASE_PATH) : -1]
|
||||
if NORMAL_REGEX.match(version):
|
||||
versions.append(parse(version))
|
||||
|
||||
list.sort(versions, reverse=True)
|
||||
return versions
|
||||
|
||||
|
||||
def fetch_firefox_last_versions():
|
||||
versions = fetch_firefox_versions()
|
||||
|
||||
result = []
|
||||
major_last = versions[0].major
|
||||
major_list = (major_last, major_last - 1)
|
||||
for version in versions:
|
||||
major_current = version.major
|
||||
minor_current = version.minor
|
||||
if major_current in major_list:
|
||||
user_agent_version = f'{major_current}.{minor_current}'
|
||||
if user_agent_version not in result:
|
||||
result.append(user_agent_version)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_useragents_filename():
|
||||
return join(join(searx_dir, "data"), "useragents.json")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
useragents["versions"] = fetch_firefox_last_versions()
|
||||
with open(get_useragents_filename(), "w", encoding='utf-8') as f:
|
||||
json.dump(useragents, f, indent=4, ensure_ascii=False)
|
||||
+218
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python
|
||||
# lint: pylint
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""Fetch OSM keys and tags.
|
||||
|
||||
To get the i18n names, the scripts uses `Wikidata Query Service`_ instead of for
|
||||
example `OSM tags API`_ (sidenote: the actual change log from
|
||||
map.atownsend.org.uk_ might be useful to normalize OSM tags).
|
||||
|
||||
Output file: :origin:`searx/data/osm_keys_tags` (:origin:`CI Update data ...
|
||||
<.github/workflows/data-update.yml>`).
|
||||
|
||||
.. _Wikidata Query Service: https://query.wikidata.org/
|
||||
.. _OSM tags API: https://taginfo.openstreetmap.org/taginfo/apidoc
|
||||
.. _map.atownsend.org.uk: https://map.atownsend.org.uk/maps/map/changelog.html
|
||||
|
||||
:py:obj:`SPARQL_TAGS_REQUEST` :
|
||||
Wikidata SPARQL query that returns *type-categories* and *types*. The
|
||||
returned tag is ``Tag:{category}={type}`` (see :py:func:`get_tags`).
|
||||
Example:
|
||||
|
||||
- https://taginfo.openstreetmap.org/tags/building=house#overview
|
||||
- https://wiki.openstreetmap.org/wiki/Tag:building%3Dhouse
|
||||
at the bottom of the infobox (right side), there is a link to wikidata:
|
||||
https://www.wikidata.org/wiki/Q3947
|
||||
see property "OpenStreetMap tag or key" (P1282)
|
||||
- https://wiki.openstreetmap.org/wiki/Tag%3Abuilding%3Dbungalow
|
||||
https://www.wikidata.org/wiki/Q850107
|
||||
|
||||
:py:obj:`SPARQL_KEYS_REQUEST` :
|
||||
Wikidata SPARQL query that returns *keys*. Example with "payment":
|
||||
|
||||
- https://wiki.openstreetmap.org/wiki/Key%3Apayment
|
||||
at the bottom of infobox (right side), there is a link to wikidata:
|
||||
https://www.wikidata.org/wiki/Q1148747
|
||||
link made using the "OpenStreetMap tag or key" property (P1282)
|
||||
to be confirm: there is a one wiki page per key ?
|
||||
- https://taginfo.openstreetmap.org/keys/payment#values
|
||||
- https://taginfo.openstreetmap.org/keys/payment:cash#values
|
||||
|
||||
``rdfs:label`` get all the labels without language selection
|
||||
(as opposed to SERVICE ``wikibase:label``).
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
import collections
|
||||
from pathlib import Path
|
||||
|
||||
from searx import searx_dir
|
||||
from searx.network import set_timeout_for_thread
|
||||
from searx.engines import wikidata, set_loggers
|
||||
from searx.sxng_locales import sxng_locales
|
||||
from searx.engines.openstreetmap import get_key_rank, VALUE_TO_LINK
|
||||
|
||||
set_loggers(wikidata, 'wikidata')
|
||||
|
||||
|
||||
SPARQL_TAGS_REQUEST = """
|
||||
SELECT ?tag ?item ?itemLabel WHERE {
|
||||
?item wdt:P1282 ?tag .
|
||||
?item rdfs:label ?itemLabel .
|
||||
FILTER(STRSTARTS(?tag, 'Tag'))
|
||||
}
|
||||
GROUP BY ?tag ?item ?itemLabel
|
||||
ORDER BY ?tag ?item ?itemLabel
|
||||
"""
|
||||
|
||||
SPARQL_KEYS_REQUEST = """
|
||||
SELECT ?key ?item ?itemLabel WHERE {
|
||||
?item wdt:P1282 ?key .
|
||||
?item rdfs:label ?itemLabel .
|
||||
FILTER(STRSTARTS(?key, 'Key'))
|
||||
}
|
||||
GROUP BY ?key ?item ?itemLabel
|
||||
ORDER BY ?key ?item ?itemLabel
|
||||
"""
|
||||
|
||||
LANGUAGES = [l[0].lower() for l in sxng_locales]
|
||||
|
||||
PRESET_KEYS = {
|
||||
('wikidata',): {'en': 'Wikidata'},
|
||||
('wikipedia',): {'en': 'Wikipedia'},
|
||||
('email',): {'en': 'Email'},
|
||||
('facebook',): {'en': 'Facebook'},
|
||||
('fax',): {'en': 'Fax'},
|
||||
('internet_access', 'ssid'): {'en': 'Wi-Fi'},
|
||||
}
|
||||
|
||||
INCLUDED_KEYS = {('addr',)}
|
||||
|
||||
|
||||
def get_preset_keys():
|
||||
results = collections.OrderedDict()
|
||||
for keys, value in PRESET_KEYS.items():
|
||||
r = results
|
||||
for k in keys:
|
||||
r = r.setdefault(k, {})
|
||||
r.setdefault('*', value)
|
||||
return results
|
||||
|
||||
|
||||
def get_keys():
|
||||
results = get_preset_keys()
|
||||
response = wikidata.send_wikidata_query(SPARQL_KEYS_REQUEST)
|
||||
|
||||
for key in response['results']['bindings']:
|
||||
keys = key['key']['value'].split(':')[1:]
|
||||
if keys[0] == 'currency' and len(keys) > 1:
|
||||
# special case in openstreetmap.py
|
||||
continue
|
||||
if keys[0] == 'contact' and len(keys) > 1:
|
||||
# label for the key "contact.email" is "Email"
|
||||
# whatever the language
|
||||
r = results.setdefault('contact', {})
|
||||
r[keys[1]] = {'*': {'en': keys[1]}}
|
||||
continue
|
||||
if tuple(keys) in PRESET_KEYS:
|
||||
# skip presets (already set above)
|
||||
continue
|
||||
if (
|
||||
get_key_rank(':'.join(keys)) is None
|
||||
and ':'.join(keys) not in VALUE_TO_LINK
|
||||
and tuple(keys) not in INCLUDED_KEYS
|
||||
):
|
||||
# keep only keys that will be displayed by openstreetmap.py
|
||||
continue
|
||||
label = key['itemLabel']['value'].lower()
|
||||
lang = key['itemLabel']['xml:lang']
|
||||
r = results
|
||||
for k in keys:
|
||||
r = r.setdefault(k, {})
|
||||
r = r.setdefault('*', {})
|
||||
if lang in LANGUAGES:
|
||||
r.setdefault(lang, label)
|
||||
|
||||
# special cases
|
||||
results['delivery']['covid19']['*'].clear()
|
||||
for k, v in results['delivery']['*'].items():
|
||||
results['delivery']['covid19']['*'][k] = v + ' (COVID19)'
|
||||
|
||||
results['opening_hours']['covid19']['*'].clear()
|
||||
for k, v in results['opening_hours']['*'].items():
|
||||
results['opening_hours']['covid19']['*'][k] = v + ' (COVID19)'
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def get_tags():
|
||||
results = collections.OrderedDict()
|
||||
response = wikidata.send_wikidata_query(SPARQL_TAGS_REQUEST)
|
||||
for tag in response['results']['bindings']:
|
||||
tag_names = tag['tag']['value'].split(':')[1].split('=')
|
||||
if len(tag_names) == 2:
|
||||
tag_category, tag_type = tag_names
|
||||
else:
|
||||
tag_category, tag_type = tag_names[0], ''
|
||||
label = tag['itemLabel']['value'].lower()
|
||||
lang = tag['itemLabel']['xml:lang']
|
||||
if lang in LANGUAGES:
|
||||
results.setdefault(tag_category, {}).setdefault(tag_type, {}).setdefault(lang, label)
|
||||
return results
|
||||
|
||||
|
||||
def optimize_data_lang(translations):
|
||||
language_to_delete = []
|
||||
# remove "zh-hk" entry if the value is the same as "zh"
|
||||
# same for "en-ca" / "en" etc...
|
||||
for language in translations:
|
||||
if '-' in language:
|
||||
base_language = language.split('-')[0]
|
||||
if translations.get(base_language) == translations.get(language):
|
||||
language_to_delete.append(language)
|
||||
|
||||
for language in language_to_delete:
|
||||
del translations[language]
|
||||
language_to_delete = []
|
||||
|
||||
# remove entries that have the same value than the "en" entry
|
||||
value_en = translations.get('en')
|
||||
if value_en:
|
||||
for language, value in translations.items():
|
||||
if language != 'en' and value == value_en:
|
||||
language_to_delete.append(language)
|
||||
|
||||
for language in language_to_delete:
|
||||
del translations[language]
|
||||
|
||||
|
||||
def optimize_tags(data):
|
||||
for v in data.values():
|
||||
for translations in v.values():
|
||||
optimize_data_lang(translations)
|
||||
return data
|
||||
|
||||
|
||||
def optimize_keys(data):
|
||||
for k, v in data.items():
|
||||
if k == '*':
|
||||
optimize_data_lang(v)
|
||||
elif isinstance(v, dict):
|
||||
optimize_keys(v)
|
||||
return data
|
||||
|
||||
|
||||
def get_osm_tags_filename():
|
||||
return Path(searx_dir) / "data" / "osm_keys_tags.json"
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
set_timeout_for_thread(60)
|
||||
result = {
|
||||
'keys': optimize_keys(get_keys()),
|
||||
'tags': optimize_tags(get_tags()),
|
||||
}
|
||||
with open(get_osm_tags_filename(), 'w', encoding="utf8") as f:
|
||||
json.dump(result, f, indent=4, ensure_ascii=False, sort_keys=True)
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
"""
|
||||
Update pygments style
|
||||
|
||||
Call this script after each upgrade of pygments
|
||||
"""
|
||||
|
||||
# pylint: disable=C0116
|
||||
|
||||
# set path
|
||||
from os.path import join
|
||||
import pygments
|
||||
from pygments.formatters import HtmlFormatter # pylint: disable=E0611
|
||||
from pygments.style import Style
|
||||
from pygments.token import Comment, Error, Generic, Keyword, Literal, Name, Operator, Text
|
||||
|
||||
from searx import searx_dir
|
||||
|
||||
CSSCLASS = '.code-highlight'
|
||||
RULE_CODE_LINENOS = """ .linenos {
|
||||
-webkit-touch-callout: none;
|
||||
-webkit-user-select: none;
|
||||
-khtml-user-select: none;
|
||||
-moz-user-select: none;
|
||||
-ms-user-select: none;
|
||||
user-select: none;
|
||||
cursor: default;
|
||||
|
||||
&::selection {
|
||||
background: transparent; /* WebKit/Blink Browsers */
|
||||
}
|
||||
&::-moz-selection {
|
||||
background: transparent; /* Gecko Browsers */
|
||||
}
|
||||
|
||||
margin-right: 8px;
|
||||
text-align: right;
|
||||
}"""
|
||||
|
||||
|
||||
def get_output_filename(relative_name):
|
||||
return join(searx_dir, relative_name)
|
||||
|
||||
|
||||
def get_css(cssclass, style):
|
||||
result = f"""/*
|
||||
this file is generated automatically by searxng_extra/update/update_pygments.py
|
||||
using pygments version {pygments.__version__}
|
||||
*/\n\n"""
|
||||
css_text = HtmlFormatter(style=style).get_style_defs(cssclass)
|
||||
result += cssclass + RULE_CODE_LINENOS + '\n\n'
|
||||
for line in css_text.splitlines():
|
||||
if ' ' in line and not line.startswith(cssclass):
|
||||
line = cssclass + ' ' + line
|
||||
result += line + '\n'
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
fname = 'static/themes/simple/src/generated/pygments.less'
|
||||
print("update: %s" % fname)
|
||||
with open(get_output_filename(fname), 'w') as f:
|
||||
f.write(get_css(CSSCLASS, 'default'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env python
|
||||
# SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
# lint: pylint
|
||||
# pylint: disable=missing-module-docstring
|
||||
|
||||
"""Fetch units from :origin:`searx/engines/wikidata.py` engine.
|
||||
|
||||
Output file: :origin:`searx/data/wikidata_units.json` (:origin:`CI Update data
|
||||
... <.github/workflows/data-update.yml>`).
|
||||
|
||||
"""
|
||||
|
||||
import json
|
||||
import collections
|
||||
|
||||
# set path
|
||||
from os.path import join
|
||||
|
||||
from searx import searx_dir
|
||||
from searx.engines import wikidata, set_loggers
|
||||
|
||||
set_loggers(wikidata, 'wikidata')
|
||||
|
||||
# the response contains duplicate ?item with the different ?symbol
|
||||
# "ORDER BY ?item DESC(?rank) ?symbol" provides a deterministic result
|
||||
# even if a ?item has different ?symbol of the same rank.
|
||||
# A deterministic result
|
||||
# see:
|
||||
# * https://www.wikidata.org/wiki/Help:Ranking
|
||||
# * https://www.mediawiki.org/wiki/Wikibase/Indexing/RDF_Dump_Format ("Statement representation" section)
|
||||
# * https://w.wiki/32BT
|
||||
# see the result for https://www.wikidata.org/wiki/Q11582
|
||||
# there are multiple symbols the same rank
|
||||
SARQL_REQUEST = """
|
||||
SELECT DISTINCT ?item ?symbol
|
||||
WHERE
|
||||
{
|
||||
?item wdt:P31/wdt:P279 wd:Q47574 .
|
||||
?item p:P5061 ?symbolP .
|
||||
?symbolP ps:P5061 ?symbol ;
|
||||
wikibase:rank ?rank .
|
||||
FILTER(LANG(?symbol) = "en").
|
||||
}
|
||||
ORDER BY ?item DESC(?rank) ?symbol
|
||||
"""
|
||||
|
||||
|
||||
def get_data():
|
||||
results = collections.OrderedDict()
|
||||
response = wikidata.send_wikidata_query(SARQL_REQUEST)
|
||||
for unit in response['results']['bindings']:
|
||||
name = unit['item']['value'].replace('http://www.wikidata.org/entity/', '')
|
||||
unit = unit['symbol']['value']
|
||||
if name not in results:
|
||||
# ignore duplicate: always use the first one
|
||||
results[name] = unit
|
||||
return results
|
||||
|
||||
|
||||
def get_wikidata_units_filename():
|
||||
return join(join(searx_dir, "data"), "wikidata_units.json")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
with open(get_wikidata_units_filename(), 'w', encoding="utf8") as f:
|
||||
json.dump(get_data(), f, indent=4, ensure_ascii=False)
|
||||
Reference in New Issue
Block a user