git.iko.soy/searx
on v1.0.0 help sign in
searx/searx_extra/update/update_engine_descriptions.py
202 lines 7.3 kB history raw
1#!/usr/bin/env python
3import sys
4import json
5from urllib.parse import quote, urlparse
6import detect_language
7from lxml.html import fromstring
9from searx.engines.wikidata import send_wikidata_query
10from searx.utils import extract_text
11import searx
12import searx.search
13import searx.poolrequests
15SPARQL_WIKIPEDIA_ARTICLE = """
16SELECT DISTINCT ?item ?name
17WHERE {
18 VALUES ?item { %IDS% }
19 ?article schema:about ?item ;
20 schema:inLanguage ?lang ;
21 schema:name ?name ;
22 schema:isPartOf [ wikibase:wikiGroup "wikipedia" ] .
23 FILTER(?lang in (%LANGUAGES_SPARQL%)) .
24 FILTER (!CONTAINS(?name, ':')) .
26"""
28SPARQL_DESCRIPTION = """
29SELECT DISTINCT ?item ?itemDescription
30WHERE {
31 VALUES ?item { %IDS% }
32 ?item schema:description ?itemDescription .
33 FILTER (lang(?itemDescription) in (%LANGUAGES_SPARQL%))
35ORDER BY ?itemLang
36"""
38LANGUAGES = searx.settings['locales'].keys()
39LANGUAGES_SPARQL = ', '.join(set(map(lambda l: repr(l.split('_')[0]), LANGUAGES)))
40IDS = None
42descriptions = {}
43wd_to_engine_name = {}
46def normalize_description(description):
47 for c in [chr(c) for c in range(0, 31)]:
48 description = description.replace(c, ' ')
49 description = ' '.join(description.strip().split())
50 return description
53def update_description(engine_name, lang, description, source, replace=True):
54 if replace or lang not in descriptions[engine_name]:
55 descriptions[engine_name][lang] = [normalize_description(description), source]
58def get_wikipedia_summary(language, pageid):
59 search_url = 'https://{language}.wikipedia.org/api/rest_v1/page/summary/{title}'
60 url = search_url.format(title=quote(pageid), language=language)
61 try:
62 response = searx.poolrequests.get(url)
63 response.raise_for_status()
64 api_result = json.loads(response.text)
65 return api_result.get('extract')
66 except:
67 return None
70def detect_language(text):
71 r = cld3.get_language(str(text)) # pylint: disable=E1101
72 if r is not None and r.probability >= 0.98 and r.is_reliable:
73 return r.language
74 return None
77def get_website_description(url, lang1, lang2=None):
78 headers = {
79 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0',
80 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
81 'DNT': '1',
82 'Upgrade-Insecure-Requests': '1',
83 'Sec-GPC': '1',
84 'Cache-Control': 'max-age=0',
85 }
86 if lang1 is not None:
87 lang_list = [lang1]
88 if lang2 is not None:
89 lang_list.append(lang2)
90 headers['Accept-Language'] = f'{",".join(lang_list)};q=0.8'
91 try:
92 response = searx.poolrequests.get(url, headers=headers, timeout=10)
93 response.raise_for_status()
94 except Exception:
95 return (None, None)
97 try:
98 html = fromstring(response.text)
99 except ValueError:
100 html = fromstring(response.content)
102 description = extract_text(html.xpath('/html/head/meta[@name="description"]/@content'))
103 if not description:
104 description = extract_text(html.xpath('/html/head/meta[@property="og:description"]/@content'))
105 if not description:
106 description = extract_text(html.xpath('/html/head/title'))
107 lang = extract_text(html.xpath('/html/@lang'))
108 if lang is None and len(lang1) > 0:
109 lang = lang1
110 lang = detect_language(description) or lang or 'en'
111 lang = lang.split('_')[0]
112 lang = lang.split('-')[0]
113 return (lang, description)
116def initialize():
117 global descriptions, wd_to_engine_name, IDS
118 searx.search.initialize()
119 for engine_name, engine in searx.engines.engines.items():
120 descriptions[engine_name] = {}
121 wikidata_id = getattr(engine, "about", {}).get('wikidata_id')
122 if wikidata_id is not None:
123 wd_to_engine_name.setdefault(wikidata_id, set()).add(engine_name)
125 IDS = ' '.join(list(map(lambda wd_id: 'wd:' + wd_id, wd_to_engine_name.keys())))
128def fetch_wikidata_descriptions():
129 global IDS
130 result = send_wikidata_query(SPARQL_DESCRIPTION
131 .replace('%IDS%', IDS)
132 .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
133 if result is not None:
134 for binding in result['results']['bindings']:
135 wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '')
136 lang = binding['itemDescription']['xml:lang']
137 description = binding['itemDescription']['value']
138 if ' ' in description: # skip unique word description (like "website")
139 for engine_name in wd_to_engine_name[wikidata_id]:
140 update_description(engine_name, lang, description, 'wikidata')
143def fetch_wikipedia_descriptions():
144 global IDS
145 result = send_wikidata_query(SPARQL_WIKIPEDIA_ARTICLE
146 .replace('%IDS%', IDS)
147 .replace('%LANGUAGES_SPARQL%', LANGUAGES_SPARQL))
148 if result is not None:
149 for binding in result['results']['bindings']:
150 wikidata_id = binding['item']['value'].replace('http://www.wikidata.org/entity/', '')
151 lang = binding['name']['xml:lang']
152 pageid = binding['name']['value']
153 description = get_wikipedia_summary(lang, pageid)
154 if description is not None and ' ' in description:
155 for engine_name in wd_to_engine_name[wikidata_id]:
156 update_description(engine_name, lang, description, 'wikipedia')
159def normalize_url(url):
160 url = url.replace('{language}', 'en')
161 url = urlparse(url)._replace(path='/', params='', query='', fragment='').geturl()
162 url = url.replace('https://api.', 'https://')
163 return url
166def fetch_website_description(engine_name, website):
167 default_lang, default_description = get_website_description(website, None, None)
168 if default_lang is None or default_description is None:
169 return
170 if default_lang not in descriptions[engine_name]:
171 descriptions[engine_name][default_lang] = [normalize_description(default_description), website]
172 for request_lang in ('en-US', 'es-US', 'fr-FR', 'zh', 'ja', 'ru', 'ar', 'ko'):
173 if request_lang.split('-')[0] not in descriptions[engine_name]:
174 lang, desc = get_website_description(website, request_lang, request_lang.split('-')[0])
175 if desc is not None and desc != default_description:
176 update_description(engine_name, lang, desc, website, replace=False)
177 else:
178 break
181def fetch_website_descriptions():
182 for engine_name, engine in searx.engines.engines.items():
183 website = getattr(engine, "about", {}).get('website')
184 if website is None:
185 website = normalize_url(getattr(engine, "search_url"))
186 if website is None:
187 website = normalize_url(getattr(engine, "base_url"))
188 if website is not None:
189 fetch_website_description(engine_name, website)
192def main():
193 initialize()
194 fetch_wikidata_descriptions()
195 fetch_wikipedia_descriptions()
196 fetch_website_descriptions()
198 sys.stdout.write(json.dumps(descriptions, indent=1, separators=(',', ':'), ensure_ascii=False))
201if __name__ == "__main__":
202 main()