git.iko.soy/searx
on master help sign in
[fix] wikidata engine: faster processing, remove one HTTP redirection.
6e0285bDalf
searx/engines/wikidata.py+141−113
compare with master
unifiedsplit
@@ -20,6 +20,7 @@ from searx.utils import match_language
20
20
21from json import loads
21from json import loads
22from lxml.html import fromstring
22from lxml.html import fromstring
23+from lxml import etree
23
24
24logger = logger.getChild('wikidata')
25logger = logger.getChild('wikidata')
25result_count = 1
26result_count = 1
@@ -27,23 +28,23 @@ result_count = 1
27# urls
28# urls
28wikidata_host = 'https://www.wikidata.org'
29wikidata_host = 'https://www.wikidata.org'
29url_search = wikidata_host \
30url_search = wikidata_host \
30− + '/w/index.php?{query}'
31+ + '/w/index.php?{query}&ns0=1'
31
32
32wikidata_api = wikidata_host + '/w/api.php'
33wikidata_api = wikidata_host + '/w/api.php'
33url_detail = wikidata_api\
34url_detail = wikidata_api\
34 + '?action=parse&format=json&{query}'\
35 + '?action=parse&format=json&{query}'\
35− + '&redirects=1&prop=text%7Cdisplaytitle%7Clanglinks%7Crevid'\
36+ + '&redirects=1&prop=text%7Cdisplaytitle%7Cparsewarnings'\
36− + '&disableeditsection=1&disabletidy=1&preview=1&sectionpreview=1&disabletoc=1&utf8=1&formatversion=2'
37+ + '&disableeditsection=1&preview=1&sectionpreview=1&disabletoc=1&utf8=1&formatversion=2'
37
38
38url_map = 'https://www.openstreetmap.org/'\
39url_map = 'https://www.openstreetmap.org/'\
39 + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
40 + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
40url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'
41url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'
41
42
42# xpaths
43# xpaths
44+div_ids_xpath = '//div[@id]'
43wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href'
45wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href'
44title_xpath = '//*[contains(@class,"wikibase-title-label")]'
46title_xpath = '//*[contains(@class,"wikibase-title-label")]'
45description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'
47description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'
46−property_xpath = '//div[@id="{propertyid}"]'
47label_xpath = './/div[contains(@class,"wikibase-statementgroupview-property-label")]/a'
48label_xpath = './/div[contains(@class,"wikibase-statementgroupview-property-label")]/a'
48url_xpath = './/a[contains(@class,"external free") or contains(@class, "wb-external-id")]'
49url_xpath = './/a[contains(@class,"external free") or contains(@class, "wb-external-id")]'
49wikilink_xpath = './/ul[contains(@class,"wikibase-sitelinklistview-listview")]'\
50wikilink_xpath = './/ul[contains(@class,"wikibase-sitelinklistview-listview")]'\
@@ -56,6 +57,31 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator
56calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
57calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'
57media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
58media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'
58
59
60+# xpath_cache
61+xpath_cache = {}
62+
63+
64+def get_xpath(xpath_str):
65+ result = xpath_cache.get(xpath_str, None)
66+ if not result:
67+ result = etree.XPath(xpath_str)
68+ xpath_cache[xpath_str] = result
69+ return result
70+
71+
72+def eval_xpath(element, xpath_str):
73+ xpath = get_xpath(xpath_str)
74+ return xpath(element)
75+
76+
77+def get_id_cache(result):
78+ id_cache = {}
79+ for e in eval_xpath(result, div_ids_xpath):
80+ id = e.get('id')
81+ if id.startswith('P'):
82+ id_cache[id] = e
83+ return id_cache
84+
59
85
60def request(query, params):
86def request(query, params):
61 params['url'] = url_search.format(
87 params['url'] = url_search.format(
@@ -65,8 +91,9 @@ def request(query, params):
65
91
66def response(resp):
92def response(resp):
67 results = []
93 results = []
94+ htmlparser = etree.HTMLParser()
68− html = fromstring(resp.text)
95+ html = fromstring(resp.content.decode("utf-8"), parser=htmlparser)
69− search_results = html.xpath(wikidata_ids_xpath)
96+ search_results = eval_xpath(html, wikidata_ids_xpath)
70
97
71 if resp.search_params['language'].split('-')[0] == 'all':
98 if resp.search_params['language'].split('-')[0] == 'all':
72 language = 'en'
99 language = 'en'
@@ -78,13 +105,13 @@ def response(resp):
78 wikidata_id = search_result.split('/')[-1]
105 wikidata_id = search_result.split('/')[-1]
79 url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))
106 url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))
80 htmlresponse = get(url)
107 htmlresponse = get(url)
81− jsonresponse = loads(htmlresponse.text)
108+ jsonresponse = loads(htmlresponse.content.decode("utf-8"))
82− results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])
109+ results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'], htmlparser)
83
110
84 return results
111 return results
85
112
86
113
87−def getDetail(jsonresponse, wikidata_id, language, locale):
114+def getDetail(jsonresponse, wikidata_id, language, locale, htmlparser):
88 results = []
115 results = []
89 urls = []
116 urls = []
90 attributes = []
117 attributes = []
@@ -95,21 +122,23 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
95 if not title or not result:
122 if not title or not result:
96 return results
123 return results
97
124
98− title = fromstring(title)
125+ title = fromstring(title, parser=htmlparser)
99− for elem in title.xpath(language_fallback_xpath):
126+ for elem in eval_xpath(title, language_fallback_xpath):
100 elem.getparent().remove(elem)
127 elem.getparent().remove(elem)
101− title = extract_text(title.xpath(title_xpath))
128+ title = extract_text(eval_xpath(title, title_xpath))
102
129
103− result = fromstring(result)
130+ result = fromstring(result, parser=htmlparser)
104− for elem in result.xpath(language_fallback_xpath):
131+ for elem in eval_xpath(result, language_fallback_xpath):
105 elem.getparent().remove(elem)
132 elem.getparent().remove(elem)
106
133
107− description = extract_text(result.xpath(description_xpath))
134+ description = extract_text(eval_xpath(result, description_xpath))
135+
136+ id_cache = get_id_cache(result)
108
137
109 # URLS
138 # URLS
110
139
111 # official website
140 # official website
112− add_url(urls, result, 'P856', results=results)
141+ add_url(urls, result, id_cache, 'P856', results=results)
113
142
114 # wikipedia
143 # wikipedia
115 wikipedia_link_count = 0
144 wikipedia_link_count = 0
@@ -130,30 +159,30 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
130 # if wikipedia_link_count == 0:
159 # if wikipedia_link_count == 0:
131
160
132 # more wikis
161 # more wikis
133− add_url(urls, result, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')
162+ add_url(urls, result, id_cache, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')
134− add_url(urls, result, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')
163+ add_url(urls, result, id_cache, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')
135− add_url(urls, result, default_label='Wikimedia Commons', link_type='commonswiki')
164+ add_url(urls, result, id_cache, default_label='Wikimedia Commons', link_type='commonswiki')
136
165
137− add_url(urls, result, 'P625', 'OpenStreetMap', link_type='geo')
166+ add_url(urls, result, id_cache, 'P625', 'OpenStreetMap', link_type='geo')
138
167
139 # musicbrainz
168 # musicbrainz
140− add_url(urls, result, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')
169+ add_url(urls, result, id_cache, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')
141− add_url(urls, result, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')
170+ add_url(urls, result, id_cache, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')
142− add_url(urls, result, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')
171+ add_url(urls, result, id_cache, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')
143− add_url(urls, result, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')
172+ add_url(urls, result, id_cache, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')
144
173
145 # IMDb
174 # IMDb
146− add_url(urls, result, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')
175+ add_url(urls, result, id_cache, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')
147 # source code repository
176 # source code repository
148− add_url(urls, result, 'P1324')
177+ add_url(urls, result, id_cache, 'P1324')
149 # blog
178 # blog
150− add_url(urls, result, 'P1581')
179+ add_url(urls, result, id_cache, 'P1581')
151 # social media links
180 # social media links
152− add_url(urls, result, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')
181+ add_url(urls, result, id_cache, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')
153− add_url(urls, result, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')
182+ add_url(urls, result, id_cache, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')
154− add_url(urls, result, 'P2002', 'Twitter', 'https://twitter.com/')
183+ add_url(urls, result, id_cache, 'P2002', 'Twitter', 'https://twitter.com/')
155− add_url(urls, result, 'P2013', 'Facebook', 'https://facebook.com/')
184+ add_url(urls, result, id_cache, 'P2013', 'Facebook', 'https://facebook.com/')
156− add_url(urls, result, 'P2003', 'Instagram', 'https://instagram.com/')
185+ add_url(urls, result, id_cache, 'P2003', 'Instagram', 'https://instagram.com/')
157
186
158 urls.append({'title': 'Wikidata',
187 urls.append({'title': 'Wikidata',
159 'url': 'https://www.wikidata.org/wiki/'
188 'url': 'https://www.wikidata.org/wiki/'
@@ -163,132 +192,132 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
163
192
164 # DATES
193 # DATES
165 # inception date
194 # inception date
166− add_attribute(attributes, result, 'P571', date=True)
195+ add_attribute(attributes, id_cache, 'P571', date=True)
167 # dissolution date
196 # dissolution date
168− add_attribute(attributes, result, 'P576', date=True)
197+ add_attribute(attributes, id_cache, 'P576', date=True)
169 # start date
198 # start date
170− add_attribute(attributes, result, 'P580', date=True)
199+ add_attribute(attributes, id_cache, 'P580', date=True)
171 # end date
200 # end date
172− add_attribute(attributes, result, 'P582', date=True)
201+ add_attribute(attributes, id_cache, 'P582', date=True)
173 # date of birth
202 # date of birth
174− add_attribute(attributes, result, 'P569', date=True)
203+ add_attribute(attributes, id_cache, 'P569', date=True)
175 # date of death
204 # date of death
176− add_attribute(attributes, result, 'P570', date=True)
205+ add_attribute(attributes, id_cache, 'P570', date=True)
177 # date of spacecraft launch
206 # date of spacecraft launch
178− add_attribute(attributes, result, 'P619', date=True)
207+ add_attribute(attributes, id_cache, 'P619', date=True)
179 # date of spacecraft landing
208 # date of spacecraft landing
180− add_attribute(attributes, result, 'P620', date=True)
209+ add_attribute(attributes, id_cache, 'P620', date=True)
181
210
182 # nationality
211 # nationality
183− add_attribute(attributes, result, 'P27')
212+ add_attribute(attributes, id_cache, 'P27')
184 # country of origin
213 # country of origin
185− add_attribute(attributes, result, 'P495')
214+ add_attribute(attributes, id_cache, 'P495')
186 # country
215 # country
187− add_attribute(attributes, result, 'P17')
216+ add_attribute(attributes, id_cache, 'P17')
188 # headquarters
217 # headquarters
189− add_attribute(attributes, result, 'Q180')
218+ add_attribute(attributes, id_cache, 'Q180')
190
219
191 # PLACES
220 # PLACES
192 # capital
221 # capital
193− add_attribute(attributes, result, 'P36', trim=True)
222+ add_attribute(attributes, id_cache, 'P36', trim=True)
194 # head of state
223 # head of state
195− add_attribute(attributes, result, 'P35', trim=True)
224+ add_attribute(attributes, id_cache, 'P35', trim=True)
196 # head of government
225 # head of government
197− add_attribute(attributes, result, 'P6', trim=True)
226+ add_attribute(attributes, id_cache, 'P6', trim=True)
198 # type of government
227 # type of government
199− add_attribute(attributes, result, 'P122')
228+ add_attribute(attributes, id_cache, 'P122')
200 # official language
229 # official language
201− add_attribute(attributes, result, 'P37')
230+ add_attribute(attributes, id_cache, 'P37')
202 # population
231 # population
203− add_attribute(attributes, result, 'P1082', trim=True)
232+ add_attribute(attributes, id_cache, 'P1082', trim=True)
204 # area
233 # area
205− add_attribute(attributes, result, 'P2046')
234+ add_attribute(attributes, id_cache, 'P2046')
206 # currency
235 # currency
207− add_attribute(attributes, result, 'P38', trim=True)
236+ add_attribute(attributes, id_cache, 'P38', trim=True)
208 # heigth (building)
237 # heigth (building)
209− add_attribute(attributes, result, 'P2048')
238+ add_attribute(attributes, id_cache, 'P2048')
210
239
211 # MEDIA
240 # MEDIA
212 # platform (videogames)
241 # platform (videogames)
213− add_attribute(attributes, result, 'P400')
242+ add_attribute(attributes, id_cache, 'P400')
214 # author
243 # author
215− add_attribute(attributes, result, 'P50')
244+ add_attribute(attributes, id_cache, 'P50')
216 # creator
245 # creator
217− add_attribute(attributes, result, 'P170')
246+ add_attribute(attributes, id_cache, 'P170')
218 # director
247 # director
219− add_attribute(attributes, result, 'P57')
248+ add_attribute(attributes, id_cache, 'P57')
220 # performer
249 # performer
221− add_attribute(attributes, result, 'P175')
250+ add_attribute(attributes, id_cache, 'P175')
222 # developer
251 # developer
223− add_attribute(attributes, result, 'P178')
252+ add_attribute(attributes, id_cache, 'P178')
224 # producer
253 # producer
225− add_attribute(attributes, result, 'P162')
254+ add_attribute(attributes, id_cache, 'P162')
226 # manufacturer
255 # manufacturer
227− add_attribute(attributes, result, 'P176')
256+ add_attribute(attributes, id_cache, 'P176')
228 # screenwriter
257 # screenwriter
229− add_attribute(attributes, result, 'P58')
258+ add_attribute(attributes, id_cache, 'P58')
230 # production company
259 # production company
231− add_attribute(attributes, result, 'P272')
260+ add_attribute(attributes, id_cache, 'P272')
232 # record label
261 # record label
233− add_attribute(attributes, result, 'P264')
262+ add_attribute(attributes, id_cache, 'P264')
234 # publisher
263 # publisher
235− add_attribute(attributes, result, 'P123')
264+ add_attribute(attributes, id_cache, 'P123')
236 # original network
265 # original network
237− add_attribute(attributes, result, 'P449')
266+ add_attribute(attributes, id_cache, 'P449')
238 # distributor
267 # distributor
239− add_attribute(attributes, result, 'P750')
268+ add_attribute(attributes, id_cache, 'P750')
240 # composer
269 # composer
241− add_attribute(attributes, result, 'P86')
270+ add_attribute(attributes, id_cache, 'P86')
242 # publication date
271 # publication date
243− add_attribute(attributes, result, 'P577', date=True)
272+ add_attribute(attributes, id_cache, 'P577', date=True)
244 # genre
273 # genre
245− add_attribute(attributes, result, 'P136')
274+ add_attribute(attributes, id_cache, 'P136')
246 # original language
275 # original language
247− add_attribute(attributes, result, 'P364')
276+ add_attribute(attributes, id_cache, 'P364')
248 # isbn
277 # isbn
249− add_attribute(attributes, result, 'Q33057')
278+ add_attribute(attributes, id_cache, 'Q33057')
250 # software license
279 # software license
251− add_attribute(attributes, result, 'P275')
280+ add_attribute(attributes, id_cache, 'P275')
252 # programming language
281 # programming language
253− add_attribute(attributes, result, 'P277')
282+ add_attribute(attributes, id_cache, 'P277')
254 # version
283 # version
255− add_attribute(attributes, result, 'P348', trim=True)
284+ add_attribute(attributes, id_cache, 'P348', trim=True)
256 # narrative location
285 # narrative location
257− add_attribute(attributes, result, 'P840')
286+ add_attribute(attributes, id_cache, 'P840')
258
287
259 # LANGUAGES
288 # LANGUAGES
260 # number of speakers
289 # number of speakers
261− add_attribute(attributes, result, 'P1098')
290+ add_attribute(attributes, id_cache, 'P1098')
262 # writing system
291 # writing system
263− add_attribute(attributes, result, 'P282')
292+ add_attribute(attributes, id_cache, 'P282')
264 # regulatory body
293 # regulatory body
265− add_attribute(attributes, result, 'P1018')
294+ add_attribute(attributes, id_cache, 'P1018')
266 # language code
295 # language code
267− add_attribute(attributes, result, 'P218')
296+ add_attribute(attributes, id_cache, 'P218')
268
297
269 # OTHER
298 # OTHER
270 # ceo
299 # ceo
271− add_attribute(attributes, result, 'P169', trim=True)
300+ add_attribute(attributes, id_cache, 'P169', trim=True)
272 # founder
301 # founder
273− add_attribute(attributes, result, 'P112')
302+ add_attribute(attributes, id_cache, 'P112')
274 # legal form (company/organization)
303 # legal form (company/organization)
275− add_attribute(attributes, result, 'P1454')
304+ add_attribute(attributes, id_cache, 'P1454')
276 # operator
305 # operator
277− add_attribute(attributes, result, 'P137')
306+ add_attribute(attributes, id_cache, 'P137')
278 # crew members (tripulation)
307 # crew members (tripulation)
279− add_attribute(attributes, result, 'P1029')
308+ add_attribute(attributes, id_cache, 'P1029')
280 # taxon
309 # taxon
281− add_attribute(attributes, result, 'P225')
310+ add_attribute(attributes, id_cache, 'P225')
282 # chemical formula
311 # chemical formula
283− add_attribute(attributes, result, 'P274')
312+ add_attribute(attributes, id_cache, 'P274')
284 # winner (sports/contests)
313 # winner (sports/contests)
285− add_attribute(attributes, result, 'P1346')
314+ add_attribute(attributes, id_cache, 'P1346')
286 # number of deaths
315 # number of deaths
287− add_attribute(attributes, result, 'P1120')
316+ add_attribute(attributes, id_cache, 'P1120')
288 # currency code
317 # currency code
289− add_attribute(attributes, result, 'P498')
318+ add_attribute(attributes, id_cache, 'P498')
290
319
291− image = add_image(result)
320+ image = add_image(id_cache)
292
321
293 if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:
322 if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:
294 results.append({
323 results.append({
@@ -310,43 +339,42 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
310
339
311
340
312# only returns first match
341# only returns first match
313−def add_image(result):
342+def add_image(id_cache):
314 # P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon
343 # P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon
315 property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910']
344 property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910']
316
345
317 for property_id in property_ids:
346 for property_id in property_ids:
318− image = result.xpath(property_xpath.replace('{propertyid}', property_id))
347+ image = id_cache.get(property_id, None)
319− if image:
348+ if image is not None:
320− image_name = image[0].xpath(media_xpath)
349+ image_name = eval_xpath(image, media_xpath)
321 image_src = url_image.replace('{filename}', extract_text(image_name[0]))
350 image_src = url_image.replace('{filename}', extract_text(image_name[0]))
322 return image_src
351 return image_src
323
352
324
353
325# setting trim will only returned high ranked rows OR the first row
354# setting trim will only returned high ranked rows OR the first row
326−def add_attribute(attributes, result, property_id, default_label=None, date=False, trim=False):
355+def add_attribute(attributes, id_cache, property_id, default_label=None, date=False, trim=False):
327− attribute = result.xpath(property_xpath.replace('{propertyid}', property_id))
356+ attribute = id_cache.get(property_id, None)
328− if attribute:
357+ if attribute is not None:
329
358
330 if default_label:
359 if default_label:
331 label = default_label
360 label = default_label
332 else:
361 else:
333− label = extract_text(attribute[0].xpath(label_xpath))
362+ label = extract_text(eval_xpath(attribute, label_xpath))
334 label = label[0].upper() + label[1:]
363 label = label[0].upper() + label[1:]
335
364
336 if date:
365 if date:
337 trim = True
366 trim = True
338 # remove calendar name
367 # remove calendar name
339− calendar_name = attribute[0].xpath(calendar_name_xpath)
368+ calendar_name = eval_xpath(attribute, calendar_name_xpath)
340 for calendar in calendar_name:
369 for calendar in calendar_name:
341 calendar.getparent().remove(calendar)
370 calendar.getparent().remove(calendar)
342
371
343 concat_values = ""
372 concat_values = ""
344 values = []
373 values = []
345 first_value = None
374 first_value = None
346− for row in attribute[0].xpath(property_row_xpath):
375+ for row in eval_xpath(attribute, property_row_xpath):
347− if not first_value or not trim or row.xpath(preferred_rank_xpath):
376+ if not first_value or not trim or eval_xpath(row, preferred_rank_xpath):
348−
349− value = row.xpath(value_xpath)
377+ value = eval_xpath(row, value_xpath)
350 if not value:
378 if not value:
351 continue
379 continue
352 value = extract_text(value)
380 value = extract_text(value)
@@ -369,18 +397,18 @@ def add_attribute(attributes, result, property_id, default_label=None, date=Fals
369
397
370
398
371# requires property_id unless it's a wiki link (defined in link_type)
399# requires property_id unless it's a wiki link (defined in link_type)
372−def add_url(urls, result, property_id=None, default_label=None, url_prefix=None, results=None, link_type=None):
400+def add_url(urls, result, id_cache, property_id=None, default_label=None, url_prefix=None, results=None,
401+ link_type=None):
373 links = []
402 links = []
374
403
375 # wiki links don't have property in wikidata page
404 # wiki links don't have property in wikidata page
376 if link_type and 'wiki' in link_type:
405 if link_type and 'wiki' in link_type:
377 links.append(get_wikilink(result, link_type))
406 links.append(get_wikilink(result, link_type))
378 else:
407 else:
379− dom_element = result.xpath(property_xpath.replace('{propertyid}', property_id))
408+ dom_element = id_cache.get(property_id, None)
380− if dom_element:
409+ if dom_element is not None:
381− dom_element = dom_element[0]
382 if not default_label:
410 if not default_label:
383− label = extract_text(dom_element.xpath(label_xpath))
411+ label = extract_text(eval_xpath(dom_element, label_xpath))
384 label = label[0].upper() + label[1:]
412 label = label[0].upper() + label[1:]
385
413
386 if link_type == 'geo':
414 if link_type == 'geo':
@@ -390,7 +418,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,
390 links.append(get_imdblink(dom_element, url_prefix))
418 links.append(get_imdblink(dom_element, url_prefix))
391
419
392 else:
420 else:
393− url_results = dom_element.xpath(url_xpath)
421+ url_results = eval_xpath(dom_element, url_xpath)
394 for link in url_results:
422 for link in url_results:
395 if link is not None:
423 if link is not None:
396 if url_prefix:
424 if url_prefix:
@@ -410,7 +438,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,
410
438
411
439
412def get_imdblink(result, url_prefix):
440def get_imdblink(result, url_prefix):
413− imdb_id = result.xpath(value_xpath)
441+ imdb_id = eval_xpath(result, value_xpath)
414 if imdb_id:
442 if imdb_id:
415 imdb_id = extract_text(imdb_id)
443 imdb_id = extract_text(imdb_id)
416 id_prefix = imdb_id[:2]
444 id_prefix = imdb_id[:2]
@@ -430,7 +458,7 @@ def get_imdblink(result, url_prefix):
430
458
431
459
432def get_geolink(result):
460def get_geolink(result):
433− coordinates = result.xpath(value_xpath)
461+ coordinates = eval_xpath(result, value_xpath)
434 if not coordinates:
462 if not coordinates:
435 return None
463 return None
436 coordinates = extract_text(coordinates[0])
464 coordinates = extract_text(coordinates[0])
@@ -477,7 +505,7 @@ def get_geolink(result):
477
505
478
506
479def get_wikilink(result, wikiid):
507def get_wikilink(result, wikiid):
480− url = result.xpath(wikilink_xpath.replace('{wikiid}', wikiid))
508+ url = eval_xpath(result, wikilink_xpath.replace('{wikiid}', wikiid))
481 if not url:
509 if not url:
482 return None
510 return None
483 url = url[0]
511 url = url[0]
tests/unit/engines/test_wikidata.py+23−15
@@ -1,5 +1,6 @@
1# -*- coding: utf-8 -*-
1# -*- coding: utf-8 -*-
2from lxml.html import fromstring
2from lxml.html import fromstring
3+from lxml import etree
3from collections import defaultdict
4from collections import defaultdict
4import mock
5import mock
5from searx.engines import wikidata
6from searx.engines import wikidata
@@ -30,12 +31,12 @@ class TestWikidataEngine(SearxTestCase):
30
31
31 wikidata.supported_languages = ['en', 'es']
32 wikidata.supported_languages = ['en', 'es']
32 wikidata.language_aliases = {}
33 wikidata.language_aliases = {}
33− response = mock.Mock(text='<html></html>', search_params={"language": "en"})
34+ response = mock.Mock(content='<html></html>'.encode("utf-8"), search_params={"language": "en"})
34 self.assertEqual(wikidata.response(response), [])
35 self.assertEqual(wikidata.response(response), [])
35
36
36 def test_getDetail(self):
37 def test_getDetail(self):
37 response = {}
38 response = {}
38− results = wikidata.getDetail(response, "Q123", "en", "en-US")
39+ results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser())
39 self.assertEqual(results, [])
40 self.assertEqual(results, [])
40
41
41 title_html = '<div><div class="wikibase-title-label">Test</div></div>'
42 title_html = '<div><div class="wikibase-title-label">Test</div></div>'
@@ -52,7 +53,7 @@ class TestWikidataEngine(SearxTestCase):
52 """
53 """
53 response = {"parse": {"displaytitle": title_html, "text": html}}
54 response = {"parse": {"displaytitle": title_html, "text": html}}
54
55
55− results = wikidata.getDetail(response, "Q123", "en", "en-US")
56+ results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser())
56 self.assertEqual(len(results), 1)
57 self.assertEqual(len(results), 1)
57 self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test')
58 self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test')
58
59
@@ -92,7 +93,7 @@ class TestWikidataEngine(SearxTestCase):
92 """
93 """
93 response = {"parse": {"displaytitle": title_html, "text": html}}
94 response = {"parse": {"displaytitle": title_html, "text": html}}
94
95
95− results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")
96+ results = wikidata.getDetail(response, "Q123", "yua", "yua_MX", etree.HTMLParser())
96 self.assertEqual(len(results), 2)
97 self.assertEqual(len(results), 2)
97 self.assertEqual(results[0]['title'], 'Official website')
98 self.assertEqual(results[0]['title'], 'Official website')
98 self.assertEqual(results[0]['url'], 'https://officialsite.com')
99 self.assertEqual(results[0]['url'], 'https://officialsite.com')
@@ -139,8 +140,8 @@ class TestWikidataEngine(SearxTestCase):
139 </div>
140 </div>
140 """
141 """
141 html_etree = fromstring(html)
142 html_etree = fromstring(html)
142−
143+ id_cache = wikidata.get_id_cache(html_etree)
143− image_src = wikidata.add_image(html_etree)
144+ image_src = wikidata.add_image(id_cache)
144 self.assertEqual(image_src,
145 self.assertEqual(image_src,
145 "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")
146 "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")
146
147
@@ -197,8 +198,9 @@ class TestWikidataEngine(SearxTestCase):
197 </div>
198 </div>
198 """
199 """
199 html_etree = fromstring(html)
200 html_etree = fromstring(html)
201+ id_cache = wikidata.get_id_cache(html_etree)
200
202
201− image_src = wikidata.add_image(html_etree)
203+ image_src = wikidata.add_image(id_cache)
202 self.assertEqual(image_src,
204 self.assertEqual(image_src,
203 "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")
205 "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")
204
206
@@ -232,11 +234,12 @@ class TestWikidataEngine(SearxTestCase):
232 """
234 """
233 attributes = []
235 attributes = []
234 html_etree = fromstring(html)
236 html_etree = fromstring(html)
237+ id_cache = wikidata.get_id_cache(html_etree)
235
238
236− wikidata.add_attribute(attributes, html_etree, "Fail")
239+ wikidata.add_attribute(attributes, id_cache, "Fail")
237 self.assertEqual(attributes, [])
240 self.assertEqual(attributes, [])
238
241
239− wikidata.add_attribute(attributes, html_etree, "P27")
242+ wikidata.add_attribute(attributes, id_cache, "P27")
240 self.assertEqual(len(attributes), 1)
243 self.assertEqual(len(attributes), 1)
241 self.assertEqual(attributes[0]["label"], "Country of citizenship")
244 self.assertEqual(attributes[0]["label"], "Country of citizenship")
242 self.assertEqual(attributes[0]["value"], "United Kingdom")
245 self.assertEqual(attributes[0]["value"], "United Kingdom")
@@ -271,7 +274,8 @@ class TestWikidataEngine(SearxTestCase):
271 """
274 """
272 attributes = []
275 attributes = []
273 html_etree = fromstring(html)
276 html_etree = fromstring(html)
277+ id_cache = wikidata.get_id_cache(html_etree)
274− wikidata.add_attribute(attributes, html_etree, "P569", date=True)
278+ wikidata.add_attribute(attributes, id_cache, "P569", date=True)
275 self.assertEqual(len(attributes), 1)
279 self.assertEqual(len(attributes), 1)
276 self.assertEqual(attributes[0]["label"], "Date of birth")
280 self.assertEqual(attributes[0]["label"], "Date of birth")
277 self.assertEqual(attributes[0]["value"], "27 January 1832")
281 self.assertEqual(attributes[0]["value"], "27 January 1832")
@@ -319,14 +323,16 @@ class TestWikidataEngine(SearxTestCase):
319 """
323 """
320 attributes = []
324 attributes = []
321 html_etree = fromstring(html)
325 html_etree = fromstring(html)
326+ id_cache = wikidata.get_id_cache(html_etree)
322− wikidata.add_attribute(attributes, html_etree, "P6")
327+ wikidata.add_attribute(attributes, id_cache, "P6")
323 self.assertEqual(len(attributes), 1)
328 self.assertEqual(len(attributes), 1)
324 self.assertEqual(attributes[0]["label"], "Head of government")
329 self.assertEqual(attributes[0]["label"], "Head of government")
325 self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")
330 self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")
326
331
327 attributes = []
332 attributes = []
328 html_etree = fromstring(html)
333 html_etree = fromstring(html)
334+ id_cache = wikidata.get_id_cache(html_etree)
329− wikidata.add_attribute(attributes, html_etree, "P6", trim=True)
335+ wikidata.add_attribute(attributes, id_cache, "P6", trim=True)
330 self.assertEqual(len(attributes), 1)
336 self.assertEqual(len(attributes), 1)
331 self.assertEqual(attributes[0]["value"], "Actual Prime Minister")
337 self.assertEqual(attributes[0]["value"], "Actual Prime Minister")
332
338
@@ -357,12 +363,13 @@ class TestWikidataEngine(SearxTestCase):
357 """
363 """
358 urls = []
364 urls = []
359 html_etree = fromstring(html)
365 html_etree = fromstring(html)
366+ id_cache = wikidata.get_id_cache(html_etree)
360− wikidata.add_url(urls, html_etree, 'P856')
367+ wikidata.add_url(urls, html_etree, id_cache, 'P856')
361 self.assertEquals(len(urls), 1)
368 self.assertEquals(len(urls), 1)
362 self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)
369 self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)
363 urls = []
370 urls = []
364 results = []
371 results = []
365− wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)
372+ wikidata.add_url(urls, html_etree, id_cache, 'P856', 'custom label', results=results)
366 self.assertEquals(len(urls), 1)
373 self.assertEquals(len(urls), 1)
367 self.assertEquals(len(results), 1)
374 self.assertEquals(len(results), 1)
368 self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls)
375 self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls)
@@ -405,7 +412,8 @@ class TestWikidataEngine(SearxTestCase):
405 """
412 """
406 urls = []
413 urls = []
407 html_etree = fromstring(html)
414 html_etree = fromstring(html)
415+ id_cache = wikidata.get_id_cache(html_etree)
408− wikidata.add_url(urls, html_etree, 'P856')
416+ wikidata.add_url(urls, html_etree, id_cache, 'P856')
409 self.assertEquals(len(urls), 2)
417 self.assertEquals(len(urls), 2)
410 self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)
418 self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)
411 self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)
419 self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)