6e0285b · searx · git.iko.soy

git.iko.soy/searx

[fix] wikidata engine: faster processing, remove one HTTP redirection.

* Search URL is https://www.wikidata.org/w/index.php?{query}&ns0=1 (with ns0=1 at the end to avoid an HTTP redirection) * url_detail: remove the disabletidy=1 deprecated parameter * Add eval_xpath function: compile once for all xpath. * Add get_id_cache: retrieve all HTML with an id, avoid the slow to procress dynamic xpath '//div[@id="{propertyid}"]'.replace('{propertyid}') * Create an etree.HTMLParser() instead of using the global one (see #1575)

commit6e0285b2db8e96b98a7348e973d399297a039f10 authorDalf <alex@al-f.net> dateThu 25 Jul 2019, 07:59 (7 years ago) parentcbc5e13

2 files changed+164−128

compare with master

unifiedsplit

searx/engines/wikidata.py+141−113 tests/unit/engines/test_wikidata.py+23−15

searx/engines/wikidata.py+141−113

@@ -20,6 +20,7 @@ from searx.utils import match_language

2020

2121from json import loads

2222from lxml.html import fromstring

23+from lxml import etree

2324

2425logger = logger.getChild('wikidata')

2526result_count = 1

@@ -27,23 +28,23 @@ result_count = 1

2728# urls

2829wikidata_host = 'https://www.wikidata.org'

2930url_search = wikidata_host \

30− + '/w/index.php?{query}'

31+ + '/w/index.php?{query}&ns0=1'

3132

3233wikidata_api = wikidata_host + '/w/api.php'

3334url_detail = wikidata_api\

3435 + '?action=parse&format=json&{query}'\

35− + '&redirects=1&prop=text%7Cdisplaytitle%7Clanglinks%7Crevid'\

36− + '&disableeditsection=1&disabletidy=1&preview=1&sectionpreview=1&disabletoc=1&utf8=1&formatversion=2'

36+ + '&redirects=1&prop=text%7Cdisplaytitle%7Cparsewarnings'\

37+ + '&disableeditsection=1&preview=1&sectionpreview=1&disabletoc=1&utf8=1&formatversion=2'

3738

3839url_map = 'https://www.openstreetmap.org/'\

3940 + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'

4041url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'

4142

4243# xpaths

44+div_ids_xpath = '//div[@id]'

4345wikidata_ids_xpath = '//ul[@class="mw-search-results"]/li//a/@href'

4446title_xpath = '//*[contains(@class,"wikibase-title-label")]'

4547description_xpath = '//div[contains(@class,"wikibase-entitytermsview-heading-description")]'

46−property_xpath = '//div[@id="{propertyid}"]'

4748label_xpath = './/div[contains(@class,"wikibase-statementgroupview-property-label")]/a'

4849url_xpath = './/a[contains(@class,"external free") or contains(@class, "wb-external-id")]'

4950wikilink_xpath = './/ul[contains(@class,"wikibase-sitelinklistview-listview")]'\

@@ -56,6 +57,31 @@ language_fallback_xpath = '//sup[contains(@class,"wb-language-fallback-indicator

5657calendar_name_xpath = './/sup[contains(@class,"wb-calendar-name")]'

5758media_xpath = value_xpath + '//div[contains(@class,"commons-media-caption")]//a'

5859

60+# xpath_cache

61+xpath_cache = {}

62+

63+

64+def get_xpath(xpath_str):

65+ result = xpath_cache.get(xpath_str, None)

66+ if not result:

67+ result = etree.XPath(xpath_str)

68+ xpath_cache[xpath_str] = result

69+ return result

70+

71+

72+def eval_xpath(element, xpath_str):

73+ xpath = get_xpath(xpath_str)

74+ return xpath(element)

75+

76+

77+def get_id_cache(result):

78+ id_cache = {}

79+ for e in eval_xpath(result, div_ids_xpath):

80+ id = e.get('id')

81+ if id.startswith('P'):

82+ id_cache[id] = e

83+ return id_cache

84+

5985

6086def request(query, params):

6187 params['url'] = url_search.format(

@@ -65,8 +91,9 @@ def request(query, params):

6591

6692def response(resp):

6793 results = []

68− html = fromstring(resp.text)

69− search_results = html.xpath(wikidata_ids_xpath)

94+ htmlparser = etree.HTMLParser()

95+ html = fromstring(resp.content.decode("utf-8"), parser=htmlparser)

96+ search_results = eval_xpath(html, wikidata_ids_xpath)

7097

7198 if resp.search_params['language'].split('-')[0] == 'all':

7299 language = 'en'

@@ -78,13 +105,13 @@ def response(resp):

78105 wikidata_id = search_result.split('/')[-1]

79106 url = url_detail.format(query=urlencode({'page': wikidata_id, 'uselang': language}))

80107 htmlresponse = get(url)

81− jsonresponse = loads(htmlresponse.text)

82− results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'])

108+ jsonresponse = loads(htmlresponse.content.decode("utf-8"))

109+ results += getDetail(jsonresponse, wikidata_id, language, resp.search_params['language'], htmlparser)

83110

84111 return results

85112

86113

87−def getDetail(jsonresponse, wikidata_id, language, locale):

114+def getDetail(jsonresponse, wikidata_id, language, locale, htmlparser):

88115 results = []

89116 urls = []

90117 attributes = []

@@ -95,21 +122,23 @@ def getDetail(jsonresponse, wikidata_id, language, locale):

95122 if not title or not result:

96123 return results

97124

98− title = fromstring(title)

99− for elem in title.xpath(language_fallback_xpath):

125+ title = fromstring(title, parser=htmlparser)

126+ for elem in eval_xpath(title, language_fallback_xpath):

100127 elem.getparent().remove(elem)

101− title = extract_text(title.xpath(title_xpath))

128+ title = extract_text(eval_xpath(title, title_xpath))

102129

103− result = fromstring(result)

104− for elem in result.xpath(language_fallback_xpath):

130+ result = fromstring(result, parser=htmlparser)

131+ for elem in eval_xpath(result, language_fallback_xpath):

105132 elem.getparent().remove(elem)

106133

107− description = extract_text(result.xpath(description_xpath))

134+ description = extract_text(eval_xpath(result, description_xpath))

135+

136+ id_cache = get_id_cache(result)

108137

109138 # URLS

110139

111140 # official website

112− add_url(urls, result, 'P856', results=results)

141+ add_url(urls, result, id_cache, 'P856', results=results)

113142

114143 # wikipedia

115144 wikipedia_link_count = 0

@@ -130,30 +159,30 @@ def getDetail(jsonresponse, wikidata_id, language, locale):

130159 # if wikipedia_link_count == 0:

131160

132161 # more wikis

133− add_url(urls, result, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')

134− add_url(urls, result, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')

135− add_url(urls, result, default_label='Wikimedia Commons', link_type='commonswiki')

162+ add_url(urls, result, id_cache, default_label='Wikivoyage (' + language + ')', link_type=language + 'wikivoyage')

163+ add_url(urls, result, id_cache, default_label='Wikiquote (' + language + ')', link_type=language + 'wikiquote')

164+ add_url(urls, result, id_cache, default_label='Wikimedia Commons', link_type='commonswiki')

136165

137− add_url(urls, result, 'P625', 'OpenStreetMap', link_type='geo')

166+ add_url(urls, result, id_cache, 'P625', 'OpenStreetMap', link_type='geo')

138167

139168 # musicbrainz

140− add_url(urls, result, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')

141− add_url(urls, result, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')

142− add_url(urls, result, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')

143− add_url(urls, result, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')

169+ add_url(urls, result, id_cache, 'P434', 'MusicBrainz', 'http://musicbrainz.org/artist/')

170+ add_url(urls, result, id_cache, 'P435', 'MusicBrainz', 'http://musicbrainz.org/work/')

171+ add_url(urls, result, id_cache, 'P436', 'MusicBrainz', 'http://musicbrainz.org/release-group/')

172+ add_url(urls, result, id_cache, 'P966', 'MusicBrainz', 'http://musicbrainz.org/label/')

144173

145174 # IMDb

146− add_url(urls, result, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')

175+ add_url(urls, result, id_cache, 'P345', 'IMDb', 'https://www.imdb.com/', link_type='imdb')

147176 # source code repository

148− add_url(urls, result, 'P1324')

177+ add_url(urls, result, id_cache, 'P1324')

149178 # blog

150− add_url(urls, result, 'P1581')

179+ add_url(urls, result, id_cache, 'P1581')

151180 # social media links

152− add_url(urls, result, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')

153− add_url(urls, result, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')

154− add_url(urls, result, 'P2002', 'Twitter', 'https://twitter.com/')

155− add_url(urls, result, 'P2013', 'Facebook', 'https://facebook.com/')

156− add_url(urls, result, 'P2003', 'Instagram', 'https://instagram.com/')

181+ add_url(urls, result, id_cache, 'P2397', 'YouTube', 'https://www.youtube.com/channel/')

182+ add_url(urls, result, id_cache, 'P1651', 'YouTube', 'https://www.youtube.com/watch?v=')

183+ add_url(urls, result, id_cache, 'P2002', 'Twitter', 'https://twitter.com/')

184+ add_url(urls, result, id_cache, 'P2013', 'Facebook', 'https://facebook.com/')

185+ add_url(urls, result, id_cache, 'P2003', 'Instagram', 'https://instagram.com/')

157186

158187 urls.append({'title': 'Wikidata',

159188 'url': 'https://www.wikidata.org/wiki/'

@@ -163,132 +192,132 @@ def getDetail(jsonresponse, wikidata_id, language, locale):

163192

164193 # DATES

165194 # inception date

166− add_attribute(attributes, result, 'P571', date=True)

195+ add_attribute(attributes, id_cache, 'P571', date=True)

167196 # dissolution date

168− add_attribute(attributes, result, 'P576', date=True)

197+ add_attribute(attributes, id_cache, 'P576', date=True)

169198 # start date

170− add_attribute(attributes, result, 'P580', date=True)

199+ add_attribute(attributes, id_cache, 'P580', date=True)

171200 # end date

172− add_attribute(attributes, result, 'P582', date=True)

201+ add_attribute(attributes, id_cache, 'P582', date=True)

173202 # date of birth

174− add_attribute(attributes, result, 'P569', date=True)

203+ add_attribute(attributes, id_cache, 'P569', date=True)

175204 # date of death

176− add_attribute(attributes, result, 'P570', date=True)

205+ add_attribute(attributes, id_cache, 'P570', date=True)

177206 # date of spacecraft launch

178− add_attribute(attributes, result, 'P619', date=True)

207+ add_attribute(attributes, id_cache, 'P619', date=True)

179208 # date of spacecraft landing

180− add_attribute(attributes, result, 'P620', date=True)

209+ add_attribute(attributes, id_cache, 'P620', date=True)

181210

182211 # nationality

183− add_attribute(attributes, result, 'P27')

212+ add_attribute(attributes, id_cache, 'P27')

184213 # country of origin

185− add_attribute(attributes, result, 'P495')

214+ add_attribute(attributes, id_cache, 'P495')

186215 # country

187− add_attribute(attributes, result, 'P17')

216+ add_attribute(attributes, id_cache, 'P17')

188217 # headquarters

189− add_attribute(attributes, result, 'Q180')

218+ add_attribute(attributes, id_cache, 'Q180')

190219

191220 # PLACES

192221 # capital

193− add_attribute(attributes, result, 'P36', trim=True)

222+ add_attribute(attributes, id_cache, 'P36', trim=True)

194223 # head of state

195− add_attribute(attributes, result, 'P35', trim=True)

224+ add_attribute(attributes, id_cache, 'P35', trim=True)

196225 # head of government

197− add_attribute(attributes, result, 'P6', trim=True)

226+ add_attribute(attributes, id_cache, 'P6', trim=True)

198227 # type of government

199− add_attribute(attributes, result, 'P122')

228+ add_attribute(attributes, id_cache, 'P122')

200229 # official language

201− add_attribute(attributes, result, 'P37')

230+ add_attribute(attributes, id_cache, 'P37')

202231 # population

203− add_attribute(attributes, result, 'P1082', trim=True)

232+ add_attribute(attributes, id_cache, 'P1082', trim=True)

204233 # area

205− add_attribute(attributes, result, 'P2046')

234+ add_attribute(attributes, id_cache, 'P2046')

206235 # currency

207− add_attribute(attributes, result, 'P38', trim=True)

236+ add_attribute(attributes, id_cache, 'P38', trim=True)

208237 # heigth (building)

209− add_attribute(attributes, result, 'P2048')

238+ add_attribute(attributes, id_cache, 'P2048')

210239

211240 # MEDIA

212241 # platform (videogames)

213− add_attribute(attributes, result, 'P400')

242+ add_attribute(attributes, id_cache, 'P400')

214243 # author

215− add_attribute(attributes, result, 'P50')

244+ add_attribute(attributes, id_cache, 'P50')

216245 # creator

217− add_attribute(attributes, result, 'P170')

246+ add_attribute(attributes, id_cache, 'P170')

218247 # director

219− add_attribute(attributes, result, 'P57')

248+ add_attribute(attributes, id_cache, 'P57')

220249 # performer

221− add_attribute(attributes, result, 'P175')

250+ add_attribute(attributes, id_cache, 'P175')

222251 # developer

223− add_attribute(attributes, result, 'P178')

252+ add_attribute(attributes, id_cache, 'P178')

224253 # producer

225− add_attribute(attributes, result, 'P162')

254+ add_attribute(attributes, id_cache, 'P162')

226255 # manufacturer

227− add_attribute(attributes, result, 'P176')

256+ add_attribute(attributes, id_cache, 'P176')

228257 # screenwriter

229− add_attribute(attributes, result, 'P58')

258+ add_attribute(attributes, id_cache, 'P58')

230259 # production company

231− add_attribute(attributes, result, 'P272')

260+ add_attribute(attributes, id_cache, 'P272')

232261 # record label

233− add_attribute(attributes, result, 'P264')

262+ add_attribute(attributes, id_cache, 'P264')

234263 # publisher

235− add_attribute(attributes, result, 'P123')

264+ add_attribute(attributes, id_cache, 'P123')

236265 # original network

237− add_attribute(attributes, result, 'P449')

266+ add_attribute(attributes, id_cache, 'P449')

238267 # distributor

239− add_attribute(attributes, result, 'P750')

268+ add_attribute(attributes, id_cache, 'P750')

240269 # composer

241− add_attribute(attributes, result, 'P86')

270+ add_attribute(attributes, id_cache, 'P86')

242271 # publication date

243− add_attribute(attributes, result, 'P577', date=True)

272+ add_attribute(attributes, id_cache, 'P577', date=True)

244273 # genre

245− add_attribute(attributes, result, 'P136')

274+ add_attribute(attributes, id_cache, 'P136')

246275 # original language

247− add_attribute(attributes, result, 'P364')

276+ add_attribute(attributes, id_cache, 'P364')

248277 # isbn

249− add_attribute(attributes, result, 'Q33057')

278+ add_attribute(attributes, id_cache, 'Q33057')

250279 # software license

251− add_attribute(attributes, result, 'P275')

280+ add_attribute(attributes, id_cache, 'P275')

252281 # programming language

253− add_attribute(attributes, result, 'P277')

282+ add_attribute(attributes, id_cache, 'P277')

254283 # version

255− add_attribute(attributes, result, 'P348', trim=True)

284+ add_attribute(attributes, id_cache, 'P348', trim=True)

256285 # narrative location

257− add_attribute(attributes, result, 'P840')

286+ add_attribute(attributes, id_cache, 'P840')

258287

259288 # LANGUAGES

260289 # number of speakers

261− add_attribute(attributes, result, 'P1098')

290+ add_attribute(attributes, id_cache, 'P1098')

262291 # writing system

263− add_attribute(attributes, result, 'P282')

292+ add_attribute(attributes, id_cache, 'P282')

264293 # regulatory body

265− add_attribute(attributes, result, 'P1018')

294+ add_attribute(attributes, id_cache, 'P1018')

266295 # language code

267− add_attribute(attributes, result, 'P218')

296+ add_attribute(attributes, id_cache, 'P218')

268297

269298 # OTHER

270299 # ceo

271− add_attribute(attributes, result, 'P169', trim=True)

300+ add_attribute(attributes, id_cache, 'P169', trim=True)

272301 # founder

273− add_attribute(attributes, result, 'P112')

302+ add_attribute(attributes, id_cache, 'P112')

274303 # legal form (company/organization)

275− add_attribute(attributes, result, 'P1454')

304+ add_attribute(attributes, id_cache, 'P1454')

276305 # operator

277− add_attribute(attributes, result, 'P137')

306+ add_attribute(attributes, id_cache, 'P137')

278307 # crew members (tripulation)

279− add_attribute(attributes, result, 'P1029')

308+ add_attribute(attributes, id_cache, 'P1029')

280309 # taxon

281− add_attribute(attributes, result, 'P225')

310+ add_attribute(attributes, id_cache, 'P225')

282311 # chemical formula

283− add_attribute(attributes, result, 'P274')

312+ add_attribute(attributes, id_cache, 'P274')

284313 # winner (sports/contests)

285− add_attribute(attributes, result, 'P1346')

314+ add_attribute(attributes, id_cache, 'P1346')

286315 # number of deaths

287− add_attribute(attributes, result, 'P1120')

316+ add_attribute(attributes, id_cache, 'P1120')

288317 # currency code

289− add_attribute(attributes, result, 'P498')

318+ add_attribute(attributes, id_cache, 'P498')

290319

291− image = add_image(result)

320+ image = add_image(id_cache)

292321

293322 if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:

294323 results.append({

@@ -310,43 +339,42 @@ def getDetail(jsonresponse, wikidata_id, language, locale):

310339

311340

312341# only returns first match

313−def add_image(result):

342+def add_image(id_cache):

314343 # P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon

315344 property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910']

316345

317346 for property_id in property_ids:

318− image = result.xpath(property_xpath.replace('{propertyid}', property_id))

319− if image:

320− image_name = image[0].xpath(media_xpath)

347+ image = id_cache.get(property_id, None)

348+ if image is not None:

349+ image_name = eval_xpath(image, media_xpath)

321350 image_src = url_image.replace('{filename}', extract_text(image_name[0]))

322351 return image_src

323352

324353

325354# setting trim will only returned high ranked rows OR the first row

326−def add_attribute(attributes, result, property_id, default_label=None, date=False, trim=False):

327− attribute = result.xpath(property_xpath.replace('{propertyid}', property_id))

328− if attribute:

355+def add_attribute(attributes, id_cache, property_id, default_label=None, date=False, trim=False):

356+ attribute = id_cache.get(property_id, None)

357+ if attribute is not None:

329358

330359 if default_label:

331360 label = default_label

332361 else:

333− label = extract_text(attribute[0].xpath(label_xpath))

362+ label = extract_text(eval_xpath(attribute, label_xpath))

334363 label = label[0].upper() + label[1:]

335364

336365 if date:

337366 trim = True

338367 # remove calendar name

339− calendar_name = attribute[0].xpath(calendar_name_xpath)

368+ calendar_name = eval_xpath(attribute, calendar_name_xpath)

340369 for calendar in calendar_name:

341370 calendar.getparent().remove(calendar)

342371

343372 concat_values = ""

344373 values = []

345374 first_value = None

346− for row in attribute[0].xpath(property_row_xpath):

347− if not first_value or not trim or row.xpath(preferred_rank_xpath):

348−

349− value = row.xpath(value_xpath)

375+ for row in eval_xpath(attribute, property_row_xpath):

376+ if not first_value or not trim or eval_xpath(row, preferred_rank_xpath):

377+ value = eval_xpath(row, value_xpath)

350378 if not value:

351379 continue

352380 value = extract_text(value)

@@ -369,18 +397,18 @@ def add_attribute(attributes, result, property_id, default_label=None, date=Fals

369397

370398

371399# requires property_id unless it's a wiki link (defined in link_type)

372−def add_url(urls, result, property_id=None, default_label=None, url_prefix=None, results=None, link_type=None):

400+def add_url(urls, result, id_cache, property_id=None, default_label=None, url_prefix=None, results=None,

401+ link_type=None):

373402 links = []

374403

375404 # wiki links don't have property in wikidata page

376405 if link_type and 'wiki' in link_type:

377406 links.append(get_wikilink(result, link_type))

378407 else:

379− dom_element = result.xpath(property_xpath.replace('{propertyid}', property_id))

380− if dom_element:

381− dom_element = dom_element[0]

408+ dom_element = id_cache.get(property_id, None)

409+ if dom_element is not None:

382410 if not default_label:

383− label = extract_text(dom_element.xpath(label_xpath))

411+ label = extract_text(eval_xpath(dom_element, label_xpath))

384412 label = label[0].upper() + label[1:]

385413

386414 if link_type == 'geo':

@@ -390,7 +418,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,

390418 links.append(get_imdblink(dom_element, url_prefix))

391419

392420 else:

393− url_results = dom_element.xpath(url_xpath)

421+ url_results = eval_xpath(dom_element, url_xpath)

394422 for link in url_results:

395423 if link is not None:

396424 if url_prefix:

@@ -410,7 +438,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,

410438

411439

412440def get_imdblink(result, url_prefix):

413− imdb_id = result.xpath(value_xpath)

441+ imdb_id = eval_xpath(result, value_xpath)

414442 if imdb_id:

415443 imdb_id = extract_text(imdb_id)

416444 id_prefix = imdb_id[:2]

@@ -430,7 +458,7 @@ def get_imdblink(result, url_prefix):

430458

431459

432460def get_geolink(result):

433− coordinates = result.xpath(value_xpath)

461+ coordinates = eval_xpath(result, value_xpath)

434462 if not coordinates:

435463 return None

436464 coordinates = extract_text(coordinates[0])

@@ -477,7 +505,7 @@ def get_geolink(result):

477505

478506

479507def get_wikilink(result, wikiid):

480− url = result.xpath(wikilink_xpath.replace('{wikiid}', wikiid))

508+ url = eval_xpath(result, wikilink_xpath.replace('{wikiid}', wikiid))

481509 if not url:

482510 return None

483511 url = url[0]

tests/unit/engines/test_wikidata.py+23−15

@@ -1,5 +1,6 @@

11# -*- coding: utf-8 -*-

22from lxml.html import fromstring

3+from lxml import etree

34from collections import defaultdict

45import mock

56from searx.engines import wikidata

@@ -30,12 +31,12 @@ class TestWikidataEngine(SearxTestCase):

3031

3132 wikidata.supported_languages = ['en', 'es']

3233 wikidata.language_aliases = {}

33− response = mock.Mock(text='<html></html>', search_params={"language": "en"})

34+ response = mock.Mock(content='<html></html>'.encode("utf-8"), search_params={"language": "en"})

3435 self.assertEqual(wikidata.response(response), [])

3536

3637 def test_getDetail(self):

3738 response = {}

38− results = wikidata.getDetail(response, "Q123", "en", "en-US")

39+ results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser())

3940 self.assertEqual(results, [])

4041

4142 title_html = '<div><div class="wikibase-title-label">Test</div></div>'

@@ -52,7 +53,7 @@ class TestWikidataEngine(SearxTestCase):

5253 """

5354 response = {"parse": {"displaytitle": title_html, "text": html}}

5455

55− results = wikidata.getDetail(response, "Q123", "en", "en-US")

56+ results = wikidata.getDetail(response, "Q123", "en", "en-US", etree.HTMLParser())

5657 self.assertEqual(len(results), 1)

5758 self.assertEqual(results[0]['url'], 'https://en.wikipedia.org/wiki/Test')

5859

@@ -92,7 +93,7 @@ class TestWikidataEngine(SearxTestCase):

9293 """

9394 response = {"parse": {"displaytitle": title_html, "text": html}}

9495

95− results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")

96+ results = wikidata.getDetail(response, "Q123", "yua", "yua_MX", etree.HTMLParser())

9697 self.assertEqual(len(results), 2)

9798 self.assertEqual(results[0]['title'], 'Official website')

9899 self.assertEqual(results[0]['url'], 'https://officialsite.com')

@@ -139,8 +140,8 @@ class TestWikidataEngine(SearxTestCase):

139140 </div>

140141 """

141142 html_etree = fromstring(html)

142−

143− image_src = wikidata.add_image(html_etree)

143+ id_cache = wikidata.get_id_cache(html_etree)

144+ image_src = wikidata.add_image(id_cache)

144145 self.assertEqual(image_src,

145146 "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")

146147

@@ -197,8 +198,9 @@ class TestWikidataEngine(SearxTestCase):

197198 </div>

198199 """

199200 html_etree = fromstring(html)

201+ id_cache = wikidata.get_id_cache(html_etree)

200202

201− image_src = wikidata.add_image(html_etree)

203+ image_src = wikidata.add_image(id_cache)

202204 self.assertEqual(image_src,

203205 "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")

204206

@@ -232,11 +234,12 @@ class TestWikidataEngine(SearxTestCase):

232234 """

233235 attributes = []

234236 html_etree = fromstring(html)

237+ id_cache = wikidata.get_id_cache(html_etree)

235238

236− wikidata.add_attribute(attributes, html_etree, "Fail")

239+ wikidata.add_attribute(attributes, id_cache, "Fail")

237240 self.assertEqual(attributes, [])

238241

239− wikidata.add_attribute(attributes, html_etree, "P27")

242+ wikidata.add_attribute(attributes, id_cache, "P27")

240243 self.assertEqual(len(attributes), 1)

241244 self.assertEqual(attributes[0]["label"], "Country of citizenship")

242245 self.assertEqual(attributes[0]["value"], "United Kingdom")

@@ -271,7 +274,8 @@ class TestWikidataEngine(SearxTestCase):

271274 """

272275 attributes = []

273276 html_etree = fromstring(html)

274− wikidata.add_attribute(attributes, html_etree, "P569", date=True)

277+ id_cache = wikidata.get_id_cache(html_etree)

278+ wikidata.add_attribute(attributes, id_cache, "P569", date=True)

275279 self.assertEqual(len(attributes), 1)

276280 self.assertEqual(attributes[0]["label"], "Date of birth")

277281 self.assertEqual(attributes[0]["value"], "27 January 1832")

@@ -319,14 +323,16 @@ class TestWikidataEngine(SearxTestCase):

319323 """

320324 attributes = []

321325 html_etree = fromstring(html)

322− wikidata.add_attribute(attributes, html_etree, "P6")

326+ id_cache = wikidata.get_id_cache(html_etree)

327+ wikidata.add_attribute(attributes, id_cache, "P6")

323328 self.assertEqual(len(attributes), 1)

324329 self.assertEqual(attributes[0]["label"], "Head of government")

325330 self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")

326331

327332 attributes = []

328333 html_etree = fromstring(html)

329− wikidata.add_attribute(attributes, html_etree, "P6", trim=True)

334+ id_cache = wikidata.get_id_cache(html_etree)

335+ wikidata.add_attribute(attributes, id_cache, "P6", trim=True)

330336 self.assertEqual(len(attributes), 1)

331337 self.assertEqual(attributes[0]["value"], "Actual Prime Minister")

332338

@@ -357,12 +363,13 @@ class TestWikidataEngine(SearxTestCase):

357363 """

358364 urls = []

359365 html_etree = fromstring(html)

360− wikidata.add_url(urls, html_etree, 'P856')

366+ id_cache = wikidata.get_id_cache(html_etree)

367+ wikidata.add_url(urls, html_etree, id_cache, 'P856')

361368 self.assertEquals(len(urls), 1)

362369 self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)

363370 urls = []

364371 results = []

365− wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)

372+ wikidata.add_url(urls, html_etree, id_cache, 'P856', 'custom label', results=results)

366373 self.assertEquals(len(urls), 1)

367374 self.assertEquals(len(results), 1)

368375 self.assertIn({'title': 'custom label', 'url': 'https://searx.me/'}, urls)

@@ -405,7 +412,8 @@ class TestWikidataEngine(SearxTestCase):

405412 """

406413 urls = []

407414 html_etree = fromstring(html)

408− wikidata.add_url(urls, html_etree, 'P856')

415+ id_cache = wikidata.get_id_cache(html_etree)

416+ wikidata.add_url(urls, html_etree, id_cache, 'P856')

409417 self.assertEquals(len(urls), 2)

410418 self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)

411419 self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)