git.iko.soy/searx
on master help sign in
[fix] merge infoboxes based on weight

also minor changes in attributes and images from wikidata

commitad58b14be7cc9a1e95858e150e9d8005734d9232 authormarc <a01200356@itesm.mx> dateMon 27 Jun 2016, 23:35 (10 years ago) parentc2e4014
4 files changed+50−16
compare with master
unifiedsplit
searx/engines/wikidata.py+19−5 searx/results.py+17−1 searx/settings.yml+2 tests/unit/engines/test_wikidata.py+12−10
searx/engines/wikidata.py+19−5
@@ -35,7 +35,7 @@ url_detail = wikidata_api\
3535
3636url_map = 'https://www.openstreetmap.org/'\
3737 + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
38−url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500'
38+url_image = 'https://commons.wikimedia.org/wiki/Special:FilePath/{filename}?width=500&height=400'
3939
4040# xpaths
4141wikidata_ids_xpath = '//div/ul[@class="wikibase-disambiguation"]/li/a/@title'
@@ -162,6 +162,7 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
162162
163163 # INFOBOX ATTRIBUTES (ROWS)
164164
165+ # DATES
165166 # inception date
166167 add_attribute(attributes, result, 'P571', date=True)
167168 # dissolution date
@@ -170,11 +171,14 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
170171 add_attribute(attributes, result, 'P580', date=True)
171172 # end date
172173 add_attribute(attributes, result, 'P582', date=True)
173−
174174 # date of birth
175175 add_attribute(attributes, result, 'P569', date=True)
176176 # date of death
177177 add_attribute(attributes, result, 'P570', date=True)
178+ # date of spacecraft launch
179+ add_attribute(attributes, result, 'P619', date=True)
180+ # date of spacecraft landing
181+ add_attribute(attributes, result, 'P620', date=True)
178182
179183 # nationality
180184 add_attribute(attributes, result, 'P27')
@@ -201,7 +205,7 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
201205 # area
202206 add_attribute(attributes, result, 'P2046')
203207 # currency
204− add_attribute(attributes, result, 'P38')
208+ add_attribute(attributes, result, 'P38', trim=True)
205209 # heigth (building)
206210 add_attribute(attributes, result, 'P2048')
207211
@@ -230,6 +234,10 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
230234 add_attribute(attributes, result, 'P264')
231235 # publisher
232236 add_attribute(attributes, result, 'P123')
237+ # original network
238+ add_attribute(attributes, result, 'P449')
239+ # distributor
240+ add_attribute(attributes, result, 'P750')
233241 # composer
234242 add_attribute(attributes, result, 'P86')
235243 # publication date
@@ -266,6 +274,10 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
266274 add_attribute(attributes, result, 'P112')
267275 # legal form (company/organization)
268276 add_attribute(attributes, result, 'P1454')
277+ # operator
278+ add_attribute(attributes, result, 'P137')
279+ # crew members (tripulation)
280+ add_attribute(attributes, result, 'P1029')
269281 # taxon
270282 add_attribute(attributes, result, 'P225')
271283 # chemical formula
@@ -300,8 +312,8 @@ def getDetail(jsonresponse, wikidata_id, language, locale):
300312
301313# only returns first match
302314def add_image(result):
303− # P18: image, P154: logo, P242: map, P41: flag, P2716: collage, P2910: icon
304− property_ids = ['P18', 'P154', 'P242', 'P41', 'P2716', 'P2910']
315+ # P15: route map, P242: locator map, P154: logo, P18: image, P242: map, P41: flag, P2716: collage, P2910: icon
316+ property_ids = ['P15', 'P242', 'P154', 'P18', 'P242', 'P41', 'P2716', 'P2910']
305317
306318 for property_id in property_ids:
307319 image = result.xpath(property_xpath.replace('{propertyid}', property_id))
@@ -320,6 +332,7 @@ def add_attribute(attributes, result, property_id, default_label=None, date=Fals
320332 label = default_label
321333 else:
322334 label = extract_text(attribute[0].xpath(label_xpath))
335+ label = label[0].upper() + label[1:]
323336
324337 if date:
325338 trim = True
@@ -369,6 +382,7 @@ def add_url(urls, result, property_id=None, default_label=None, url_prefix=None,
369382 dom_element = dom_element[0]
370383 if not default_label:
371384 label = extract_text(dom_element.xpath(label_xpath))
385+ label = label[0].upper() + label[1:]
372386
373387 if link_type == 'geo':
374388 links.append(get_geolink(dom_element))
searx/results.py+17−1
@@ -43,6 +43,19 @@ def compare_urls(url_a, url_b):
4343
4444
4545def merge_two_infoboxes(infobox1, infobox2):
46+ # get engines weights
47+ if hasattr(engines[infobox1['engine']], 'weight'):
48+ weight1 = engines[infobox1['engine']].weight
49+ else:
50+ weight1 = 1
51+ if hasattr(engines[infobox2['engine']], 'weight'):
52+ weight2 = engines[infobox2['engine']].weight
53+ else:
54+ weight2 = 1
55+
56+ if weight2 > weight1:
57+ infobox1['engine'] = infobox2['engine']
58+
4659 if 'urls' in infobox2:
4760 urls1 = infobox1.get('urls', None)
4861 if urls1 is None:
@@ -64,6 +77,8 @@ def merge_two_infoboxes(infobox1, infobox2):
6477 img2 = infobox2.get('img_src')
6578 if img1 is None:
6679 infobox1['img_src'] = img2
80+ elif weight2 > weight1:
81+ infobox1['img_src'] = img2
6782
6883 if 'attributes' in infobox2:
6984 attributes1 = infobox1.get('attributes', None)
@@ -77,7 +92,8 @@ def merge_two_infoboxes(infobox1, infobox2):
7792 attributeSet.add(attribute.get('label', None))
7893
7994 for attribute in infobox2.get('attributes', []):
80− attributes1.append(attribute)
95+ if attribute.get('label', None) not in attributeSet:
96+ attributes1.append(attribute)
8197
8298 if 'content' in infobox2:
8399 content1 = infobox1.get('content', None)
searx/settings.yml+2
@@ -105,6 +105,7 @@ engines:
105105 - name : ddg definitions
106106 engine : duckduckgo_definitions
107107 shortcut : ddd
108+ weight : 2
108109 disabled : True
109110
110111 - name : digg
@@ -127,6 +128,7 @@ engines:
127128 - name : wikidata
128129 engine : wikidata
129130 shortcut : wd
131+ weight : 2
130132
131133 - name : duckduckgo
132134 engine : duckduckgo
tests/unit/engines/test_wikidata.py+12−10
@@ -95,14 +95,14 @@ class TestWikidataEngine(SearxTestCase):
9595
9696 results = wikidata.getDetail(response, "Q123", "yua", "yua_MX")
9797 self.assertEqual(len(results), 2)
98− self.assertEqual(results[0]['title'], 'official website')
98+ self.assertEqual(results[0]['title'], 'Official website')
9999 self.assertEqual(results[0]['url'], 'https://officialsite.com')
100100
101101 self.assertEqual(results[1]['infobox'], 'Test')
102102 self.assertEqual(results[1]['id'], None)
103103 self.assertEqual(results[1]['content'], 'Description')
104104 self.assertEqual(results[1]['attributes'], [])
105− self.assertEqual(results[1]['urls'][0]['title'], 'official website')
105+ self.assertEqual(results[1]['urls'][0]['title'], 'Official website')
106106 self.assertEqual(results[1]['urls'][0]['url'], 'https://officialsite.com')
107107 self.assertEqual(results[1]['urls'][1]['title'], 'Wikipedia (en)')
108108 self.assertEqual(results[1]['urls'][1]['url'], 'https://en.wikipedia.org/wiki/Test')
@@ -141,7 +141,8 @@ class TestWikidataEngine(SearxTestCase):
141141 html_etree = fromstring(html)
142142
143143 image_src = wikidata.add_image(html_etree)
144− self.assertEqual(image_src, "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500")
144+ self.assertEqual(image_src,
145+ "https://commons.wikimedia.org/wiki/Special:FilePath/image.png?width=500&height=400")
145146
146147 html = u"""
147148 <div>
@@ -196,7 +197,8 @@ class TestWikidataEngine(SearxTestCase):
196197 html_etree = fromstring(html)
197198
198199 image_src = wikidata.add_image(html_etree)
199− self.assertEqual(image_src, "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500")
200+ self.assertEqual(image_src,
201+ "https://commons.wikimedia.org/wiki/Special:FilePath/logo.png?width=500&height=400")
200202
201203 def test_add_attribute(self):
202204 html = u"""
@@ -234,7 +236,7 @@ class TestWikidataEngine(SearxTestCase):
234236
235237 wikidata.add_attribute(attributes, html_etree, "P27")
236238 self.assertEqual(len(attributes), 1)
237− self.assertEqual(attributes[0]["label"], "country of citizenship")
239+ self.assertEqual(attributes[0]["label"], "Country of citizenship")
238240 self.assertEqual(attributes[0]["value"], "United Kingdom")
239241
240242 html = u"""
@@ -269,7 +271,7 @@ class TestWikidataEngine(SearxTestCase):
269271 html_etree = fromstring(html)
270272 wikidata.add_attribute(attributes, html_etree, "P569", date=True)
271273 self.assertEqual(len(attributes), 1)
272− self.assertEqual(attributes[0]["label"], "date of birth")
274+ self.assertEqual(attributes[0]["label"], "Date of birth")
273275 self.assertEqual(attributes[0]["value"], "27 January 1832")
274276
275277 html = u"""
@@ -317,7 +319,7 @@ class TestWikidataEngine(SearxTestCase):
317319 html_etree = fromstring(html)
318320 wikidata.add_attribute(attributes, html_etree, "P6")
319321 self.assertEqual(len(attributes), 1)
320− self.assertEqual(attributes[0]["label"], "head of government")
322+ self.assertEqual(attributes[0]["label"], "Head of government")
321323 self.assertEqual(attributes[0]["value"], "Old Prime Minister, Actual Prime Minister")
322324
323325 attributes = []
@@ -355,7 +357,7 @@ class TestWikidataEngine(SearxTestCase):
355357 html_etree = fromstring(html)
356358 wikidata.add_url(urls, html_etree, 'P856')
357359 self.assertEquals(len(urls), 1)
358− self.assertIn({'title': 'official website', 'url': 'https://searx.me/'}, urls)
360+ self.assertIn({'title': 'Official website', 'url': 'https://searx.me/'}, urls)
359361 urls = []
360362 results = []
361363 wikidata.add_url(urls, html_etree, 'P856', 'custom label', results=results)
@@ -403,8 +405,8 @@ class TestWikidataEngine(SearxTestCase):
403405 html_etree = fromstring(html)
404406 wikidata.add_url(urls, html_etree, 'P856')
405407 self.assertEquals(len(urls), 2)
406− self.assertIn({'title': 'official website', 'url': 'http://www.worldofwarcraft.com'}, urls)
407− self.assertIn({'title': 'official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)
408+ self.assertIn({'title': 'Official website', 'url': 'http://www.worldofwarcraft.com'}, urls)
409+ self.assertIn({'title': 'Official website', 'url': 'http://eu.battle.net/wow/en/'}, urls)
408410
409411 def test_get_imdblink(self):
410412 html = u"""