on master help sign in
Merge pull request #139 from dalf/master

pep8 : engines

commit9517f7a6e775aded6f1673f4a991594b08750467 authorAdam Tauber <asciimoo@gmail.com> dateSun 7 Dec 2014, 17:49 (12 years ago) parent611f4e27c13d63
24 files changed+277−187
searx/engines/bing.py+8−7 searx/engines/bing_images.py+11−9 searx/engines/bing_news.py+10−9 searx/engines/currency_convert.py+1−1 searx/engines/dailymotion.py+2−3 searx/engines/deviantart.py+2−2 searx/engines/duckduckgo.py+7−5 searx/engines/duckduckgo_definitions.py+46−31 searx/engines/dummy.py+1−1 searx/engines/faroo.py+6−5 searx/engines/generalfile.py+2−2 searx/engines/github.py+2−2 searx/engines/google_images.py+4−3 searx/engines/google_news.py+4−3 searx/engines/openstreetmap.py+15−15 searx/engines/piratebay.py+3−3 searx/engines/soundcloud.py+2−2 searx/engines/stackoverflow.py+4−4 searx/engines/twitter.py+2−2 searx/engines/vimeo.py+7−5 searx/engines/wikidata.py+122−59 searx/engines/yacy.py+6−5 searx/engines/yahoo.py+8−7 searx/engines/youtube.py+2−2
searx/engines/bing.py+8−7
@@ -1,8 +1,9 @@
11## Bing (Web)
2−#
2+#
33# @website https://www.bing.com
4−# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month
5−#
4+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
5+# max. 5000 query/month
6+#
67# @using-api no (because of query limit)
78# @results HTML (using search portal)
89# @stable no (HTML can change)
@@ -58,8 +59,8 @@ def response(resp):
5859 content = escape(' '.join(result.xpath('.//p//text()')))
5960
6061 # append result
61− results.append({'url': url,
62− 'title': title,
62+ results.append({'url': url,
63+ 'title': title,
6364 'content': content})
6465
6566 # return results if something is found
@@ -74,8 +75,8 @@ def response(resp):
7475 content = escape(' '.join(result.xpath('.//p//text()')))
7576
7677 # append result
77− results.append({'url': url,
78− 'title': title,
78+ results.append({'url': url,
79+ 'title': title,
7980 'content': content})
8081
8182 # return results
searx/engines/bing_images.py+11−9
@@ -1,17 +1,19 @@
11## Bing (Images)
2−#
2+#
33# @website https://www.bing.com/images
4−# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month
5−#
4+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
5+# max. 5000 query/month
6+#
67# @using-api no (because of query limit)
78# @results HTML (using search portal)
89# @stable no (HTML can change)
910# @parse url, title, img_src
1011#
11−# @todo currently there are up to 35 images receive per page, because bing does not parse count=10. limited response to 10 images
12+# @todo currently there are up to 35 images receive per page,
13+# because bing does not parse count=10.
14+# limited response to 10 images
1215
1316from urllib import urlencode
14−from cgi import escape
1517from lxml import html
1618from yaml import load
1719import re
@@ -51,15 +53,15 @@ def response(resp):
5153 dom = html.fromstring(resp.content)
5254
5355 # init regex for yaml-parsing
54− p = re.compile( '({|,)([a-z]+):(")')
56+ p = re.compile('({|,)([a-z]+):(")')
5557
5658 # parse results
5759 for result in dom.xpath('//div[@class="dg_u"]'):
5860 link = result.xpath('./a')[0]
5961
6062 # parse yaml-data (it is required to add a space, to make it parsable)
61− yaml_data = load(p.sub( r'\1\2: \3', link.attrib.get('m')))
62−
63+ yaml_data = load(p.sub(r'\1\2: \3', link.attrib.get('m')))
64+
6365 title = link.attrib.get('t1')
6466 #url = 'http://' + link.attrib.get('t3')
6567 url = yaml_data.get('surl')
@@ -69,7 +71,7 @@ def response(resp):
6971 results.append({'template': 'images.html',
7072 'url': url,
7173 'title': title,
72− 'content': '',
74+ 'content': '',
7375 'img_src': img_src})
7476
7577 # TODO stop parsing if 10 images are found
searx/engines/bing_news.py+10−9
@@ -1,8 +1,9 @@
11## Bing (News)
2−#
2+#
33# @website https://www.bing.com/news
4−# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month
5−#
4+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),
5+# max. 5000 query/month
6+#
67# @using-api no (because of query limit)
78# @results HTML (using search portal)
89# @stable no (HTML can change)
@@ -57,12 +58,12 @@ def response(resp):
5758 url = link.attrib.get('href')
5859 title = ' '.join(link.xpath('.//text()'))
5960 contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()')
60− if contentXPath != None:
61+ if contentXPath is not None:
6162 content = escape(' '.join(contentXPath))
62−
63+
6364 # parse publishedDate
6465 publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()')
65− if publishedDateXPath != None:
66+ if publishedDateXPath is not None:
6667 publishedDate = escape(' '.join(publishedDateXPath))
6768
6869 if re.match("^[0-9]+ minute(s|) ago$", publishedDate):
@@ -89,10 +90,10 @@ def response(resp):
8990 except TypeError:
9091 # FIXME
9192 publishedDate = datetime.now()
92−
93+
9394 # append result
94− results.append({'url': url,
95− 'title': title,
95+ results.append({'url': url,
96+ 'title': title,
9697 'publishedDate': publishedDate,
9798 'content': content})
9899
searx/engines/currency_convert.py+1−1
@@ -55,6 +55,6 @@ def response(resp):
5555 resp.search_params['to'].lower()
5656 )
5757
58− results.append({'answer' : answer, 'url': url})
58+ results.append({'answer': answer, 'url': url})
5959
6060 return results
searx/engines/dailymotion.py+2−3
@@ -1,8 +1,8 @@
11## Dailymotion (Videos)
2−#
2+#
33# @website https://www.dailymotion.com
44# @provide-api yes (http://www.dailymotion.com/developer)
5−#
5+#
66# @using-api yes
77# @results JSON
88# @stable yes
@@ -12,7 +12,6 @@
1212
1313from urllib import urlencode
1414from json import loads
15−from lxml import html
1615
1716# engine dependent config
1817categories = ['videos']
searx/engines/deviantart.py+2−2
@@ -1,8 +1,8 @@
11## Deviantart (Images)
2−#
2+#
33# @website https://www.deviantart.com/
44# @provide-api yes (https://www.deviantart.com/developers/) (RSS)
5−#
5+#
66# @using-api no (TODO, rewrite to api)
77# @results HTML
88# @stable no (HTML can change)
searx/engines/duckduckgo.py+7−5
@@ -1,15 +1,17 @@
11## DuckDuckGo (Web)
2−#
2+#
33# @website https://duckduckgo.com/
4−# @provide-api yes (https://duckduckgo.com/api), but not all results from search-site
5−#
4+# @provide-api yes (https://duckduckgo.com/api),
5+# but not all results from search-site
6+#
67# @using-api no
78# @results HTML (using search portal)
89# @stable no (HTML can change)
910# @parse url, title, content
1011#
1112# @todo rewrite to api
12−# @todo language support (the current used site does not support language-change)
13+# @todo language support
14+# (the current used site does not support language-change)
1315
1416from urllib import urlencode
1517from lxml.html import fromstring
@@ -37,7 +39,7 @@ def request(query, params):
3739 if params['language'] == 'all':
3840 locale = 'en-us'
3941 else:
40− locale = params['language'].replace('_','-').lower()
42+ locale = params['language'].replace('_', '-').lower()
4143
4244 params['url'] = url.format(
4345 query=urlencode({'q': query, 'kl': locale}),
searx/engines/duckduckgo_definitions.py+46−31
@@ -3,21 +3,25 @@ from urllib import urlencode
33from lxml import html
44from searx.engines.xpath import extract_text
55
6−url = 'https://api.duckduckgo.com/?{query}&format=json&pretty=0&no_redirect=1&d=1'
6+url = 'https://api.duckduckgo.com/'\
7+ + '?{query}&format=json&pretty=0&no_redirect=1&d=1'
8+
79
810def result_to_text(url, text, htmlResult):
911 # TODO : remove result ending with "Meaning" or "Category"
1012 dom = html.fromstring(htmlResult)
1113 a = dom.xpath('//a')
12− if len(a)>=1:
14+ if len(a) >= 1:
1315 return extract_text(a[0])
1416 else:
1517 return text
1618
19+
1720def html_to_text(htmlFragment):
1821 dom = html.fromstring(htmlFragment)
1922 return extract_text(dom)
2023
24+
2125def request(query, params):
2226 # TODO add kl={locale}
2327 params['url'] = url.format(query=urlencode({'q': query}))
@@ -38,16 +42,15 @@ def response(resp):
3842 # add answer if there is one
3943 answer = search_res.get('Answer', '')
4044 if answer != '':
41− results.append({ 'answer' : html_to_text(answer) })
45+ results.append({'answer': html_to_text(answer)})
4246
4347 # add infobox
4448 if 'Definition' in search_res:
45− content = content + search_res.get('Definition', '')
49+ content = content + search_res.get('Definition', '')
4650
4751 if 'Abstract' in search_res:
4852 content = content + search_res.get('Abstract', '')
4953
50−
5154 # image
5255 image = search_res.get('Image', '')
5356 image = None if image == '' else image
@@ -55,29 +58,35 @@ def response(resp):
5558 # attributes
5659 if 'Infobox' in search_res:
5760 infobox = search_res.get('Infobox', None)
58− if 'content' in infobox:
61+ if 'content' in infobox:
5962 for info in infobox.get('content'):
60− attributes.append({'label': info.get('label'), 'value': info.get('value')})
63+ attributes.append({'label': info.get('label'),
64+ 'value': info.get('value')})
6165
6266 # urls
6367 for ddg_result in search_res.get('Results', []):
6468 if 'FirstURL' in ddg_result:
6569 firstURL = ddg_result.get('FirstURL', '')
6670 text = ddg_result.get('Text', '')
67− urls.append({'title':text, 'url':firstURL})
68− results.append({'title':heading, 'url': firstURL})
71+ urls.append({'title': text, 'url': firstURL})
72+ results.append({'title': heading, 'url': firstURL})
6973
7074 # related topics
7175 for ddg_result in search_res.get('RelatedTopics', None):
7276 if 'FirstURL' in ddg_result:
73− suggestion = result_to_text(ddg_result.get('FirstURL', None), ddg_result.get('Text', None), ddg_result.get('Result', None))
77+ suggestion = result_to_text(ddg_result.get('FirstURL', None),
78+ ddg_result.get('Text', None),
79+ ddg_result.get('Result', None))
7480 if suggestion != heading:
7581 results.append({'suggestion': suggestion})
7682 elif 'Topics' in ddg_result:
7783 suggestions = []
78− relatedTopics.append({ 'name' : ddg_result.get('Name', ''), 'suggestions': suggestions })
84+ relatedTopics.append({'name': ddg_result.get('Name', ''),
85+ 'suggestions': suggestions})
7986 for topic_result in ddg_result.get('Topics', []):
80− suggestion = result_to_text(topic_result.get('FirstURL', None), topic_result.get('Text', None), topic_result.get('Result', None))
87+ suggestion = result_to_text(topic_result.get('FirstURL', None),
88+ topic_result.get('Text', None),
89+ topic_result.get('Result', None))
8190 if suggestion != heading:
8291 suggestions.append(suggestion)
8392
@@ -86,21 +95,26 @@ def response(resp):
8695 if abstractURL != '':
8796 # add as result ? problem always in english
8897 infobox_id = abstractURL
89− urls.append({'title': search_res.get('AbstractSource'), 'url': abstractURL})
98+ urls.append({'title': search_res.get('AbstractSource'),
99+ 'url': abstractURL})
90100
91101 # definition
92102 definitionURL = search_res.get('DefinitionURL', '')
93103 if definitionURL != '':
94104 # add as result ? as answer ? problem always in english
95105 infobox_id = definitionURL
96− urls.append({'title': search_res.get('DefinitionSource'), 'url': definitionURL})
106+ urls.append({'title': search_res.get('DefinitionSource'),
107+ 'url': definitionURL})
97108
98109 # entity
99110 entity = search_res.get('Entity', None)
100− # TODO continent / country / department / location / waterfall / mountain range : link to map search, get weather, near by locations
111+ # TODO continent / country / department / location / waterfall /
112+ # mountain range :
113+ # link to map search, get weather, near by locations
101114 # TODO musician : link to music search
102115 # TODO concert tour : ??
103− # TODO film / actor / television / media franchise : links to IMDB / rottentomatoes (or scrap result)
116+ # TODO film / actor / television / media franchise :
117+ # links to IMDB / rottentomatoes (or scrap result)
104118 # TODO music : link tu musicbrainz / last.fm
105119 # TODO book : ??
106120 # TODO artist / playwright : ??
@@ -114,24 +128,25 @@ def response(resp):
114128 # TODO programming language : ??
115129 # TODO file format : ??
116130
117− if len(heading)>0:
131+ if len(heading) > 0:
118132 # TODO get infobox.meta.value where .label='article_title'
119− if image==None and len(attributes)==0 and len(urls)==1 and len(relatedTopics)==0 and len(content)==0:
133+ if image is None and len(attributes) == 0 and len(urls) == 1 and\
134+ len(relatedTopics) == 0 and len(content) == 0:
120135 results.append({
121− 'url': urls[0]['url'],
122− 'title': heading,
123− 'content': content
124− })
136+ 'url': urls[0]['url'],
137+ 'title': heading,
138+ 'content': content
139+ })
125140 else:
126141 results.append({
127− 'infobox': heading,
128− 'id': infobox_id,
129− 'entity': entity,
130− 'content': content,
131− 'img_src' : image,
132− 'attributes': attributes,
133− 'urls': urls,
134− 'relatedTopics': relatedTopics
135− })
142+ 'infobox': heading,
143+ 'id': infobox_id,
144+ 'entity': entity,
145+ 'content': content,
146+ 'img_src': image,
147+ 'attributes': attributes,
148+ 'urls': urls,
149+ 'relatedTopics': relatedTopics
150+ })
136151
137152 return results
searx/engines/dummy.py+1−1
@@ -1,5 +1,5 @@
11## Dummy
2−#
2+#
33# @results empty array
44# @stable yes
55
searx/engines/faroo.py+6−5
@@ -1,8 +1,8 @@
11## Faroo (Web, News)
2−#
2+#
33# @website http://www.faroo.com
44# @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key
5−#
5+#
66# @using-api yes
77# @results JSON
88# @stable yes
@@ -24,9 +24,10 @@ api_key = None
2424url = 'http://www.faroo.com/'
2525search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}'
2626
27−search_category = {'general': 'web',
27+search_category = {'general': 'web',
2828 'news': 'news'}
2929
30+
3031# do search-request
3132def request(query, params):
3233 offset = (params['pageno']-1) * number_of_results + 1
@@ -48,7 +49,7 @@ def request(query, params):
4849 query=urlencode({'q': query}),
4950 language=language,
5051 categorie=categorie,
51− api_key=api_key )
52+ api_key=api_key)
5253
5354 # using searx User-Agent
5455 params['headers']['User-Agent'] = searx_useragent()
@@ -101,7 +102,7 @@ def response(resp):
101102 results.append({'template': 'images.html',
102103 'url': result['url'],
103104 'title': result['title'],
104− 'content': result['kwic'],
105+ 'content': result['kwic'],
105106 'img_src': result['iurl']})
106107
107108 # return results
searx/engines/generalfile.py+2−2
@@ -1,8 +1,8 @@
11## General Files (Files)
2−#
2+#
33# @website http://www.general-files.org
44# @provide-api no (nothing found)
5−#
5+#
66# @using-api no (because nothing found)
77# @results HTML (using search portal)
88# @stable no (HTML can change)
searx/engines/github.py+2−2
@@ -1,8 +1,8 @@
11## Github (It)
2−#
2+#
33# @website https://github.com/
44# @provide-api yes (https://developer.github.com/v3/)
5−#
5+#
66# @using-api yes
77# @results JSON
88# @stable yes (using api)
searx/engines/google_images.py+4−3
@@ -1,8 +1,9 @@
11## Google (Images)
2−#
2+#
33# @website https://www.google.com
4−# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated!
5−#
4+# @provide-api yes (https://developers.google.com/web-search/docs/),
5+# deprecated!
6+#
67# @using-api yes
78# @results JSON
89# @stable yes (but deprecated)
searx/engines/google_news.py+4−3
@@ -1,8 +1,9 @@
11## Google (News)
2−#
2+#
33# @website https://www.google.com
4−# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated!
5−#
4+# @provide-api yes (https://developers.google.com/web-search/docs/),
5+# deprecated!
6+#
67# @using-api yes
78# @results JSON
89# @stable yes (but deprecated)
searx/engines/openstreetmap.py+15−15
@@ -39,16 +39,16 @@ def response(resp):
3939 url = result_base_url.format(osm_type=osm_type,
4040 osm_id=r['osm_id'])
4141
42− osm = {'type':osm_type,
43− 'id':r['osm_id']}
42+ osm = {'type': osm_type,
43+ 'id': r['osm_id']}
4444
45− geojson = r.get('geojson')
45+ geojson = r.get('geojson')
4646
4747 # if no geojson is found and osm_type is a node, add geojson Point
4848 if not geojson and\
4949 osm_type == 'node':
50− geojson = {u'type':u'Point',
51− u'coordinates':[r['lon'],r['lat']]}
50+ geojson = {u'type': u'Point',
51+ u'coordinates': [r['lon'], r['lat']]}
5252
5353 address_raw = r.get('address')
5454 address = {}
@@ -59,20 +59,20 @@ def response(resp):
5959 r['class'] == 'tourism' or\
6060 r['class'] == 'leisure':
6161 if address_raw.get('address29'):
62− address = {'name':address_raw.get('address29')}
62+ address = {'name': address_raw.get('address29')}
6363 else:
64− address = {'name':address_raw.get(r['type'])}
64+ address = {'name': address_raw.get(r['type'])}
6565
6666 # add rest of adressdata, if something is already found
6767 if address.get('name'):
68− address.update({'house_number':address_raw.get('house_number'),
69− 'road':address_raw.get('road'),
70− 'locality':address_raw.get('city',
71− address_raw.get('town',
72− address_raw.get('village'))),
73− 'postcode':address_raw.get('postcode'),
74− 'country':address_raw.get('country'),
75− 'country_code':address_raw.get('country_code')})
68+ address.update({'house_number': address_raw.get('house_number'),
69+ 'road': address_raw.get('road'),
70+ 'locality': address_raw.get('city',
71+ address_raw.get('town',
72+ address_raw.get('village'))),
73+ 'postcode': address_raw.get('postcode'),
74+ 'country': address_raw.get('country'),
75+ 'country_code': address_raw.get('country_code')})
7676 else:
7777 address = None
7878
searx/engines/piratebay.py+3−3
@@ -1,8 +1,8 @@
11## Piratebay (Videos, Music, Files)
2−#
2+#
33# @website https://thepiratebay.se
44# @provide-api no (nothing found)
5−#
5+#
66# @using-api no
77# @results HTML (using search portal)
88# @stable yes (HTML can change)
@@ -23,7 +23,7 @@ url = 'https://thepiratebay.se/'
2323search_url = url + 'search/{search_term}/{pageno}/99/{search_type}'
2424
2525# piratebay specific type-definitions
26−search_types = {'files': '0',
26+search_types = {'files': '0',
2727 'music': '100',
2828 'videos': '200'}
2929
searx/engines/soundcloud.py+2−2
@@ -1,8 +1,8 @@
11## Soundcloud (Music)
2−#
2+#
33# @website https://soundcloud.com
44# @provide-api yes (https://developers.soundcloud.com/)
5−#
5+#
66# @using-api yes
77# @results JSON
88# @stable yes
searx/engines/stackoverflow.py+4−4
@@ -1,8 +1,8 @@
11## Stackoverflow (It)
2−#
2+#
33# @website https://stackoverflow.com/
44# @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)
5−#
5+#
66# @using-api no
77# @results HTML
88# @stable no (HTML can change)
@@ -50,8 +50,8 @@ def response(resp):
5050 content = escape(' '.join(result.xpath(content_xpath)))
5151
5252 # append result
53− results.append({'url': href,
54− 'title': title,
53+ results.append({'url': href,
54+ 'title': title,
5555 'content': content})
5656
5757 # return results
searx/engines/twitter.py+2−2
@@ -1,8 +1,8 @@
11## Twitter (Social media)
2−#
2+#
33# @website https://www.bing.com/news
44# @provide-api yes (https://dev.twitter.com/docs/using-search)
5−#
5+#
66# @using-api no
77# @results HTML (using search portal)
88# @stable no (HTML can change)
searx/engines/vimeo.py+7−5
@@ -1,8 +1,9 @@
11## Vimeo (Videos)
2−#
2+#
33# @website https://vimeo.com/
4−# @provide-api yes (http://developer.vimeo.com/api), they have a maximum count of queries/hour
5−#
4+# @provide-api yes (http://developer.vimeo.com/api),
5+# they have a maximum count of queries/hour
6+#
67# @using-api no (TODO, rewrite to api)
78# @results HTML (using search portal)
89# @stable no (HTML can change)
@@ -35,11 +36,12 @@ publishedDate_xpath = './/p[@class="meta"]//attribute::datetime'
3536
3637# do search-request
3738def request(query, params):
38− params['url'] = search_url.format(pageno=params['pageno'] ,
39+ params['url'] = search_url.format(pageno=params['pageno'],
3940 query=urlencode({'q': query}))
4041
4142 # TODO required?
42− params['cookies']['__utma'] = '00000000.000#0000000.0000000000.0000000000.0000000000.0'
43+ params['cookies']['__utma'] =\
44+ '00000000.000#0000000.0000000000.0000000000.0000000000.0'
4345
4446 return params
4547
searx/engines/wikidata.py+122−59
@@ -2,13 +2,25 @@ import json
22from requests import get
33from urllib import urlencode
44
5−resultCount=1
6−urlSearch = 'https://www.wikidata.org/w/api.php?action=query&list=search&format=json&srnamespace=0&srprop=sectiontitle&{query}'
7−urlDetail = 'https://www.wikidata.org/w/api.php?action=wbgetentities&format=json&props=labels%7Cinfo%7Csitelinks%7Csitelinks%2Furls%7Cdescriptions%7Cclaims&{query}'
8−urlMap = 'https://www.openstreetmap.org/?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
5+result_count = 1
6+wikidata_host = 'https://www.wikidata.org'
7+wikidata_api = wikidata_host + '/w/api.php'
8+url_search = wikidata_api \
9+ + '?action=query&list=search&format=json'\
10+ + '&srnamespace=0&srprop=sectiontitle&{query}'
11+url_detail = wikidata_api\
12+ + '?action=wbgetentities&format=json'\
13+ + '&props=labels%7Cinfo%7Csitelinks'\
14+ + '%7Csitelinks%2Furls%7Cdescriptions%7Cclaims'\
15+ + '&{query}'
16+url_map = 'https://www.openstreetmap.org/'\
17+ + '?lat={latitude}&lon={longitude}&zoom={zoom}&layers=M'
18+
919
1020def request(query, params):
11− params['url'] = urlSearch.format(query=urlencode({'srsearch': query, 'srlimit': resultCount}))
21+ params['url'] = url_search.format(
22+ query=urlencode({'srsearch': query,
23+ 'srlimit': result_count}))
1224 return params
1325
1426
@@ -23,7 +35,8 @@ def response(resp):
2335 language = resp.search_params['language'].split('_')[0]
2436 if language == 'all':
2537 language = 'en'
26− url = urlDetail.format(query=urlencode({'ids': '|'.join(wikidata_ids), 'languages': language + '|en'}))
38+ url = url_detail.format(query=urlencode({'ids': '|'.join(wikidata_ids),
39+ 'languages': language + '|en'}))
2740
2841 htmlresponse = get(url)
2942 jsonresponse = json.loads(htmlresponse.content)
@@ -32,6 +45,7 @@ def response(resp):
3245
3346 return results
3447
48+
3549def getDetail(jsonresponse, wikidata_id, language):
3650 results = []
3751 urls = []
@@ -40,60 +54,103 @@ def getDetail(jsonresponse, wikidata_id, language):
4054 result = jsonresponse.get('entities', {}).get(wikidata_id, {})
4155
4256 title = result.get('labels', {}).get(language, {}).get('value', None)
43− if title == None:
57+ if title is None:
4458 title = result.get('labels', {}).get('en', {}).get('value', None)
45− if title == None:
59+ if title is None:
4660 return results
4761
48− description = result.get('descriptions', {}).get(language, {}).get('value', None)
49− if description == None:
50− description = result.get('descriptions', {}).get('en', {}).get('value', '')
62+ description = result\
63+ .get('descriptions', {})\
64+ .get(language, {})\
65+ .get('value', None)
66+
67+ if description is None:
68+ description = result\
69+ .get('descriptions', {})\
70+ .get('en', {})\
71+ .get('value', '')
5172
5273 claims = result.get('claims', {})
5374 official_website = get_string(claims, 'P856', None)
54− if official_website != None:
55− urls.append({ 'title' : 'Official site', 'url': official_website })
56− results.append({ 'title': title, 'url' : official_website })
75+ if official_website is not None:
76+ urls.append({'title': 'Official site', 'url': official_website})
77+ results.append({'title': title, 'url': official_website})
5778
5879 wikipedia_link_count = 0
5980 if language != 'en':
60− wikipedia_link_count += add_url(urls, 'Wikipedia (' + language + ')', get_wikilink(result, language + 'wiki'))
81+ wikipedia_link_count += add_url(urls,
82+ 'Wikipedia (' + language + ')',
83+ get_wikilink(result, language +
84+ 'wiki'))
6185 wikipedia_en_link = get_wikilink(result, 'enwiki')
62− wikipedia_link_count += add_url(urls, 'Wikipedia (en)', wikipedia_en_link)
86+ wikipedia_link_count += add_url(urls,
87+ 'Wikipedia (en)',
88+ wikipedia_en_link)
6389 if wikipedia_link_count == 0:
6490 misc_language = get_wiki_firstlanguage(result, 'wiki')
65− if misc_language != None:
66− add_url(urls, 'Wikipedia (' + misc_language + ')', get_wikilink(result, misc_language + 'wiki'))
91+ if misc_language is not None:
92+ add_url(urls,
93+ 'Wikipedia (' + misc_language + ')',
94+ get_wikilink(result, misc_language + 'wiki'))
6795
6896 if language != 'en':
69− add_url(urls, 'Wiki voyage (' + language + ')', get_wikilink(result, language + 'wikivoyage'))
70− add_url(urls, 'Wiki voyage (en)', get_wikilink(result, 'enwikivoyage'))
97+ add_url(urls,
98+ 'Wiki voyage (' + language + ')',
99+ get_wikilink(result, language + 'wikivoyage'))
100+
101+ add_url(urls,
102+ 'Wiki voyage (en)',
103+ get_wikilink(result, 'enwikivoyage'))
71104
72105 if language != 'en':
73− add_url(urls, 'Wikiquote (' + language + ')', get_wikilink(result, language + 'wikiquote'))
74− add_url(urls, 'Wikiquote (en)', get_wikilink(result, 'enwikiquote'))
106+ add_url(urls,
107+ 'Wikiquote (' + language + ')',
108+ get_wikilink(result, language + 'wikiquote'))
75109
76− add_url(urls, 'Commons wiki', get_wikilink(result, 'commonswiki'))
110+ add_url(urls,
111+ 'Wikiquote (en)',
112+ get_wikilink(result, 'enwikiquote'))
77113
78− add_url(urls, 'Location', get_geolink(claims, 'P625', None))
114+ add_url(urls,
115+ 'Commons wiki',
116+ get_wikilink(result, 'commonswiki'))
79117
80− add_url(urls, 'Wikidata', 'https://www.wikidata.org/wiki/' + wikidata_id + '?uselang='+ language)
118+ add_url(urls,
119+ 'Location',
120+ get_geolink(claims, 'P625', None))
121+
122+ add_url(urls,
123+ 'Wikidata',
124+ 'https://www.wikidata.org/wiki/'
125+ + wikidata_id + '?uselang=' + language)
81126
82127 musicbrainz_work_id = get_string(claims, 'P435')
83− if musicbrainz_work_id != None:
84− add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/work/' + musicbrainz_work_id)
128+ if musicbrainz_work_id is not None:
129+ add_url(urls,
130+ 'MusicBrainz',
131+ 'http://musicbrainz.org/work/'
132+ + musicbrainz_work_id)
85133
86134 musicbrainz_artist_id = get_string(claims, 'P434')
87− if musicbrainz_artist_id != None:
88− add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/artist/' + musicbrainz_artist_id)
135+ if musicbrainz_artist_id is not None:
136+ add_url(urls,
137+ 'MusicBrainz',
138+ 'http://musicbrainz.org/artist/'
139+ + musicbrainz_artist_id)
89140
90141 musicbrainz_release_group_id = get_string(claims, 'P436')
91− if musicbrainz_release_group_id != None:
92− add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/release-group/' + musicbrainz_release_group_id)
142+ if musicbrainz_release_group_id is not None:
143+ add_url(urls,
144+ 'MusicBrainz',
145+ 'http://musicbrainz.org/release-group/'
146+ + musicbrainz_release_group_id)
93147
94148 musicbrainz_label_id = get_string(claims, 'P966')
95− if musicbrainz_label_id != None:
96− add_url(urls, 'MusicBrainz', 'http://musicbrainz.org/label/' + musicbrainz_label_id)
149+ if musicbrainz_label_id is not None:
150+ add_url(urls,
151+ 'MusicBrainz',
152+ 'http://musicbrainz.org/label/'
153+ + musicbrainz_label_id)
97154
98155 # musicbrainz_area_id = get_string(claims, 'P982')
99156 # P1407 MusicBrainz series ID
@@ -102,42 +159,43 @@ def getDetail(jsonresponse, wikidata_id, language):
102159 # P1407 MusicBrainz series ID
103160
104161 postal_code = get_string(claims, 'P281', None)
105− if postal_code != None:
106− attributes.append({'label' : 'Postal code(s)', 'value' : postal_code})
162+ if postal_code is not None:
163+ attributes.append({'label': 'Postal code(s)', 'value': postal_code})
107164
108165 date_of_birth = get_time(claims, 'P569', None)
109− if date_of_birth != None:
110− attributes.append({'label' : 'Date of birth', 'value' : date_of_birth})
166+ if date_of_birth is not None:
167+ attributes.append({'label': 'Date of birth', 'value': date_of_birth})
111168
112169 date_of_death = get_time(claims, 'P570', None)
113− if date_of_death != None:
114− attributes.append({'label' : 'Date of death', 'value' : date_of_death})
170+ if date_of_death is not None:
171+ attributes.append({'label': 'Date of death', 'value': date_of_death})
115172
116− if len(attributes)==0 and len(urls)==2 and len(description)==0:
173+ if len(attributes) == 0 and len(urls) == 2 and len(description) == 0:
117174 results.append({
118− 'url': urls[0]['url'],
119− 'title': title,
120− 'content': description
121− })
175+ 'url': urls[0]['url'],
176+ 'title': title,
177+ 'content': description
178+ })
122179 else:
123180 results.append({
124− 'infobox' : title,
125− 'id' : wikipedia_en_link,
126− 'content' : description,
127− 'attributes' : attributes,
128− 'urls' : urls
129− })
181+ 'infobox': title,
182+ 'id': wikipedia_en_link,
183+ 'content': description,
184+ 'attributes': attributes,
185+ 'urls': urls
186+ })
130187
131188 return results
132189
133190
134191def add_url(urls, title, url):
135− if url != None:
136− urls.append({'title' : title, 'url' : url})
192+ if url is not None:
193+ urls.append({'title': title, 'url': url})
137194 return 1
138195 else:
139196 return 0
140197
198+
141199def get_mainsnak(claims, propertyName):
142200 propValue = claims.get(propertyName, {})
143201 if len(propValue) == 0:
@@ -157,7 +215,7 @@ def get_string(claims, propertyName, defaultValue=None):
157215 mainsnak = e.get('mainsnak', {})
158216
159217 datavalue = mainsnak.get('datavalue', {})
160− if datavalue != None:
218+ if datavalue is not None:
161219 result.append(datavalue.get('value', ''))
162220
163221 if len(result) == 0:
@@ -177,7 +235,7 @@ def get_time(claims, propertyName, defaultValue=None):
177235 mainsnak = e.get('mainsnak', {})
178236
179237 datavalue = mainsnak.get('datavalue', {})
180− if datavalue != None:
238+ if datavalue is not None:
181239 value = datavalue.get('value', '')
182240 result.append(value.get('time', ''))
183241
@@ -190,7 +248,7 @@ def get_time(claims, propertyName, defaultValue=None):
190248def get_geolink(claims, propertyName, defaultValue=''):
191249 mainsnak = get_mainsnak(claims, propertyName)
192250
193− if mainsnak == None:
251+ if mainsnak is None:
194252 return defaultValue
195253
196254 datatype = mainsnak.get('datatype', '')
@@ -209,21 +267,25 @@ def get_geolink(claims, propertyName, defaultValue=''):
209267 # 1 --> 6
210268 # 0.016666666666667 --> 9
211269 # 0.00027777777777778 --> 19
212− # wolframalpha : quadratic fit { {13, 5}, {1, 6}, {0.0166666, 9}, {0.0002777777,19}}
270+ # wolframalpha :
271+ # quadratic fit { {13, 5}, {1, 6}, {0.0166666, 9}, {0.0002777777,19}}
213272 # 14.1186-8.8322 x+0.625447 x^2
214273 if precision < 0.0003:
215274 zoom = 19
216275 else:
217276 zoom = int(15 - precision*8.8322 + precision*precision*0.625447)
218277
219− url = urlMap.replace('{latitude}', str(value.get('latitude',0))).replace('{longitude}', str(value.get('longitude',0))).replace('{zoom}', str(zoom))
278+ url = url_map\
279+ .replace('{latitude}', str(value.get('latitude', 0)))\
280+ .replace('{longitude}', str(value.get('longitude', 0)))\
281+ .replace('{zoom}', str(zoom))
220282
221283 return url
222284
223285
224286def get_wikilink(result, wikiid):
225287 url = result.get('sitelinks', {}).get(wikiid, {}).get('url', None)
226− if url == None:
288+ if url is None:
227289 return url
228290 elif url.startswith('http://'):
229291 url = url.replace('http://', 'https://')
@@ -231,8 +293,9 @@ def get_wikilink(result, wikiid):
231293 url = 'https:' + url
232294 return url
233295
296+
234297def get_wiki_firstlanguage(result, wikipatternid):
235298 for k in result.get('sitelinks', {}).keys():
236− if k.endswith(wikipatternid) and len(k)==(2+len(wikipatternid)):
299+ if k.endswith(wikipatternid) and len(k) == (2+len(wikipatternid)):
237300 return k[0:2]
238301 return None
searx/engines/yacy.py+6−5
@@ -1,8 +1,9 @@
11## Yacy (Web, Images, Videos, Music, Files)
2−#
2+#
33# @website http://yacy.net
4−# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
5−#
4+# @provide-api yes
5+# (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)
6+#
67# @using-api yes
78# @results JSON
89# @stable yes
@@ -16,7 +17,7 @@ from urllib import urlencode
1617from dateutil import parser
1718
1819# engine dependent config
19−categories = ['general', 'images'] #TODO , 'music', 'videos', 'files'
20+categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'
2021paging = True
2122language_support = True
2223number_of_results = 5
@@ -28,7 +29,7 @@ search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limi
2829# yacy specific type-definitions
2930search_types = {'general': 'text',
3031 'images': 'image',
31− 'files': 'app',
32+ 'files': 'app',
3233 'music': 'audio',
3334 'videos': 'video'}
3435
searx/engines/yahoo.py+8−7
@@ -1,8 +1,9 @@
11## Yahoo (Web)
2−#
2+#
33# @website https://search.yahoo.com/web
4−# @provide-api yes (https://developer.yahoo.com/boss/search/), $0.80/1000 queries
5−#
4+# @provide-api yes (https://developer.yahoo.com/boss/search/),
5+# $0.80/1000 queries
6+#
67# @using-api no (because pricing)
78# @results HTML (using search portal)
89# @stable no (HTML can change)
@@ -40,8 +41,8 @@ def parse_url(url_string):
4041 if endpos > -1:
4142 endpositions.append(endpos)
4243
43− if start==0 or len(endpositions) == 0:
44− return url_string
44+ if start == 0 or len(endpositions) == 0:
45+ return url_string
4546 else:
4647 end = min(endpositions)
4748 return unquote(url_string[start:end])
@@ -84,8 +85,8 @@ def response(resp):
8485 content = extract_text(result.xpath(content_xpath)[0])
8586
8687 # append result
87− results.append({'url': url,
88− 'title': title,
88+ results.append({'url': url,
89+ 'title': title,
8990 'content': content})
9091
9192 # if no suggestion found, return results
searx/engines/youtube.py+2−2
@@ -1,8 +1,8 @@
11## Youtube (Videos)
2−#
2+#
33# @website https://www.youtube.com/
44# @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/)
5−#
5+#
66# @using-api yes
77# @results JSON
88# @stable yes