7c13d63 · searx · git.iko.soy

searx/engines/bing.py+8−7

@@ -1,8 +1,9 @@

11## Bing (Web)

2−#

2+#

33# @website https://www.bing.com

4−# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month

5−#

4+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),

5+# max. 5000 query/month

6+#

67# @using-api no (because of query limit)

78# @results HTML (using search portal)

89# @stable no (HTML can change)

@@ -58,8 +59,8 @@ def response(resp):

5859 content = escape(' '.join(result.xpath('.//p//text()')))

5960

6061 # append result

61− results.append({'url': url,

62− 'title': title,

62+ results.append({'url': url,

63+ 'title': title,

6364 'content': content})

6465

6566 # return results if something is found

@@ -74,8 +75,8 @@ def response(resp):

7475 content = escape(' '.join(result.xpath('.//p//text()')))

7576

7677 # append result

77− results.append({'url': url,

78− 'title': title,

78+ results.append({'url': url,

79+ 'title': title,

7980 'content': content})

8081

8182 # return results

searx/engines/bing_images.py+11−9

@@ -1,17 +1,19 @@

11## Bing (Images)

2−#

2+#

33# @website https://www.bing.com/images

4−# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month

5−#

4+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),

5+# max. 5000 query/month

6+#

67# @using-api no (because of query limit)

78# @results HTML (using search portal)

89# @stable no (HTML can change)

910# @parse url, title, img_src

1011#

11−# @todo currently there are up to 35 images receive per page, because bing does not parse count=10. limited response to 10 images

12+# @todo currently there are up to 35 images receive per page,

13+# because bing does not parse count=10.

14+# limited response to 10 images

1215

1316from urllib import urlencode

14−from cgi import escape

1517from lxml import html

1618from yaml import load

1719import re

@@ -51,15 +53,15 @@ def response(resp):

5153 dom = html.fromstring(resp.content)

5254

5355 # init regex for yaml-parsing

54− p = re.compile( '({|,)([a-z]+):(")')

56+ p = re.compile('({|,)([a-z]+):(")')

5557

5658 # parse results

5759 for result in dom.xpath('//div[@class="dg_u"]'):

5860 link = result.xpath('./a')[0]

5961

6062 # parse yaml-data (it is required to add a space, to make it parsable)

61− yaml_data = load(p.sub( r'\1\2: \3', link.attrib.get('m')))

62−

63+ yaml_data = load(p.sub(r'\1\2: \3', link.attrib.get('m')))

64+

6365 title = link.attrib.get('t1')

6466 #url = 'http://' + link.attrib.get('t3')

6567 url = yaml_data.get('surl')

@@ -69,7 +71,7 @@ def response(resp):

6971 results.append({'template': 'images.html',

7072 'url': url,

7173 'title': title,

72− 'content': '',

74+ 'content': '',

7375 'img_src': img_src})

7476

7577 # TODO stop parsing if 10 images are found

searx/engines/bing_news.py+10−9

@@ -1,8 +1,9 @@

11## Bing (News)

2−#

2+#

33# @website https://www.bing.com/news

4−# @provide-api yes (http://datamarket.azure.com/dataset/bing/search), max. 5000 query/month

5−#

4+# @provide-api yes (http://datamarket.azure.com/dataset/bing/search),

5+# max. 5000 query/month

6+#

67# @using-api no (because of query limit)

78# @results HTML (using search portal)

89# @stable no (HTML can change)

@@ -57,12 +58,12 @@ def response(resp):

5758 url = link.attrib.get('href')

5859 title = ' '.join(link.xpath('.//text()'))

5960 contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()')

60− if contentXPath != None:

61+ if contentXPath is not None:

6162 content = escape(' '.join(contentXPath))

62−

63+

6364 # parse publishedDate

6465 publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()')

65− if publishedDateXPath != None:

66+ if publishedDateXPath is not None:

6667 publishedDate = escape(' '.join(publishedDateXPath))

6768

6869 if re.match("^[0-9]+ minute(s|) ago$", publishedDate):

@@ -89,10 +90,10 @@ def response(resp):

8990 except TypeError:

9091 # FIXME

9192 publishedDate = datetime.now()

92−

93+

9394 # append result

94− results.append({'url': url,

95− 'title': title,

95+ results.append({'url': url,

96+ 'title': title,

9697 'publishedDate': publishedDate,

9798 'content': content})

9899

searx/engines/currency_convert.py+1−1

@@ -55,6 +55,6 @@ def response(resp):

5555 resp.search_params['to'].lower()

5656 )

5757

58− results.append({'answer' : answer, 'url': url})

58+ results.append({'answer': answer, 'url': url})

5959

6060 return results

searx/engines/dailymotion.py+2−3

@@ -1,8 +1,8 @@

11## Dailymotion (Videos)

2−#

2+#

33# @website https://www.dailymotion.com

44# @provide-api yes (http://www.dailymotion.com/developer)

5−#

5+#

66# @using-api yes

77# @results JSON

88# @stable yes

@@ -12,7 +12,6 @@

1212

1313from urllib import urlencode

1414from json import loads

15−from lxml import html

1615

1716# engine dependent config

1817categories = ['videos']

searx/engines/deviantart.py+2−2

@@ -1,8 +1,8 @@

11## Deviantart (Images)

2−#

2+#

33# @website https://www.deviantart.com/

44# @provide-api yes (https://www.deviantart.com/developers/) (RSS)

5−#

5+#

66# @using-api no (TODO, rewrite to api)

77# @results HTML

88# @stable no (HTML can change)

searx/engines/duckduckgo.py+7−5

@@ -1,15 +1,17 @@

11## DuckDuckGo (Web)

2−#

2+#

33# @website https://duckduckgo.com/

4−# @provide-api yes (https://duckduckgo.com/api), but not all results from search-site

5−#

4+# @provide-api yes (https://duckduckgo.com/api),

5+# but not all results from search-site

6+#

67# @using-api no

78# @results HTML (using search portal)

89# @stable no (HTML can change)

910# @parse url, title, content

1011#

1112# @todo rewrite to api

12−# @todo language support (the current used site does not support language-change)

13+# @todo language support

14+# (the current used site does not support language-change)

1315

1416from urllib import urlencode

1517from lxml.html import fromstring

@@ -37,7 +39,7 @@ def request(query, params):

3739 if params['language'] == 'all':

3840 locale = 'en-us'

3941 else:

40− locale = params['language'].replace('_','-').lower()

42+ locale = params['language'].replace('_', '-').lower()

4143

4244 params['url'] = url.format(

4345 query=urlencode({'q': query, 'kl': locale}),

searx/engines/dummy.py+1−1

@@ -1,5 +1,5 @@

11## Dummy

2−#

2+#

33# @results empty array

44# @stable yes

55

searx/engines/faroo.py+6−5

@@ -1,8 +1,8 @@

11## Faroo (Web, News)

2−#

2+#

33# @website http://www.faroo.com

44# @provide-api yes (http://www.faroo.com/hp/api/api.html), require API-key

5−#

5+#

66# @using-api yes

77# @results JSON

88# @stable yes

@@ -24,9 +24,10 @@ api_key = None

2424url = 'http://www.faroo.com/'

2525search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}'

2626

27−search_category = {'general': 'web',

27+search_category = {'general': 'web',

2828 'news': 'news'}

2929

30+

3031# do search-request

3132def request(query, params):

3233 offset = (params['pageno']-1) * number_of_results + 1

@@ -48,7 +49,7 @@ def request(query, params):

4849 query=urlencode({'q': query}),

4950 language=language,

5051 categorie=categorie,

51− api_key=api_key )

52+ api_key=api_key)

5253

5354 # using searx User-Agent

5455 params['headers']['User-Agent'] = searx_useragent()

@@ -101,7 +102,7 @@ def response(resp):

101102 results.append({'template': 'images.html',

102103 'url': result['url'],

103104 'title': result['title'],

104− 'content': result['kwic'],

105+ 'content': result['kwic'],

105106 'img_src': result['iurl']})

106107

107108 # return results

searx/engines/generalfile.py+2−2

@@ -1,8 +1,8 @@

11## General Files (Files)

2−#

2+#

33# @website http://www.general-files.org

44# @provide-api no (nothing found)

5−#

5+#

66# @using-api no (because nothing found)

77# @results HTML (using search portal)

88# @stable no (HTML can change)

searx/engines/github.py+2−2

@@ -1,8 +1,8 @@

11## Github (It)

2−#

2+#

33# @website https://github.com/

44# @provide-api yes (https://developer.github.com/v3/)

5−#

5+#

66# @using-api yes

77# @results JSON

88# @stable yes (using api)

searx/engines/google_images.py+4−3

@@ -1,8 +1,9 @@

11## Google (Images)

2−#

2+#

33# @website https://www.google.com

4−# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated!

5−#

4+# @provide-api yes (https://developers.google.com/web-search/docs/),

5+# deprecated!

6+#

67# @using-api yes

78# @results JSON

89# @stable yes (but deprecated)

searx/engines/google_news.py+4−3

@@ -1,8 +1,9 @@

11## Google (News)

2−#

2+#

33# @website https://www.google.com

4−# @provide-api yes (https://developers.google.com/web-search/docs/), deprecated!

5−#

4+# @provide-api yes (https://developers.google.com/web-search/docs/),

5+# deprecated!

6+#

67# @using-api yes

78# @results JSON

89# @stable yes (but deprecated)

searx/engines/openstreetmap.py+15−15

@@ -39,16 +39,16 @@ def response(resp):

3939 url = result_base_url.format(osm_type=osm_type,

4040 osm_id=r['osm_id'])

4141

42− osm = {'type':osm_type,

43− 'id':r['osm_id']}

42+ osm = {'type': osm_type,

43+ 'id': r['osm_id']}

4444

45− geojson = r.get('geojson')

45+ geojson = r.get('geojson')

4646

4747 # if no geojson is found and osm_type is a node, add geojson Point

4848 if not geojson and\

4949 osm_type == 'node':

50− geojson = {u'type':u'Point',

51− u'coordinates':[r['lon'],r['lat']]}

50+ geojson = {u'type': u'Point',

51+ u'coordinates': [r['lon'], r['lat']]}

5252

5353 address_raw = r.get('address')

5454 address = {}

@@ -59,20 +59,20 @@ def response(resp):

5959 r['class'] == 'tourism' or\

6060 r['class'] == 'leisure':

6161 if address_raw.get('address29'):

62− address = {'name':address_raw.get('address29')}

62+ address = {'name': address_raw.get('address29')}

6363 else:

64− address = {'name':address_raw.get(r['type'])}

64+ address = {'name': address_raw.get(r['type'])}

6565

6666 # add rest of adressdata, if something is already found

6767 if address.get('name'):

68− address.update({'house_number':address_raw.get('house_number'),

69− 'road':address_raw.get('road'),

70− 'locality':address_raw.get('city',

71− address_raw.get('town',

72− address_raw.get('village'))),

73− 'postcode':address_raw.get('postcode'),

74− 'country':address_raw.get('country'),

75− 'country_code':address_raw.get('country_code')})

68+ address.update({'house_number': address_raw.get('house_number'),

69+ 'road': address_raw.get('road'),

70+ 'locality': address_raw.get('city',

71+ address_raw.get('town',

72+ address_raw.get('village'))),

73+ 'postcode': address_raw.get('postcode'),

74+ 'country': address_raw.get('country'),

75+ 'country_code': address_raw.get('country_code')})

7676 else:

7777 address = None

7878

searx/engines/piratebay.py+3−3

@@ -1,8 +1,8 @@

11## Piratebay (Videos, Music, Files)

2−#

2+#

33# @website https://thepiratebay.se

44# @provide-api no (nothing found)

5−#

5+#

66# @using-api no

77# @results HTML (using search portal)

88# @stable yes (HTML can change)

@@ -23,7 +23,7 @@ url = 'https://thepiratebay.se/'

2323search_url = url + 'search/{search_term}/{pageno}/99/{search_type}'

2424

2525# piratebay specific type-definitions

26−search_types = {'files': '0',

26+search_types = {'files': '0',

2727 'music': '100',

2828 'videos': '200'}

2929

searx/engines/soundcloud.py+2−2

@@ -1,8 +1,8 @@

11## Soundcloud (Music)

2−#

2+#

33# @website https://soundcloud.com

44# @provide-api yes (https://developers.soundcloud.com/)

5−#

5+#

66# @using-api yes

77# @results JSON

88# @stable yes

searx/engines/stackoverflow.py+4−4

@@ -1,8 +1,8 @@

11## Stackoverflow (It)

2−#

2+#

33# @website https://stackoverflow.com/

44# @provide-api not clear (https://api.stackexchange.com/docs/advanced-search)

5−#

5+#

66# @using-api no

77# @results HTML

88# @stable no (HTML can change)

@@ -50,8 +50,8 @@ def response(resp):

5050 content = escape(' '.join(result.xpath(content_xpath)))

5151

5252 # append result

53− results.append({'url': href,

54− 'title': title,

53+ results.append({'url': href,

54+ 'title': title,

5555 'content': content})

5656

5757 # return results

searx/engines/twitter.py+2−2

@@ -1,8 +1,8 @@

11## Twitter (Social media)

2−#

2+#

33# @website https://www.bing.com/news

44# @provide-api yes (https://dev.twitter.com/docs/using-search)

5−#

5+#

66# @using-api no

77# @results HTML (using search portal)

88# @stable no (HTML can change)

searx/engines/vimeo.py+7−5

@@ -1,8 +1,9 @@

11## Vimeo (Videos)

2−#

2+#

33# @website https://vimeo.com/

4−# @provide-api yes (http://developer.vimeo.com/api), they have a maximum count of queries/hour

5−#

4+# @provide-api yes (http://developer.vimeo.com/api),

5+# they have a maximum count of queries/hour

6+#

67# @using-api no (TODO, rewrite to api)

78# @results HTML (using search portal)

89# @stable no (HTML can change)

@@ -35,11 +36,12 @@ publishedDate_xpath = './/p[@class="meta"]//attribute::datetime'

3536

3637# do search-request

3738def request(query, params):

38− params['url'] = search_url.format(pageno=params['pageno'] ,

39+ params['url'] = search_url.format(pageno=params['pageno'],

3940 query=urlencode({'q': query}))

4041

4142 # TODO required?

42− params['cookies']['__utma'] = '00000000.000#0000000.0000000000.0000000000.0000000000.0'

43+ params['cookies']['__utma'] =\

44+ '00000000.000#0000000.0000000000.0000000000.0000000000.0'

4345

4446 return params

4547

searx/engines/yacy.py+6−5

@@ -1,8 +1,9 @@

11## Yacy (Web, Images, Videos, Music, Files)

2−#

2+#

33# @website http://yacy.net

4−# @provide-api yes (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)

5−#

4+# @provide-api yes

5+# (http://www.yacy-websuche.de/wiki/index.php/Dev:APIyacysearch)

6+#

67# @using-api yes

78# @results JSON

89# @stable yes

@@ -16,7 +17,7 @@ from urllib import urlencode

1617from dateutil import parser

1718

1819# engine dependent config

19−categories = ['general', 'images'] #TODO , 'music', 'videos', 'files'

20+categories = ['general', 'images'] # TODO , 'music', 'videos', 'files'

2021paging = True

2122language_support = True

2223number_of_results = 5

@@ -28,7 +29,7 @@ search_url = '/yacysearch.json?{query}&startRecord={offset}&maximumRecords={limi

2829# yacy specific type-definitions

2930search_types = {'general': 'text',

3031 'images': 'image',

31− 'files': 'app',

32+ 'files': 'app',

3233 'music': 'audio',

3334 'videos': 'video'}

3435

searx/engines/yahoo.py+8−7

@@ -1,8 +1,9 @@

11## Yahoo (Web)

2−#

2+#

33# @website https://search.yahoo.com/web

4−# @provide-api yes (https://developer.yahoo.com/boss/search/), $0.80/1000 queries

5−#

4+# @provide-api yes (https://developer.yahoo.com/boss/search/),

5+# $0.80/1000 queries

6+#

67# @using-api no (because pricing)

78# @results HTML (using search portal)

89# @stable no (HTML can change)

@@ -40,8 +41,8 @@ def parse_url(url_string):

4041 if endpos > -1:

4142 endpositions.append(endpos)

4243

43− if start==0 or len(endpositions) == 0:

44− return url_string

44+ if start == 0 or len(endpositions) == 0:

45+ return url_string

4546 else:

4647 end = min(endpositions)

4748 return unquote(url_string[start:end])

@@ -84,8 +85,8 @@ def response(resp):

8485 content = extract_text(result.xpath(content_xpath)[0])

8586

8687 # append result

87− results.append({'url': url,

88− 'title': title,

88+ results.append({'url': url,

89+ 'title': title,

8990 'content': content})

9091

9192 # if no suggestion found, return results

searx/engines/youtube.py+2−2

@@ -1,8 +1,8 @@

11## Youtube (Videos)

2−#

2+#

33# @website https://www.youtube.com/

44# @provide-api yes (http://gdata-samples-youtube-search-py.appspot.com/)

5−#

5+#

66# @using-api yes

77# @results JSON

88# @stable yes