469e088 · searx · git.iko.soy

searx/engines/500px.py+2−2

compare with master

unifiedsplit

@@ -35,9 +35,9 @@ def request(query, params):

35# get response from search-request

36def response(resp):

37 results = []

38−

38+

39 dom = html.fromstring(resp.text)

40−

40+

41 # parse results

42 for result in dom.xpath('//div[@class="photo"]'):

43 link = result.xpath('.//a')[0]

searx/engines/__init__.py+2−2

@@ -81,7 +81,7 @@ def load_engine(engine_data):

81 if engine_attr.startswith('_'):

82 continue

83 if getattr(engine, engine_attr) is None:

84− print('[E] Engine config error: Missing attribute "{0}.{1}"'\

84+ print('[E] Engine config error: Missing attribute "{0}.{1}"'

85 .format(engine.name, engine_attr))

86 sys.exit(1)

87

@@ -102,7 +102,7 @@ def load_engine(engine_data):

102 if engine.shortcut:

103 # TODO check duplications

104 if engine.shortcut in engine_shortcuts:

105− print('[E] Engine config error: ambigious shortcut: {0}'\

105+ print('[E] Engine config error: ambigious shortcut: {0}'

106 .format(engine.shortcut))

107 sys.exit(1)

108 engine_shortcuts[engine.shortcut] = engine.name

searx/engines/digg.pynew file+67

@@ -0,0 +1,67 @@

1+## Digg (News, Social media)

2+#

3+# @website https://digg.com/

4+# @provide-api no

5+#

6+# @using-api no

7+# @results HTML (using search portal)

8+# @stable no (HTML can change)

9+# @parse url, title, content, publishedDate, thumbnail

10+

11+from urllib import quote_plus

12+from json import loads

13+from lxml import html

14+from cgi import escape

15+from dateutil import parser

16+

17+# engine dependent config

18+categories = ['news', 'social media']

19+paging = True

20+

21+# search-url

22+base_url = 'https://digg.com/'

23+search_url = base_url+'api/search/{query}.json?position={position}&format=html'

24+

25+# specific xpath variables

26+results_xpath = '//article'

27+link_xpath = './/small[@class="time"]//a'

28+title_xpath = './/h2//a//text()'

29+content_xpath = './/p//text()'

30+pubdate_xpath = './/time'

31+

32+

33+# do search-request

34+def request(query, params):

35+ offset = (params['pageno'] - 1) * 10

36+ params['url'] = search_url.format(position=offset,

37+ query=quote_plus(query))

38+ return params

39+

40+

41+# get response from search-request

42+def response(resp):

43+ results = []

44+

45+ search_result = loads(resp.text)

46+

47+ dom = html.fromstring(search_result['html'])

48+

49+ # parse results

50+ for result in dom.xpath(results_xpath):

51+ url = result.attrib.get('data-contenturl')

52+ thumbnail = result.xpath('.//img')[0].attrib.get('src')

53+ title = ''.join(result.xpath(title_xpath))

54+ content = escape(''.join(result.xpath(content_xpath)))

55+ pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')

56+ publishedDate = parser.parse(pubdate)

57+

58+ # append result

59+ results.append({'url': url,

60+ 'title': title,

61+ 'content': content,

62+ 'template': 'videos.html',

63+ 'publishedDate': publishedDate,

64+ 'thumbnail': thumbnail})

65+

66+ # return results

67+ return results

searx/engines/flickr-noapi.py+9−3

@@ -53,7 +53,8 @@ def response(resp):

53

54 for photo in photos:

55

56− # In paged configuration, the first pages' photos are represented by a None object

56+ # In paged configuration, the first pages' photos

57+ # are represented by a None object

57 if photo is None:

58 if photo is None:

58 continue

59 continue

59

60

@@ -74,10 +75,15 @@ def response(resp):

74

75

75 title = photo['title']

76 title = photo['title']

76

77

77− content = '' + photo['owner']['username'] + ' '

78+ content = '' +\

79+ photo['owner']['username'] +\

80+ ' '

78

81

79 if 'description' in photo:

82 if 'description' in photo:

83+ content = content +\

80− content = content + '' + photo['description'] + ''

84+ '' +\

85+ photo['description'] +\

86+ ''

81

87

82 # append result

88 # append result

83 results.append({'url': url,

89 results.append({'url': url,

searx/engines/flickr.py+19−12

@@ -1,10 +1,10 @@

1#!/usr/bin/env python

2

3## Flickr (Images)

4−#

4+#

5# @website https://www.flickr.com

6−# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)

6+# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)

7−#

7+#

8# @using-api yes

9# @results JSON

10# @stable yes

@@ -18,16 +18,20 @@ categories = ['images']

18

19nb_per_page = 15

20paging = True

21−api_key= None

21+api_key = None

22

23

24−url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key={api_key}&{text}&sort=relevance&extras=description%2C+owner_name%2C+url_o%2C+url_z&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}'

24+url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search' +\

25+ '&api_key={api_key}&{text}&sort=relevance' +\

26+ '&extras=description%2C+owner_name%2C+url_o%2C+url_z' +\

27+ '&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}'

25photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'

28photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'

26

29

27paging = True

30paging = True

28

31

32+

29def build_flickr_url(user_id, photo_id):

33def build_flickr_url(user_id, photo_id):

30− return photo_url.format(userid=user_id,photoid=photo_id)

34+ return photo_url.format(userid=user_id, photoid=photo_id)

31

35

32

36

33def request(query, params):

37def request(query, params):

@@ -40,7 +44,7 @@ def request(query, params):

40

44

41def response(resp):

45def response(resp):

42 results = []

46 results = []

43−

47+

44 search_results = loads(resp.text)

48 search_results = loads(resp.text)

45

49

46 # return empty array if there are no results

50 # return empty array if there are no results

@@ -64,11 +68,14 @@ def response(resp):

64 url = build_flickr_url(photo['owner'], photo['id'])

68 url = build_flickr_url(photo['owner'], photo['id'])

65

69

66 title = photo['title']

70 title = photo['title']

67−

71+

68− content = ''+ photo['ownername'] +' '

72+ content = '' +\

69−

73+ photo['ownername'] +\

74+ ' ' +\

75+ '' +\

70− content = content + '' + photo['description']['_content'] + ''

76+ photo['description']['_content'] +\

71−

77+ ''

78+

72 # append result

79 # append result

73 results.append({'url': url,

80 results.append({'url': url,

74 'title': title,

81 'title': title,

searx/engines/kickass.py+3−2

@@ -24,7 +24,7 @@ search_url = url + 'search/{search_term}/{pageno}/'

24

25# specific xpath variables

26magnet_xpath = './/a[@title="Torrent magnet link"]'

27−#content_xpath = './/font[@class="detDesc"]//text()'

27+content_xpath = './/span[@class="font11px lightgrey block"]'

28

29

30# do search-request

@@ -56,7 +56,8 @@ def response(resp):

56 link = result.xpath('.//a[@class="cellMainLink"]')[0]

57 href = urljoin(url, link.attrib['href'])

58 title = ' '.join(link.xpath('.//text()'))

59− content = escape(html.tostring(result.xpath('.//span[@class="font11px lightgrey block"]')[0], method="text"))

59+ content = escape(html.tostring(result.xpath(content_xpath)[0],

60+ method="text"))

60 seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]

61 seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]

61 leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]

62 leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]

62

63

searx/engines/searchcode_code.py+9−9

@@ -11,7 +11,6 @@

11from urllib import urlencode

12from json import loads

13import cgi

14−import re

15

14

16# engine dependent config

15# engine dependent config

17categories = ['it']

16categories = ['it']

@@ -33,7 +32,7 @@ def request(query, params):

33# get response from search-request

32# get response from search-request

34def response(resp):

33def response(resp):

35 results = []

34 results = []

36−

35+

37 search_results = loads(resp.text)

36 search_results = loads(resp.text)

38

37

39 # parse results

38 # parse results

@@ -41,21 +40,22 @@ def response(resp):

41 href = result['url']

40 href = result['url']

42 title = "" + result['name'] + " - " + result['filename']

41 title = "" + result['name'] + " - " + result['filename']

43 content = result['repo'] + " "

42 content = result['repo'] + " "

44−

43+

45 lines = dict()

44 lines = dict()

46 for line, code in result['lines'].items():

45 for line, code in result['lines'].items():

47 lines[int(line)] = code

46 lines[int(line)] = code

48

47

49 content = content + '<pre class="code-formatter"><table class="code">'

48 content = content + '<pre class="code-formatter"><table class="code">'

50 for line, code in sorted(lines.items()):

49 for line, code in sorted(lines.items()):

51− content = content + '<tr><td class="line-number" style="padding-right:5px;">'

50+ content = content + '<tr><td class="line-number" style="padding-right:5px;">'

52− content = content + str(line) + '</td><td class="code-snippet">'

51+ content = content + str(line) + '</td><td class="code-snippet">'

52+ # Replace every two spaces with ' &nbps;' to keep formatting

53− # Replace every two spaces with ' &nbps;' to keep formatting while allowing the browser to break the line if necessary

53+ # while allowing the browser to break the line if necessary

54− content = content + cgi.escape(code).replace('\t', ' ').replace(' ', '  ').replace(' ', '  ')

54+ content = content + cgi.escape(code).replace('\t', ' ').replace(' ', '  ').replace(' ', '  ')

55 content = content + "</td></tr>"

56−

56+

57 content = content + "</table></pre>"

58−

58+

59 # append result

60 results.append({'url': href,

61 'title': title,

searx/engines/searchcode_doc.py+11−4

@@ -31,15 +31,22 @@ def request(query, params):

31# get response from search-request

32def response(resp):

33 results = []

34−

34+

35 search_results = loads(resp.text)

36

37 # parse results

38 for result in search_results['results']:

39 href = result['url']

40− title = "[" + result['type'] + "] " + result['namespace'] + " " + result['name']

40+ title = "[" + result['type'] + "] " +\

41+ result['namespace'] +\

42+ " " + result['name']

41− content = '[' + result['type'] + "] " + result['name'] + " " + result['synopsis'] + " " + result['description']

43+ content = '[' +\

42−

44+ result['type'] + "] " +\

45+ result['name'] + " " +\

46+ result['synopsis'] +\

47+ " " +\

48+ result['description']

49+

43 # append result

50 # append result

44 results.append({'url': href,

51 results.append({'url': href,

45 'title': title,

52 'title': title,

searx/engines/subtitleseeker.py+6−2

@@ -60,10 +60,14 @@ def response(resp):

60

61 content = result.xpath('.//div[contains(@class,"red")]//text()')[0]

62 content = content + " - "

63− content = content + html.tostring(result.xpath('.//div[contains(@class,"grey-web")]')[0], method='text')

63+ text = result.xpath('.//div[contains(@class,"grey-web")]')[0]

64+ content = content + html.tostring(text, method='text')

64

65

65 if result.xpath(".//span") != []:

66 if result.xpath(".//span") != []:

67+ content = content +\

68+ " - (" +\

66− content = content + " - (" + result.xpath(".//span//text()")[0].strip() + ")"

69+ result.xpath(".//span//text()")[0].strip() +\

70+ ")"

67

71

68 # append result

72 # append result

69 results.append({'url': href,

73 results.append({'url': href,

searx/engines/twitter.py+19−8

@@ -1,6 +1,6 @@

1## Twitter (Social media)

2#

3−# @website https://www.bing.com/news

3+# @website https://twitter.com/

4# @provide-api yes (https://dev.twitter.com/docs/using-search)

5#

6# @using-api no

@@ -14,6 +14,7 @@ from urlparse import urljoin

14from urllib import urlencode

15from lxml import html

16from cgi import escape

17+from datetime import datetime

17

18

18# engine dependent config

19# engine dependent config

19categories = ['social media']

20categories = ['social media']

@@ -27,7 +28,8 @@ search_url = base_url+'search?'

27results_xpath = '//li[@data-item-type="tweet"]'

28results_xpath = '//li[@data-item-type="tweet"]'

28link_xpath = './/small[@class="time"]//a'

29link_xpath = './/small[@class="time"]//a'

29title_xpath = './/span[@class="username js-action-profile-name"]//text()'

30title_xpath = './/span[@class="username js-action-profile-name"]//text()'

30−content_xpath = './/p[@class="js-tweet-text tweet-text"]//text()'

31+content_xpath = './/p[@class="js-tweet-text tweet-text"]'

32+timestamp_xpath = './/span[contains(@class,"_timestamp")]'

31

33

32

34

33# do search-request

35# do search-request

@@ -52,12 +54,21 @@ def response(resp):

52 link = tweet.xpath(link_xpath)[0]

54 link = tweet.xpath(link_xpath)[0]

53 url = urljoin(base_url, link.attrib.get('href'))

55 url = urljoin(base_url, link.attrib.get('href'))

54 title = ''.join(tweet.xpath(title_xpath))

56 title = ''.join(tweet.xpath(title_xpath))

55− content = escape(''.join(tweet.xpath(content_xpath)))

57+ content = escape(html.tostring(tweet.xpath(content_xpath)[0], method='text', encoding='UTF-8').decode("utf-8"))

56−

58+ pubdate = tweet.xpath(timestamp_xpath)

59+ if len(pubdate) > 0:

60+ timestamp = float(pubdate[0].attrib.get('data-time'))

61+ publishedDate = datetime.fromtimestamp(timestamp, None)

57− # append result

62+ # append result

63+ results.append({'url': url,

64+ 'title': title,

65+ 'content': content,

66+ 'publishedDate': publishedDate})

67+ else:

68+ # append result

58− results.append({'url': url,

69+ results.append({'url': url,

59− 'title': title,

70+ 'title': title,

60− 'content': content})

71+ 'content': content})

61

72

62 # return results

73 # return results

63 return results

74 return results

searx/https_rewrite.py−1

@@ -154,7 +154,6 @@ def load_https_rules(rules_path):

154 print(' * {n} https-rules loaded'.format(n=len(https_rules)))

155

156

157−

158def https_url_rewrite(result):

157def https_url_rewrite(result):

159 skip_https_rewrite = False

158 skip_https_rewrite = False

160 # check if HTTPS rewrite is possible

159 # check if HTTPS rewrite is possible

searx/search.py+10−4

@@ -69,11 +69,16 @@ def threaded_requests(requests):

69 print('engine timeout: {0}'.format(th._engine_name))

70

71

72−

73# get default reqest parameter

72# get default reqest parameter

74def default_request_params():

73def default_request_params():

75 return {

74 return {

75+ 'method': 'GET',

76+ 'headers': {},

77+ 'data': {},

78+ 'url': '',

79+ 'cookies': {},

76− 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}, 'verify': True}

80+ 'verify': True

81+ }

77

82

78

83

79# create a callback wrapper for the search engine results

84# create a callback wrapper for the search engine results

@@ -487,14 +492,15 @@ class Search(object):

487 continue

492 continue

488

493

489 # append request to list

494 # append request to list

490− requests.append((req, request_params['url'], request_args, selected_engine['name']))

495+ requests.append((req, request_params['url'],

496+ request_args,

497+ selected_engine['name']))

491

498

492 if not requests:

499 if not requests:

493 return results, suggestions, answers, infoboxes

500 return results, suggestions, answers, infoboxes

494 # send all search-request

501 # send all search-request

495 threaded_requests(requests)

502 threaded_requests(requests)

496

503

497−

498 while not results_queue.empty():

504 while not results_queue.empty():

499 engine_name, engine_results = results_queue.get_nowait()

505 engine_name, engine_results = results_queue.get_nowait()

500

506

searx/settings.yml+31

@@ -44,6 +44,10 @@ engines:

44 - name : ddg definitions

45 engine : duckduckgo_definitions

46 shortcut : ddd

47+

48+ - name : digg

49+ engine : digg

50+ shortcut : dg

47

51

48 - name : wikidata

52 - name : wikidata

49 engine : wikidata

53 engine : wikidata

@@ -99,6 +103,33 @@ engines:

99 engine : google_news

103 engine : google_news

100 shortcut : gon

104 shortcut : gon

101

105

106+ - name : google play apps

107+ engine : xpath

108+ search_url : https://play.google.com/store/search?q={query}&c=apps

109+ url_xpath : //a[@class="title"]/@href

110+ title_xpath : //a[@class="title"]

111+ content_xpath : //a[@class="subtitle"]

112+ categories : files

113+ shortcut : gpa

114+

115+ - name : google play movies

116+ engine : xpath

117+ search_url : https://play.google.com/store/search?q={query}&c=movies

118+ url_xpath : //a[@class="title"]/@href

119+ title_xpath : //a[@class="title"]

120+ content_xpath : //a[@class="subtitle"]

121+ categories : videos

122+ shortcut : gpm

123+

124+ - name : google play music

125+ engine : xpath

126+ search_url : https://play.google.com/store/search?q={query}&c=music

127+ url_xpath : //a[@class="title"]/@href

128+ title_xpath : //a[@class="title"]

129+ content_xpath : //a[@class="subtitle"]

130+ categories : music

131+ shortcut : gps

132+

102 - name : openstreetmap

133 - name : openstreetmap

103 engine : openstreetmap

134 engine : openstreetmap

104 shortcut : osm

135 shortcut : osm

searx/utils.py+3−2

@@ -30,8 +30,9 @@ def gen_useragent():

30

31

32def searx_useragent():

33− return 'searx/{searx_version} {suffix}'.format(searx_version=VERSION_STRING,

33+ return 'searx/{searx_version} {suffix}'.format(

34+ searx_version=VERSION_STRING,

34− suffix=settings['server'].get('useragent_suffix', ''))

35+ suffix=settings['server'].get('useragent_suffix', ''))

35

36

37

37def highlight_content(content, query):

38def highlight_content(content, query):