5d97705 · searx · git.iko.soy

searx/engines/500px.py+2−2

compare with master

unifiedsplit

@@ -35,9 +35,9 @@ def request(query, params):

35# get response from search-request

36def response(resp):

37 results = []

38−

38+

39 dom = html.fromstring(resp.text)

40−

40+

41 # parse results

42 for result in dom.xpath('//div[@class="photo"]'):

43 link = result.xpath('.//a')[0]

searx/engines/__init__.py+2−2

@@ -81,7 +81,7 @@ def load_engine(engine_data):

81 if engine_attr.startswith('_'):

82 continue

83 if getattr(engine, engine_attr) is None:

84− print('[E] Engine config error: Missing attribute "{0}.{1}"'\

84+ print('[E] Engine config error: Missing attribute "{0}.{1}"'

85 .format(engine.name, engine_attr))

86 sys.exit(1)

87

@@ -102,7 +102,7 @@ def load_engine(engine_data):

102 if engine.shortcut:

103 # TODO check duplications

104 if engine.shortcut in engine_shortcuts:

105− print('[E] Engine config error: ambigious shortcut: {0}'\

105+ print('[E] Engine config error: ambigious shortcut: {0}'

106 .format(engine.shortcut))

107 sys.exit(1)

108 engine_shortcuts[engine.shortcut] = engine.name

searx/engines/digg.py+2−1

@@ -52,7 +52,8 @@ def response(resp):

52 thumbnail = result.xpath('.//img')[0].attrib.get('src')

53 title = ''.join(result.xpath(title_xpath))

54 content = escape(''.join(result.xpath(content_xpath)))

55− publishedDate = parser.parse(result.xpath(pubdate_xpath)[0].attrib.get('datetime'))

55+ pubdate = result.xpath(pubdate_xpath)[0].attrib.get('datetime')

56+ publishedDate = parser.parse(pubdate)

56

57

57 # append result

58 # append result

58 results.append({'url': url,

59 results.append({'url': url,

searx/engines/flickr-noapi.py+9−3

@@ -53,7 +53,8 @@ def response(resp):

53

54 for photo in photos:

55

56− # In paged configuration, the first pages' photos are represented by a None object

56+ # In paged configuration, the first pages' photos

57+ # are represented by a None object

57 if photo is None:

58 if photo is None:

58 continue

59 continue

59

60

@@ -74,10 +75,15 @@ def response(resp):

74

75

75 title = photo['title']

76 title = photo['title']

76

77

77− content = '' + photo['owner']['username'] + ' '

78+ content = '' +\

79+ photo['owner']['username'] +\

80+ ' '

78

81

79 if 'description' in photo:

82 if 'description' in photo:

83+ content = content +\

80− content = content + '' + photo['description'] + ''

84+ '' +\

85+ photo['description'] +\

86+ ''

81

87

82 # append result

88 # append result

83 results.append({'url': url,

89 results.append({'url': url,

searx/engines/flickr.py+19−12

@@ -1,10 +1,10 @@

1#!/usr/bin/env python

2

3## Flickr (Images)

4−#

4+#

5# @website https://www.flickr.com

6−# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)

6+# @provide-api yes (https://secure.flickr.com/services/api/flickr.photos.search.html)

7−#

7+#

8# @using-api yes

9# @results JSON

10# @stable yes

@@ -18,16 +18,20 @@ categories = ['images']

18

19nb_per_page = 15

20paging = True

21−api_key= None

21+api_key = None

22

23

24−url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search&api_key={api_key}&{text}&sort=relevance&extras=description%2C+owner_name%2C+url_o%2C+url_z&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}'

24+url = 'https://api.flickr.com/services/rest/?method=flickr.photos.search' +\

25+ '&api_key={api_key}&{text}&sort=relevance' +\

26+ '&extras=description%2C+owner_name%2C+url_o%2C+url_z' +\

27+ '&per_page={nb_per_page}&format=json&nojsoncallback=1&page={page}'

25photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'

28photo_url = 'https://www.flickr.com/photos/{userid}/{photoid}'

26

29

27paging = True

30paging = True

28

31

32+

29def build_flickr_url(user_id, photo_id):

33def build_flickr_url(user_id, photo_id):

30− return photo_url.format(userid=user_id,photoid=photo_id)

34+ return photo_url.format(userid=user_id, photoid=photo_id)

31

35

32

36

33def request(query, params):

37def request(query, params):

@@ -40,7 +44,7 @@ def request(query, params):

40

44

41def response(resp):

45def response(resp):

42 results = []

46 results = []

43−

47+

44 search_results = loads(resp.text)

48 search_results = loads(resp.text)

45

49

46 # return empty array if there are no results

50 # return empty array if there are no results

@@ -64,11 +68,14 @@ def response(resp):

64 url = build_flickr_url(photo['owner'], photo['id'])

68 url = build_flickr_url(photo['owner'], photo['id'])

65

69

66 title = photo['title']

70 title = photo['title']

67−

71+

68− content = ''+ photo['ownername'] +' '

72+ content = '' +\

69−

73+ photo['ownername'] +\

74+ ' ' +\

75+ '' +\

70− content = content + '' + photo['description']['_content'] + ''

76+ photo['description']['_content'] +\

71−

77+ ''

78+

72 # append result

79 # append result

73 results.append({'url': url,

80 results.append({'url': url,

74 'title': title,

81 'title': title,

searx/engines/kickass.py+3−2

@@ -24,7 +24,7 @@ search_url = url + 'search/{search_term}/{pageno}/'

24

25# specific xpath variables

26magnet_xpath = './/a[@title="Torrent magnet link"]'

27−#content_xpath = './/font[@class="detDesc"]//text()'

27+content_xpath = './/span[@class="font11px lightgrey block"]'

28

29

30# do search-request

@@ -56,7 +56,8 @@ def response(resp):

56 link = result.xpath('.//a[@class="cellMainLink"]')[0]

57 href = urljoin(url, link.attrib['href'])

58 title = ' '.join(link.xpath('.//text()'))

59− content = escape(html.tostring(result.xpath('.//span[@class="font11px lightgrey block"]')[0], method="text"))

59+ content = escape(html.tostring(result.xpath(content_xpath)[0],

60+ method="text"))

60 seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]

61 seed = result.xpath('.//td[contains(@class, "green")]/text()')[0]

61 leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]

62 leech = result.xpath('.//td[contains(@class, "red")]/text()')[0]

62

63

searx/engines/searchcode_code.py+9−9

@@ -11,7 +11,6 @@

11from urllib import urlencode

12from json import loads

13import cgi

14−import re

15

14

16# engine dependent config

15# engine dependent config

17categories = ['it']

16categories = ['it']

@@ -33,7 +32,7 @@ def request(query, params):

33# get response from search-request

32# get response from search-request

34def response(resp):

33def response(resp):

35 results = []

34 results = []

36−

35+

37 search_results = loads(resp.text)

36 search_results = loads(resp.text)

38

37

39 # parse results

38 # parse results

@@ -41,21 +40,22 @@ def response(resp):

41 href = result['url']

40 href = result['url']

42 title = "" + result['name'] + " - " + result['filename']

41 title = "" + result['name'] + " - " + result['filename']

43 content = result['repo'] + " "

42 content = result['repo'] + " "

44−

43+

45 lines = dict()

44 lines = dict()

46 for line, code in result['lines'].items():

45 for line, code in result['lines'].items():

47 lines[int(line)] = code

46 lines[int(line)] = code

48

47

49 content = content + '<pre class="code-formatter"><table class="code">'

48 content = content + '<pre class="code-formatter"><table class="code">'

50 for line, code in sorted(lines.items()):

49 for line, code in sorted(lines.items()):

51− content = content + '<tr><td class="line-number" style="padding-right:5px;">'

50+ content = content + '<tr><td class="line-number" style="padding-right:5px;">'

52− content = content + str(line) + '</td><td class="code-snippet">'

51+ content = content + str(line) + '</td><td class="code-snippet">'

52+ # Replace every two spaces with ' &nbps;' to keep formatting

53− # Replace every two spaces with ' &nbps;' to keep formatting while allowing the browser to break the line if necessary

53+ # while allowing the browser to break the line if necessary

54− content = content + cgi.escape(code).replace('\t', ' ').replace(' ', '  ').replace(' ', '  ')

54+ content = content + cgi.escape(code).replace('\t', ' ').replace(' ', '  ').replace(' ', '  ')

55 content = content + "</td></tr>"

56−

56+

57 content = content + "</table></pre>"

58−

58+

59 # append result

60 results.append({'url': href,

61 'title': title,

searx/engines/searchcode_doc.py+11−4

@@ -31,15 +31,22 @@ def request(query, params):

31# get response from search-request

32def response(resp):

33 results = []

34−

34+

35 search_results = loads(resp.text)

36

37 # parse results

38 for result in search_results['results']:

39 href = result['url']

40− title = "[" + result['type'] + "] " + result['namespace'] + " " + result['name']

40+ title = "[" + result['type'] + "] " +\

41+ result['namespace'] +\

42+ " " + result['name']

41− content = '[' + result['type'] + "] " + result['name'] + " " + result['synopsis'] + " " + result['description']

43+ content = '[' +\

42−

44+ result['type'] + "] " +\

45+ result['name'] + " " +\

46+ result['synopsis'] +\

47+ " " +\

48+ result['description']

49+

43 # append result

50 # append result

44 results.append({'url': href,

51 results.append({'url': href,

45 'title': title,

52 'title': title,

searx/engines/subtitleseeker.py+6−2

@@ -60,10 +60,14 @@ def response(resp):

60

61 content = result.xpath('.//div[contains(@class,"red")]//text()')[0]

62 content = content + " - "

63− content = content + html.tostring(result.xpath('.//div[contains(@class,"grey-web")]')[0], method='text')

63+ text = result.xpath('.//div[contains(@class,"grey-web")]')[0]

64+ content = content + html.tostring(text, method='text')

64

65

65 if result.xpath(".//span") != []:

66 if result.xpath(".//span") != []:

67+ content = content +\

68+ " - (" +\

66− content = content + " - (" + result.xpath(".//span//text()")[0].strip() + ")"

69+ result.xpath(".//span//text()")[0].strip() +\

70+ ")"

67

71

68 # append result

72 # append result

69 results.append({'url': href,

73 results.append({'url': href,

searx/engines/twitter.py+4−3

@@ -28,7 +28,7 @@ search_url = base_url+'search?'

28results_xpath = '//li[@data-item-type="tweet"]'

29link_xpath = './/small[@class="time"]//a'

30title_xpath = './/span[@class="username js-action-profile-name"]//text()'

31−content_xpath = './/p[@class="js-tweet-text tweet-text"]//text()'

31+content_xpath = './/p[@class="js-tweet-text tweet-text"]'

32timestamp_xpath = './/span[contains(@class,"_timestamp")]'

33

34

@@ -54,10 +54,11 @@ def response(resp):

54 link = tweet.xpath(link_xpath)[0]

55 url = urljoin(base_url, link.attrib.get('href'))

56 title = ''.join(tweet.xpath(title_xpath))

57− content = escape(''.join(tweet.xpath(content_xpath)))

57+ content = escape(html.tostring(tweet.xpath(content_xpath)[0], method='text', encoding='UTF-8').decode("utf-8"))

58 pubdate = tweet.xpath(timestamp_xpath)

59 if len(pubdate) > 0:

60− publishedDate = datetime.fromtimestamp(float(pubdate[0].attrib.get('data-time')), None)

60+ timestamp = float(pubdate[0].attrib.get('data-time'))

61+ publishedDate = datetime.fromtimestamp(timestamp, None)

61 # append result

62 # append result

62 results.append({'url': url,

63 results.append({'url': url,

63 'title': title,

64 'title': title,

searx/https_rewrite.py−1

@@ -154,7 +154,6 @@ def load_https_rules(rules_path):

154 print(' * {n} https-rules loaded'.format(n=len(https_rules)))

155

156

157−

158def https_url_rewrite(result):

157def https_url_rewrite(result):

159 skip_https_rewrite = False

158 skip_https_rewrite = False

160 # check if HTTPS rewrite is possible

159 # check if HTTPS rewrite is possible

searx/search.py+10−4

@@ -69,11 +69,16 @@ def threaded_requests(requests):

69 print('engine timeout: {0}'.format(th._engine_name))

70

71

72−

73# get default reqest parameter

72# get default reqest parameter

74def default_request_params():

73def default_request_params():

75 return {

74 return {

75+ 'method': 'GET',

76+ 'headers': {},

77+ 'data': {},

78+ 'url': '',

79+ 'cookies': {},

76− 'method': 'GET', 'headers': {}, 'data': {}, 'url': '', 'cookies': {}, 'verify': True}

80+ 'verify': True

81+ }

77

82

78

83

79# create a callback wrapper for the search engine results

84# create a callback wrapper for the search engine results

@@ -487,14 +492,15 @@ class Search(object):

487 continue

492 continue

488

493

489 # append request to list

494 # append request to list

490− requests.append((req, request_params['url'], request_args, selected_engine['name']))

495+ requests.append((req, request_params['url'],

496+ request_args,

497+ selected_engine['name']))

491

498

492 if not requests:

499 if not requests:

493 return results, suggestions, answers, infoboxes

500 return results, suggestions, answers, infoboxes

494 # send all search-request

501 # send all search-request

495 threaded_requests(requests)

502 threaded_requests(requests)

496

503

497−

498 while not results_queue.empty():

504 while not results_queue.empty():

499 engine_name, engine_results = results_queue.get_nowait()

505 engine_name, engine_results = results_queue.get_nowait()

500

506

searx/utils.py+3−2

@@ -30,8 +30,9 @@ def gen_useragent():

30

31

32def searx_useragent():

33− return 'searx/{searx_version} {suffix}'.format(searx_version=VERSION_STRING,

33+ return 'searx/{searx_version} {suffix}'.format(

34+ searx_version=VERSION_STRING,

34− suffix=settings['server'].get('useragent_suffix', ''))

35+ suffix=settings['server'].get('useragent_suffix', ''))

35

36

37

37def highlight_content(content, query):

38def highlight_content(content, query):