a508d54 · searx · git.iko.soy

searx/engines/bing_news.py+8−3

compare with master

unifiedsplit

@@ -57,12 +57,16 @@ def response(resp):

57 link = result.xpath('.//div[@class="newstitle"]/a')[0]

58 url = link.attrib.get('href')

59 title = ' '.join(link.xpath('.//text()'))

60− contentXPath = result.xpath('.//div[@class="sn_txt"]/div//span[@class="sn_snip"]//text()')

60+ contentXPath = result.xpath('.//div[@class="sn_txt"]/div'

61+ '//span[@class="sn_snip"]//text()')

61 if contentXPath is not None:

62 if contentXPath is not None:

62 content = escape(' '.join(contentXPath))

63 content = escape(' '.join(contentXPath))

63

64

64 # parse publishedDate

65 # parse publishedDate

65− publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div//span[contains(@class,"sn_ST")]//span[contains(@class,"sn_tm")]//text()')

66+ publishedDateXPath = result.xpath('.//div[@class="sn_txt"]/div'

67+ '//span[contains(@class,"sn_ST")]'

68+ '//span[contains(@class,"sn_tm")]'

69+ '//text()')

66 if publishedDateXPath is not None:

70 if publishedDateXPath is not None:

67 publishedDate = escape(' '.join(publishedDateXPath))

71 publishedDate = escape(' '.join(publishedDateXPath))

68

72

@@ -74,7 +78,8 @@ def response(resp):

74 timeNumbers = re.findall(r'\d+', publishedDate)

78 timeNumbers = re.findall(r'\d+', publishedDate)

75 publishedDate = datetime.now()\

79 publishedDate = datetime.now()\

76 - timedelta(hours=int(timeNumbers[0]))

80 - timedelta(hours=int(timeNumbers[0]))

81+ elif re.match("^[0-9]+ hour(s|),"

77− elif re.match("^[0-9]+ hour(s|), [0-9]+ minute(s|) ago$", publishedDate):

82+ " [0-9]+ minute(s|) ago$", publishedDate):

78 timeNumbers = re.findall(r'\d+', publishedDate)

83 timeNumbers = re.findall(r'\d+', publishedDate)

79 publishedDate = datetime.now()\

84 publishedDate = datetime.now()\

80 - timedelta(hours=int(timeNumbers[0]))\

85 - timedelta(hours=int(timeNumbers[0]))\

searx/engines/faroo.py+11−4

@@ -22,10 +22,17 @@ api_key = None

22

23# search-url

24url = 'http://www.faroo.com/'

25−search_url = url + 'api?{query}&start={offset}&length={number_of_results}&l={language}&src={categorie}&i=false&f=json&key={api_key}'

25+search_url = url + 'api?{query}'\

26+ '&start={offset}'\

27+ '&length={number_of_results}'\

28+ '&l={language}'\

29+ '&src={categorie}'\

30+ '&i=false'\

31+ '&f=json'\

32+ '&key={api_key}' # noqa

26

33

27search_category = {'general': 'web',

34search_category = {'general': 'web',

28− 'news': 'news'}

35+ 'news': 'news'}

29

36

30

37

31# do search-request

38# do search-request

@@ -80,8 +87,8 @@ def response(resp):

80 # parse results

87 # parse results

81 for result in search_res['results']:

88 for result in search_res['results']:

82 if result['news']:

89 if result['news']:

83− # timestamp (how many milliseconds have passed between now and the beginning of 1970)

90+ # timestamp (milliseconds since 1970)

84− publishedDate = datetime.datetime.fromtimestamp(result['date']/1000.0)

91+ publishedDate = datetime.datetime.fromtimestamp(result['date']/1000.0) # noqa

85

92

86 # append news result

93 # append news result

87 results.append({'url': result['url'],

94 results.append({'url': result['url'],

searx/engines/google_images.py+1−1

@@ -9,7 +9,7 @@

9# @stable yes (but deprecated)

10# @parse url, title, img_src

11

12−from urllib import urlencode,unquote

12+from urllib import urlencode, unquote

13from json import loads

14

15# engine dependent config

searx/engines/kickass.py+4−4

@@ -1,8 +1,8 @@

1## Kickass Torrent (Videos, Music, Files)

2−#

2+#

3# @website https://kickass.so

4# @provide-api no (nothing found)

5−#

5+#

6# @using-api no

7# @results HTML (using search portal)

8# @stable yes (HTML can change)

@@ -13,7 +13,6 @@ from cgi import escape

13from urllib import quote

14from lxml import html

15from operator import itemgetter

16−from dateutil import parser

17

16

18# engine dependent config

17# engine dependent config

19categories = ['videos', 'music', 'files']

18categories = ['videos', 'music', 'files']

@@ -33,7 +32,8 @@ def request(query, params):

33 params['url'] = search_url.format(search_term=quote(query),

32 params['url'] = search_url.format(search_term=quote(query),

34 pageno=params['pageno'])

33 pageno=params['pageno'])

35

34

35+ # FIX: SSLError: hostname 'kickass.so'

36− # FIX: SSLError: hostname 'kickass.so' doesn't match either of '*.kickass.to', 'kickass.to'

36+ # doesn't match either of '*.kickass.to', 'kickass.to'

37 params['verify'] = False

38

39 return params

searx/engines/soundcloud.py+6−1

@@ -20,7 +20,12 @@ guest_client_id = 'b45b1aa10f1ac2941910a7f0d10f8e28'

20

21# search-url

22url = 'https://api.soundcloud.com/'

23−search_url = url + 'search?{query}&facet=model&limit=20&offset={offset}&linked_partitioning=1&client_id={client_id}'

23+search_url = url + 'search?{query}'\

24+ '&facet=model'\

25+ '&limit=20'\

26+ '&offset={offset}'\

27+ '&linked_partitioning=1'\

28+ '&client_id={client_id}' # noqa

24

29

25

30

26# do search-request

31# do search-request

searx/engines/yahoo.py+5−4

@@ -20,7 +20,8 @@ paging = True

20language_support = True

21

22# search-url

23+base_url = 'https://search.yahoo.com/'

23−search_url = 'https://search.yahoo.com/search?{query}&b={offset}&fl=1&vl=lang_{lang}'

24+search_url = 'search?{query}&b={offset}&fl=1&vl=lang_{lang}'

24

25

25# specific xpath variables

26# specific xpath variables

26results_xpath = '//div[@class="res"]'

27results_xpath = '//div[@class="res"]'

@@ -57,9 +58,9 @@ def request(query, params):

57 else:

58 else:

58 language = params['language'].split('_')[0]

59 language = params['language'].split('_')[0]

59

60

60− params['url'] = search_url.format(offset=offset,

61+ params['url'] = base_url + search_url.format(offset=offset,

61− query=urlencode({'p': query}),

62+ query=urlencode({'p': query}),

62− lang=language)

63+ lang=language)

63

64

64 # TODO required?

65 # TODO required?

65 params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\

66 params['cookies']['sB'] = 'fl=1&vl=lang_{lang}&sh=1&rw=new&v=1'\