489bf0f · searx · git.iko.soy

git.iko.soy/searx

Merge pull request #1009 from misnyo/torrentz

[fix]torrentz search engine fixed for new version

commit489bf0f6fd1b153d20aad2ca34706cc214d91558 authorAdam Tauber <asciimoo@gmail.com> dateWed 30 Aug 2017, 21:08 (9 years ago) parent700baf2 bf3f9a9

3 files changed+42−34

compare with master

unifiedsplit

searx/engines/torrentz.py+22−16 searx/settings.yml+6 tests/unit/engines/test_torrentz.py+14−18

searx/engines/torrentz.py+22−16

@@ -1,7 +1,7 @@

11"""

2− Torrentz.eu (BitTorrent meta-search engine)

2+ Torrentz2.eu (BitTorrent meta-search engine)

4− @website https://torrentz.eu/

4+ @website https://torrentz2.eu/

55 @provide-api no

77 @using-api no

@@ -14,24 +14,24 @@

1414import re

1515from lxml import html

1616from datetime import datetime

17−from searx.engines.nyaa import int_or_zero, get_filesize_mul

1817from searx.engines.xpath import extract_text

1918from searx.url_utils import urlencode

19+from searx.utils import get_torrent_size

2020

2121# engine dependent config

2222categories = ['files', 'videos', 'music']

2323paging = True

2424

2525# search-url

26−# https://torrentz.eu/search?f=EXAMPLE&p=6

27−base_url = 'https://torrentz.eu/'

26+# https://torrentz2.eu/search?f=EXAMPLE&p=6

27+base_url = 'https://torrentz2.eu/'

2828search_url = base_url + 'search?{query}'

2929

3030

3131# do search-request

3232def request(query, params):

3333 page = params['pageno'] - 1

34− query = urlencode({'q': query, 'p': page})

34+ query = urlencode({'f': query, 'p': page})

3535 params['url'] = search_url.format(query=query)

3636 return params

3737

@@ -54,22 +54,29 @@ def response(resp):

5454 # extract url and remove a slash in the beginning

5555 link = links[0].attrib.get('href').lstrip('/')

5656

57− seed = result.xpath('./dd/span[@class="u"]/text()')[0].replace(',', '')

58− leech = result.xpath('./dd/span[@class="d"]/text()')[0].replace(',', '')

57+ seed = 0

58+ leech = 0

59+ try:

60+ seed = int(result.xpath('./dd/span[4]/text()')[0].replace(',', ''))

61+ leech = int(result.xpath('./dd/span[5]/text()')[0].replace(',', ''))

62+ except:

63+ pass

5964

6065 params = {

6166 'url': base_url + link,

6267 'title': title,

63− 'seed': int_or_zero(seed),

64− 'leech': int_or_zero(leech),

68+ 'seed': seed,

69+ 'leech': leech,

6570 'template': 'torrent.html'

6671 }

6772

6873 # let's try to calculate the torrent size

6974 try:

70− size_str = result.xpath('./dd/span[@class="s"]/text()')[0]

71− size, suffix = size_str.split()

72− params['filesize'] = int(size) * get_filesize_mul(suffix)

75+ filesize_info = result.xpath('./dd/span[3]/text()')[0]

76+ filesize, filesize_multiplier = filesize_info.split()

77+ filesize = get_torrent_size(filesize, filesize_multiplier)

78+

79+ params['filesize'] = filesize

7380 except:

7481 pass

7582

@@ -80,9 +87,8 @@ def response(resp):

8087

8188 # extract and convert creation date

8289 try:

83− date_str = result.xpath('./dd/span[@class="a"]/span')[0].attrib.get('title')

84− # Fri, 25 Mar 2016 16:29:01

85− date = datetime.strptime(date_str, '%a, %d %b %Y %H:%M:%S')

90+ date_ts = result.xpath('./dd/span[2]')[0].attrib.get('title')

91+ date = datetime.fromtimestamp(float(date_ts))

8692 params['publishedDate'] = date

8793 except:

8894 pass

searx/settings.yml+6

@@ -552,6 +552,12 @@ engines:

552552 timeout : 6.0

553553 disabled : True

554554

555+ - name : torrentz

556+ engine : torrentz

557+ shortcut : tor

558+ url: https://torrentz2.eu/

559+ timeout : 3.0

560+

555561 - name : twitter

556562 engine : twitter

557563 shortcut : tw

tests/unit/engines/test_torrentz.py+14−18

@@ -14,7 +14,7 @@ class TestTorrentzEngine(SearxTestCase):

1414 params = torrentz.request(query, dic)

1515 self.assertTrue('url' in params)

1616 self.assertTrue(query in params['url'])

17− self.assertTrue('torrentz.eu' in params['url'])

17+ self.assertTrue('torrentz2.eu' in params['url'])

1818

1919 def test_response(self):

2020 resp = mock.Mock(text='<html></html>')

@@ -30,13 +30,11 @@ class TestTorrentzEngine(SearxTestCase):

3030 books ebooks

3131 </dt>

3232 <dd>

33− 1

34−

35− 4 months

36−

37− 30 MB

38− 14

39− 1

33+ 1

34+ 5 hours

35+ 30 MB

36+ 14

37+ 1

4038 </dd>

4139 </dl>

4240

@@ -48,13 +46,11 @@ class TestTorrentzEngine(SearxTestCase):

4846 books ebooks

4947 </dt>

5048 <dd>

51− 1

52−

53− 4 months

54−

55− 30MB

56− 5,555

57− 1,234,567

49+ 1

50+ 5 hours

51+ 30MB

52+ 5,555

53+ 1,234,567

5854 </dd>

5955 </dl>

6056 </div>

@@ -68,10 +64,10 @@ class TestTorrentzEngine(SearxTestCase):

6864

6965 # testing against the first result

7066 r = results[0]

71− self.assertEqual(r['url'], 'https://torrentz.eu/4362e08b1d80e1820fb2550b752f9f3126fe76d6')

67+ self.assertEqual(r['url'], 'https://torrentz2.eu/4362e08b1d80e1820fb2550b752f9f3126fe76d6')

7268 self.assertEqual(r['title'], 'Completely valid info books ebooks')

7369 # 22 Nov 2015 03:01:42

74− self.assertEqual(r['publishedDate'], datetime(2015, 11, 22, 3, 1, 42))

70+ self.assertEqual(r['publishedDate'], datetime.fromtimestamp(1503595924))

7571 self.assertEqual(r['seed'], 14)

7672 self.assertEqual(r['leech'], 1)

7773 self.assertEqual(r['filesize'], 30 * 1024 * 1024)

@@ -79,7 +75,7 @@ class TestTorrentzEngine(SearxTestCase):

7975

8076 # testing against the second result

8177 r = results[1]

82− self.assertEqual(r['url'], 'https://torrentz.eu/poaskdpokaspod')

78+ self.assertEqual(r['url'], 'https://torrentz2.eu/poaskdpokaspod')

8379 self.assertEqual(r['title'], 'Invalid hash and date and filesize books ebooks')

8480 self.assertEqual(r['seed'], 5555)

8581 self.assertEqual(r['leech'], 1234567)