git.iko.soy/searx
on master help sign in
Merge pull request #1022 from misnyo/nyaa

nyaa.si fixed

commitf4df27fa59c21f6c776081364afb2ab1a8b83b74 authorAdam Tauber <asciimoo@gmail.com> dateThu 7 Sep 2017, 19:33 (9 years ago) parent688801033fd938
4 files changed+142−85
compare with master
unifiedsplit
searx/engines/nyaa.py+40−49 searx/engines/tokyotoshokan.py+2−3 searx/utils.py+9 tests/unit/engines/test_nyaa.py+91−33
searx/engines/nyaa.py+40−49
@@ -1,7 +1,7 @@
11"""
2− Nyaa.se (Anime Bittorrent tracker)
2+ Nyaa.si (Anime Bittorrent tracker)
33
4− @website http://www.nyaa.se/
4+ @website http://www.nyaa.si/
55 @provide-api no
66 @using-api no
77 @results HTML
@@ -12,50 +12,25 @@
1212from lxml import html
1313from searx.engines.xpath import extract_text
1414from searx.url_utils import urlencode
15+from searx.utils import get_torrent_size, int_or_zero
1516
1617# engine dependent config
1718categories = ['files', 'images', 'videos', 'music']
1819paging = True
1920
2021# search-url
21−base_url = 'http://www.nyaa.se/'
22+base_url = 'http://www.nyaa.si/'
2223search_url = base_url + '?page=search&{query}&offset={offset}'
2324
2425# xpath queries
25−xpath_results = '//table[@class="tlist"]//tr[contains(@class, "tlistrow")]'
26−xpath_category = './/td[@class="tlisticon"]/a'
27−xpath_title = './/td[@class="tlistname"]/a'
28−xpath_torrent_file = './/td[@class="tlistdownload"]/a'
29−xpath_filesize = './/td[@class="tlistsize"]/text()'
30−xpath_seeds = './/td[@class="tlistsn"]/text()'
31−xpath_leeches = './/td[@class="tlistln"]/text()'
32−xpath_downloads = './/td[@class="tlistdn"]/text()'
33−
34−
35−# convert a variable to integer or return 0 if it's not a number
36−def int_or_zero(num):
37− if isinstance(num, list):
38− if len(num) < 1:
39− return 0
40− num = num[0]
41− if num.isdigit():
42− return int(num)
43− return 0
44−
45−
46−# get multiplier to convert torrent size to bytes
47−def get_filesize_mul(suffix):
48− return {
49− 'KB': 1024,
50− 'MB': 1024 ** 2,
51− 'GB': 1024 ** 3,
52− 'TB': 1024 ** 4,
53−
54− 'KIB': 1024,
55− 'MIB': 1024 ** 2,
56− 'GIB': 1024 ** 3,
57− 'TIB': 1024 ** 4
58− }[str(suffix).upper()]
26+xpath_results = '//table[contains(@class, "torrent-list")]//tr[not(th)]'
27+xpath_category = './/td[1]/a[1]'
28+xpath_title = './/td[2]/a[last()]'
29+xpath_torrent_links = './/td[3]/a'
30+xpath_filesize = './/td[4]/text()'
31+xpath_seeds = './/td[6]/text()'
32+xpath_leeches = './/td[7]/text()'
33+xpath_downloads = './/td[8]/text()'
5934
6035
6136# do search-request
@@ -72,25 +47,32 @@ def response(resp):
7247 dom = html.fromstring(resp.text)
7348
7449 for result in dom.xpath(xpath_results):
50+ # defaults
51+ filesize = 0
52+ magnet_link = ""
53+ torrent_link = ""
54+
7555 # category in which our torrent belongs
76− category = result.xpath(xpath_category)[0].attrib.get('title')
56+ try:
57+ category = result.xpath(xpath_category)[0].attrib.get('title')
58+ except:
59+ pass
7760
7861 # torrent title
7962 page_a = result.xpath(xpath_title)[0]
8063 title = extract_text(page_a)
8164
8265 # link to the page
83− href = page_a.attrib.get('href')
66+ href = base_url + page_a.attrib.get('href')
8467
85− # link to the torrent file
86− torrent_link = result.xpath(xpath_torrent_file)[0].attrib.get('href')
87−
88− # torrent size
89− try:
90− file_size, suffix = result.xpath(xpath_filesize)[0].split(' ')
91− file_size = int(float(file_size) * get_filesize_mul(suffix))
92− except:
93− file_size = None
68+ for link in result.xpath(xpath_torrent_links):
69+ url = link.attrib.get('href')
70+ if 'magnet' in url:
71+ # link to the magnet
72+ magnet_link = url
73+ else:
74+ # link to the torrent file
75+ torrent_link = url
9476
9577 # seed count
9678 seed = int_or_zero(result.xpath(xpath_seeds))
@@ -101,6 +83,14 @@ def response(resp):
10183 # torrent downloads count
10284 downloads = int_or_zero(result.xpath(xpath_downloads))
10385
86+ # let's try to calculate the torrent size
87+ try:
88+ filesize_info = result.xpath(xpath_filesize)[0]
89+ filesize, filesize_multiplier = filesize_info.split()
90+ filesize = get_torrent_size(filesize, filesize_multiplier)
91+ except:
92+ pass
93+
10494 # content string contains all information not included into template
10595 content = 'Category: "{category}". Downloaded {downloads} times.'
10696 content = content.format(category=category, downloads=downloads)
@@ -110,8 +100,9 @@ def response(resp):
110100 'content': content,
111101 'seed': seed,
112102 'leech': leech,
113− 'filesize': file_size,
103+ 'filesize': filesize,
114104 'torrentfile': torrent_link,
105+ 'magnetlink': magnet_link,
115106 'template': 'torrent.html'})
116107
117108 return results
searx/engines/tokyotoshokan.py+2−3
@@ -14,8 +14,8 @@ import re
1414from lxml import html
1515from searx.engines.xpath import extract_text
1616from datetime import datetime
17−from searx.engines.nyaa import int_or_zero, get_filesize_mul
1817from searx.url_utils import urlencode
18+from searx.utils import get_torrent_size, int_or_zero
1919
2020# engine dependent config
2121categories = ['files', 'videos', 'music']
@@ -76,8 +76,7 @@ def response(resp):
7676 try:
7777 # ('1.228', 'GB')
7878 groups = size_re.match(item).groups()
79− multiplier = get_filesize_mul(groups[1])
80− params['filesize'] = int(multiplier * float(groups[0]))
79+ params['filesize'] = get_torrent_size(groups[0], groups[1])
8180 except:
8281 pass
8382 elif item.startswith('Date:'):
searx/utils.py+9
@@ -290,6 +290,15 @@ def convert_str_to_int(number_str):
290290 return 0
291291
292292
293+# convert a variable to integer or return 0 if it's not a number
294+def int_or_zero(num):
295+ if isinstance(num, list):
296+ if len(num) < 1:
297+ return 0
298+ num = num[0]
299+ return convert_str_to_int(num)
300+
301+
293302def is_valid_lang(lang):
294303 is_abbr = (len(lang) == 2)
295304 if is_abbr:
tests/unit/engines/test_nyaa.py+91−33
@@ -13,38 +13,92 @@ class TestNyaaEngine(SearxTestCase):
1313 params = nyaa.request(query, dic)
1414 self.assertTrue('url' in params)
1515 self.assertTrue(query in params['url'])
16− self.assertTrue('nyaa.se' in params['url'])
16+ self.assertTrue('nyaa.si' in params['url'])
1717
1818 def test_response(self):
1919 resp = mock.Mock(text='<html></html>')
2020 self.assertEqual(nyaa.response(resp), [])
2121
2222 html = """
23− <table class="tlist">
24− <tbody>
25− <tr class="trusted tlistrow">
26− <td class="tlisticon">
27− <a href="//www.nyaa.se" title="English-translated Anime">
28− <img src="//files.nyaa.se" alt="English-translated Anime">
29− </a>
30− </td>
31− <td class="tlistname">
32− <a href="//www.nyaa.se/?page3">
33− Sample torrent title
34− </a>
35− </td>
36− <td class="tlistdownload">
37− <a href="//www.nyaa.se/?page_dl" title="Download">
38− <img src="//files.nyaa.se/www-dl.png" alt="DL">
39− </a>
40− </td>
41− <td class="tlistsize">10 MiB</td>
42− <td class="tlistsn">1</td>
43− <td class="tlistln">3</td>
44− <td class="tlistdn">666</td>
45− <td class="tlistmn">0</td>
46− </tr>
47− </tbody>
23+ <table class="table table-bordered table-hover table-striped torrent-list">
24+ <thead>
25+ <tr>
26+ <th class="hdr-category text-center" style="width:80px;">
27+ <div>Category</div>
28+ </th>
29+ <th class="hdr-name" style="width:auto;">
30+ <div>Name</div>
31+ </th>
32+ <th class="hdr-comments sorting text-center" title="Comments" style="width:50px;">
33+ <a href="/?f=0&amp;c=0_0&amp;q=Death+Parade&amp;s=comments&amp;o=desc"></a>
34+ <i class="fa fa-comments-o"></i>
35+ </th>
36+ <th class="hdr-link text-center" style="width:70px;">
37+ <div>Link</div>
38+ </th>
39+ <th class="hdr-size sorting text-center" style="width:100px;">
40+ <a href="/?f=0&amp;c=0_0&amp;q=Death+Parade&amp;s=size&amp;o=desc"></a>
41+ <div>Size</div>
42+ </th>
43+ <th class="hdr-date sorting_desc text-center" title="In local time" style="width:140px;">
44+ <a href="/?f=0&amp;c=0_0&amp;q=Death+Parade&amp;s=id&amp;o=asc"></a>
45+ <div>Date</div>
46+ </th>
47+ <th class="hdr-seeders sorting text-center" title="Seeders" style="width:50px;">
48+ <a href="/?f=0&amp;c=0_0&amp;q=Death+Parade&amp;s=seeders&amp;o=desc"></a>
49+ <i class="fa fa-arrow-up" aria-hidden="true"></i>
50+ </th>
51+ <th class="hdr-leechers sorting text-center" title="Leechers" style="width:50px;">
52+ <a href="/?f=0&amp;c=0_0&amp;q=Death+Parade&amp;s=leechers&amp;o=desc"></a>
53+ <i class="fa fa-arrow-down" aria-hidden="true"></i>
54+ </th>
55+ <th class="hdr-downloads sorting text-center" title="Completed downloads" style="width:50px;">
56+ <a href="/?f=0&amp;c=0_0&amp;q=Death+Parade&amp;s=downloads&amp;o=desc"></a>
57+ <i class="fa fa-check" aria-hidden="true"></i>
58+ </th>
59+ </tr>
60+ </thead>
61+ <tbody>
62+ <tr class="default">
63+ <td style="padding:0 4px;">
64+ <a href="/?c=1_2" title="Anime - English-translated">
65+ <img src="/static/img/icons/nyaa/1_2.png" alt="Anime - English-translated">
66+ </a>
67+ </td>
68+ <td colspan="2">
69+ <a href="/view/1" title="Sample title 1">Sample title 1</a>
70+ </td>
71+ <td class="text-center" style="white-space: nowrap;">
72+ <a href="/download/1.torrent"><i class="fa fa-fw fa-download"></i></a>
73+ <a href="magnet:?xt=urn:btih:2"><i class="fa fa-fw fa-magnet"></i></a>
74+ </td>
75+ <td class="text-center">723.7 MiB</td>
76+ <td class="text-center" data-timestamp="1503307456" title="1 week 3
77+ days 9 hours 44 minutes 39 seconds ago">2017-08-21 11:24</td>
78+ <td class="text-center" style="color: green;">1</td>
79+ <td class="text-center" style="color: red;">3</td>
80+ <td class="text-center">12</td>
81+ </tr>
82+ <tr class="default">
83+ <td style="padding:0 4px;">
84+ <a href="/?c=1_2" title="Anime - English-translated">
85+ <img src="/static/img/icons/nyaa/1_2.png" alt="Anime - English-translated">
86+ </a>
87+ </td>
88+ <td colspan="2">
89+ <a href="/view/2" title="Sample title 2">Sample title 2</a>
90+ </td>
91+ <td class="text-center" style="white-space: nowrap;">
92+ <a href="magnet:?xt=urn:btih:2"><i class="fa fa-fw fa-magnet"></i></a>
93+ </td>
94+ <td class="text-center">8.2 GiB</td>
95+ <td class="text-center" data-timestamp="1491608400" title="4 months 3
96+ weeks 4 days 19 hours 28 minutes 55 seconds ago">2017-04-08 01:40</td>
97+ <td class="text-center" style="color: green;">10</td>
98+ <td class="text-center" style="color: red;">1</td>
99+ <td class="text-center">206</td>
100+ </tr>
101+ </tbody>
48102 </table>
49103 """
50104
@@ -52,15 +106,19 @@ class TestNyaaEngine(SearxTestCase):
52106 results = nyaa.response(resp)
53107
54108 self.assertEqual(type(results), list)
55− self.assertEqual(len(results), 1)
109+ self.assertEqual(len(results), 2)
56110
57111 r = results[0]
58− self.assertTrue(r['url'].find('www.nyaa.se/?page3') >= 0)
59− self.assertTrue(r['torrentfile'].find('www.nyaa.se/?page_dl') >= 0)
60− self.assertTrue(r['content'].find('English-translated Anime') >= 0)
61− self.assertTrue(r['content'].find('Downloaded 666 times.') >= 0)
112+ self.assertTrue(r['url'].find('1') >= 0)
113+ self.assertTrue(r['torrentfile'].find('1.torrent') >= 0)
114+ self.assertTrue(r['content'].find('Anime - English-translated') >= 0)
115+ self.assertTrue(r['content'].find('Downloaded 12 times.') >= 0)
62116
63− self.assertEqual(r['title'], 'Sample torrent title')
117+ self.assertEqual(r['title'], 'Sample title 1')
64118 self.assertEqual(r['seed'], 1)
65119 self.assertEqual(r['leech'], 3)
66− self.assertEqual(r['filesize'], 10 * 1024 * 1024)
120+ self.assertEqual(r['filesize'], 723700000)
121+
122+ r = results[1]
123+ self.assertTrue(r['url'].find('2') >= 0)
124+ self.assertTrue(r['magnetlink'].find('magnet:') >= 0)