git.iko.soy/searx
on master help sign in
Merge pull request #677 from pydo/feature/seedpeer-engine-integration

Feature/seedpeer engine integration

commite7d005c621a0ce08d3fcdfc6122d27c809a5dff3 authorAdam Tauber <asciimoo@gmail.com> dateMon 3 Oct 2016, 13:21 (10 years ago) parentdceb903beff8c6
5 files changed+246
compare with master
unifiedsplit
AUTHORS.rst+1 searx/engines/seedpeer.py+78 searx/settings.yml+6 tests/unit/engines/seedpeer_fixture.html+110 tests/unit/engines/test_seedpeer.py+51
AUTHORS.rst+1
@@ -58,3 +58,4 @@ generally made searx better:
5858- marc @a01200356
5959- Harry Wood @harry-wood
6060- Thomas Renard @threnard
61+- Pydo `<https://github.com/pydo>`_
searx/engines/seedpeer.pynew file+78
@@ -0,0 +1,78 @@
1+# Seedpeer (Videos, Music, Files)
2+#
3+# @website http://seedpeer.eu
4+# @provide-api no (nothing found)
5+#
6+# @using-api no
7+# @results HTML (using search portal)
8+# @stable yes (HTML can change)
9+# @parse url, title, content, seed, leech, magnetlink
10+
11+from urlparse import urljoin
12+from cgi import escape
13+from urllib import quote
14+from lxml import html
15+from operator import itemgetter
16+from searx.engines.xpath import extract_text
17+
18+
19+url = 'http://www.seedpeer.eu/'
20+search_url = url + 'search/{search_term}/7/{page_no}.html'
21+# specific xpath variables
22+torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
23+alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
24+title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
25+alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
26+seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
27+alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
28+peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
29+alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
30+age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
31+alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
32+size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
33+alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
34+
35+
36+# do search-request
37+def request(query, params):
38+ params['url'] = search_url.format(search_term=quote(query),
39+ page_no=params['pageno'] - 1)
40+ return params
41+
42+
43+# get response from search-request
44+def response(resp):
45+ results = []
46+ dom = html.fromstring(resp.text)
47+ torrent_links = dom.xpath(torrent_xpath)
48+ if len(torrent_links) > 0:
49+ seeds = dom.xpath(seeds_xpath)
50+ peers = dom.xpath(peers_xpath)
51+ titles = dom.xpath(title_xpath)
52+ sizes = dom.xpath(size_xpath)
53+ ages = dom.xpath(age_xpath)
54+ else: # under ~5 results uses a different xpath
55+ torrent_links = dom.xpath(alternative_torrent_xpath)
56+ seeds = dom.xpath(alternative_seeds_xpath)
57+ peers = dom.xpath(alternative_peers_xpath)
58+ titles = dom.xpath(alternative_title_xpath)
59+ sizes = dom.xpath(alternative_size_xpath)
60+ ages = dom.xpath(alternative_age_xpath)
61+ # return empty array if nothing is found
62+ if not torrent_links:
63+ return []
64+
65+ # parse results
66+ for index, result in enumerate(torrent_links):
67+ link = result.attrib.get('href')
68+ href = urljoin(url, link)
69+ results.append({'url': href,
70+ 'title': titles[index].text_content(),
71+ 'content': '{}, {}'.format(sizes[index], ages[index]),
72+ 'seed': seeds[index],
73+ 'leech': peers[index],
74+
75+ 'template': 'torrent.html'})
76+
77+ # return results sorted by seeder
78+ return sorted(results, key=itemgetter('seed'), reverse=True)
searx/settings.yml+6
@@ -495,6 +495,12 @@ engines:
495495 timeout: 6.0
496496 categories : science
497497
498+ - name : seedpeer
499+ engine : seedpeer
500+ shortcut: speu
501+ categories: files, music, videos
502+ disabled: True
503+
498504 - name : dictzone
499505 engine : dictzone
500506 shortcut : dc
tests/unit/engines/seedpeer_fixture.htmlnew file+110
@@ -0,0 +1,110 @@
1+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
2+ <html xmlns="http://www.w3.org/1999/xhtml">
3+ <head>
4+ </head>
5+ <body>
6+ <div id="header">
7+ <div id="whoIsYou">
8+ <a href="/lang.php"><small>SeedPeer in your own language?</small></a>&nbsp;&nbsp;&nbsp;<a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a>
9+ </div>
10+
11+ <script type="text/javascript">
12+ whoIsYou();
13+ </script>
14+ <div id="search">
15+ <form action="/search.php" method="get">
16+ <input id="topsearchbar" name="search" value="narcos season 2" />
17+ <input type="submit" class="searchbutton" value="Torrents" />
18+ <input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" />
19+ </form>
20+ <div id="suggestion"></div>
21+ </div>
22+ <div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div>
23+ <div id="subtext"><a href="/">Home</a> &gt; <a href="/search.html">Torrent search</a> &gt; Narcos season 2 | page 1</div>
24+ </div>
25+ <div id="nav">
26+ <ul>
27+ <!--
28+ <li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li>
29+
30+ -->
31+ <li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a>
32+ <ul>
33+ <li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li>
34+ <li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li>
35+ <li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li>
36+ <li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li>
37+ <li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li>
38+ <li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li>
39+ <li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li>
40+ </ul>
41+ </li>
42+ <li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li>
43+ <li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li>
44+ <li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li>
45+ <li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li>
46+ <li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a>
47+ <ul>
48+ <li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li>
49+ <li><a title="Contact Us" href="/contact.html">Contact</a></li>
50+ <li><a title="Frequently Asked Questions" href="/faq.html">FAQ</a></li>
51+ <li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li>
52+ <li><a title="SeedPeer Blog" href="/blog">Blog</a></li>
53+ </ul>
54+ </li>
55+ <!--<li><a href="/toolbar.php">Our Toolbar</a></li>-->
56+ </ul>
57+ <div class="clear"></div>
58+ </div>
59+ <div id="body"><div id="pageTop"></div>
60+ <div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th>
61+ <span style="float:right">
62+ <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
63+ <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
64+ </span>
65+ <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a>&nbsp;2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th>
66+ <span style="float:right">
67+ <a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
68+ <a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
69+ </span>
70+ <a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe>&nbsp;<iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer">
71+ <table width="100%">
72+ <tr>
73+ <td width="30%">
74+ <h2>Torrents Download</h2>
75+ <a href="/">Torrent search</a><br />
76+ <a href="/browse.html">Browse categories</a><br />
77+ <a href="/verified.html">Verified Torrents</a><br />
78+ <a href="/order-date.html">Today's torrents</a><br />
79+ <a href="/yesterday.html">Yesterday's torrents</a><br />
80+ <a href="/stats.html">Statistics</a><br />
81+ <br />
82+ <a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a>
83+ </td>
84+ <td width="30%"><h2>Cool Stuff</h2>
85+ <a href="/promotional.php">Promotional</a><br />
86+ <a href="/contact.html">Advertising Information</a><br />
87+ <strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br />
88+ <a href="http://www.utorrent.com">&micro;Torrent Client</a><br />
89+ <a href="/blog">Seedpeer Blog</a><br />
90+ </td>
91+ <td width="30%"><h2>Links</h2>
92+ <a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br />
93+ <a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br />
94+ <a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br />
95+ <a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br />
96+ <a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br />
97+ <a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br />
98+ <a href="https://monova.org" target="_blank">Monova</a><br />
99+ <a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br />
100+ <a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br />
101+ </td>
102+ <td width="10%"><div id="bottomlogo"></div></td>
103+ </tr>
104+ </table>
105+ <br />
106+ <br />
107+ </div>
108+ </div>
109+ </body>
110+ </html>
tests/unit/engines/test_seedpeer.pynew file+51
@@ -0,0 +1,51 @@
1+import mock
2+from collections import defaultdict
3+from searx.engines import seedpeer
4+from searx.testing import SearxTestCase
5+from datetime import datetime
6+
7+
8+class TestSeedPeerEngine(SearxTestCase):
9+
10+ html = ''
11+ with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
12+ html += fixture.read()
13+
14+ def test_request(self):
15+ query = 'test_query'
16+ dicto = defaultdict(dict)
17+ dicto['pageno'] = 1
18+ params = seedpeer.request(query, dicto)
19+ self.assertIn('url', params)
20+ self.assertIn(query, params['url'])
21+ self.assertIn('seedpeer.eu', params['url'])
22+
23+ def test_response_raises_attr_error_on_empty_response(self):
24+ self.assertRaises(AttributeError, seedpeer.response, None)
25+ self.assertRaises(AttributeError, seedpeer.response, [])
26+ self.assertRaises(AttributeError, seedpeer.response, '')
27+ self.assertRaises(AttributeError, seedpeer.response, '[]')
28+
29+ def test_response_returns_empty_list(self):
30+ response = mock.Mock(text='<html></html>')
31+ self.assertEqual(seedpeer.response(response), [])
32+
33+ def test_response_returns_all_results(self):
34+ response = mock.Mock(text=self.html)
35+ results = seedpeer.response(response)
36+ self.assertTrue(isinstance(results, list))
37+ self.assertEqual(len(results), 2)
38+
39+ def test_response_returns_correct_results(self):
40+ response = mock.Mock(text=self.html)
41+ results = seedpeer.response(response)
42+ self.assertEqual(
43+ results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
44+ )
45+ self.assertEqual(
46+ results[0]['url'],
47+ 'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
48+ )
49+ self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
50+ self.assertEqual(results[0]['seed'], '861')
51+ self.assertEqual(results[0]['leech'], '332')