mirror of
https://github.com/searxng/searxng.git
synced 2025-10-30 18:22:31 -04:00
Merge pull request #677 from pydo/feature/seedpeer-engine-integration
Feature/seedpeer engine integration
This commit is contained in:
Adam Tauber
GitHub
commit
e7d005c621
@ -58,3 +58,4 @@ generally made searx better:
|
||||
- marc @a01200356
|
||||
- Harry Wood @harry-wood
|
||||
- Thomas Renard @threnard
|
||||
- Pydo `<https://github.com/pydo>`_
|
||||
|
||||
78
searx/engines/seedpeer.py
Normal file
78
searx/engines/seedpeer.py
Normal file
@ -0,0 +1,78 @@
|
||||
# Seedpeer (Videos, Music, Files)
|
||||
#
|
||||
# @website http://seedpeer.eu
|
||||
# @provide-api no (nothing found)
|
||||
#
|
||||
# @using-api no
|
||||
# @results HTML (using search portal)
|
||||
# @stable yes (HTML can change)
|
||||
# @parse url, title, content, seed, leech, magnetlink
|
||||
|
||||
from urlparse import urljoin
|
||||
from cgi import escape
|
||||
from urllib import quote
|
||||
from lxml import html
|
||||
from operator import itemgetter
|
||||
from searx.engines.xpath import extract_text
|
||||
|
||||
|
||||
url = 'http://www.seedpeer.eu/'
|
||||
search_url = url + 'search/{search_term}/7/{page_no}.html'
|
||||
# specific xpath variables
|
||||
torrent_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a'
|
||||
alternative_torrent_xpath = '//*[@id="body"]/center/center/table[1]/tr/td/a'
|
||||
title_xpath = '//*[@id="body"]/center/center/table[2]/tr/td/a/text()'
|
||||
alternative_title_xpath = '//*[@id="body"]/center/center/table/tr/td/a'
|
||||
seeds_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[4]/font/text()'
|
||||
alternative_seeds_xpath = '//*[@id="body"]/center/center/table/tr/td[4]/font/text()'
|
||||
peers_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[5]/font/text()'
|
||||
alternative_peers_xpath = '//*[@id="body"]/center/center/table/tr/td[5]/font/text()'
|
||||
age_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[2]/text()'
|
||||
alternative_age_xpath = '//*[@id="body"]/center/center/table/tr/td[2]/text()'
|
||||
size_xpath = '//*[@id="body"]/center/center/table[2]/tr/td[3]/text()'
|
||||
alternative_size_xpath = '//*[@id="body"]/center/center/table/tr/td[3]/text()'
|
||||
|
||||
|
||||
# do search-request
|
||||
def request(query, params):
|
||||
params['url'] = search_url.format(search_term=quote(query),
|
||||
page_no=params['pageno'] - 1)
|
||||
return params
|
||||
|
||||
|
||||
# get response from search-request
|
||||
def response(resp):
|
||||
results = []
|
||||
dom = html.fromstring(resp.text)
|
||||
torrent_links = dom.xpath(torrent_xpath)
|
||||
if len(torrent_links) > 0:
|
||||
seeds = dom.xpath(seeds_xpath)
|
||||
peers = dom.xpath(peers_xpath)
|
||||
titles = dom.xpath(title_xpath)
|
||||
sizes = dom.xpath(size_xpath)
|
||||
ages = dom.xpath(age_xpath)
|
||||
else: # under ~5 results uses a different xpath
|
||||
torrent_links = dom.xpath(alternative_torrent_xpath)
|
||||
seeds = dom.xpath(alternative_seeds_xpath)
|
||||
peers = dom.xpath(alternative_peers_xpath)
|
||||
titles = dom.xpath(alternative_title_xpath)
|
||||
sizes = dom.xpath(alternative_size_xpath)
|
||||
ages = dom.xpath(alternative_age_xpath)
|
||||
# return empty array if nothing is found
|
||||
if not torrent_links:
|
||||
return []
|
||||
|
||||
# parse results
|
||||
for index, result in enumerate(torrent_links):
|
||||
link = result.attrib.get('href')
|
||||
href = urljoin(url, link)
|
||||
results.append({'url': href,
|
||||
'title': titles[index].text_content(),
|
||||
'content': '{}, {}'.format(sizes[index], ages[index]),
|
||||
'seed': seeds[index],
|
||||
'leech': peers[index],
|
||||
|
||||
'template': 'torrent.html'})
|
||||
|
||||
# return results sorted by seeder
|
||||
return sorted(results, key=itemgetter('seed'), reverse=True)
|
||||
@ -495,6 +495,12 @@ engines:
|
||||
timeout: 6.0
|
||||
categories : science
|
||||
|
||||
- name : seedpeer
|
||||
engine : seedpeer
|
||||
shortcut: speu
|
||||
categories: files, music, videos
|
||||
disabled: True
|
||||
|
||||
- name : dictzone
|
||||
engine : dictzone
|
||||
shortcut : dc
|
||||
|
||||
110
tests/unit/engines/seedpeer_fixture.html
Normal file
110
tests/unit/engines/seedpeer_fixture.html
Normal file
@ -0,0 +1,110 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
</head>
|
||||
<body>
|
||||
<div id="header">
|
||||
<div id="whoIsYou">
|
||||
<a href="/lang.php"><small>SeedPeer in your own language?</small></a> <a href="http://www.seedpeer.eu"><img src="/images/flags/uk.gif" width="16px" alt="Torrents EN" /></a> <a href="http://spanish.seedpeer.eu"><img src="/images/flags/es.gif" width="16px" alt="Torrents ES" /></a> <a href="http://german.seedpeer.eu"><img src="/images/flags/de.gif" width="16px" alt="Torrents DE" /></a> <a href="http://french.seedpeer.eu"><img src="/images/flags/fr.gif" width="16px" alt="Torrents FR" /></a> <a href="http://portuguese.seedpeer.eu"><img src="/images/flags/pt.gif" width="16px" alt="Torrents Portuguese" /></a> <a href="http://swedish.seedpeer.eu"><img src="/images/flags/se.gif" width="16px" alt="Torrents Sweden" /></a>
|
||||
</div>
|
||||
|
||||
<script type="text/javascript">
|
||||
whoIsYou();
|
||||
</script>
|
||||
<div id="search">
|
||||
<form action="/search.php" method="get">
|
||||
<input id="topsearchbar" name="search" value="narcos season 2" />
|
||||
<input type="submit" class="searchbutton" value="Torrents" />
|
||||
<input style="color:#000" type="submit" class="searchbutton" name="usenet" value="Usenet Binaries" />
|
||||
</form>
|
||||
<div id="suggestion"></div>
|
||||
</div>
|
||||
<div id="logo"><a href="/"><img src="/images/logo2.gif" alt="Seedpeer homepage" width="415" height="143" /></a></div>
|
||||
<div id="subtext"><a href="/">Home</a> > <a href="/search.html">Torrent search</a> > Narcos season 2 | page 1</div>
|
||||
</div>
|
||||
<div id="nav">
|
||||
<ul>
|
||||
<!--
|
||||
<li><font style="color:red;font-size:9px;font-weight:bold;">NEW</font><a title="Download TOP Games for FREE" rel="nofollow" href="http://www.bigrebelads.com/affiliate/index?ref=9301" target="_blank">FREE Games</a></li>
|
||||
|
||||
-->
|
||||
<li style="border-left:none" id="categories"><a title="Browse Torrent Categories" href="/browse.html">Categories</a>
|
||||
<ul>
|
||||
<li><a title="Browse Anime Torrents" href="/browse.html#6">Anime</a></li>
|
||||
<li><a title="Browse Game Torrents" href="/browse.html#4">Games</a></li>
|
||||
<li><a title="Browse Movie Torrents" href="/browse.html#1">Movies</a></li>
|
||||
<li><a title="Browse Music Torrents" href="/browse.html#3">Music</a></li>
|
||||
<li><a title="Browse Software Torrents" href="/browse.html#5">Software</a></li>
|
||||
<li><a title="Browse TV Torrents" href="/browse.html#2">TV Shows</a></li>
|
||||
<li><a title="Browse Other Torrents" href="/browse.html#7">Others</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<li><a title="Upload A Torrents" href="/upload.html">Upload torrent</a></li>
|
||||
<li id="verified"><a title="Verified Torrents" href="/verified.html">Verified</a></li>
|
||||
<li id="searchoptions"><a title="Search Torrents" href="/search.html">Torrent search</a></li>
|
||||
<li id="newsgroups"><a style="color:#212b3e" title="News Groups" href="/usenet.html">Usenet Binaries</a></li>
|
||||
<li id="about" style="border-right:none"><a rel="nofollow" href="/faq.html">About Us</a>
|
||||
<ul>
|
||||
<li><a title="SeedPeer Statistics" href="/stats.html">Statistics</a></li>
|
||||
<li><a title="Contact Us" href="/contact.html">Contact</a></li>
|
||||
<li><a title="Frequently Asked Questions" href="/faq.html">FAQ</a></li>
|
||||
<li><a title="SeedPeer API" href="http://api.seedpeer.eu">Our API</a></li>
|
||||
<li><a title="SeedPeer Blog" href="/blog">Blog</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
<!--<li><a href="/toolbar.php">Our Toolbar</a></li>-->
|
||||
</ul>
|
||||
<div class="clear"></div>
|
||||
</div>
|
||||
<div id="body"><div id="pageTop"></div>
|
||||
<div id="headerbox"><h1>Verified <font class="colored">Narcos season 2</font> torrents</h1></div><table width="100%"><tr><th>
|
||||
<span style="float:right">
|
||||
<a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
|
||||
<a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
|
||||
</span>
|
||||
<a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th><td class="tableAd" rowspan="6"><iframe src="http://creative.wwwpromoter.com/13689?d=300x250" width="300" height="250" style="border: none;" frameborder="0" scrolling="no"></iframe></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_1" href="" data-tad="431726" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Full Version</a></td><td class="right">20 hours</td><td class="right">681.3 MB</td><td class="right"><font color="green">28</font> </td><td class="right"><font color="navy">654</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_2" href="" data-tad="431727" data-url="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> Trusted Source</a></td><td class="right">12 hours</td><td class="right">787.1 MB</td><td class="right"><font color="green">64</font> </td><td class="right"><font color="navy">220</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_3" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Full Narcos season 2 Download</strong></a> <small><a class="pblink" id="pblink_table_item_4" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Usenet</a></small></td><td class="right">24 hours</td><td class="right">775.5 MB</td><td class="right"><font color="green">60</font> </td><td class="right"><font color="navy">236</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class="tdark"><td><a class="pblink" id="pblink_table_item_5" href="" data-tad="431730" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014 - DIRECT STREAMING</a> <small><a class="pblink" id="pblink_table_item_6" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">17 hours</td><td class="right">654.1 MB</td><td class="right"><font color="green">2</font> </td><td class="right"><font color="navy">391</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr><tr class=""><td><a class="pblink" id="pblink_table_item_7" href="" data-tad="431731" data-last-search="narcos+season+2" target="_blank" rel="nofollow"><strong class='colored'>Narcos season 2</strong> 2014</a> <small><a class="pblink" id="pblink_table_item_8" href="" data-tad="431729" data-last-search="narcos+season+2" target="_blank" rel="nofollow">Movies</a></small></td><td class="right">20 hours</td><td class="right">754.5 MB</td><td class="right"><font color="green">21</font> </td><td class="right"><font color="navy">919</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" /></td></tr></table><br /><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=728x90' width='728' height='90' style='border: none;' frameborder='0' scrolling='no'></iframe><center><span style="float:right;margin:1em .2em 0 0"><a title="Download at the speed of your connection" href="/usenet.php?search=narcos+season+2"><img src="/images/dlf.gif" alt="Search Binaries" /></a></span><div style="margin-bottom:1em;margin-right:290px" id="headerbox"><h1><a href="/searchfeed/narcos+season+2.xml" target="_blank" title="SeedPeer RSS Torrent Search Feed fornarcos season 2"><img src="/images/feedIcon.png" border="0" /></a> 2 <font class="colored">Narcos season 2</font> Torrents were found</h1></div><table width="100%"><tr><th>
|
||||
<span style="float:right">
|
||||
<a href="/search/narcos-season-2/8/1.html"><img style="vertical-align:middle" src="/images/comments.gif" alt="comments" /></a> |
|
||||
<a href="/search/narcos-season-2/7/1.html"><img style="vertical-align:middle" src="/images/ver.gif" alt="verified" /></a>
|
||||
</span>
|
||||
<a href="/search/narcos-season-2/1/1.html">Torrent name</a></th><th class="right"><a href="/search/narcos-season-2/2/1.html">Age</a></th><th class="right"><a href="/search/narcos-season-2/3/1.html">Size</a></th><th class="right"><a href="/search/narcos-season-2/4/1.html">Seeds</a></th><th class="right"><a href="/search/narcos-season-2/5/1.html">Peers</a></th><th class="center"><a href="/search/narcos-season-2/6/1.html">Health</a></th></tr><tr class=""><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong>%20 torrent&u=http://seedpeer.seedpeer.eu/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11686840/Narcos-Season-2-Complete-720p-WebRip-EN-SUB-x264-[MULVAcoded]-S02.html"><strong class='colored'>Narcos</strong> <strong class='colored'>Season</strong> <strong class='colored'>2</strong> Complete 7<strong class='colored'>2</strong>0p WebRip EN-SUB x<strong class='colored'>2</strong>64-[MULVAcoded] S0<strong class='colored'>2</strong> <small><a href="/browse.html#11686840"></a></small></a></td><td class="right">19 hours</td><td class="right">4.39 GB</td><td class="right"><font color="green">715</font> </td><td class="right"><font color="navy">183</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr><tr class="tdark"><td><small class="comments"><a href="http://www.facebook.com/sharer.php?t=Download%20<strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG%20 torrent&u=http://seedpeer.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><img src="/images/facebook.png" alt="Add to Facebook" width="14" height="14" /></a></small><a href="/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html"><strong class='colored'>Narcos</strong> - <strong class='colored'>Season</strong> <strong class='colored'>2</strong> - 7<strong class='colored'>2</strong>0p WEBRiP - x<strong class='colored'>2</strong>65 HEVC - ShAaNiG <small><a href="/browse.html#11685972"></a></small></a></td><td class="right">1 day</td><td class="right">2.48 GB</td><td class="right"><font color="green">861</font> </td><td class="right"><font color="navy">332</font> </td><td class="center"><img src="/images/h5.gif" alt="Health" width="40" height="11" /></td></tr></table><div id="headerbox"><h1>Related searches for: <font class="colored">Narcos season 2</font></h1></div><div id="search_suggestions"><br />Other suggested searches: </div><br /><a href="http://torrentz2.eu/search?f=narcos-season-2">Search for "narcos-season-2" on Torrentz2.eu</a><br /><a href="http://torrent-finder.info/show.php?q=narcos-season-2">Search for "narcos-season-2" on Torrent-Finder</a><br /><center><iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe> <iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe> <iframe src='http://creative.wwwpromoter.com/13689?d=300x250' width='300' height='250' style='border: none;' frameborder='0' scrolling='no'></iframe></center><div id="footer">
|
||||
<table width="100%">
|
||||
<tr>
|
||||
<td width="30%">
|
||||
<h2>Torrents Download</h2>
|
||||
<a href="/">Torrent search</a><br />
|
||||
<a href="/browse.html">Browse categories</a><br />
|
||||
<a href="/verified.html">Verified Torrents</a><br />
|
||||
<a href="/order-date.html">Today's torrents</a><br />
|
||||
<a href="/yesterday.html">Yesterday's torrents</a><br />
|
||||
<a href="/stats.html">Statistics</a><br />
|
||||
<br />
|
||||
<a href="/faq.html#copyright"><strong>Copyright & Removal</strong></a>
|
||||
</td>
|
||||
<td width="30%"><h2>Cool Stuff</h2>
|
||||
<a href="/promotional.php">Promotional</a><br />
|
||||
<a href="/contact.html">Advertising Information</a><br />
|
||||
<strong><a href="/plugins.php" title="Add a search plugin to Firefox or Internet Explorer">Search Plugin <span style="color:red">*</span></a></strong><br />
|
||||
<a href="http://www.utorrent.com">µTorrent Client</a><br />
|
||||
<a href="/blog">Seedpeer Blog</a><br />
|
||||
</td>
|
||||
<td width="30%"><h2>Links</h2>
|
||||
<a href="http://www.sumotorrent.com" target="_blank"><strong>SumoTorrent</strong></a><br />
|
||||
<a href="http://www.torrent-finder.info" target="_blank"><strong>Torrent Finder</strong></a><br />
|
||||
<a href="http://www.torrentpond.com" target="_blank"><strong>TorrentPond</strong></a><br />
|
||||
<a href="https://www.limetorrents.cc" target="_blank">LimeTorrents.cc</a><br />
|
||||
<a href="http://www.torrents.to/" target="_blank">Torrents.to</a><br />
|
||||
<a href="http://www.torrentfunk.com" target="_blank">TorrentFunk</a><br />
|
||||
<a href="https://monova.org" target="_blank">Monova</a><br />
|
||||
<a href="http://www.torrentroom.com" target="_blank">TorrentRoom</a><br />
|
||||
<a href="http://www.katcr.co/" target="_blank">Kickass Torrents Community</a><br />
|
||||
</td>
|
||||
<td width="10%"><div id="bottomlogo"></div></td>
|
||||
</tr>
|
||||
</table>
|
||||
<br />
|
||||
<br />
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
51
tests/unit/engines/test_seedpeer.py
Normal file
51
tests/unit/engines/test_seedpeer.py
Normal file
@ -0,0 +1,51 @@
|
||||
import mock
|
||||
from collections import defaultdict
|
||||
from searx.engines import seedpeer
|
||||
from searx.testing import SearxTestCase
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class TestSeedPeerEngine(SearxTestCase):
|
||||
|
||||
html = ''
|
||||
with open('./tests/unit/engines/seedpeer_fixture.html') as fixture:
|
||||
html += fixture.read()
|
||||
|
||||
def test_request(self):
|
||||
query = 'test_query'
|
||||
dicto = defaultdict(dict)
|
||||
dicto['pageno'] = 1
|
||||
params = seedpeer.request(query, dicto)
|
||||
self.assertIn('url', params)
|
||||
self.assertIn(query, params['url'])
|
||||
self.assertIn('seedpeer.eu', params['url'])
|
||||
|
||||
def test_response_raises_attr_error_on_empty_response(self):
|
||||
self.assertRaises(AttributeError, seedpeer.response, None)
|
||||
self.assertRaises(AttributeError, seedpeer.response, [])
|
||||
self.assertRaises(AttributeError, seedpeer.response, '')
|
||||
self.assertRaises(AttributeError, seedpeer.response, '[]')
|
||||
|
||||
def test_response_returns_empty_list(self):
|
||||
response = mock.Mock(text='<html></html>')
|
||||
self.assertEqual(seedpeer.response(response), [])
|
||||
|
||||
def test_response_returns_all_results(self):
|
||||
response = mock.Mock(text=self.html)
|
||||
results = seedpeer.response(response)
|
||||
self.assertTrue(isinstance(results, list))
|
||||
self.assertEqual(len(results), 2)
|
||||
|
||||
def test_response_returns_correct_results(self):
|
||||
response = mock.Mock(text=self.html)
|
||||
results = seedpeer.response(response)
|
||||
self.assertEqual(
|
||||
results[0]['title'], 'Narcos - Season 2 - 720p WEBRiP - x265 HEVC - ShAaNiG '
|
||||
)
|
||||
self.assertEqual(
|
||||
results[0]['url'],
|
||||
'http://www.seedpeer.eu/details/11685972/Narcos---Season-2---720p-WEBRiP---x265-HEVC---ShAaNiG.html'
|
||||
)
|
||||
self.assertEqual(results[0]['content'], '2.48 GB, 1 day')
|
||||
self.assertEqual(results[0]['seed'], '861')
|
||||
self.assertEqual(results[0]['leech'], '332')
|
||||
Loading…
x
Reference in New Issue
Block a user