load more than 10 results in gandalf

This commit is contained in:
Tomasz Długosz 2012-01-30 22:43:19 +01:00
parent d974ec57f6
commit 135d61b008

View File

@ -3,7 +3,7 @@
from __future__ import (unicode_literals, division, absolute_import, print_function) from __future__ import (unicode_literals, division, absolute_import, print_function)
__license__ = 'GPL 3' __license__ = 'GPL 3'
__copyright__ = '2011, Tomasz Długosz <tomek3d@gmail.com>' __copyright__ = '2011-2012, Tomasz Długosz <tomek3d@gmail.com>'
__docformat__ = 'restructuredtext en' __docformat__ = 'restructuredtext en'
import re import re
@ -35,16 +35,14 @@ class GandalfStore(BasicStoreConfig, StorePlugin):
d.exec_() d.exec_()
def search(self, query, max_results=10, timeout=60): def search(self, query, max_results=10, timeout=60):
url = 'http://www.gandalf.com.pl/s/' counter = max_results
values={ page = 1
'search': query.decode('utf-8').encode('iso8859_2'), url = 'http://www.gandalf.com.pl/we/' + urllib.quote_plus(query.decode('utf-8').encode('iso8859_2')) + '/bdb'
'dzialx':'11'
}
br = browser() br = browser()
counter = max_results while counter:
with closing(br.open(url, data=urllib.urlencode(values), timeout=timeout)) as f: with closing(br.open((url + str(page-1) + '/#s') if (page-1) else (url + '/#s'), timeout=timeout)) as f:
doc = html.fromstring(f.read()) doc = html.fromstring(f.read())
for data in doc.xpath('//div[@class="box"]'): for data in doc.xpath('//div[@class="box"]'):
if counter <= 0: if counter <= 0:
@ -79,3 +77,6 @@ class GandalfStore(BasicStoreConfig, StorePlugin):
s.formats = formats.upper().strip() s.formats = formats.upper().strip()
yield s yield s
if not doc.xpath('boolean(//div[@class="wyszukiwanie_podstawowe_header"]//div[@class="box"])'):
break
page+=1