Get Books: Update the Kobo Store plugin for website changes

src/calibre/gui2/store/stores/kobo_plugin.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import (unicode_literals, division, absolute_import, print_function)
-store_version = 3 # Needed for dynamic plugin loading
+store_version = 4  # Needed for dynamic plugin loading
 
 __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
@@ -11,19 +11,80 @@ import random
 import urllib
 from contextlib import closing
 
-from lxml import html
+from lxml import html, etree
 
 from PyQt5.Qt import QUrl
 
 from calibre import browser, url_slash_cleaner
+from calibre.ebooks.metadata import authors_to_string
 from calibre.gui2 import open_url
 from calibre.gui2.store import StorePlugin
 from calibre.gui2.store.basic_config import BasicStoreConfig
 from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
+def search_kobo(query, max_results=10, timeout=60, write_html_to=None):
+    from css_selectors import Select
+    url = 'http://www.kobobooks.com/search/search.html?q=' + urllib.quote_plus(query)
+
+    br = browser()
+
+    with closing(br.open(url, timeout=timeout)) as f:
+        raw = f.read()
+        if write_html_to is not None:
+            with open(write_html_to, 'wb') as f:
+                f.write(raw)
+        doc = html.fromstring(raw)
+        select = Select(doc)
+        for i, item in enumerate(select('.result-items .item-wrapper.book')):
+            if i == max_results:
+                break
+            for img in select('.item-image img[src]', item):
+                cover_url = img.get('src')
+                if cover_url.startswith('//'):
+                    cover_url = 'http:' + cover_url
+                break
+            else:
+                cover_url = None
+
+            for p in select('p.title', item):
+                title = etree.tostring(p, method='text', encoding=unicode).strip()
+                for a in select('a[href]', p):
+                    url = 'http://store.kobobooks.com' + a.get('href')
+                    break
+                else:
+                    url = None
+                break
+            else:
+                title = None
+
+            authors = []
+            for a in select('p.author a.contributor', item):
+                authors.append(etree.tostring(a, method='text', encoding=unicode).strip())
+            authors = authors_to_string(authors)
+
+            for p in select('p.price', item):
+                price = etree.tostring(p, method='text', encoding=unicode).strip()
+                break
+            else:
+                price = None
+
+            if title and authors and url:
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title
+                s.author = authors
+                s.price = price
+                s.detail_item = url
+                s.formats = 'EPUB'
+                s.drm = SearchResult.DRM_UNKNOWN
+
+            yield s
+
 class KoboStore(BasicStoreConfig, StorePlugin):
 
+    minimum_calibre_version = (2, 21, 0)
+
     def open(self, parent=None, detail_item=None, external=False):
         pub_id = 'sHa5EXvYOwA'
         # Use Kovid's affiliate id 30% of the time.
@@ -48,40 +109,8 @@ class KoboStore(BasicStoreConfig, StorePlugin):
             d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://www.kobobooks.com/search/search.html?q=' + urllib.quote_plus(query)
-
-        br = browser()
-
-        counter = max_results
-        with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())
-            for data in doc.xpath('//ul[contains(@class, "flowview-items")]/li'):
-                if counter <= 0:
-                    break
-
-                id = ''.join(data.xpath('./a[contains(@class, "block-link")]/@href'))
-                if not id:
-                    continue
-                id = id[1:]
-
-                price = ''.join(data.xpath('.//a[contains(@class, "primary-button")]//text()'))
-
-                cover_url = ''.join(data.xpath('.//img[1]/@src'))
-                cover_url = 'http:%s' % cover_url
-
-                title = ''.join(data.xpath('.//p[contains(@class, "flowview-item-title")]//text()'))
-
-                counter -= 1
-
-                s = SearchResult()
-                s.cover_url = cover_url
-                s.title = title.strip()
-                s.price = price.strip()
-                s.detail_item = 'http://store.kobobooks.com/' + id.strip()
-                s.formats = 'EPUB'
-                s.drm = SearchResult.DRM_UNKNOWN
-
-                yield s
+        for result in search_kobo(query, max_results=max_results, timeout=timeout):
+            yield result
 
     def get_details(self, search_result, timeout):
         br = browser()
@@ -89,3 +118,8 @@ class KoboStore(BasicStoreConfig, StorePlugin):
             idata = html.fromstring(nf.read())
             search_result.author = ', '.join(idata.xpath('.//h2[contains(@class, "author")]//a/text()'))
         return True
+
+if __name__ == '__main__':
+    import sys
+    for result in search_kobo(' '.join(sys.argv[1:]), write_html_to='/t/kobo.html'):
+        print (result)