Merge from trunk

manual/conversion.rst

@@ -436,8 +436,8 @@ generate a Table of Contents in the converted ebook, based on the actual content
 
 .. note:: Using these options can be a little challenging to get exactly right.
     If you prefer creating/editing the Table of Contents by hand, convert to
-    the EPUB or AZW3 formats and select the checkbox at the bottom of the
+    the EPUB or AZW3 formats and select the checkbox at the bottom of the Table
-    screen that says 
+    of Contents section of the conversion dialog that says 
     :guilabel:`Manually fine-tune the Table of Contents after conversion`. 
     This will launch the ToC Editor tool after the conversion. It allows you to
     create entries in the Table of Contents by simply clicking the place in the

manual/server.rst

@@ -91,7 +91,11 @@ First, we have to create a WSGI *adapter* for the calibre content server. Here i
             # Path to the calibre library to be served
             # The server process must have write permission for all files/dirs
             # in this directory or BAD things will happen
-            path_to_library='/home/kovid/documents/demo library'
+            path_to_library='/home/kovid/documents/demo library',
+
+            # The virtual library (restriction) to be used when serving this
+            # library. 
+            virtual_library=None
     )
 
     del create_wsgi_app

recipes/the_escapist.recipe

@@ -1,8 +1,11 @@
 #!/usr/bin/env  python
 __license__   = 'GPL v3'
-__author__    = 'Lorenzo Vigentini'
+__author__    = 'Lorenzo Vigentini and Tom Surace'
-__copyright__ = '2009, Lorenzo Vigentini <l.vigentini at gmail.com>'
+__copyright__ = '2009, Lorenzo Vigentini <l.vigentini at gmail.com>, 2013 Tom Surace <tekhedd@byteheaven.net>'
-description   = 'the Escapist Magazine - v1.02 (09, January 2010)'
+description   = 'The Escapist Magazine - v1.3 (2013, April 2013)'
+
+#
+# Based on 'the Escapist Magazine - v1.02 (09, January 2010)'
 
 '''
 http://www.escapistmagazine.com/
@@ -11,12 +14,11 @@ http://www.escapistmagazine.com/
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class al(BasicNewsRecipe):
-    author        = 'Lorenzo Vigentini'
+    author        = 'Lorenzo Vigentini and Tom Surace'
     description   = 'The Escapist Magazine'
-
     cover_url      = 'http://cdn.themis-media.com/themes/escapistmagazine/default/images/logo.png'
     title          = u'The Escapist Magazine'
-    publisher      = 'Themis media'
+    publisher      = 'Themis Media'
     category       = 'Video games news, lifestyle, gaming culture'
 
     language       = 'en'
@@ -36,18 +38,19 @@ class al(BasicNewsRecipe):
                      ]
 
     def print_version(self,url):
+        # Expect article url in the format:
+        # http://www.escapistmagazine.com/news/view/123198-article-name?utm_source=rss&utm_medium=rss&utm_campaign=news
+        # 
         baseURL='http://www.escapistmagazine.com'
         segments = url.split('/')
-        #basename = '/'.join(segments[:3]) + '/'
         subPath= '/'+ segments[3] + '/'
-        articleURL=(segments[len(segments)-1])[0:5]
 
-        if articleURL[4] =='-':
+        # The article number is the "number" that starts the name
-            articleURL=articleURL[:4]
+        articleNumber = segments[len(segments)-1];   # the "article name"
+        articleNumber = articleNumber.split('-')[0]; # keep part before hyphen
 
-        printVerString='print/'+ articleURL
+        fullUrl =  baseURL + subPath + 'print/' + articleNumber
-        s=  baseURL + subPath + printVerString
+        return fullUrl
-        return s
 
     keep_only_tags     = [
                             dict(name='div', attrs={'id':'article'})

src/calibre/customize/builtins.py

@@ -1468,6 +1468,17 @@ class StoreKoboStore(StoreBase):
     formats = ['EPUB']
     affiliate = True
 
+class StoreKoobeStore(StoreBase):
+    name = 'Koobe'
+    author = u'Tomasz Długosz'
+    description = u'Księgarnia internetowa oferuje ebooki (książki elektroniczne) w postaci plików epub, mobi i pdf.'
+    actual_plugin = 'calibre.gui2.store.stores.koobe_plugin:KoobeStore'
+
+    drm_free_only = True
+    headquarters = 'PL'
+    formats = ['EPUB', 'MOBI', 'PDF']
+    affiliate = True
+
 class StoreLegimiStore(StoreBase):
     name = 'Legimi'
     author = u'Tomasz Długosz'
@@ -1650,6 +1661,7 @@ class StoreWoblinkStore(StoreBase):
 
     headquarters = 'PL'
     formats = ['EPUB', 'MOBI', 'PDF', 'WOBLINK']
+    affiliate = True
 
 class XinXiiStore(StoreBase):
     name = 'XinXii'
@@ -1687,6 +1699,7 @@ plugins += [
     StoreGoogleBooksStore,
     StoreGutenbergStore,
     StoreKoboStore,
+    StoreKoobeStore,
     StoreLegimiStore,
     StoreLibreDEStore,
     StoreLitResStore,

src/calibre/gui2/store/stores/gutenberg_plugin.py

@@ -1,91 +1,104 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import (unicode_literals, division, absolute_import, print_function)
-store_version = 2 # Needed for dynamic plugin loading
+store_version = 3 # Needed for dynamic plugin loading
 
 __license__ = 'GPL 3'
-__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__copyright__ = '2011, 2013, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import base64
 import mimetypes
+import re
 import urllib
 from contextlib import closing
 
-from lxml import html
+from lxml import etree
 
-from PyQt4.Qt import QUrl
+from calibre import browser, url_slash_cleaner
-
+from calibre.constants import __version__
-from calibre import browser, random_user_agent, url_slash_cleaner
-from calibre.gui2 import open_url
-from calibre.gui2.store import StorePlugin
 from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore
 from calibre.gui2.store.search_result import SearchResult
-from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
-class GutenbergStore(BasicStoreConfig, StorePlugin):
+class GutenbergStore(BasicStoreConfig, OpenSearchOPDSStore):
 
-    def open(self, parent=None, detail_item=None, external=False):
+    open_search_url = 'http://www.gutenberg.org/catalog/osd-books.xml'
-        url = 'http://gutenberg.org/'
+    web_url = 'http://m.gutenberg.org/'
-
-        if detail_item:
-            detail_item = url_slash_cleaner(url + detail_item)
-
-        if external or self.config.get('open_external', False):
-            open_url(QUrl(detail_item if detail_item else url))
-        else:
-            d = WebStoreDialog(self.gui, url, parent, detail_item)
-            d.setWindowTitle(self.name)
-            d.set_tags(self.config.get('tags', ''))
-            d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://m.gutenberg.org/ebooks/search.mobile/?default_prefix=all&sort_order=title&query=' + urllib.quote_plus(query)
+        '''
+        Gutenberg's ODPS feed is poorly implmented and has a number of issues
+        which require very special handling to fix the results.
 
-        br = browser(user_agent=random_user_agent())
+        Issues:
+          * "Sort Alphabetically" and "Sort by Release Date" are returned
+            as book entries.
+          * The author is put into a "content" tag and not the author tag.
+          * The link to the book itself goes to an odps page which we need
+            to turn into a link to a web page.
+          * acquisition links are not part of the search result so we have
+            to go to the odps item itself. Detail item pages have a nasty
+            note saying:
+              DON'T USE THIS PAGE FOR SCRAPING. 
+              Seriously. You'll only get your IP blocked.
+            We're using the ODPS feed because people are getting blocked with
+            the previous implementation so due to this using ODPS probably
+            won't solve this issue.
+          * Images are not links but base64 encoded strings. They are also not
+            real cover images but a little blue book thumbnail.
+        '''
+
+        url = 'http://m.gutenberg.org/ebooks/search.opds/?query=' + urllib.quote_plus(query)
 
         counter = max_results
+        br = browser(user_agent='calibre/'+__version__)
         with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())
+            doc = etree.fromstring(f.read())
-            for data in doc.xpath('//ol[@class="results"]/li[@class="booklink"]'):
+            for data in doc.xpath('//*[local-name() = "entry"]'):
                 if counter <= 0:
                     break
 
-                id = ''.join(data.xpath('./a/@href'))
-                id = id.split('.mobile')[0]
-
-                title = ''.join(data.xpath('.//span[@class="title"]/text()'))
-                author = ''.join(data.xpath('.//span[@class="subtitle"]/text()'))
-
                 counter -= 1
 
                 s = SearchResult()
-                s.cover_url = ''
 
-                s.detail_item = id.strip()
+                # We could use the <link rel="alternate" type="text/html" ...> tag from the
-                s.title = title.strip()
+                # detail odps page but this is easier.
-                s.author = author.strip()
+                id = ''.join(data.xpath('./*[local-name() = "id"]/text()')).strip()
-                s.price = '$0.00'
+                s.detail_item = url_slash_cleaner('%s/ebooks/%s' % (self.web_url, re.sub('[^\d]', '', id)))
-                s.drm = SearchResult.DRM_UNLOCKED
+                if not s.detail_item:
+                    continue
 
-                yield s
+                s.title = ' '.join(data.xpath('./*[local-name() = "title"]//text()')).strip()
-
+                s.author = ', '.join(data.xpath('./*[local-name() = "content"]//text()')).strip()
-    def get_details(self, search_result, timeout):
+                if not s.title or not s.author:
-        url = url_slash_cleaner('http://m.gutenberg.org/' + search_result.detail_item)
+                    continue
-
-        br = browser(user_agent=random_user_agent())
-        with closing(br.open(url, timeout=timeout)) as nf:
-            doc = html.fromstring(nf.read())
-
-            for save_item in doc.xpath('//li[contains(@class, "icon_save")]/a'):
-                type = save_item.get('type')
-                href = save_item.get('href')
 
+                # Get the formats and direct download links.
+                with closing(br.open(id, timeout=timeout/4)) as nf:
+                    ndoc = etree.fromstring(nf.read())
+                    for link in ndoc.xpath('//*[local-name() = "link" and @rel = "http://opds-spec.org/acquisition"]'):
+                        type = link.get('type')
+                        href = link.get('href')
                         if type:
                             ext = mimetypes.guess_extension(type)
                             if ext:
                                 ext = ext[1:].upper().strip()
-                        search_result.downloads[ext] = href
+                                s.downloads[ext] = href
 
-                search_result.formats = ', '.join(search_result.downloads.keys())
+                s.formats = ', '.join(s.downloads.keys())
+                if not s.formats:
+                    continue
 
-        return True
+                for link in data.xpath('./*[local-name() = "link"]'):
+                    rel = link.get('rel')
+                    href = link.get('href')
+                    type = link.get('type')
+
+                    if rel and href and type:
+                        if rel in ('http://opds-spec.org/thumbnail', 'http://opds-spec.org/image/thumbnail'):
+                            if href.startswith('data:image/png;base64,'):
+                                s.cover_data = base64.b64decode(href.replace('data:image/png;base64,', ''))
+
+                yield s

src/calibre/gui2/store/stores/koobe_plugin.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (division, absolute_import, print_function)
+store_version = 1 # Needed for dynamic plugin loading
+
+__license__ = 'GPL 3'
+__copyright__ = '2013, Tomasz Długosz <tomek3d@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import urllib
+from base64 import b64encode
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class KoobeStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/15/58/'
+        url = 'http://www.koobe.pl/'
+
+        aff_url = aff_root + str(b64encode(url))
+
+        detail_url = None
+        if detail_item:
+            detail_url = aff_root + str(b64encode(detail_item))
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url if detail_url else aff_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+
+        br = browser()
+        page=1
+
+        counter = max_results
+        while counter:
+            with closing(br.open('http://www.koobe.pl/s,p,' + str(page) + ',szukaj/fraza:' + urllib.quote(query), timeout=timeout)) as f:
+                doc = html.fromstring(f.read().decode('utf-8'))
+                for data in doc.xpath('//div[@class="seach_result"]/div[@class="result"]'):
+                    if counter <= 0:
+                        break
+
+                    id = ''.join(data.xpath('.//div[@class="cover"]/a/@href'))
+                    if not id:
+                        continue
+
+                    cover_url = ''.join(data.xpath('.//div[@class="cover"]/a/img/@src'))
+                    price = ''.join(data.xpath('.//span[@class="current_price"]/text()'))
+                    title = ''.join(data.xpath('.//h2[@class="title"]/a/text()'))
+                    author = ''.join(data.xpath('.//h3[@class="book_author"]/a/text()'))
+                    formats = ', '.join(data.xpath('.//div[@class="formats"]/div/div/@title'))
+
+                    counter -= 1
+
+                    s = SearchResult()
+                    s.cover_url =  'http://koobe.pl/' + cover_url
+                    s.title = title.strip()
+                    s.author = author.strip()
+                    s.price = price
+                    s.detail_item = 'http://koobe.pl' + id[1:]
+                    s.formats = formats.upper()
+                    s.drm = SearchResult.DRM_UNLOCKED
+
+                    yield s
+                if not doc.xpath('//div[@class="site_bottom"]//a[@class="right"]'):
+                    break
+            page+=1

src/calibre/gui2/store/stores/woblink_plugin.py

@@ -1,14 +1,15 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import (unicode_literals, division, absolute_import, print_function)
-store_version = 1 # Needed for dynamic plugin loading
+store_version = 2 # Needed for dynamic plugin loading
 
 __license__ = 'GPL 3'
-__copyright__ = '2011-2012, Tomasz Długosz <tomek3d@gmail.com>'
+__copyright__ = '2011-2013, Tomasz Długosz <tomek3d@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import re
 import urllib
+from base64 import b64encode
 from contextlib import closing
 
 from lxml import html
@@ -25,17 +26,19 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class WoblinkStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-
+        aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/16/58/'
         url = 'http://woblink.com/publication'
+
+        aff_url = aff_root + str(b64encode(url))
         detail_url = None
 
         if detail_item:
-            detail_url = 'http://woblink.com' + detail_item
+            detail_url = aff_root + str(b64encode('http://woblink.com' + detail_item))
 
         if external or self.config.get('open_external', False):
-            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))
         else:
-            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d = WebStoreDialog(self.gui, url, parent, detail_url if detail_url else aff_url)
             d.setWindowTitle(self.name)
             d.set_tags(self.config.get('tags', ''))
             d.exec_()

src/calibre/library/server/main.py

@@ -24,7 +24,7 @@ def stop_threaded_server(server):
     server.exit()
     server.thread = None
 
-def create_wsgi_app(path_to_library=None, prefix=''):
+def create_wsgi_app(path_to_library=None, prefix='', virtual_library=None):
     'WSGI entry point'
     from calibre.library import db
     cherrypy.config.update({'environment': 'embedded'})
@@ -32,6 +32,7 @@ def create_wsgi_app(path_to_library=None, prefix=''):
     parser = option_parser()
     opts, args = parser.parse_args(['calibre-server'])
     opts.url_prefix = prefix
+    opts.restriction = virtual_library
     server = LibraryServer(db, opts, wsgi=True, show_tracebacks=True)
     return cherrypy.Application(server, script_name=None, config=server.config)
 
@@ -98,7 +99,6 @@ def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
     os.dup2(se.fileno(), sys.stderr.fileno())
 
 
-
 def main(args=sys.argv):
     from calibre.library.database2 import LibraryDatabase2
     parser = option_parser()