Merge from virt_lib

Changelog.yaml

@@ -1,4 +1,4 @@
-# vim:fileencoding=UTF-8:ts=2:sw=2:sta:et:sts=2:ai
+# vim:fileencoding=utf-8:ts=2:sw=2:sta:et:sts=2:ai
 # Each release can have new features and bug fixes. Each of which
 # must have a title and can optionally have linked tickets and a description.
 # In addition they can have a type field which defaults to minor, but should be major
@@ -20,6 +20,66 @@
 #   new recipes:
 #     - title: 
 
+- version: 0.9.27
+  date: 2013-04-12
+
+  new features:
+    - title: "Metadata download: Add two new sources for covers: Google Image Search and bigbooksearch.com."
+      description: "To enable them go to Preferences->Metadata download and enable the 'Google Image' and 'Big Book Search' sources. Google Images is useful for finding larger covers as well as alternate versions of the cover. Big Book Search searches for alternate covers from amazon.com. It can occasionally find nicer covers than the direct Amazon source. Note that both these sources download multiple covers for a single book. Some of these covers can be wrong (i.e. they may be of a different book or not covers at all, so you should inspect the results and manually pick the best match). When bulk downloading, these sources are only used if the other sources find no covers."
+      type: major
+
+    - title: "Content server: Allow specifying a reestriction to use for the server when embedding it as a WSGI app."
+      tickets: [1167951]
+
+    - title: "Get Books: Add a plugin for the Koobe Polish book store"
+
+    - title: "calibredb add_format: Add an option to not replace existing formats. Also pep8 compliance."
+
+    - title: "Allow restoring of the ORIGINAL_XXX format by right-clicking it in the book details panel"
+ 
+  bug fixes:
+    - title: "AZW3 Input: Do not fail to identify JPEG images with 8BIM headers created with Adobe Photoshop."
+      tickets: [1167985]
+
+    - title: "Amazon metadata download: Ignore Spanish edition entries when searching for a book on amazon.com"
+
+    - title: "TXT Input: When converting a txt file with a Byte Order Mark, remove the Byte Order Mark before further processing as it can cause the first line of the text to be mis-interpreted."
+
+    - title: "Get Books: Fix searching for current book/title/author by right clicking the get books icon"
+
+    - title: "Get Books: Update nexto, gutenberg, and virtualo store plugins for website changes"
+
+    - title: "Amazon metadata download: When downloading from amazon.co.jp handle the 'Black curtain redirect' for adult titles."
+      tickets: [1165628]
+
+    - title: "When extracting zip files do not allow maliciously created zip files to overwrite other files on the system"
+
+    - title: "RTF Input: Handle RTF files with invalid border style specifications"
+      tickets: [1021270]
+
+  improved recipes:
+    - The Escapist
+    - San Francisco Chronicle
+    - The Onion
+    - Fronda
+    - Tom's Hardware
+    - New Yorker
+    - Financial Times UK
+    - Business Week Magazine
+    - Victoria Times
+    - tvxs
+    - The Independent
+
+  new recipes:
+    - title: Economia 
+      author: Manish Bhattarai
+
+    - title: Universe Today 
+      author: seird
+
+    - title: The Galaxy's Edge 
+      author: Krittika Goyal
+
 - version: 0.9.26
   date: 2013-04-05
 

manual/conversion.rst

@@ -436,8 +436,8 @@ generate a Table of Contents in the converted ebook, based on the actual content
 
 .. note:: Using these options can be a little challenging to get exactly right.
     If you prefer creating/editing the Table of Contents by hand, convert to
-    the EPUB or AZW3 formats and select the checkbox at the bottom of the
+    the EPUB or AZW3 formats and select the checkbox at the bottom of the Table
-    screen that says 
+    of Contents section of the conversion dialog that says 
     :guilabel:`Manually fine-tune the Table of Contents after conversion`. 
     This will launch the ToC Editor tool after the conversion. It allows you to
     create entries in the Table of Contents by simply clicking the place in the

manual/faq.rst

@@ -802,6 +802,12 @@ Downloading from the Internet can sometimes result in a corrupted download. If t
     * Try temporarily disabling your antivirus program (Microsoft Security Essentials, or Kaspersky or Norton or McAfee or whatever). This is most likely the culprit if the upgrade process is hanging in the middle.
     * Try rebooting your computer and running a registry cleaner like `Wise registry cleaner <http://www.wisecleaner.com>`_.
     * Try downloading the installer with an alternate browser. For example if you are using Internet Explorer, try using Firefox or Chrome instead.
+    * If you get an error about a missing DLL on windows, then most likely, the
+      permissions on your temporary folder are incorrect. Go to the folder
+      :file:`C:\\Users\\USERNAME\\AppData\\Local` in Windows explorer and then
+      right click on the :file:`Temp` folder and select :guilabel:`Properties` and go to
+      the :guilabel:`Security` tab. Make sure that your user account has full control 
+      for this folder.
       
 If you still cannot get the installer to work and you are on windows, you can use the `calibre portable install <http://calibre-ebook.com/download_portable>`_, which does not need an installer (it is just a zip file).
 

manual/server.rst

@@ -91,7 +91,11 @@ First, we have to create a WSGI *adapter* for the calibre content server. Here i
             # Path to the calibre library to be served
             # The server process must have write permission for all files/dirs
             # in this directory or BAD things will happen
-            path_to_library='/home/kovid/documents/demo library'
+            path_to_library='/home/kovid/documents/demo library',
+
+            # The virtual library (restriction) to be used when serving this
+            # library. 
+            virtual_library=None
     )
 
     del create_wsgi_app

recipes/independent.recipe

@@ -41,6 +41,7 @@ class TheIndependentNew(BasicNewsRecipe):
     publication_type        = 'newspaper'
     masthead_url            = 'http://www.independent.co.uk/independent.co.uk/editorial/logo/independent_Masthead.png'
     encoding                = 'utf-8'
+    compress_news_images    = True
     remove_tags             =[
                                dict(attrs={'id' : ['RelatedArtTag','renderBiography']}),
                                dict(attrs={'class' : ['autoplay','openBiogPopup']}),
@@ -343,7 +344,7 @@ class TheIndependentNew(BasicNewsRecipe):
             if 'class' in subtitle_div:
                 clazz = subtitle_div['class'] + ' '
             clazz = clazz + 'subtitle'
-            subtitle_div['class'] = clazz    
+            subtitle_div['class'] = clazz
 
         #find broken images and remove captions
         items_to_extract = []

recipes/the_escapist.recipe

@@ -1,8 +1,11 @@
 #!/usr/bin/env  python
 __license__   = 'GPL v3'
-__author__    = 'Lorenzo Vigentini'
+__author__    = 'Lorenzo Vigentini and Tom Surace'
-__copyright__ = '2009, Lorenzo Vigentini <l.vigentini at gmail.com>'
+__copyright__ = '2009, Lorenzo Vigentini <l.vigentini at gmail.com>, 2013 Tom Surace <tekhedd@byteheaven.net>'
-description   = 'the Escapist Magazine - v1.02 (09, January 2010)'
+description   = 'The Escapist Magazine - v1.3 (2013, April 2013)'
+
+#
+# Based on 'the Escapist Magazine - v1.02 (09, January 2010)'
 
 '''
 http://www.escapistmagazine.com/
@@ -11,12 +14,11 @@ http://www.escapistmagazine.com/
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class al(BasicNewsRecipe):
-    author        = 'Lorenzo Vigentini'
+    author        = 'Lorenzo Vigentini and Tom Surace'
     description   = 'The Escapist Magazine'
-
     cover_url      = 'http://cdn.themis-media.com/themes/escapistmagazine/default/images/logo.png'
     title          = u'The Escapist Magazine'
-    publisher      = 'Themis media'
+    publisher      = 'Themis Media'
     category       = 'Video games news, lifestyle, gaming culture'
 
     language       = 'en'
@@ -36,18 +38,19 @@ class al(BasicNewsRecipe):
                      ]
 
     def print_version(self,url):
+        # Expect article url in the format:
+        # http://www.escapistmagazine.com/news/view/123198-article-name?utm_source=rss&utm_medium=rss&utm_campaign=news
+        # 
         baseURL='http://www.escapistmagazine.com'
         segments = url.split('/')
-        #basename = '/'.join(segments[:3]) + '/'
         subPath= '/'+ segments[3] + '/'
-        articleURL=(segments[len(segments)-1])[0:5]
 
-        if articleURL[4] =='-':
+        # The article number is the "number" that starts the name
-            articleURL=articleURL[:4]
+        articleNumber = segments[len(segments)-1];   # the "article name"
+        articleNumber = articleNumber.split('-')[0]; # keep part before hyphen
 
-        printVerString='print/'+ articleURL
+        fullUrl =  baseURL + subPath + 'print/' + articleNumber
-        s=  baseURL + subPath + printVerString
+        return fullUrl
-        return s
 
     keep_only_tags     = [
                             dict(name='div', attrs={'id':'article'})

recipes/tvxs.recipe

@@ -1,5 +1,6 @@
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 
+import re
 from calibre.web.feeds.recipes import BasicNewsRecipe
 
 class TVXS(BasicNewsRecipe):
@@ -8,19 +9,30 @@ class TVXS(BasicNewsRecipe):
     description            = 'News from Greece'
     max_articles_per_feed  = 100
     oldest_article         = 3
-    simultaneous_downloads = 1
     publisher              = 'TVXS'
-    category               = 'news, GR'
+    category               = 'news, sport, greece'
     language               = 'el'
     encoding               = None
     use_embedded_content = False
     remove_empty_feeds = True
-    #conversion_options     = { 'linearize_tables': True}
+    conversion_options = {'smarten_punctuation': True}
     no_stylesheets         = True
+    publication_type = 'newspaper'
     remove_tags_before     = dict(name='h1',attrs={'class':'print-title'})
     remove_tags_after      = dict(name='div',attrs={'class':'field field-type-relevant-content field-field-relevant-articles'})
-    remove_attributes      = ['width', 'src', 'header', 'footer']
+    remove_tags      = [dict(name='div',attrs={'class':'field field-type-relevant-content field-field-relevant-articles'}),
-
+                        dict(name='div',attrs={'class':'field field-type-filefield field-field-image-gallery'}),
+                        dict(name='div',attrs={'class':'filefield-file'})]
+    remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan', 'valign', 'vspace', 'hspace', 'alt', 'width', 'height']
+    extra_css = 'body { font-family: verdana, helvetica, sans-serif; } \
+                 table { width: 100%; } \
+                 td img { display: block; margin: 5px auto; } \
+                 ul { padding-top: 10px; } \
+                 ol { padding-top: 10px; } \
+                 li { padding-top: 5px; padding-bottom: 5px; } \
+                 h1 { text-align: center; font-size: 125%; font-weight: bold; } \
+                 h2, h3, h4, h5, h6 { text-align: center; font-size: 100%; font-weight: bold; }'
+    preprocess_regexps     = [(re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''), (re.compile(r'<br[ ]*clear.*/>', re.IGNORECASE), lambda m: '')]
 
     feeds = [(u'Ελλάδα', 'http://tvxs.gr/feeds/2/feed.xml'),
         (u'Κόσμος', 'http://tvxs.gr/feeds/5/feed.xml'),
@@ -35,17 +47,10 @@ class TVXS(BasicNewsRecipe):
         (u'Ιστορία', 'http://tvxs.gr/feeds/1573/feed.xml'),
         (u'Χιούμορ', 'http://tvxs.gr/feeds/692/feed.xml')]
 
-
     def print_version(self, url):
-        import urllib2, urlparse, StringIO, gzip
+        br = self.get_browser()
-
+        response = br.open(url)
-        fp = urllib2.urlopen(url)
+        data = response.read()
-        data = fp.read()
-        if fp.info()['content-encoding'] == 'gzip':
-            gzip_data = StringIO.StringIO(data)
-            gzipper = gzip.GzipFile(fileobj=gzip_data)
-            data = gzipper.read()
-        fp.close()
 
         pos_1 = data.find('<a href="/print/')
         if pos_1 == -1:
@@ -57,5 +62,5 @@ class TVXS(BasicNewsRecipe):
         pos_1 += len('<a href="')
         new_url = data[pos_1:pos_2]
 
-        print_url = urlparse.urljoin(url, new_url)
+        print_url = "http://tvxs.gr" + new_url
         return print_url

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 9, 26)
+numeric_version = (0, 9, 27)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/customize/builtins.py

@@ -1468,6 +1468,17 @@ class StoreKoboStore(StoreBase):
     formats = ['EPUB']
     affiliate = True
 
+class StoreKoobeStore(StoreBase):
+    name = 'Koobe'
+    author = u'Tomasz Długosz'
+    description = u'Księgarnia internetowa oferuje ebooki (książki elektroniczne) w postaci plików epub, mobi i pdf.'
+    actual_plugin = 'calibre.gui2.store.stores.koobe_plugin:KoobeStore'
+
+    drm_free_only = True
+    headquarters = 'PL'
+    formats = ['EPUB', 'MOBI', 'PDF']
+    affiliate = True
+
 class StoreLegimiStore(StoreBase):
     name = 'Legimi'
     author = u'Tomasz Długosz'
@@ -1650,6 +1661,7 @@ class StoreWoblinkStore(StoreBase):
 
     headquarters = 'PL'
     formats = ['EPUB', 'MOBI', 'PDF', 'WOBLINK']
+    affiliate = True
 
 class XinXiiStore(StoreBase):
     name = 'XinXii'
@@ -1687,6 +1699,7 @@ plugins += [
     StoreGoogleBooksStore,
     StoreGutenbergStore,
     StoreKoboStore,
+    StoreKoobeStore,
     StoreLegimiStore,
     StoreLibreDEStore,
     StoreLitResStore,

src/calibre/db/backend.py

@@ -422,6 +422,8 @@ class DB(object):
         ('uuid', False), ('comments', True), ('id', False), ('pubdate', False),
         ('last_modified', False), ('size', False), ('languages', False),
         ]
+        defs['virtual_libraries'] = {}
+        defs['virtual_lib_on_startup'] = defs['cs_virtual_lib_on_startup'] = ''
 
         # Migrate the bool tristate tweak
         defs['bools_are_tristate'] = \
@@ -470,6 +472,24 @@ class DB(object):
             except:
                 pass
 
+        # migrate the gui_restriction preference to a virtual library
+        gr_pref = self.prefs.get('gui_restriction', None)
+        if gr_pref:
+            virt_libs = self.prefs.get('virtual_libraries', {})
+            virt_libs[gr_pref] = 'search:"' + gr_pref + '"'
+            self.prefs['virtual_libraries'] = virt_libs
+            self.prefs['gui_restriction'] = ''
+            self.prefs['virtual_lib_on_startup'] = gr_pref
+
+        # migrate the cs_restriction preference to a virtual library
+        gr_pref = self.prefs.get('cs_restriction', None)
+        if gr_pref:
+            virt_libs = self.prefs.get('virtual_libraries', {})
+            virt_libs[gr_pref] = 'search:"' + gr_pref + '"'
+            self.prefs['virtual_libraries'] = virt_libs
+            self.prefs['cs_restriction'] = ''
+            self.prefs['cs_virtual_lib_on_startup'] = gr_pref
+
         # Rename any user categories with names that differ only in case
         user_cats = self.prefs.get('user_categories', [])
         catmap = {}

src/calibre/db/view.py

@@ -49,7 +49,8 @@ class View(object):
         self.cache = cache
         self.marked_ids = {}
         self.search_restriction_book_count = 0
-        self.search_restriction = ''
+        self.search_restriction = self.base_restriction = ''
+        self.search_restriction_name = self.base_restriction_name = ''
         self._field_getters = {}
         for col, idx in cache.backend.FIELD_MAP.iteritems():
             if isinstance(col, int):
@@ -168,8 +169,19 @@ class View(object):
             return ans
         self._map_filtered = tuple(ans)
 
+    def _build_restriction_string(self, restriction):
+        if self.base_restriction:
+            if restriction:
+                return u'(%s) and (%s)' % (self.base_restriction, restriction)
+            else:
+                return self.base_restriction
+        else:
+            return restriction
+
     def search_getting_ids(self, query, search_restriction,
-                           set_restriction_count=False):
+                           set_restriction_count=False, use_virtual_library=True):
+        if use_virtual_library:
+            search_restriction = self._build_restriction_string(search_restriction)
         q = ''
         if not query or not query.strip():
             q = search_restriction
@@ -188,11 +200,32 @@ class View(object):
             self.search_restriction_book_count = len(rv)
         return rv
 
+    def get_search_restriction(self):
+        return self.search_restriction
+
     def set_search_restriction(self, s):
         self.search_restriction = s
 
+    def get_base_restriction(self):
+        return self.base_restriction
+
+    def set_base_restriction(self, s):
+        self.base_restriction = s
+
+    def get_base_restriction_name(self):
+        return self.base_restriction_name
+
+    def set_base_restriction_name(self, s):
+        self.base_restriction_name = s
+
+    def get_search_restriction_name(self):
+        return self.search_restriction_name
+
+    def set_search_restriction_name(self, s):
+        self.search_restriction_name = s
+
     def search_restriction_applied(self):
-        return bool(self.search_restriction)
+        return bool(self.search_restriction) or bool(self.base_restriction)
 
     def get_search_restriction_book_count(self):
         return self.search_restriction_book_count

src/calibre/ebooks/metadata/opf2.py

@@ -24,7 +24,7 @@ from calibre import prints, guess_type
 from calibre.utils.cleantext import clean_ascii_chars
 from calibre.utils.config import tweaks
 
-class Resource(object): # {{{
+class Resource(object):  # {{{
     '''
     Represents a resource (usually a file on the filesystem or a URL pointing
     to the web. Such resources are commonly referred to in OPF files.
@@ -68,7 +68,6 @@ class Resource(object): # {{{
                 self.path = os.path.abspath(os.path.join(basedir, pc.replace('/', os.sep)))
                 self.fragment = url[-1]
 
-
     def href(self, basedir=None):
         '''
         Return a URL pointing to this resource. If it is a file on the filesystem
@@ -90,7 +89,7 @@ class Resource(object): # {{{
             return ''+frag
         try:
             rpath = os.path.relpath(self.path, basedir)
-        except ValueError: # On windows path and basedir could be on different drives
+        except ValueError:  # On windows path and basedir could be on different drives
             rpath = self.path
         if isinstance(rpath, unicode):
             rpath = rpath.encode('utf-8')
@@ -107,7 +106,7 @@ class Resource(object): # {{{
 
 # }}}
 
-class ResourceCollection(object): # {{{
+class ResourceCollection(object):  # {{{
 
     def __init__(self):
         self._resources = []
@@ -160,7 +159,7 @@ class ResourceCollection(object): # {{{
 
 # }}}
 
-class ManifestItem(Resource): # {{{
+class ManifestItem(Resource):  # {{{
 
     @staticmethod
     def from_opf_manifest_item(item, basedir):
@@ -180,7 +179,6 @@ class ManifestItem(Resource): # {{{
             self.mime_type = val
         return property(fget=fget, fset=fset)
 
-
     def __unicode__(self):
         return u'<item id="%s" href="%s" media-type="%s" />'%(self.id, self.href(), self.media_type)
 
@@ -190,7 +188,6 @@ class ManifestItem(Resource): # {{{
     def __repr__(self):
         return unicode(self)
 
-
     def __getitem__(self, index):
         if index == 0:
             return self.href()
@@ -200,7 +197,7 @@ class ManifestItem(Resource): # {{{
 
 # }}}
 
-class Manifest(ResourceCollection): # {{{
+class Manifest(ResourceCollection):  # {{{
 
     @staticmethod
     def from_opf_manifest_element(items, dir):
@@ -245,7 +242,6 @@ class Manifest(ResourceCollection): # {{{
         ResourceCollection.__init__(self)
         self.next_id = 1
 
-
     def item(self, id):
         for i in self:
             if i.id == id:
@@ -269,7 +265,7 @@ class Manifest(ResourceCollection): # {{{
 
 # }}}
 
-class Spine(ResourceCollection): # {{{
+class Spine(ResourceCollection):  # {{{
 
     class Item(Resource):
 
@@ -309,13 +305,10 @@ class Spine(ResourceCollection): # {{{
                 continue
         return s
 
-
-
     def __init__(self, manifest):
         ResourceCollection.__init__(self)
         self.manifest = manifest
 
-
     def replace(self, start, end, ids):
         '''
         Replace the items between start (inclusive) and end (not inclusive) with
@@ -345,7 +338,7 @@ class Spine(ResourceCollection): # {{{
 
 # }}}
 
-class Guide(ResourceCollection): # {{{
+class Guide(ResourceCollection):  # {{{
 
     class Reference(Resource):
 
@@ -363,7 +356,6 @@ class Guide(ResourceCollection): # {{{
                 ans += 'title="%s" '%self.title
             return ans + '/>'
 
-
     @staticmethod
     def from_opf_guide(references, base_dir=os.getcwdu()):
         coll = Guide()
@@ -484,14 +476,14 @@ def dump_dict(cats):
     return json.dumps(object_to_unicode(cats), ensure_ascii=False,
             skipkeys=True)
 
-class OPF(object): # {{{
+class OPF(object):  # {{{
 
     MIMETYPE         = 'application/oebps-package+xml'
     PARSER           = etree.XMLParser(recover=True)
     NAMESPACES       = {
-                        None  : "http://www.idpf.org/2007/opf",
+                        None: "http://www.idpf.org/2007/opf",
-                        'dc'  : "http://purl.org/dc/elements/1.1/",
+                        'dc': "http://purl.org/dc/elements/1.1/",
-                        'opf' : "http://www.idpf.org/2007/opf",
+                        'opf': "http://www.idpf.org/2007/opf",
                        }
     META             = '{%s}meta' % NAMESPACES['opf']
     xpn = NAMESPACES.copy()
@@ -501,9 +493,10 @@ class OPF(object): # {{{
     CONTENT          = XPath('self::*[re:match(name(), "meta$", "i")]/@content')
     TEXT             = XPath('string()')
 
-
     metadata_path   = XPath('descendant::*[re:match(name(), "metadata", "i")]')
-    metadata_elem_path = XPath('descendant::*[re:match(name(), concat($name, "$"), "i") or (re:match(name(), "meta$", "i") and re:match(@name, concat("^calibre:", $name, "$"), "i"))]')
+    metadata_elem_path = XPath(
+        'descendant::*[re:match(name(), concat($name, "$"), "i") or (re:match(name(), "meta$", "i") '
+        'and re:match(@name, concat("^calibre:", $name, "$"), "i"))]')
     title_path      = XPath('descendant::*[re:match(name(), "title", "i")]')
     authors_path    = XPath('descendant::*[re:match(name(), "creator", "i") and (@role="aut" or @opf:role="aut" or (not(@role) and not(@opf:role)))]')
     bkp_path        = XPath('descendant::*[re:match(name(), "contributor", "i") and (@role="bkp" or @opf:role="bkp")]')
@@ -640,7 +633,8 @@ class OPF(object): # {{{
                     if 'toc' in item.href().lower():
                         toc = item.path
 
-            if toc is None: return
+            if toc is None:
+                return
             self.toc = TOC(base_path=self.base_dir)
             is_ncx = getattr(self, 'manifest', None) is not None and \
                      self.manifest.type_for_id(toc) is not None and \
@@ -976,7 +970,6 @@ class OPF(object): # {{{
 
         return property(fget=fget, fset=fset)
 
-
     @dynamic_property
     def language(self):
 
@@ -990,7 +983,6 @@ class OPF(object): # {{{
 
         return property(fget=fget, fset=fset)
 
-
     @dynamic_property
     def languages(self):
 
@@ -1015,7 +1007,6 @@ class OPF(object): # {{{
 
         return property(fget=fget, fset=fset)
 
-
     @dynamic_property
     def book_producer(self):
 
@@ -1196,7 +1187,6 @@ class OPFCreator(Metadata):
         if self.cover:
             self.guide.set_cover(self.cover)
 
-
     def create_manifest(self, entries):
         '''
         Create <manifest>
@@ -1615,9 +1605,9 @@ def test_user_metadata():
     from cStringIO import StringIO
     mi = Metadata('Test title', ['test author1', 'test author2'])
     um = {
-        '#myseries': { '#value#': u'test series\xe4', 'datatype':'text',
+        '#myseries': {'#value#': u'test series\xe4', 'datatype':'text',
             'is_multiple': None, 'name': u'My Series'},
-        '#myseries_index': { '#value#': 2.45, 'datatype': 'float',
+        '#myseries_index': {'#value#': 2.45, 'datatype': 'float',
             'is_multiple': None},
         '#mytags': {'#value#':['t1','t2','t3'], 'datatype':'text',
             'is_multiple': '|', 'name': u'My Tags'}

src/calibre/ebooks/mobi/reader/mobi8.py

@@ -51,9 +51,11 @@ def reverse_tag_iter(block):
     end = len(block)
     while True:
         pgt = block.rfind(b'>', 0, end)
-        if pgt == -1: break
+        if pgt == -1:
+            break
         plt = block.rfind(b'<', 0, pgt)
-        if plt == -1: break
+        if plt == -1:
+            break
         yield block[plt:pgt+1]
         end = plt
 
@@ -231,12 +233,12 @@ class Mobi8Reader(object):
             flowpart = self.flows[j]
             nstr = '%04d' % j
             m = svg_tag_pattern.search(flowpart)
-            if m != None:
+            if m is not None:
                 # svg
                 typ = 'svg'
                 start = m.start()
                 m2 = image_tag_pattern.search(flowpart)
-                if m2 != None:
+                if m2 is not None:
                     format = 'inline'
                     dir = None
                     fname = None
@@ -320,7 +322,7 @@ class Mobi8Reader(object):
                 if len(pos_fid) != 2:
                     continue
             except TypeError:
-                continue # thumbnailstandard record, ignore it
+                continue  # thumbnailstandard record, ignore it
             linktgt, idtext = self.get_id_tag_by_pos_fid(*pos_fid)
             if idtext:
                 linktgt += b'#' + idtext
@@ -389,7 +391,7 @@ class Mobi8Reader(object):
             href = None
             if typ in {b'FLIS', b'FCIS', b'SRCS', b'\xe9\x8e\r\n',
                     b'RESC', b'BOUN', b'FDST', b'DATP', b'AUDI', b'VIDE'}:
-                pass # Ignore these records
+                pass  # Ignore these records
             elif typ == b'FONT':
                 font = read_font_record(data)
                 href = "fonts/%05d.%s" % (fname_idx, font['ext'])
@@ -406,7 +408,11 @@ class Mobi8Reader(object):
             else:
                 imgtype = what(None, data)
                 if imgtype is None:
-                    imgtype = 'unknown'
+                    from calibre.utils.magick.draw import identify_data
+                    try:
+                        imgtype = identify_data(data)[2]
+                    except Exception:
+                        imgtype = 'unknown'
                 href = 'images/%05d.%s'%(fname_idx, imgtype)
                 with open(href.replace('/', os.sep), 'wb') as f:
                     f.write(data)

src/calibre/ebooks/mobi/tweak.py

@@ -19,7 +19,7 @@ from calibre.ebooks.mobi.reader.mobi8 import Mobi8Reader
 from calibre.ebooks.conversion.plumber import Plumber, create_oebbook
 from calibre.customize.ui import (plugin_for_input_format,
         plugin_for_output_format)
-from calibre.utils.ipc.simple_worker  import fork_job
+from calibre.utils.ipc.simple_worker import fork_job
 
 class BadFormat(ValueError):
     pass
@@ -72,7 +72,8 @@ def explode(path, dest, question=lambda x:True):
             dest), no_output=True)['result']
 
 def set_cover(oeb):
-    if 'cover' not in oeb.guide or oeb.metadata['cover']: return
+    if 'cover' not in oeb.guide or oeb.metadata['cover']:
+        return
     cover = oeb.guide['cover']
     if cover.href in oeb.manifest.hrefs:
         item = oeb.manifest.hrefs[cover.href]
@@ -95,8 +96,9 @@ def rebuild(src_dir, dest_path):
     if not opf:
         raise ValueError('No OPF file found in %s'%src_dir)
     opf = opf[0]
-    # For debugging, uncomment the following line
+    # For debugging, uncomment the following two lines
-    # def fork_job(a, b, args=None, no_output=True): do_rebuild(*args)
+    # def fork_job(a, b, args=None, no_output=True):
+    #     do_rebuild(*args)
     fork_job('calibre.ebooks.mobi.tweak', 'do_rebuild', args=(opf, dest_path),
             no_output=True)
 

src/calibre/ebooks/mobi/writer2/resources.py

@@ -69,7 +69,8 @@ class Resources(object):
             cover_href = item.href
 
         for item in self.oeb.manifest.values():
-            if item.media_type not in OEB_RASTER_IMAGES: continue
+            if item.media_type not in OEB_RASTER_IMAGES:
+                continue
             try:
                 data = self.process_image(item.data)
             except:
@@ -116,8 +117,8 @@ class Resources(object):
         Add any images that were created after the call to add_resources()
         '''
         for item in self.oeb.manifest.values():
-            if (item.media_type not in OEB_RASTER_IMAGES or item.href in
+            if (item.media_type not in OEB_RASTER_IMAGES or item.href in self.item_map):
-                    self.item_map): continue
+                continue
             try:
                 data = self.process_image(item.data)
             except:

src/calibre/ebooks/oeb/base.py

@@ -270,7 +270,7 @@ BINARY_MIME    = 'application/octet-stream'
 
 XHTML_CSS_NAMESPACE = u'@namespace "%s";\n' % XHTML_NS
 
-OEB_STYLES        = set([CSS_MIME, OEB_CSS_MIME, 'text/x-oeb-css'])
+OEB_STYLES        = set([CSS_MIME, OEB_CSS_MIME, 'text/x-oeb-css', 'xhtml/css'])
 OEB_DOCS          = set([XHTML_MIME, 'text/html', OEB_DOC_MIME,
                          'text/x-oeb-document'])
 OEB_RASTER_IMAGES = set([GIF_MIME, JPEG_MIME, PNG_MIME])

src/calibre/gui2/layout.py

@@ -16,11 +16,10 @@ from calibre.constants import __appname__
 from calibre.gui2.search_box import SearchBox2, SavedSearchBox
 from calibre.gui2.throbber import ThrobbingButton
 from calibre.gui2.bars import BarsManager
-from calibre.gui2.widgets import ComboBoxWithHelp
 from calibre.utils.config_base import tweaks
 from calibre import human_readable
 
-class LocationManager(QObject): # {{{
+class LocationManager(QObject):  # {{{
 
     locations_changed = pyqtSignal()
     unmount_device = pyqtSignal()
@@ -165,7 +164,7 @@ class LocationManager(QObject): # {{{
 
 # }}}
 
-class SearchBar(QWidget): # {{{
+class SearchBar(QWidget):  # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
@@ -174,8 +173,10 @@ class SearchBar(QWidget): # {{{
         self._layout.setContentsMargins(0,5,0,0)
 
         x = QToolButton(self)
-        x.setText(_('Virtual Libraries'))
+        x.setText(_('Virtual Library'))
+        x.setIcon(QIcon(I('lt.png')))
         x.setObjectName("virtual_library")
+        x.setToolButtonStyle(Qt.ToolButtonTextBesideIcon)
         l.addWidget(x)
         parent.virtual_library = x
 
@@ -243,7 +244,7 @@ class SearchBar(QWidget): # {{{
 
 # }}}
 
-class Spacer(QWidget): # {{{
+class Spacer(QWidget):  # {{{
 
     def __init__(self, parent):
         QWidget.__init__(self, parent)
@@ -252,7 +253,7 @@ class Spacer(QWidget): # {{{
         self.l.addStretch(10)
 # }}}
 
-class MainWindowMixin(object): # {{{
+class MainWindowMixin(object):  # {{{
 
     def __init__(self, db):
         self.setObjectName('MainWindow')

src/calibre/gui2/preferences/behavior.py

@@ -47,8 +47,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         choices = [(x.upper(), x) for x in output_formats]
         r('output_format', prefs, choices=choices, setting=OutputFormatSetting)
 
-        restrictions = sorted(db.prefs.get('virtual_libraries').keys(), key=sort_key)
+        restrictions = sorted(db.prefs['virtual_libraries'].iterkeys(), key=sort_key)
         choices = [('', '')] + [(x, x) for x in restrictions]
+        # check that the virtual library still exists
+        vls = db.prefs['virtual_lib_on_startup']
+        if vls and vls not in restrictions:
+            db.prefs['virtual_lib_on_startup'] = ''
         r('virtual_lib_on_startup', db.prefs, choices=choices)
         self.reset_confirmation_button.clicked.connect(self.reset_confirmation_dialogs)
 

src/calibre/gui2/preferences/server.py

@@ -12,7 +12,6 @@ from PyQt4.Qt import Qt, QUrl, QDialog, QSize, QVBoxLayout, QLabel, \
 
 from calibre.gui2.preferences import ConfigWidgetBase, test_widget
 from calibre.gui2.preferences.server_ui import Ui_Form
-from calibre.utils.search_query_parser import saved_searches
 from calibre.library.server import server_config
 from calibre.utils.config import ConfigProxy
 from calibre.gui2 import error_dialog, config, open_url, warning_dialog, \
@@ -44,12 +43,12 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                          else self.opt_password.Password))
         self.opt_password.setEchoMode(self.opt_password.Password)
 
-        restrictions = sorted(db.prefs.get('virtual_libraries').keys(), key=sort_key)
+        restrictions = sorted(db.prefs['virtual_libraries'].iterkeys(), key=sort_key)
-        # verify that the current restriction still exists. If not, clear it.
-        csr = db.prefs.get('cs_virtual_lib_on_startup', None)
-        if csr and csr not in restrictions:
-            db.prefs.set('cs_restriction', '')
         choices = [('', '')] + [(x, x) for x in restrictions]
+        # check that the virtual library still exists
+        vls = db.prefs['cs_virtual_lib_on_startup']
+        if vls and vls not in restrictions:
+            db.prefs['cs_virtual_lib_on_startup'] = ''
         r('cs_virtual_lib_on_startup', db.prefs, choices=choices)
 
         self.start_button.setEnabled(not getattr(self.server, 'is_running', False))

src/calibre/gui2/search_box.py

@@ -19,9 +19,8 @@ from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.saved_search_editor import SavedSearchEditor
 from calibre.gui2.dialogs.search import SearchDialog
 from calibre.utils.search_query_parser import saved_searches
-from calibre.utils.icu import sort_key
 
-class SearchLineEdit(QLineEdit): # {{{
+class SearchLineEdit(QLineEdit):  # {{{
     key_pressed = pyqtSignal(object)
 
     def keyPressEvent(self, event):
@@ -42,7 +41,7 @@ class SearchLineEdit(QLineEdit): # {{{
         return QLineEdit.paste(self)
 # }}}
 
-class SearchBox2(QComboBox): # {{{
+class SearchBox2(QComboBox):  # {{{
 
     '''
     To use this class:
@@ -59,7 +58,7 @@ class SearchBox2(QComboBox): # {{{
           accurate.
     '''
 
-    INTERVAL = 1500 #: Time to wait before emitting search signal
+    INTERVAL = 1500  #: Time to wait before emitting search signal
     MAX_COUNT = 25
 
     search  = pyqtSignal(object)
@@ -254,7 +253,7 @@ class SearchBox2(QComboBox): # {{{
 
     # }}}
 
-class SavedSearchBox(QComboBox): # {{{
+class SavedSearchBox(QComboBox):  # {{{
 
     '''
     To use this class:
@@ -343,7 +342,7 @@ class SavedSearchBox(QComboBox): # {{{
         # references the new search instead of the text in the search.
         self.clear()
         self.setCurrentIndex(self.findText(name))
-        self.saved_search_selected (name)
+        self.saved_search_selected(name)
         self.changed.emit()
 
     def delete_current_search(self):
@@ -365,15 +364,15 @@ class SavedSearchBox(QComboBox): # {{{
         self.changed.emit()
 
     # SIGNALed from the main UI
-    def copy_search_button_clicked (self):
+    def copy_search_button_clicked(self):
-        idx = self.currentIndex();
+        idx = self.currentIndex()
         if idx < 0:
             return
         self.search_box.set_search_string(saved_searches().lookup(unicode(self.currentText())))
 
     # }}}
 
-class SearchBoxMixin(object): # {{{
+class SearchBoxMixin(object):  # {{{
 
     def __init__(self):
         self.search.initialize('main_search_history', colorize=True,
@@ -447,7 +446,7 @@ class SearchBoxMixin(object): # {{{
 
     # }}}
 
-class SavedSearchBoxMixin(object): # {{{
+class SavedSearchBoxMixin(object):  # {{{
 
     def __init__(self):
         self.saved_search.changed.connect(self.saved_searches_changed)
@@ -456,7 +455,7 @@ class SavedSearchBoxMixin(object): # {{{
                                 self.saved_search.save_search_button_clicked)
         self.copy_search_button.clicked.connect(
                                 self.saved_search.copy_search_button_clicked)
-#         self.saved_searches_changed()
+        # self.saved_searches_changed()
         self.saved_search.initialize(self.search, colorize=True,
                 help_text=_('Saved Searches'))
         self.saved_search.setToolTip(
@@ -486,7 +485,7 @@ class SavedSearchBoxMixin(object): # {{{
         self.build_search_restriction_list()
         if recount:
             self.tags_view.recount()
-        if set_restriction: # redo the search restriction if there was one
+        if set_restriction:  # redo the search restriction if there was one
             self.apply_named_search_restriction(set_restriction)
 
     def do_saved_search_edit(self, search):

src/calibre/gui2/search_restriction_mixin.py

@@ -6,38 +6,118 @@ Created on 10 Jun 2010
 
 from functools import partial
 
-from PyQt4.Qt import (Qt, QMenu, QPoint, QIcon, QDialog, QGridLayout, QLabel,
+from PyQt4.Qt import (
-                      QLineEdit, QDialogButtonBox, QEvent, QToolTip)
+    Qt, QMenu, QPoint, QIcon, QDialog, QGridLayout, QLabel, QLineEdit,
+    QDialogButtonBox, QSize, QVBoxLayout, QListWidget, QStringList)
+
 from calibre.gui2 import error_dialog, question_dialog
 from calibre.gui2.widgets import ComboBoxWithHelp
 from calibre.utils.icu import sort_key
 from calibre.utils.pyparsing import ParseException
 from calibre.utils.search_query_parser import saved_searches
 
-class CreateVirtualLibrary(QDialog):
+class SelectNames(QDialog):  # {{{
+
+    def __init__(self, names, txt, parent=None):
+        QDialog.__init__(self, parent)
+        self.l = l = QVBoxLayout(self)
+        self.setLayout(l)
+
+        self.la = la = QLabel(_('Create a Virtual Library based on %s') % txt)
+        l.addWidget(la)
+
+        self._names = QListWidget(self)
+        self._names.addItems(QStringList(sorted(names, key=sort_key)))
+        self._names.setSelectionMode(self._names.ExtendedSelection)
+        l.addWidget(self._names)
+
+        self.bb = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        self.bb.accepted.connect(self.accept)
+        self.bb.rejected.connect(self.reject)
+        l.addWidget(self.bb)
+
+        self.resize(self.sizeHint())
+
+    @property
+    def names(self):
+        for item in self._names.selectedItems():
+            yield unicode(item.data(Qt.DisplayRole).toString())
+# }}}
+
+class CreateVirtualLibrary(QDialog):  # {{{
+
     def __init__(self, gui, existing_names):
-        QDialog.__init__(self, None, Qt.WindowSystemMenuHint | Qt.WindowTitleHint)
+        QDialog.__init__(self, gui)
 
         self.gui = gui
         self.existing_names = existing_names
 
         self.setWindowTitle(_('Create virtual library'))
+        self.setWindowIcon(QIcon(I('lt.png')))
+
         gl = QGridLayout()
         self.setLayout(gl)
-        gl.addWidget(QLabel(_('Virtual library name')), 0, 0)
+        self.la1 = la1 = QLabel(_('Virtual library &name:'))
+        gl.addWidget(la1, 0, 0)
         self.vl_name = QLineEdit()
-        self.vl_name.setMinimumWidth(400)
+        la1.setBuddy(self.vl_name)
         gl.addWidget(self.vl_name, 0, 1)
-        gl.addWidget(QLabel(_('Search expression')), 1, 0)
+
+        self.la2 = la2 = QLabel(_('&Search expression:'))
+        gl.addWidget(la2, 1, 0)
         self.vl_text = QLineEdit()
+        la2.setBuddy(self.vl_text)
         gl.addWidget(self.vl_text, 1, 1)
         self.vl_text.setText(self.build_full_search_string())
-        bb = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
-        bb.accepted.connect(self.accepted)
-        bb.rejected.connect(self.rejected)
-        gl.addWidget(bb, 2, 0, 1, 0)
 
-    search_templates = [
+        self.sl = sl = QLabel('<p>'+_('Create a virtual library based on: ')+
+            ('<a href="author.{0}">{0}</a>, '
+            '<a href="tag.{1}">{1}</a>, '
+            '<a href="publisher.{2}">{2}</a>, '
+            '<a href="series.{3}">{3}</a>.').format(_('Authors'), _('Tags'), _('Publishers'), _('Series')))
+        sl.setWordWrap(True)
+        sl.setTextInteractionFlags(Qt.LinksAccessibleByMouse)
+        sl.linkActivated.connect(self.link_activated)
+        gl.addWidget(sl, 2, 0, 1, 2)
+
+        self.hl = hl = QLabel(_('''
+            <h2>Virtual Libraries</h2>
+
+            <p>Using <i>virtual libraries</i> you can restrict calibre to only show
+            you books that match a search. When a virtual library is in effect, calibre
+            behaves as though the library contains only the matched books. The Tag Browser
+            display only the tags/authors/series/etc. that belong to the matched books and any searches
+            you do will only search within the books in the virtual library. This
+            is a good way to partition your large library into smaller and easier to work with subsets.</p>
+
+            <p>For example you can use a Virtual Library to only show you books with the Tag <i>"Unread"</i>
+            or only books by <i>"My Favorite Author"</i> or only books in a particular series.</p>
+            '''))
+        hl.setWordWrap(True)
+        hl.setFrameStyle(hl.StyledPanel)
+        gl.addWidget(hl, 0, 3, 4, 1)
+
+        bb = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel)
+        bb.accepted.connect(self.accept)
+        bb.rejected.connect(self.reject)
+        gl.addWidget(bb, 4, 0, 1, 0)
+
+        self.resize(self.sizeHint()+QSize(150, 25))
+
+    def link_activated(self, url):
+        db = self.gui.current_db
+        f, txt = unicode(url).partition('.')[0::2]
+        names = getattr(db, 'all_%s_names'%f)()
+        d = SelectNames(names, txt, parent=self)
+        if d.exec_() == d.Accepted:
+            prefix = f+'s' if f in {'tag', 'author'} else f
+            search = ['%s:"=%s"'%(prefix, x.replace('"', '\\"')) for x in d.names]
+            if search:
+                self.vl_name.setText(d.names.next())
+                self.vl_text.setText(' or '.join(search))
+
+    def build_full_search_string(self):
+        search_templates = (
             '',
             '{cl}',
             '{cr}',
@@ -46,11 +126,10 @@ class CreateVirtualLibrary(QDialog):
             '(({cl}) and ({sb}))',
             '(({cr}) and ({sb}))',
             '(({cl}) and ({cr}) and ({sb}))'
-        ]
+        )
 
-    def build_full_search_string(self):
         sb = self.gui.search.current_text
-        db = self.gui.library_view.model().db
+        db = self.gui.current_db
         cr = db.data.get_search_restriction()
         cl = db.data.get_base_restriction()
         dex = 0
@@ -60,10 +139,10 @@ class CreateVirtualLibrary(QDialog):
             dex += 2
         if cl:
             dex += 1
-        template = self.search_templates[dex]
+        template = search_templates[dex]
         return template.format(cl=cl, cr=cr, sb=sb)
 
-    def accepted(self):
+    def accept(self):
         n = unicode(self.vl_name.text())
         if not n:
             error_dialog(self.gui, _('No name'),
@@ -91,45 +170,20 @@ class CreateVirtualLibrary(QDialog):
         except ParseException as e:
             error_dialog(self.gui, _('Invalid search string'),
                          _('The search string is not a valid search expression'),
-                         det_msg = e.msg, show=True)
+                         det_msg=e.msg, show=True)
             return
 
-        if not recs:
+        if not recs and not question_dialog(
-            if question_dialog(self.gui, _('Search found no books'),
+                self.gui, _('Search found no books'),
-                         _('The search found no books, so the virtual library '
+                _('The search found no books, so the virtual library '
-                           'will be empty. Do you really want to use that search?'),
+                'will be empty. Do you really want to use that search?'),
-                            default_yes=False) == self.Rejected:
+                default_yes=False):
                 return
 
         self.library_name = n
         self.library_search = v
-        self.accept()
+        QDialog.accept(self)
-
+# }}}
-    def rejected(self):
-        self.reject()
-
-class VirtLibMenu(QMenu):
-
-    def __init__(self):
-        QMenu.__init__(self)
-        self.show_tt_for = []
-
-    def event(self, e):
-        QMenu.event(self, e)
-        if e.type() == QEvent.ToolTip:
-            a = self.activeAction()
-            if a and a in self.show_tt_for:
-                tt = a.toolTip()
-                if tt:
-                    QToolTip.showText(e.globalPos(), tt)
-        return True
-
-    def clear(self):
-        self.show_tt_for = []
-        QMenu.clear(self)
-
-    def show_tooltip_for_action(self, a):
-        self.show_tt_for.append(a)
 
 class SearchRestrictionMixin(object):
 
@@ -139,7 +193,7 @@ class SearchRestrictionMixin(object):
         self.checked = QIcon(I('ok.png'))
         self.empty = QIcon()
 
-        self.virtual_library_menu = VirtLibMenu()
+        self.virtual_library_menu = QMenu()
 
         self.virtual_library.clicked.connect(self.virtual_library_clicked)
 
@@ -161,8 +215,7 @@ class SearchRestrictionMixin(object):
         db = self.library_view.model().db
         virt_libs = db.prefs.get('virtual_libraries', {})
         cd = CreateVirtualLibrary(self, virt_libs.keys())
-        ret = cd.exec_()
+        if cd.exec_() == cd.Accepted:
-        if ret == cd.Accepted:
             self.add_virtual_library(db, cd.library_name, cd.library_search)
             self.apply_virtual_library(cd.library_name)
 
@@ -180,11 +233,10 @@ class SearchRestrictionMixin(object):
         a = m.addAction(_('Create Virtual Library'))
         a.triggered.connect(self.do_create)
         a.setToolTip(_('Create a new virtual library from the results of a search'))
-        m.show_tooltip_for_action(a)
 
-        self.rm_menu = a = VirtLibMenu()
+        self.rm_menu = a = QMenu()
         a.setTitle(_('Remove Virtual Library'))
-        a.aboutToShow.connect(self.build_virtual_library_list);
+        a.aboutToShow.connect(self.build_virtual_library_list)
         m.addMenu(a)
 
         m.addSeparator()
@@ -194,7 +246,7 @@ class SearchRestrictionMixin(object):
         a = self.ar_menu
         a.clear()
         a.setIcon(self.checked if db.data.get_search_restriction_name() else self.empty)
-        a.aboutToShow.connect(self.build_search_restriction_list);
+        a.aboutToShow.connect(self.build_search_restriction_list)
         m.addMenu(a)
 
         m.addSeparator()
@@ -212,12 +264,11 @@ class SearchRestrictionMixin(object):
             a = m.addAction(self.checked if vl == current_lib else self.empty, vl)
             a.setToolTip(virt_libs[vl])
             a.triggered.connect(partial(self.apply_virtual_library, library=vl))
-            m.show_tooltip_for_action(a)
 
         p = QPoint(0, self.virtual_library.height())
         self.virtual_library_menu.popup(self.virtual_library.mapToGlobal(p))
 
-    def apply_virtual_library(self, library = None):
+    def apply_virtual_library(self, library=None):
         db = self.library_view.model().db
         virt_libs = db.prefs.get('virtual_libraries', {})
         if not library:
@@ -238,7 +289,6 @@ class SearchRestrictionMixin(object):
         def add_action(name, search):
             a = m.addAction(name)
             a.setToolTip(search)
-            m.show_tooltip_for_action(a)
             a.triggered.connect(partial(self.remove_vl_triggered, name=name))
 
         for n in sorted(virt_libs.keys(), key=sort_key):
@@ -269,7 +319,6 @@ class SearchRestrictionMixin(object):
                 current_restriction_text = txt
         self.search_restriction.clear()
 
-
         current_restriction = self.library_view.model().db.data.get_search_restriction_name()
         m.setIcon(self.checked if current_restriction else self.empty)
 
@@ -359,9 +408,9 @@ class SearchRestrictionMixin(object):
             rows = self.current_view().row_count()
             rbc = max(rows, db.data.get_search_restriction_book_count())
             t = _("({0} of {1})").format(rows, rbc)
-            self.search_count.setStyleSheet \
+            self.search_count.setStyleSheet(
-                   ('QLabel { border-radius: 8px; background-color: yellow; }')
+                'QLabel { border-radius: 8px; background-color: yellow; }')
-        else: # No restriction or not library view
+        else:  # No restriction or not library view
             if not self.search.in_a_search():
                 t = _("(all books)")
             else:
@@ -369,3 +418,14 @@ class SearchRestrictionMixin(object):
             self.search_count.setStyleSheet(
                     'QLabel { background-color: transparent; }')
         self.search_count.setText(t)
+
+if __name__ == '__main__':
+    from calibre.gui2 import Application
+    from calibre.gui2.preferences import init_gui
+    app = Application([])
+    app
+    gui = init_gui()
+    d = CreateVirtualLibrary(gui, [])
+    d.exec_()
+
+

src/calibre/gui2/store/stores/gutenberg_plugin.py

@@ -1,91 +1,104 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import (unicode_literals, division, absolute_import, print_function)
-store_version = 2 # Needed for dynamic plugin loading
+store_version = 3 # Needed for dynamic plugin loading
 
 __license__ = 'GPL 3'
-__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__copyright__ = '2011, 2013, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
+import base64
 import mimetypes
+import re
 import urllib
 from contextlib import closing
 
-from lxml import html
+from lxml import etree
 
-from PyQt4.Qt import QUrl
+from calibre import browser, url_slash_cleaner
-
+from calibre.constants import __version__
-from calibre import browser, random_user_agent, url_slash_cleaner
-from calibre.gui2 import open_url
-from calibre.gui2.store import StorePlugin
 from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore
 from calibre.gui2.store.search_result import SearchResult
-from calibre.gui2.store.web_store_dialog import WebStoreDialog
 
-class GutenbergStore(BasicStoreConfig, StorePlugin):
+class GutenbergStore(BasicStoreConfig, OpenSearchOPDSStore):
 
-    def open(self, parent=None, detail_item=None, external=False):
+    open_search_url = 'http://www.gutenberg.org/catalog/osd-books.xml'
-        url = 'http://gutenberg.org/'
+    web_url = 'http://m.gutenberg.org/'
-
-        if detail_item:
-            detail_item = url_slash_cleaner(url + detail_item)
-
-        if external or self.config.get('open_external', False):
-            open_url(QUrl(detail_item if detail_item else url))
-        else:
-            d = WebStoreDialog(self.gui, url, parent, detail_item)
-            d.setWindowTitle(self.name)
-            d.set_tags(self.config.get('tags', ''))
-            d.exec_()
 
     def search(self, query, max_results=10, timeout=60):
-        url = 'http://m.gutenberg.org/ebooks/search.mobile/?default_prefix=all&sort_order=title&query=' + urllib.quote_plus(query)
+        '''
+        Gutenberg's ODPS feed is poorly implmented and has a number of issues
+        which require very special handling to fix the results.
 
-        br = browser(user_agent=random_user_agent())
+        Issues:
+          * "Sort Alphabetically" and "Sort by Release Date" are returned
+            as book entries.
+          * The author is put into a "content" tag and not the author tag.
+          * The link to the book itself goes to an odps page which we need
+            to turn into a link to a web page.
+          * acquisition links are not part of the search result so we have
+            to go to the odps item itself. Detail item pages have a nasty
+            note saying:
+              DON'T USE THIS PAGE FOR SCRAPING. 
+              Seriously. You'll only get your IP blocked.
+            We're using the ODPS feed because people are getting blocked with
+            the previous implementation so due to this using ODPS probably
+            won't solve this issue.
+          * Images are not links but base64 encoded strings. They are also not
+            real cover images but a little blue book thumbnail.
+        '''
+
+        url = 'http://m.gutenberg.org/ebooks/search.opds/?query=' + urllib.quote_plus(query)
 
         counter = max_results
+        br = browser(user_agent='calibre/'+__version__)
         with closing(br.open(url, timeout=timeout)) as f:
-            doc = html.fromstring(f.read())
+            doc = etree.fromstring(f.read())
-            for data in doc.xpath('//ol[@class="results"]/li[@class="booklink"]'):
+            for data in doc.xpath('//*[local-name() = "entry"]'):
                 if counter <= 0:
                     break
 
-                id = ''.join(data.xpath('./a/@href'))
-                id = id.split('.mobile')[0]
-
-                title = ''.join(data.xpath('.//span[@class="title"]/text()'))
-                author = ''.join(data.xpath('.//span[@class="subtitle"]/text()'))
-
                 counter -= 1
 
                 s = SearchResult()
-                s.cover_url = ''
 
-                s.detail_item = id.strip()
+                # We could use the <link rel="alternate" type="text/html" ...> tag from the
-                s.title = title.strip()
+                # detail odps page but this is easier.
-                s.author = author.strip()
+                id = ''.join(data.xpath('./*[local-name() = "id"]/text()')).strip()
-                s.price = '$0.00'
+                s.detail_item = url_slash_cleaner('%s/ebooks/%s' % (self.web_url, re.sub('[^\d]', '', id)))
-                s.drm = SearchResult.DRM_UNLOCKED
+                if not s.detail_item:
+                    continue
+
+                s.title = ' '.join(data.xpath('./*[local-name() = "title"]//text()')).strip()
+                s.author = ', '.join(data.xpath('./*[local-name() = "content"]//text()')).strip()
+                if not s.title or not s.author:
+                    continue
+
+                # Get the formats and direct download links.
+                with closing(br.open(id, timeout=timeout/4)) as nf:
+                    ndoc = etree.fromstring(nf.read())
+                    for link in ndoc.xpath('//*[local-name() = "link" and @rel = "http://opds-spec.org/acquisition"]'):
+                        type = link.get('type')
+                        href = link.get('href')
+                        if type:
+                            ext = mimetypes.guess_extension(type)
+                            if ext:
+                                ext = ext[1:].upper().strip()
+                                s.downloads[ext] = href
+
+                s.formats = ', '.join(s.downloads.keys())
+                if not s.formats:
+                    continue
+
+                for link in data.xpath('./*[local-name() = "link"]'):
+                    rel = link.get('rel')
+                    href = link.get('href')
+                    type = link.get('type')
+
+                    if rel and href and type:
+                        if rel in ('http://opds-spec.org/thumbnail', 'http://opds-spec.org/image/thumbnail'):
+                            if href.startswith('data:image/png;base64,'):
+                                s.cover_data = base64.b64decode(href.replace('data:image/png;base64,', ''))
 
                 yield s
-
-    def get_details(self, search_result, timeout):
-        url = url_slash_cleaner('http://m.gutenberg.org/' + search_result.detail_item)
-
-        br = browser(user_agent=random_user_agent())
-        with closing(br.open(url, timeout=timeout)) as nf:
-            doc = html.fromstring(nf.read())
-
-            for save_item in doc.xpath('//li[contains(@class, "icon_save")]/a'):
-                type = save_item.get('type')
-                href = save_item.get('href')
-
-                if type:
-                    ext = mimetypes.guess_extension(type)
-                    if ext:
-                        ext = ext[1:].upper().strip()
-                        search_result.downloads[ext] = href
-
-                search_result.formats = ', '.join(search_result.downloads.keys())
-
-        return True

src/calibre/gui2/store/stores/koobe_plugin.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (division, absolute_import, print_function)
+store_version = 1  # Needed for dynamic plugin loading
+
+__license__ = 'GPL 3'
+__copyright__ = '2013, Tomasz Długosz <tomek3d@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib
+from base64 import b64encode
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class KoobeStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/15/58/'
+        url = 'http://www.koobe.pl/'
+
+        aff_url = aff_root + str(b64encode(url))
+
+        detail_url = None
+        if detail_item:
+            detail_url = aff_root + str(b64encode(detail_item))
+
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url if detail_url else aff_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(self.config.get('tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+
+        br = browser()
+        page=1
+
+        counter = max_results
+        while counter:
+            with closing(br.open('http://www.koobe.pl/s,p,' + str(page) + ',szukaj/fraza:' + urllib.quote(query), timeout=timeout)) as f:
+                doc = html.fromstring(f.read().decode('utf-8'))
+                for data in doc.xpath('//div[@class="seach_result"]/div[@class="result"]'):
+                    if counter <= 0:
+                        break
+
+                    id = ''.join(data.xpath('.//div[@class="cover"]/a/@href'))
+                    if not id:
+                        continue
+
+                    cover_url = ''.join(data.xpath('.//div[@class="cover"]/a/img/@src'))
+                    price = ''.join(data.xpath('.//span[@class="current_price"]/text()'))
+                    title = ''.join(data.xpath('.//h2[@class="title"]/a/text()'))
+                    author = ''.join(data.xpath('.//h3[@class="book_author"]/a/text()'))
+                    formats = ', '.join(data.xpath('.//div[@class="formats"]/div/div/@title'))
+
+                    counter -= 1
+
+                    s = SearchResult()
+                    s.cover_url =  'http://koobe.pl/' + cover_url
+                    s.title = title.strip()
+                    s.author = author.strip()
+                    s.price = price
+                    s.detail_item = 'http://koobe.pl' + id[1:]
+                    s.formats = formats.upper()
+                    s.drm = SearchResult.DRM_UNLOCKED
+
+                    yield s
+                if not doc.xpath('//div[@class="site_bottom"]//a[@class="right"]'):
+                    break
+            page+=1

src/calibre/gui2/store/stores/woblink_plugin.py

@@ -1,14 +1,15 @@
 # -*- coding: utf-8 -*-
 
 from __future__ import (unicode_literals, division, absolute_import, print_function)
-store_version = 1 # Needed for dynamic plugin loading
+store_version = 2 # Needed for dynamic plugin loading
 
 __license__ = 'GPL 3'
-__copyright__ = '2011-2012, Tomasz Długosz <tomek3d@gmail.com>'
+__copyright__ = '2011-2013, Tomasz Długosz <tomek3d@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
 import re
 import urllib
+from base64 import b64encode
 from contextlib import closing
 
 from lxml import html
@@ -25,17 +26,19 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog
 class WoblinkStore(BasicStoreConfig, StorePlugin):
 
     def open(self, parent=None, detail_item=None, external=False):
-
+        aff_root = 'https://www.a4b-tracking.com/pl/stat-click-text-link/16/58/'
         url = 'http://woblink.com/publication'
+
+        aff_url = aff_root + str(b64encode(url))
         detail_url = None
 
         if detail_item:
-            detail_url = 'http://woblink.com' + detail_item
+            detail_url = aff_root + str(b64encode('http://woblink.com' + detail_item))
 
         if external or self.config.get('open_external', False):
-            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else aff_url)))
         else:
-            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d = WebStoreDialog(self.gui, url, parent, detail_url if detail_url else aff_url)
             d.setWindowTitle(self.name)
             d.set_tags(self.config.get('tags', ''))
             d.exec_()

src/calibre/gui2/ui.py

@@ -15,7 +15,7 @@ from threading import Thread
 from collections import OrderedDict
 
 from PyQt4.Qt import (Qt, SIGNAL, QTimer, QHelpEvent, QAction,
-                     QMenu, QIcon, pyqtSignal, QUrl,
+                     QMenu, QIcon, pyqtSignal, QUrl, QFont,
                      QDialog, QSystemTrayIcon, QApplication)
 
 from calibre import prints, force_unicode
@@ -47,7 +47,7 @@ from calibre.gui2.proceed import ProceedQuestion
 from calibre.gui2.dialogs.message_box import JobError
 from calibre.gui2.job_indicator import Pointer
 
-class Listener(Thread): # {{{
+class Listener(Thread):  # {{{
 
     def __init__(self, listener):
         Thread.__init__(self)
@@ -76,7 +76,7 @@ class Listener(Thread): # {{{
 
 # }}}
 
-class SystemTrayIcon(QSystemTrayIcon): # {{{
+class SystemTrayIcon(QSystemTrayIcon):  # {{{
 
     tooltip_requested = pyqtSignal(object)
 
@@ -98,7 +98,7 @@ _gui = None
 def get_gui():
     return _gui
 
-class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
+class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin,  # {{{
         TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin,
         SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin,
         EbookDownloadMixin
@@ -187,7 +187,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         else:
             stmap[st.name] = st
 
-
     def initialize(self, library_path, db, listener, actions, show_gui=True):
         opts = self.opts
         self.preferences_action, self.quit_action = actions
@@ -339,7 +338,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         if config['autolaunch_server']:
             self.start_content_server()
 
-
         self.keyboard_interrupt.connect(self.quit, type=Qt.QueuedConnection)
 
         self.read_settings()
@@ -393,7 +391,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             if not self.device_manager.is_running('Wireless Devices'):
                     error_dialog(self, _('Problem starting the wireless device'),
                                  _('The wireless device driver did not start. '
-                                   'It said "%s"')%message,  show=True)
+                                   'It said "%s"')%message, show=True)
         self.iactions['Connect Share'].set_smartdevice_action_state()
 
     def start_content_server(self, check_started=True):
@@ -494,7 +492,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 path = os.path.abspath(argv[1])
                 if os.access(path, os.R_OK):
                     self.iactions['Add Books'].add_filesystem_book(path)
-            self.setWindowState(self.windowState() & \
+            self.setWindowState(self.windowState() &
                     ~Qt.WindowMinimized|Qt.WindowActive)
             self.show_windows()
             self.raise_()
@@ -526,7 +524,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
 
     def library_moved(self, newloc, copy_structure=False, call_close=True,
             allow_rebuild=False):
-        if newloc is None: return
+        if newloc is None:
+            return
         default_prefs = None
         try:
             olddb = self.library_view.model().db
@@ -537,7 +536,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         try:
             db = LibraryDatabase2(newloc, default_prefs=default_prefs)
         except (DatabaseException, sqlite.Error):
-            if not allow_rebuild: raise
+            if not allow_rebuild:
+                raise
             import traceback
             repair = question_dialog(self, _('Corrupted database'),
                     _('The library database at %s appears to be corrupted. Do '
@@ -571,8 +571,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         db = self.library_view.model().db
         self.iactions['Choose Library'].count_changed(db.count())
         self.set_window_title()
-        self.apply_named_search_restriction('') # reset restriction to null
+        self.apply_named_search_restriction('')  # reset restriction to null
-        self.saved_searches_changed(recount=False) # reload the search restrictions combo box
+        self.saved_searches_changed(recount=False)  # reload the search restrictions combo box
         self.apply_named_search_restriction(db.prefs['gui_restriction'])
         for action in self.iactions.values():
             action.library_changed(db)
@@ -596,13 +596,18 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         # interface later
         gc.collect()
 
-
     def set_window_title(self):
-        title = u'{0} - || {1} :: {2} :: {3} ||'.format(
+        db = self.current_db
-                                __appname__,
+        restrictions = [x for x in (db.data.get_base_restriction_name(),
-                                self.iactions['Choose Library'].library_name(),
+                        db.data.get_search_restriction_name()) if x]
-                                self.library_view.model().db.data.get_base_restriction_name(),
+        restrictions = ' :: '.join(restrictions)
-                                self.library_view.model().db.data.get_search_restriction_name())
+        font = QFont()
+        if restrictions:
+            restrictions = ' :: ' + restrictions
+            font.setBold(True)
+        self.virtual_library.setFont(font)
+        title = u'{0} - || {1}{2} ||'.format(
+                __appname__, self.iactions['Choose Library'].library_name(), restrictions)
         self.setWindowTitle(title)
 
     def location_selected(self, location):
@@ -627,8 +632,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
             self.current_view().reset()
         self.set_number_of_books_shown()
 
-
-
     def job_exception(self, job, dialog_title=_('Conversion Error')):
         if not hasattr(self, '_modeless_dialogs'):
             self._modeless_dialogs = []
@@ -720,7 +723,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.read_layout_settings()
 
     def write_settings(self):
-        with gprefs: # Only write to gprefs once
+        with gprefs:  # Only write to gprefs once
             config.set('main_window_geometry', self.saveGeometry())
             dynamic.set('sort_history', self.library_view.model().sort_history)
             self.save_layout_state()
@@ -753,7 +756,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 return False
         return True
 
-
     def shutdown(self, write_settings=True):
         try:
             db = self.library_view.model().db
@@ -813,13 +815,11 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 pass
             QApplication.instance().quit()
 
-
-
     def closeEvent(self, e):
         self.write_settings()
         if self.system_tray_icon.isVisible():
             if not dynamic['systray_msg'] and not isosx:
-                info_dialog(self, 'calibre', 'calibre '+ \
+                info_dialog(self, 'calibre', 'calibre '+
                         _('will keep running in the system tray. To close it, '
                         'choose <b>Quit</b> in the context menu of the '
                         'system tray.'), show_copy_button=False).exec_()

src/calibre/library/server/main.py

@@ -24,7 +24,7 @@ def stop_threaded_server(server):
     server.exit()
     server.thread = None
 
-def create_wsgi_app(path_to_library=None, prefix=''):
+def create_wsgi_app(path_to_library=None, prefix='', virtual_library=None):
     'WSGI entry point'
     from calibre.library import db
     cherrypy.config.update({'environment': 'embedded'})
@@ -32,6 +32,7 @@ def create_wsgi_app(path_to_library=None, prefix=''):
     parser = option_parser()
     opts, args = parser.parse_args(['calibre-server'])
     opts.url_prefix = prefix
+    opts.restriction = virtual_library
     server = LibraryServer(db, opts, wsgi=True, show_tracebacks=True)
     return cherrypy.Application(server, script_name=None, config=server.config)
 
@@ -98,7 +99,6 @@ def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
     os.dup2(se.fileno(), sys.stderr.fileno())
 
 
-
 def main(args=sys.argv):
     from calibre.library.database2 import LibraryDatabase2
     parser = option_parser()