Sync to trunk.

Changelog.yaml

@@ -19,6 +19,66 @@
 #  new recipes:
 #    - title: 
 
+- version: 0.8.7
+  date: 2011-06-24
+
+  new features:
+    - title: "Connect to iTunes: You now need to tell iTunes to keep its own copy of every ebook. Do this in iTunes by going to Preferences->Advanced and setting the 'Copy files to iTunes Media folder when adding to library' option. To learn about why this is necessary, see: http://www.mobileread.com/forums/showthread.php?t=140260"
+      type: major
+
+    - title: "Add a couple of date related functions to the calibre template langauge to get 'todays' date and create text based on the value of a date type field"
+
+    - title: "Improved reading of metadata from FB2 files, with support for reading isbns, tags, published date, etc."
+
+    - title: "Driver for the Imagine IMEB5"
+      tickets: [800642]
+
+    - title: "Show the currently used network proxies in Preferences->Miscellaneous"
+
+    - title: "Kobo Touch driver: Show Favorites as a device collection. Various other minor fixes."
+
+    - title: "Content server now sends the Content-Disposition header when sending ebook files."
+
+    - title: "Allow search and replace on comments custom columns."
+
+    - title: "Add a new action 'Quick View' to show the books in your library by the author/tags/series/etc. of the currently selected book, in a separate window. You can add it to your toolbar or right click menu by going to Preferences->Toolbars."
+
+    - title: "Get Books: Add libri.de as a book source. Fix a bug that caused some books downloads to fail. Fixes to the Legimi and  beam-ebooks.de stores"
+      tickets: [799367]
+  
+  bug fixes:
+    - title: "Fix a memory leak that could result in the leaking of several MB of memory with large libraries"
+      tickets: [800952]
+
+    - title: "Fix the read metadata from format button in the edit metadata dialog using the wrong timezone when setting published date"
+      tickets: [799777]
+
+    - title: "Generating catalog: Fix occassional file in use errors when generating catalogs on windows"
+
+    - title: "Fix clicking on News in Tag Browser not working in non English locales."
+      tickets: [799471]
+
+    - title: "HTML Input: Fix a regression in 0.8.6 that caused CSS stylesheets to be ignored"
+      tickets: [799171]
+
+    - title: "Fix a regression that caused restore database to stop working on some windows sytems"
+
+    - title: "EPUB Output: Convert <br> tags with text in them into <divs> as ADE cannot handle them."
+      tickets: [794427]
+
+  improved recipes:
+    - Le Temps
+    - Perfil
+    - Financial Times UK
+
+  new recipes:
+    - title: "Daytona Beach Journal"
+      author: BRGriff
+
+    - title: "El club del ebook and Frontline"
+      author: Darko Miletic
+
+
 - version: 0.8.6
   date: 2011-06-17
 

recipes/financial_times_uk.recipe

@@ -1,15 +1,17 @@
 __license__   = 'GPL v3'
 __copyright__ = '2010-2011, Darko Miletic <darko.miletic at gmail.com>'
 '''
-ft.com
+www.ft.com/uk-edition
 '''
 from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class FinancialTimes(BasicNewsRecipe):
-    title                 = u'Financial Times - UK printed edition'
+    title                 = 'Financial Times - UK printed edition'
     __author__            = 'Darko Miletic'
-    description           = 'Financial world news'
+    description           = "The Financial Times (FT) is one of the world's leading business news and information organisations, recognised internationally for its authority, integrity and accuracy."
+    publisher             = 'The Financial Times Ltd.'
+    category              = 'news, finances, politics, UK, World'
     oldest_article        = 2
     language              = 'en_GB'
     max_articles_per_feed = 250
@@ -17,14 +19,24 @@ class FinancialTimes(BasicNewsRecipe):
     use_embedded_content  = False
     needs_subscription    = True
     encoding              = 'utf8'
-    simultaneous_downloads= 1
+    publication_type      = 'newspaper'
-    delay                 = 1
+    cover_url             = strftime('http://specials.ft.com/vtf_pdf/%d%m%y_FRONT1_LON.pdf')
+    masthead_url          = 'http://im.media.ft.com/m/img/masthead_main.jpg'
     LOGIN                 = 'https://registration.ft.com/registration/barrier/login'
     INDEX                 = 'http://www.ft.com/uk-edition'
     PREFIX                = 'http://www.ft.com'
 
+    conversion_options = {
+                          'comment'          : description
+                        , 'tags'             : category
+                        , 'publisher'        : publisher
+                        , 'language'         : language
+                        , 'linearize_tables' : True
+                        }
+
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
+        br.open(self.INDEX)
         if self.username is not None and self.password is not None:
             br.open(self.LOGIN)
             br.select_form(name='loginForm')
@@ -33,29 +45,34 @@ class FinancialTimes(BasicNewsRecipe):
             br.submit()
         return br
 
-    keep_only_tags    = [ dict(name='div', attrs={'id':'cont'}) ]
+    keep_only_tags = [dict(name='div', attrs={'class':['fullstory fullstoryHeader','fullstory fullstoryBody','ft-story-header','ft-story-body','index-detail']})]
-    remove_tags_after = dict(name='p', attrs={'class':'copyright'})
     remove_tags = [
                       dict(name='div', attrs={'id':'floating-con'})
                      ,dict(name=['meta','iframe','base','object','embed','link'])
+                     ,dict(attrs={'class':['storyTools','story-package','screen-copy','story-package separator','expandable-image']})
                   ]
     remove_attributes = ['width','height','lang']
 
     extra_css = """
-                body{font-family:Arial,Helvetica,sans-serif;}
+                body{font-family: Georgia,Times,"Times New Roman",serif}
-                h2{font-size:large;}
+                h2{font-size:large}
-                .ft-story-header{font-size:xx-small;}
+                .ft-story-header{font-size: x-small}
-                .ft-story-body{font-size:small;}
-                a{color:#003399;}
                 .container{font-size:x-small;}
                 h3{font-size:x-small;color:#003399;}
                 .copyright{font-size: x-small}
+                img{margin-top: 0.8em; display: block}
+                .lastUpdated{font-family: Arial,Helvetica,sans-serif; font-size: x-small}
+                .byline,.ft-story-body,.ft-story-header{font-family: Arial,Helvetica,sans-serif}
                 """
 
     def get_artlinks(self, elem):
         articles = []
         for item in elem.findAll('a',href=True):
-            url   = self.PREFIX + item['href']
+            rawlink = item['href']
+            if rawlink.startswith('http://'):
+               url = rawlink
+            else:
+               url   = self.PREFIX + rawlink
             title = self.tag_to_string(item)
             date = strftime(self.timefmt)
             articles.append({
@@ -65,7 +82,7 @@ class FinancialTimes(BasicNewsRecipe):
                              ,'description':''
                             })
         return articles
-                
+
     def parse_index(self):
         feeds = []
         soup = self.index_to_soup(self.INDEX)
@@ -80,11 +97,34 @@ class FinancialTimes(BasicNewsRecipe):
            strest.insert(0,st)
         for item in strest:
             ftitle   = self.tag_to_string(item)
-            self.report_progress(0, _('Fetching feed')+' %s...'%(ftitle))            
+            self.report_progress(0, _('Fetching feed')+' %s...'%(ftitle))
             feedarts = self.get_artlinks(item.parent.ul)
             feeds.append((ftitle,feedarts))
         return feeds
 
     def preprocess_html(self, soup):
-        return self.adeify_images(soup)
+        items = ['promo-box','promo-title',
-
+                 'promo-headline','promo-image',
+                 'promo-intro','promo-link','subhead']
+        for item in items:
+            for it in soup.findAll(item):
+                it.name = 'div'
+                it.attrs = []
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll('a'):
+            limg = item.find('img')
+            if item.string is not None:
+               str = item.string
+               item.replaceWith(str)
+            else:
+               if limg:
+                  item.name = 'div'
+                  item.attrs = []
+               else:
+                   str = self.tag_to_string(item)
+                   item.replaceWith(str)
+        for item in soup.findAll('img'):
+            if not item.has_key('alt'):
+               item['alt'] = 'image'
+        return soup

recipes/le_temps.recipe

@@ -14,7 +14,7 @@ class LeTemps(BasicNewsRecipe):
      title          = u'Le Temps'
      oldest_article = 7
      max_articles_per_feed = 100
-     __author__ = 'Sujata Raman'
+     __author__ = 'Kovid Goyal'
      description = 'French news. Needs a subscription from http://www.letemps.ch'
      no_stylesheets = True
      remove_javascript = True
@@ -27,6 +27,7 @@ class LeTemps(BasicNewsRecipe):
      def get_browser(self):
          br = BasicNewsRecipe.get_browser(self)
          br.open('http://www.letemps.ch/login')
+         br.select_form(nr=1)
          br['username'] = self.username
          br['password'] = self.password
          raw = br.submit().read()

src/calibre/constants.py

@@ -4,7 +4,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = u'calibre'
-numeric_version = (0, 8, 6)
+numeric_version = (0, 8, 7)
 __version__   = u'.'.join(map(unicode, numeric_version))
 __author__    = u"Kovid Goyal <kovid@kovidgoyal.net>"
 

src/calibre/devices/android/driver.py

@@ -45,8 +45,11 @@ class ANDROID(USBMS):
             0xfce : { 0xd12e : [0x0100]},
 
             # Google
-            0x18d1 : { 0x4e11 : [0x0100, 0x226, 0x227], 0x4e12: [0x0100, 0x226,
+            0x18d1 : {
-                0x227], 0x4e21: [0x0100, 0x226, 0x227], 0xb058: [0x0222]},
+                0x4e11 : [0x0100, 0x226, 0x227],
+                0x4e12: [0x0100, 0x226, 0x227],
+                0x4e21: [0x0100, 0x226, 0x227],
+                0xb058: [0x0222, 0x226, 0x227]},
 
             # Samsung
             0x04e8 : { 0x681d : [0x0222, 0x0223, 0x0224, 0x0400],
@@ -107,7 +110,7 @@ class ANDROID(USBMS):
     VENDOR_NAME      = ['HTC', 'MOTOROLA', 'GOOGLE_', 'ANDROID', 'ACER',
             'GT-I5700', 'SAMSUNG', 'DELL', 'LINUX', 'GOOGLE', 'ARCHOS',
             'TELECHIP', 'HUAWEI', 'T-MOBILE', 'SEMC', 'LGE', 'NVIDIA',
-            'GENERIC-', 'ZTE']
+            'GENERIC-', 'ZTE', 'MID']
     WINDOWS_MAIN_MEM = ['ANDROID_PHONE', 'A855', 'A853', 'INC.NEXUS_ONE',
             '__UMS_COMPOSITE', '_MB200', 'MASS_STORAGE', '_-_CARD', 'SGH-I897',
             'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID',
@@ -116,7 +119,7 @@ class ANDROID(USBMS):
             'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD',
             '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2',
             'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB', 'STREAK',
-            'MB525']
+            'MB525', 'ANDROID2.3']
     WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897',
             'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD',
             'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD',

src/calibre/devices/apple/driver.py

@@ -1211,7 +1211,8 @@ class ITUNES(DriverBase):
         '''
         windows assumes pythoncom wrapper
         '''
-        self.log.info(" ITUNES._add_library_book()")
+        if DEBUG:
+            self.log.info(" ITUNES._add_library_book()")
         if isosx:
             added = self.iTunes.add(appscript.mactypes.File(file))
 
@@ -1335,7 +1336,8 @@ class ITUNES(DriverBase):
         assumes pythoncom wrapper for db_added
         as of iTunes 9.2, iBooks 1.1, can't set artwork for PDF files via automation
         '''
-        self.log.info(" ITUNES._cover_to_thumb()")
+        if DEBUG:
+            self.log.info(" ITUNES._cover_to_thumb()")
 
         thumb = None
         if metadata.cover:
@@ -2489,7 +2491,8 @@ class ITUNES(DriverBase):
         '''
         Windows assumes pythoncom wrapper
         '''
-        self.log.info(" ITUNES._remove_from_device()")
+        if DEBUG:
+            self.log.info(" ITUNES._remove_from_device()")
         if isosx:
             if DEBUG:
                 self.log.info("  deleting '%s' from iDevice" % cached_book['title'])
@@ -2616,7 +2619,8 @@ class ITUNES(DriverBase):
     def _update_epub_metadata(self, fpath, metadata):
         '''
         '''
-        self.log.info(" ITUNES._update_epub_metadata()")
+        if DEBUG:
+            self.log.info(" ITUNES._update_epub_metadata()")
 
         # Fetch plugboard updates
         metadata_x = self._xform_metadata_via_plugboard(metadata, 'epub')
@@ -2983,7 +2987,8 @@ class ITUNES(DriverBase):
                 self.log.info("        tags: %s %s" % (book.tags, ">>> %s" %
                                            newmi.tags if book.tags != newmi.tags else ''))
             else:
-                self.log("  matching plugboard not found")
+                if DEBUG:
+                    self.log("  matching plugboard not found")
 
         else:
             newmi = book

src/calibre/ebooks/conversion/plumber.py

@@ -59,6 +59,8 @@ class CompositeProgressReporter(object):
                 (self.global_max - self.global_min)
         self.global_reporter(global_frac, msg)
 
+ARCHIVE_FMTS = ('zip', 'rar', 'oebzip')
+
 class Plumber(object):
     '''
     The `Plumber` manages the conversion pipeline. An UI should call the methods
@@ -594,7 +596,7 @@ OptionRecommendation(name='sr3_replace',
             raise ValueError('Input file must have an extension')
         input_fmt = input_fmt[1:].lower()
         self.archive_input_tdir = None
-        if input_fmt in ('zip', 'rar', 'oebzip'):
+        if input_fmt in ARCHIVE_FMTS:
             self.log('Processing archive...')
             tdir = PersistentTemporaryDirectory('_plumber_archive')
             self.input, input_fmt = self.unarchive(self.input, tdir)

src/calibre/ebooks/metadata/fb2.py

@@ -1,96 +1,235 @@
 #!/usr/bin/env python
 from __future__ import with_statement
 __license__   = 'GPL v3'
-__copyright__ = '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
+__copyright__ = '2011, Roman Mukhin <ramses_ru at hotmail.com>, '\
-
+                '2008, Anatoly Shipitsin <norguhtar at gmail.com>'
 '''Read meta information from fb2 files'''
 
 import os
+import datetime
+from functools import partial
 from base64 import b64decode
 from lxml import etree
-from calibre.ebooks.metadata import MetaInformation
+from calibre.utils.date import parse_date
+from calibre import guess_all_extensions, prints, force_unicode
+from calibre.ebooks.metadata import MetaInformation, check_isbn
 from calibre.ebooks.chardet import xml_to_unicode
-from calibre import guess_all_extensions
 
-XLINK_NS     = 'http://www.w3.org/1999/xlink'
-def XLINK(name):
-    return '{%s}%s' % (XLINK_NS, name)
 
+NAMESPACES = {
+    'fb2'   :   'http://www.gribuser.ru/xml/fictionbook/2.0',
+    'xlink' :   'http://www.w3.org/1999/xlink'  }
+
+XPath = partial(etree.XPath, namespaces=NAMESPACES)
+tostring = partial(etree.tostring, method='text', encoding=unicode)
 
 def get_metadata(stream):
-    """ Return metadata as a L{MetaInfo} object """
+    """ Return fb2 metadata as a L{MetaInformation} object """
-    XPath = lambda x : etree.XPath(x,
+
-            namespaces={'fb2':'http://www.gribuser.ru/xml/fictionbook/2.0',
+    root = _get_fbroot(stream)
-                'xlink':XLINK_NS})
+
-    tostring = lambda x : etree.tostring(x, method='text',
+    book_title = _parse_book_title(root)
-            encoding=unicode).strip()
+    authors = _parse_authors(root)
+
+    # fallback for book_title
+    if book_title:
+        book_title = unicode(book_title)
+    else:
+        book_title = force_unicode(os.path.splitext(
+            os.path.basename(getattr(stream, 'name',
+                _('Unknown'))))[0])
+    mi = MetaInformation(book_title, authors)
+
+    try:
+        _parse_cover(root, mi)
+    except:
+        pass
+    try:
+        _parse_comments(root, mi)
+    except:
+        pass
+    try:
+        _parse_tags(root, mi)
+    except:
+        pass
+    try:
+        _parse_series(root, mi)
+    except:
+        pass
+    try:
+        _parse_isbn(root, mi)
+    except:
+        pass
+    try:
+        _parse_publisher(root, mi)
+    except:
+        pass
+    try:
+        _parse_pubdate(root, mi)
+    except:
+        pass
+    try:
+        _parse_timestamp(root, mi)
+    except:
+        pass
+
+    try:
+        _parse_language(root, mi)
+    except:
+        pass
+    #_parse_uuid(root, mi)
+
+    #if DEBUG:
+    #   prints(mi)
+    return mi
+
+def _parse_authors(root):
+    authors = []
+    # pick up authors but only from 1 secrion <title-info>; otherwise it is not consistent!
+    # Those are fallbacks: <src-title-info>, <document-info>
+    for author_sec in ['title-info', 'src-title-info', 'document-info']:
+        for au in XPath('//fb2:%s/fb2:author'%author_sec)(root):
+            author = _parse_author(au)
+            if author:
+                authors.append(author)
+        if author:
+            break
+
+    # if no author so far
+    if not authors:
+        authors.append(_('Unknown'))
+
+    return authors
+
+def _parse_author(elm_author):
+    """ Returns a list of display author and sortable author"""
+
+    xp_templ = 'normalize-space(fb2:%s/text())'
+
+    author = XPath(xp_templ % 'first-name')(elm_author)
+    lname = XPath(xp_templ % 'last-name')(elm_author)
+    mname = XPath(xp_templ % 'middle-name')(elm_author)
+
+    if mname:
+        author = (author + ' ' + mname).strip()
+    if lname:
+        author = (author + ' ' + lname).strip()
+
+    # fallback to nickname
+    if not author:
+        nname = XPath(xp_templ % 'nickname')(elm_author)
+        if nname:
+            author = nname
+
+    return author
+
+
+def _parse_book_title(root):
+    # <title-info> has a priority.   (actually <title-info>  is mandatory)
+    # other are backup solution (sequence is important. other then in fb2-doc)
+    xp_ti = '//fb2:title-info/fb2:book-title/text()'
+    xp_pi = '//fb2:publish-info/fb2:book-title/text()'
+    xp_si = '//fb2:src-title-info/fb2:book-title/text()'
+    book_title = XPath('normalize-space(%s|%s|%s)' % (xp_ti, xp_pi, xp_si))(root)
+
+    return book_title
+
+def _parse_cover(root, mi):
+    # pickup from <title-info>, if not exists it fallbacks to <src-title-info>
+    imgid = XPath('substring-after(string(//fb2:coverpage/fb2:image/@xlink:href), "#")')(root)
+    if imgid:
+        try:
+            _parse_cover_data(root, imgid, mi)
+        except:
+            pass
+
+def _parse_cover_data(root, imgid, mi):
+    elm_binary = XPath('//fb2:binary[@id="%s"]'%imgid)(root)
+    if elm_binary:
+        mimetype = elm_binary[0].get('content-type', 'image/jpeg')
+        mime_extensions = guess_all_extensions(mimetype)
+        if mime_extensions:
+            pic_data = elm_binary[0].text
+            if pic_data:
+                mi.cover_data = (mime_extensions[0][1:], b64decode(pic_data))
+        else:
+            prints("WARNING: Unsupported coverpage mime-type '%s' (id=#%s)" % (mimetype, imgid) )
+
+def _parse_tags(root, mi):
+    # pick up genre but only from 1 secrion <title-info>; otherwise it is not consistent!
+    # Those are fallbacks: <src-title-info>
+    for genre_sec in ['title-info', 'src-title-info']:
+        # -- i18n Translations-- ?
+        tags = XPath('//fb2:%s/fb2:genre/text()' % genre_sec)(root)
+        if tags:
+            mi.tags = list(map(unicode, tags))
+            break
+
+def _parse_series(root, mi):
+    #calibri supports only 1 series: use the 1-st one
+    # pick up sequence but only from 1 secrion in prefered order
+    # except <src-title-info>
+    xp_ti = '//fb2:title-info/fb2:sequence[1]'
+    xp_pi = '//fb2:publish-info/fb2:sequence[1]'
+
+    elms_sequence = XPath('%s|%s' % (xp_ti, xp_pi))(root)
+    if elms_sequence:
+        mi.series = elms_sequence[0].get('name', None)
+        if mi.series:
+            mi.series_index = elms_sequence[0].get('number', None)
+
+def _parse_isbn(root, mi):
+    # some people try to put several isbn in this field, but it is not allowed.  try to stick to the 1-st one in this case
+    isbn = XPath('normalize-space(//fb2:publish-info/fb2:isbn/text())')(root)
+    # some people try to put several isbn in this field, but it is not allowed.  try to stick to the 1-st one in this case
+    if ',' in isbn:
+        isbn = isbn[:isbn.index(',')]
+    if check_isbn(isbn):
+        mi.isbn = isbn
+
+def _parse_comments(root, mi):
+    # pick up annotation but only from 1 secrion <title-info>;  fallback: <src-title-info>
+    for annotation_sec in ['title-info', 'src-title-info']:
+        elms_annotation = XPath('//fb2:%s/fb2:annotation' % annotation_sec)(root)
+        if elms_annotation:
+            mi.comments = tostring(elms_annotation[0])
+            # TODO: tags i18n, xslt?
+            break
+
+def _parse_publisher(root, mi):
+    publisher = XPath('string(//fb2:publish-info/fb2:publisher/text())')(root)
+    if publisher:
+        mi.publisher = publisher
+
+def _parse_pubdate(root, mi):
+    year = XPath('number(//fb2:publish-info/fb2:year/text())')(root)
+    if float.is_integer(year):
+        # only year is available, so use 1-st of Jan
+        mi.pubdate = datetime.date(int(year), 1, 1)
+
+def _parse_timestamp(root, mi):
+    #<date value="1996-12-03">03.12.1996</date>
+    xp ='//fb2:document-info/fb2:date/@value|'\
+        '//fb2:document-info/fb2:date/text()'
+    docdate = XPath('string(%s)' % xp)(root)
+    if docdate:
+        mi.timestamp = parse_date(docdate)
+
+def _parse_language(root, mi):
+    language = XPath('string(//fb2:title-info/fb2:lang/text())')(root)
+    if language:
+        mi.language = language
+        mi.languages = [ language ]
+
+def _parse_uuid(root, mi):
+    uuid = XPath('normalize-space(//document-info/fb2:id/text())')(root)
+    if uuid:
+        mi.uuid = uuid
+
+def _get_fbroot(stream):
     parser = etree.XMLParser(recover=True, no_network=True)
     raw = stream.read()
-    raw = xml_to_unicode(raw, strip_encoding_pats=True,
+    raw = xml_to_unicode(raw, strip_encoding_pats=True)[0]
-            assume_utf8=True)[0]
     root = etree.fromstring(raw, parser=parser)
-    authors, author_sort = [], None
+    return root
-    for au in XPath('//fb2:author')(root):
-        fname = lname = author = None
-        fe = XPath('descendant::fb2:first-name')(au)
-        if fe:
-            fname = tostring(fe[0])
-            author = fname
-        le = XPath('descendant::fb2:last-name')(au)
-        if le:
-            lname = tostring(le[0])
-            if author:
-                author += ' '+lname
-            else:
-                author = lname
-        if author:
-            authors.append(author)
-        if len(authors) == 1 and author is not None:
-            if lname:
-                author_sort = lname
-            if fname:
-                if author_sort: author_sort += ', '+fname
-                else: author_sort = fname
-    title = os.path.splitext(os.path.basename(getattr(stream, 'name',
-        _('Unknown'))))[0]
-    for x in XPath('//fb2:book-title')(root):
-        title = tostring(x)
-        break
-    comments = ''
-    for x in XPath('//fb2:annotation')(root):
-        comments += tostring(x)
-    if not comments:
-        comments = None
-    tags = list(map(tostring, XPath('//fb2:genre')(root)))
 
-    cp = XPath('//fb2:coverpage')(root)
-    cdata = None
-    if cp:
-        cimage = XPath('descendant::fb2:image[@xlink:href]')(cp[0])
-        if cimage:
-            id = cimage[0].get(XLINK('href')).replace('#', '')
-            binary = XPath('//fb2:binary[@id="%s"]'%id)(root)
-            if binary:
-                mt = binary[0].get('content-type', 'image/jpeg')
-                exts = guess_all_extensions(mt)
-                if not exts:
-                    exts = ['.jpg']
-                cdata = (exts[0][1:], b64decode(tostring(binary[0])))
-
-    series = None
-    series_index = 1.0
-    for x in XPath('//fb2:sequence')(root):
-        series = x.get('name', None)
-        if series is not None:
-            series_index = x.get('number', 1.0)
-            break
-    mi = MetaInformation(title, authors)
-    mi.comments = comments
-    mi.author_sort = author_sort
-    if tags:
-        mi.tags = tags
-    mi.series = series
-    mi.series_index = series_index
-    if cdata:
-        mi.cover_data = cdata
-    return mi

src/calibre/gui2/__init__.py

@@ -248,10 +248,11 @@ def error_dialog(parent, title, msg, det_msg='', show=False,
         return d.exec_()
     return d
 
-def question_dialog(parent, title, msg, det_msg='', show_copy_button=False):
+def question_dialog(parent, title, msg, det_msg='', show_copy_button=False,
+        default_yes=True):
     from calibre.gui2.dialogs.message_box import MessageBox
     d = MessageBox(MessageBox.QUESTION, title, msg, det_msg, parent=parent,
-                    show_copy_button=show_copy_button)
+                    show_copy_button=show_copy_button, default_yes=default_yes)
     return d.exec_() == d.Accepted
 
 def info_dialog(parent, title, msg, det_msg='', show=False,

src/calibre/gui2/actions/choose_library.py

@@ -252,11 +252,12 @@ class ChooseLibraryAction(InterfaceAction):
 
     def delete_requested(self, name, location):
         loc = location.replace('/', os.sep)
-        if not question_dialog(self.gui, _('Are you sure?'), '<p>'+
+        if not question_dialog(self.gui, _('Are you sure?'),
+                _('<h1 style="color:red">WARNING</h1>')+
                 _('<b style="color: red">All files</b> (not just ebooks) '
                     'from <br><br><b>%s</b><br><br> will be '
                 '<b>permanently deleted</b>. Are you sure?') % loc,
-                show_copy_button=False):
+                show_copy_button=False, default_yes=False):
             return
         exists = self.gui.library_view.model().db.exists_at(loc)
         if exists:

src/calibre/gui2/convert/regex_builder.py

@@ -139,7 +139,12 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
         try:
             self.open_book(fpath)
         finally:
-            os.remove(fpath)
+            try:
+                os.remove(fpath)
+            except:
+                # Fails on windows if the input plugin for this format keeps the file open
+                # Happens for LIT files
+                pass
         return True
 
     def open_book(self, pathtoebook):
@@ -148,7 +153,8 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
         text = [u'']
         preprocessor = HTMLPreProcessor(None, False)
         for path in self.iterator.spine:
-            html = open(path, 'rb').read().decode('utf-8', 'replace')
+            with open(path, 'rb') as f:
+                html = f.read().decode('utf-8', 'replace')
             html = preprocessor(html, get_preprocess_html=True)
             text.append(html)
         self.preview.setPlainText('\n---\n'.join(text))

src/calibre/gui2/convert/single.py

@@ -11,8 +11,8 @@ import sys, cPickle, shutil, importlib
 from PyQt4.Qt import QString, SIGNAL, QAbstractListModel, Qt, QVariant, QFont
 
 from calibre.gui2 import ResizableDialog, NONE
-from calibre.ebooks.conversion.config import GuiRecommendations, save_specifics, \
+from calibre.ebooks.conversion.config import (GuiRecommendations, save_specifics,
-        load_specifics
+        load_specifics)
 from calibre.gui2.convert.single_ui import Ui_Dialog
 from calibre.gui2.convert.metadata import MetadataWidget
 from calibre.gui2.convert.look_and_feel import LookAndFeelWidget
@@ -24,7 +24,8 @@ from calibre.gui2.convert.toc import TOCWidget
 from calibre.gui2.convert.debug import DebugWidget
 
 
-from calibre.ebooks.conversion.plumber import Plumber, supported_input_formats
+from calibre.ebooks.conversion.plumber import (Plumber,
+        supported_input_formats, ARCHIVE_FMTS)
 from calibre.ebooks.conversion.config import delete_specifics
 from calibre.customize.ui import available_output_formats
 from calibre.customize.conversion import OptionRecommendation
@@ -158,7 +159,10 @@ class Config(ResizableDialog, Ui_Dialog):
         output_path = 'dummy.'+output_format
         log = Log()
         log.outputs = []
-        self.plumber = Plumber('dummy.'+input_format, output_path, log)
+        input_file = 'dummy.'+input_format
+        if input_format in ARCHIVE_FMTS:
+            input_file = 'dummy.html'
+        self.plumber = Plumber(input_file, output_path, log)
 
         def widget_factory(cls):
             return cls(self.stack, self.plumber.get_option_by_name,

src/calibre/gui2/dialogs/choose_library.ui

@@ -41,7 +41,7 @@
    <item row="4" column="0" colspan="4">
     <widget class="QRadioButton" name="existing_library">
      <property name="text">
-      <string>Use &amp;existing library at the new location</string>
+      <string>Use the previously &amp;existing library at the new location</string>
      </property>
      <property name="checked">
       <bool>true</bool>

src/calibre/gui2/dialogs/message_box.py

@@ -23,7 +23,7 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
                  det_msg='',
                  q_icon=None,
                  show_copy_button=True,
-                 parent=None):
+                 parent=None, default_yes=True):
         QDialog.__init__(self, parent)
         if q_icon is None:
             icon = {
@@ -65,7 +65,9 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
         self.is_question = type_ == self.QUESTION
         if self.is_question:
             self.bb.setStandardButtons(self.bb.Yes|self.bb.No)
-            self.bb.button(self.bb.Yes).setDefault(True)
+            self.bb.button(self.bb.Yes if default_yes else self.bb.No
+                    ).setDefault(True)
+            self.default_yes = default_yes
         else:
             self.bb.button(self.bb.Ok).setDefault(True)
 
@@ -101,7 +103,8 @@ class MessageBox(QDialog, Ui_Dialog): # {{{
         ret = QDialog.showEvent(self, ev)
         if self.is_question:
             try:
-                self.bb.button(self.bb.Yes).setFocus(Qt.OtherFocusReason)
+                self.bb.button(self.bb.Yes if self.default_yes else self.bb.No
+                        ).setFocus(Qt.OtherFocusReason)
             except:
                 pass# Buttons were changed
         else:

src/calibre/gui2/dialogs/quickview.py

@@ -5,11 +5,13 @@ __docformat__ = 'restructuredtext en'
 
 
 from PyQt4.Qt import (Qt, QDialog, QAbstractItemView, QTableWidgetItem,
-                      QListWidgetItem, QByteArray, QModelIndex, QCoreApplication)
+                      QListWidgetItem, QByteArray, QCoreApplication,
+                      QApplication)
 
+from calibre.customize.ui import find_plugin
+from calibre.gui2 import gprefs
 from calibre.gui2.dialogs.quickview_ui import Ui_Quickview
 from calibre.utils.icu import sort_key
-from calibre.gui2 import gprefs
 
 class TableItem(QTableWidgetItem):
     '''
@@ -55,8 +57,9 @@ class Quickview(QDialog, Ui_Quickview):
         self.is_closed = False
         self.current_book_id = None
         self.current_key = None
-        self.use_current_key_for_next_refresh = False
         self.last_search = None
+        self.current_column = None
+        self.current_item = None
 
         self.items.setSelectionMode(QAbstractItemView.SingleSelection)
         self.items.currentTextChanged.connect(self.item_selected)
@@ -87,16 +90,24 @@ class Quickview(QDialog, Ui_Quickview):
         # Add the data
         self.refresh(row)
 
-        self.view.selectionModel().currentChanged[QModelIndex,QModelIndex].connect(self.slave)
+        self.view.clicked.connect(self.slave)
         QCoreApplication.instance().aboutToQuit.connect(self.save_state)
         self.search_button.clicked.connect(self.do_search)
+        view.model().new_bookdisplay_data.connect(self.book_was_changed)
 
     # search button
     def do_search(self):
         if self.last_search is not None:
-            self.use_current_key_for_next_refresh = True
             self.gui.search.set_search_string(self.last_search)
 
+    # Called when book information is changed in the library view. Make that
+    # book current. This means that prev and next in edit metadata will move
+    # the current book.
+    def book_was_changed(self, mi):
+        if self.is_closed or self.current_column is None:
+            return
+        self.refresh(self.view.model().index(self.db.row(mi.id), self.current_column))
+
     # clicks on the items listWidget
     def item_selected(self, txt):
         self.fill_in_books_box(unicode(txt))
@@ -104,22 +115,15 @@ class Quickview(QDialog, Ui_Quickview):
     # Given a cell in the library view, display the information
     def refresh(self, idx):
         bv_row = idx.row()
-        key = self.view.model().column_map[idx.column()]
+        self.current_column = idx.column()
-
+        key = self.view.model().column_map[self.current_column]
         book_id = self.view.model().id(bv_row)
 
-        # Double-clicking on a book to show it in the library view will result
+        # Only show items for categories
-        # in a signal emitted for column 1 of the book row. Use the original
+        if not self.db.field_metadata[key]['is_category']:
-        # column for this signal.
+            if self.current_key is None:
-        if self.use_current_key_for_next_refresh:
+                return
             key = self.current_key
-            self.use_current_key_for_next_refresh = False
-        else:
-            # Only show items for categories
-            if not self.db.field_metadata[key]['is_category']:
-                if self.current_key is None:
-                    return
-                key = self.current_key
         self.items_label.setText('{0} ({1})'.format(
                                     self.db.field_metadata[key]['name'], key))
 
@@ -147,6 +151,7 @@ class Quickview(QDialog, Ui_Quickview):
         self.items.blockSignals(False)
 
     def fill_in_books_box(self, selected_item):
+        self.current_item = selected_item
         # Do a bit of fix-up on the items so that the search works.
         if selected_item.startswith('.'):
             sv = '.' + selected_item
@@ -162,19 +167,26 @@ class Quickview(QDialog, Ui_Quickview):
 
         select_item = None
         self.books_table.setSortingEnabled(False)
+        tt = ('<p>' +
+            _('Double-click on a book to change the selection in the library view. '
+               'Shift- or control-double-click to edit the metadata of a book')
+              + '</p>')
         for row, b in enumerate(books):
             mi = self.db.get_metadata(b, index_is_id=True, get_user_categories=False)
             a = TableItem(mi.title, mi.title_sort)
             a.setData(Qt.UserRole, b)
+            a.setToolTip(tt)
             self.books_table.setItem(row, 0, a)
             if b == self.current_book_id:
                 select_item = a
             a = TableItem(' & '.join(mi.authors), mi.author_sort)
+            a.setToolTip(tt)
             self.books_table.setItem(row, 1, a)
             series = mi.format_field('series')[1]
             if series is None:
                 series = ''
             a = TableItem(series, series)
+            a.setToolTip(tt)
             self.books_table.setItem(row, 2, a)
             self.books_table.setRowHeight(row, self.books_table_row_height)
 
@@ -201,11 +213,16 @@ class Quickview(QDialog, Ui_Quickview):
         self.save_state()
 
     def book_doubleclicked(self, row, column):
-        self.use_current_key_for_next_refresh = True
+        book_id = self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0]
-        self.view.select_rows([self.books_table.item(row, 0).data(Qt.UserRole).toInt()[0]])
+        self.view.select_rows([book_id])
+        modifiers = int(QApplication.keyboardModifiers())
+        if modifiers in (Qt.CTRL, Qt.SHIFT):
+            em = find_plugin('Edit Metadata')
+            if em is not None:
+                em.actual_plugin_.edit_metadata(None)
 
     # called when a book is clicked on the library view
-    def slave(self, current, previous):
+    def slave(self, current):
         if self.is_closed:
             return
         self.refresh(current)

src/calibre/gui2/metadata/basic_widgets.py

@@ -173,9 +173,20 @@ class TitleSortEdit(TitleEdit):
 
     def auto_generate(self, *args):
         self.current_val = title_sort(self.title_edit.current_val)
-        self.title_edit.textChanged.disconnect()
+
-        self.textChanged.disconnect()
+    def break_cycles(self):
-        self.autogen_button.clicked.disconnect()
+        try:
+            self.title_edit.textChanged.disconnect()
+        except:
+            pass
+        try:
+            self.textChanged.disconnect()
+        except:
+            pass
+        try:
+            self.autogen_button.clicked.disconnect()
+        except:
+            pass
 
 # }}}
 
@@ -280,7 +291,10 @@ class AuthorsEdit(MultiCompleteComboBox):
 
     def break_cycles(self):
         self.db = self.dialog = None
-        self.manage_authors_signal.triggered.disconnect()
+        try:
+            self.manage_authors_signal.triggered.disconnect()
+        except:
+            pass
 
 class AuthorSortEdit(EnLineEdit):
 
@@ -387,11 +401,26 @@ class AuthorSortEdit(EnLineEdit):
 
     def break_cycles(self):
         self.db = None
-        self.authors_edit.editTextChanged.disconnect()
+        try:
-        self.textChanged.disconnect()
+            self.authors_edit.editTextChanged.disconnect()
-        self.autogen_button.clicked.disconnect()
+        except:
-        self.copy_a_to_as_action.triggered.disconnect()
+            pass
-        self.copy_as_to_a_action.triggered.disconnect()
+        try:
+            self.textChanged.disconnect()
+        except:
+            pass
+        try:
+            self.autogen_button.clicked.disconnect()
+        except:
+            pass
+        try:
+            self.copy_a_to_as_action.triggered.disconnect()
+        except:
+            pass
+        try:
+            self.copy_as_to_a_action.triggered.disconnect()
+        except:
+            pass
         self.authors_edit = None
 
 # }}}
@@ -519,9 +548,18 @@ class SeriesIndexEdit(QDoubleSpinBox):
                 traceback.print_exc()
 
     def break_cycles(self):
-        self.series_edit.currentIndexChanged.disconnect()
+        try:
-        self.series_edit.editTextChanged.disconnect()
+            self.series_edit.currentIndexChanged.disconnect()
-        self.series_edit.lineEdit().editingFinished.disconnect()
+        except:
+            pass
+        try:
+            self.series_edit.editTextChanged.disconnect()
+        except:
+            pass
+        try:
+            self.series_edit.lineEdit().editingFinished.disconnect()
+        except:
+            pass
         self.db = self.series_edit = self.dialog = None
 
 # }}}
@@ -898,7 +936,10 @@ class Cover(ImageView): # {{{
         return True
 
     def break_cycles(self):
-        self.cover_changed.disconnect()
+        try:
+            self.cover_changed.disconnect()
+        except:
+            pass
         self.dialog = self._cdata = self.current_val = self.original_val = None
 
 # }}}

src/calibre/utils/formatter_functions.py

@@ -12,7 +12,7 @@ import inspect, re, traceback
 
 from calibre.utils.titlecase import titlecase
 from calibre.utils.icu import capitalize, strcmp, sort_key
-from calibre.utils.date import parse_date, format_date
+from calibre.utils.date import parse_date, format_date, now, UNDEFINED_DATE
 
 
 class FormatterFunctions(object):
@@ -579,7 +579,7 @@ class BuiltinSubitems(BuiltinFormatterFunction):
 class BuiltinFormatDate(BuiltinFormatterFunction):
     name = 'format_date'
     arg_count = 2
-    category = 'Get values from metadata'
+    category = 'Date functions'
     __doc__ = doc = _('format_date(val, format_string) -- format the value, '
             'which must be a date, using the format_string, returning a string. '
             'The formatting codes are: '
@@ -754,6 +754,39 @@ class BuiltinMergeLists(BuiltinFormatterFunction):
                 res.append(i)
         return ', '.join(sorted(res, key=sort_key))
 
+class BuiltinToday(BuiltinFormatterFunction):
+    name = 'today'
+    arg_count = 0
+    category = 'Date functions'
+    __doc__ = doc = _('today() -- '
+            'return a date string for today. This value is designed for use in '
+            'format_date or days_between, but can be manipulated like any '
+            'other string. The date is in ISO format.')
+    def evaluate(self, formatter, kwargs, mi, locals):
+        return format_date(now(), 'iso')
+
+class BuiltinDaysBetween(BuiltinFormatterFunction):
+    name = 'days_between'
+    arg_count = 2
+    category = 'Date functions'
+    __doc__ = doc = _('days_between(date1, date2) -- '
+            'return the number of days between date1 and date2. The number is '
+            'positive if date1 is greater than date2, otherwise negative. If '
+            'either date1 or date2 are not dates, the function returns the '
+            'empty string.')
+    def evaluate(self, formatter, kwargs, mi, locals, date1, date2):
+        try:
+            d1 = parse_date(date1)
+            if d1 == UNDEFINED_DATE:
+                return ''
+            d2 = parse_date(date2)
+            if d2 == UNDEFINED_DATE:
+                return ''
+        except:
+            return ''
+        i = d1 - d2
+        return str(i.days)
+
 
 builtin_add         = BuiltinAdd()
 builtin_and         = BuiltinAnd()
@@ -763,6 +796,7 @@ builtin_capitalize  = BuiltinCapitalize()
 builtin_cmp         = BuiltinCmp()
 builtin_contains    = BuiltinContains()
 builtin_count       = BuiltinCount()
+builtin_days_between= BuiltinDaysBetween()
 builtin_divide      = BuiltinDivide()
 builtin_eval        = BuiltinEval()
 builtin_first_non_empty = BuiltinFirstNonEmpty()
@@ -795,6 +829,7 @@ builtin_switch      = BuiltinSwitch()
 builtin_template    = BuiltinTemplate()
 builtin_test        = BuiltinTest()
 builtin_titlecase   = BuiltinTitlecase()
+builtin_today       = BuiltinToday()
 builtin_uppercase   = BuiltinUppercase()
 
 class FormatterUserFunction(FormatterFunction):

src/calibre/utils/ipc/proxy.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, cPickle, struct
+from threading import Thread
+from Queue import Queue, Empty
+from multiprocessing.connection import arbitrary_address, Listener
+from functools import partial
+
+from calibre import as_unicode, prints
+from calibre.constants import iswindows, DEBUG
+
+def _encode(msg):
+    raw = cPickle.dumps(msg, -1)
+    size = len(raw)
+    header = struct.pack('!Q', size)
+    return header + raw
+
+def _decode(raw):
+    sz = struct.calcsize('!Q')
+    if len(raw) < sz:
+        return 'invalid', None
+    header, = struct.unpack('!Q', raw[:sz])
+    if len(raw) != sz + header or header == 0:
+        return 'invalid', None
+    return cPickle.loads(raw[sz:])
+
+
+class Writer(Thread):
+
+    TIMEOUT = 60 #seconds
+
+    def __init__(self, conn):
+        Thread.__init__(self)
+        self.daemon = True
+        self.dataq, self.resultq = Queue(), Queue()
+        self.conn = conn
+        self.start()
+        self.data_written = False
+
+    def close(self):
+        self.dataq.put(None)
+
+    def flush(self):
+        pass
+
+    def write(self, raw_data):
+        self.dataq.put(raw_data)
+
+        try:
+            ex = self.resultq.get(True, self.TIMEOUT)
+        except Empty:
+            raise IOError('Writing to socket timed out')
+        else:
+            if ex is not None:
+                raise IOError('Writing to socket failed with error: %s' % ex)
+
+    def run(self):
+        while True:
+            x = self.dataq.get()
+            if x is None:
+                break
+            try:
+                self.data_written = True
+                self.conn.send_bytes(x)
+            except Exception as e:
+                self.resultq.put(as_unicode(e))
+            else:
+                self.resultq.put(None)
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.close()
+
+class Server(Thread):
+
+    def __init__(self, dispatcher):
+        Thread.__init__(self)
+        self.daemon = True
+
+        self.auth_key = os.urandom(32)
+        self.address = arbitrary_address('AF_PIPE' if iswindows else 'AF_UNIX')
+        if iswindows and self.address[1] == ':':
+            self.address = self.address[2:]
+        self.listener = Listener(address=self.address,
+                authkey=self.auth_key, backlog=4)
+
+        self.keep_going = True
+        self.dispatcher = dispatcher
+
+    @property
+    def connection_information(self):
+        if not self.is_alive():
+            self.start()
+        return (self.address, self.auth_key)
+
+    def stop(self):
+        self.keep_going = False
+        try:
+            self.listener.close()
+        except:
+            pass
+
+    def run(self):
+        while self.keep_going:
+            try:
+                conn = self.listener.accept()
+                self.handle_client(conn)
+            except:
+                pass
+
+    def handle_client(self, conn):
+        t = Thread(target=partial(self._handle_client, conn))
+        t.daemon = True
+        t.start()
+
+    def _handle_client(self, conn):
+        while True:
+            try:
+                func_name, args, kwargs = conn.recv()
+            except EOFError:
+                try:
+                    conn.close()
+                except:
+                    pass
+                return
+            else:
+                try:
+                    self.call_func(func_name, args, kwargs, conn)
+                except:
+                    try:
+                        conn.close()
+                    except:
+                        pass
+                    prints('Proxy function: %s with args: %r and'
+                            ' kwargs: %r failed')
+                    if DEBUG:
+                        import traceback
+                        traceback.print_exc()
+                    break
+
+    def call_func(self, func_name, args, kwargs, conn):
+        with Writer(conn) as f:
+            try:
+                self.dispatcher(f, func_name, args, kwargs)
+            except Exception as e:
+                if not f.data_written:
+                    import traceback
+                    # Try to tell the client process what error happened
+                    try:
+                        conn.send_bytes(_encode(('failed', (unicode(e),
+                            as_unicode(traceback.format_exc())))))
+                    except:
+                        pass
+                raise
+
+