Merge to the latest trunk

recipes/stars_and_stripes.recipe

@@ -18,7 +18,7 @@ class AdvancedUserRecipe1308791026(BasicNewsRecipe):
     encoding               = 'utf8'
     publisher              = 'stripes.com'
     category               = 'news, US, world'
-    language               = 'en_US'
+    language               = 'en'
     publication_type       = 'newsportal'
     preprocess_regexps     = [(re.compile(r'<!--.*?-->', re.DOTALL), lambda m: '')]
     conversion_options = {

session.vim

@@ -1,40 +1,9 @@
 " Project wide builtins
 let $PYFLAKES_BUILTINS = "_,dynamic_property,__,P,I,lopen,icu_lower,icu_upper,icu_title,ngettext"
 
-python << EOFPY
-import os, sys
-
-import vipy
-
-source_file = vipy.vipy.eval('expand("<sfile>")')
-project_dir = os.path.dirname(source_file)
-src_dir = os.path.abspath(os.path.join(project_dir, 'src'))
-base_dir = os.path.join(src_dir, 'calibre')
-
-sys.path.insert(0, src_dir)
-sys.resources_location = os.path.join(project_dir, 'resources')
-sys.extensions_location = os.path.join(base_dir, 'plugins')
-sys.executables_location = os.environ.get('CALIBRE_EXECUTABLES_PATH', '/usr/bin')
-
-vipy.session.initialize(project_name='calibre', src_dir=src_dir,
-            project_dir=project_dir, base_dir=project_dir)
-
-def recipe_title_callback(raw):
-    try:
-        raw = eval(raw)
-        if isinstance(raw, bytes):
-            raw = raw.decode('utf-8')
-        return raw.replace(u' ', u'_')
-    except:
-        print ('Failed to decode recipe title: %r'%raw)
-        raise
-
-vipy.session.add_content_browser('<leader>r', 'Recipe',
-    vipy.session.glob_based_iterator(os.path.join(project_dir, 'recipes', '*.recipe')),
-    vipy.session.regexp_based_matcher(r'title\s*=\s*(?P<title>.+)', 'title', recipe_title_callback))
-EOFPY
-
 fun! CalibreLog()
+    " Setup buffers to edit the calibre changelog and version info prior to
+    " making a release.
     enew
     read ! bzr log -l 500
     set nomodifiable noswapfile buftype=nofile

src/calibre/devices/kindle/driver.py

@@ -13,6 +13,7 @@ import datetime, os, re, sys, json, hashlib
 from calibre.devices.kindle.bookmark import Bookmark
 from calibre.devices.usbms.driver import USBMS
 from calibre import strftime
+from calibre.utils.logging import default_log
 
 '''
 Notes on collections:
@@ -324,6 +325,7 @@ class KINDLE2(KINDLE):
     OPT_APNX           = 0
     OPT_APNX_ACCURATE  = 1
     OPT_APNX_CUST_COL  = 2
+    THUMBNAIL_HEIGHT = 180
 
     def formats_to_scan_for(self):
         ans = USBMS.formats_to_scan_for(self) | {'azw3'}
@@ -375,8 +377,36 @@ class KINDLE2(KINDLE):
 
     def upload_cover(self, path, filename, metadata, filepath):
         '''
-        Hijacking this function to write the apnx file.
+        Upload sidecar files: cover thumbnails and page count
         '''
+        # Upload the cover thumbnail
+        try:
+            self.upload_kindle_thumbnail(metadata, filepath)
+        except:
+            import traceback
+            traceback.print_exc()
+        # Upload the apnx file
+        self.upload_apnx(filename, metadata, filepath)
+
+    def upload_kindle_thumbnail(self, metadata, filepath):
+        coverdata = getattr(metadata, 'thumbnail', None)
+        if not coverdata or not coverdata[2]:
+            return
+        thumb_dir = os.path.join(self._main_prefix, 'system', 'thumbnails')
+        if not os.path.exists(thumb_dir): return
+
+        from calibre.ebooks.mobi.reader.headers import MetadataHeader
+        with lopen(filepath, 'rb') as f:
+            mh = MetadataHeader(f, default_log)
+        if mh.exth is None or not mh.exth.uuid or not mh.exth.cdetype:
+            return
+        thumbfile = os.path.join(thumb_dir,
+                'thumbnail_{uuid}_{cdetype}_portrait.jpg'.format(
+                    uuid=mh.exth.uuid, cdetype=mh.exth.cdetype))
+        with open(thumbfile, 'wb') as f:
+            f.write(coverdata[2])
+
+    def upload_apnx(self, filename, metadata, filepath):
         from calibre.devices.kindle.apnx import APNXBuilder
 
         opts = self.settings()
@@ -422,6 +452,9 @@ class KINDLE_DX(KINDLE2):
     PRODUCT_ID = [0x0003]
     BCD        = [0x0100]
 
+    def upload_kindle_thumbnail(self, metadata, filepath):
+        pass
+
 class KINDLE_FIRE(KINDLE2):
 
     name = 'Kindle Fire Device Interface'
@@ -440,4 +473,6 @@ class KINDLE_FIRE(KINDLE2):
     VENDOR_NAME = 'AMAZON'
     WINDOWS_MAIN_MEM = 'KINDLE'
 
+    def upload_kindle_thumbnail(self, metadata, filepath):
+        pass
 

src/calibre/ebooks/metadata/fb2.py

@@ -22,6 +22,9 @@ NAMESPACES = {
 XPath = partial(etree.XPath, namespaces=NAMESPACES)
 tostring = partial(etree.tostring, method='text', encoding=unicode)
 
+def FB2(tag):
+    return '{%s}%s'%(NAMESPACES['fb2'], tag)
+
 def get_metadata(stream):
     ''' Return fb2 metadata as a L{MetaInformation} object '''
 
@@ -85,6 +88,7 @@ def _parse_authors(root):
     authors = []
     # pick up authors but only from 1 secrion <title-info>; otherwise it is not consistent!
     # Those are fallbacks: <src-title-info>, <document-info>
+    author = None
     for author_sec in ['title-info', 'src-title-info', 'document-info']:
         for au in XPath('//fb2:%s/fb2:author'%author_sec)(root):
             author = _parse_author(au)
@@ -211,8 +215,8 @@ def _parse_publisher(root, mi):
 def _parse_pubdate(root, mi):
     year = XPath('number(//fb2:publish-info/fb2:year/text())')(root)
     if float.is_integer(year):
-        # only year is available, so use 1-st of Jan
+        # only year is available, so use 2nd of June
-        mi.pubdate = datetime.date(int(year), 1, 1)
+        mi.pubdate = datetime.date(int(year), 6, 2)
 
 def _parse_timestamp(root, mi):
     #<date value="1996-12-03">03.12.1996</date>
@@ -240,70 +244,92 @@ def _get_fbroot(stream):
     root = etree.fromstring(raw, parser=parser)
     return root
 
-def _get_child_or_create_and_insert_before(doc, root, tag, index, location):
+def _clear_meta_tags(doc, tag):
-    nodes = root.getElementsByTagName(tag)
+    for parent in ('title-info', 'src-title-info', 'publish-info'):
-    node = nodes[index] if nodes else root.insertBefore(doc.createElement(tag), location)
+        for x in XPath('//fb2:%s/fb2:%s'%(parent, tag))(doc):
-    return node
+            x.getparent().remove(x)
 
-def _get_first_child_or_create_and_insert_before(doc, root, tag, location):
+def _set_title(title_info, mi):
-    return _get_child_or_create_and_insert_before(doc, root, tag, 0, location)
-
-def _get_first_child_or_create_and_insert_first(doc, root, tag):
-    return _get_first_child_or_create_and_insert_before(doc, root, tag, root.firstChild)
-
-def _get_first_child_or_create_and_append(doc, root, tag):
-    nodes = root.getElementsByTagName(tag)
-    node = nodes[0] if nodes else root.appendChild(doc.createElement(tag))
-    return node
-
-def _set_title(doc, title_info, mi):
     if not mi.is_null('title'):
-        xml_title = _get_first_child_or_create_and_append(doc, title_info, 'book-title')
+        _clear_meta_tags(title_info, 'book-title')
-        xml_title.childNodes = []
+        title = _get_or_create(title_info, 'book-title')
-        xml_title.appendChild(doc.createTextNode(mi.title))
+        title.text = mi.title
 
-def _set_authors(doc, title_info, mi):
+def _set_comments(title_info, mi):
+    if not mi.is_null('comments'):
+        _clear_meta_tags(title_info, 'annotation')
+        title = _get_or_create(title_info, 'annotation')
+        title.text = mi.comments
+
+
+def _set_authors(title_info, mi):
     if not mi.is_null('authors'):
-        xml_authors = title_info.getElementsByTagName('author')
+        _clear_meta_tags(title_info, 'author')
-        count = len(xml_authors)
-        i = 0
         for author in mi.authors:
-            xml_author = xml_authors[i] if i < count else title_info.insertBefore(doc.createElement('author'), xml_authors[-1].nextSibling)
+            author_parts = author.split()
-            i += 1
+            if not author_parts: continue
-            xml_author.childNodes = []
+            atag = title_info.makeelement(FB2('author'))
-            author_parts = author.split(' ')
+            title_info.insert(0, atag)
-            c = len(author_parts)
+            if len(author_parts) == 1:
-            name_tags = ['nickname'] if c == 1 else \
+                _get_or_create(atag, 'nickname').text = author
-                        ['first-name', 'last-name'] if c == 2 else \
+            else:
-                        ['first-name', 'middle-name', 'last-name'] if c == 3 else \
+                _get_or_create(atag, 'first-name').text = author_parts[0]
-                        ['first-name', 'middle-name', 'last-name', 'nickname']
+                author_parts = author_parts[1:]
-            for tag, part in zip(name_tags, author_parts):
+                if len(author_parts) > 1:
-                xml_author_part = xml_author.appendChild(doc.createElement(tag))
+                    _get_or_create(atag, 'middle-name', at_start=False).text = author_parts[0]
-                xml_author_part.appendChild(doc.createTextNode(part))
+                    author_parts = author_parts[1:]
-        if i < count:
+                if author_parts:
-            for ind in range(i, count):
+                    _get_or_create(atag, 'last-name', at_start=False).text = ' '.join(author_parts)
-                title_info.removeChild(xml_authors[ind])
 
-def _set_series(doc, title_info, mi):
+def _set_tags(title_info, mi):
+    if not mi.is_null('tags'):
+        _clear_meta_tags(title_info, 'genre')
+        for t in mi.tags:
+            tag = title_info.makeelement(FB2('genre'))
+            tag.text = t
+            title_info.insert(0, tag)
+
+def _set_series(title_info, mi):
     if not mi.is_null('series'):
-        xml_sequence = _get_first_child_or_create_and_append(doc, title_info, 'sequence')
+        _clear_meta_tags(title_info, 'sequence')
-        xml_sequence.setAttribute('name', mi.series)
+        seq = _get_or_create(title_info, 'sequence')
-        if not mi.is_null('series_index'):
+        seq.set('name', mi.series)
-            xml_sequence.setAttribute('number', str(int(float(mi.series_index))))
+        try:
+            seq.set('number', '%g'%mi.series_index)
+        except:
+            seq.set('number', '1')
+
+def _get_or_create(parent, tag, at_start=True):
+    ans = XPath('./fb2:'+tag)(parent)
+    if ans:
+        ans = ans[0]
+    else:
+        ans = parent.makeelement(FB2(tag))
+        if at_start:
+            parent.insert(0, ans)
+        else:
+            parent.append(ans)
+    return ans
 
 def set_metadata(stream, mi, apply_null=False, update_timestamp=False):
-    import xml.dom.minidom as md
-    from calibre.ebooks.metadata import MetaInformation
     stream.seek(0)
-    raw = stream.read()
+    root = _get_fbroot(stream)
-    xml_doc = md.parseString(raw)
+    desc = _get_or_create(root, 'description')
-    xml_fiction_book = xml_doc.getElementsByTagName('FictionBook')[0]
+    ti = _get_or_create(desc, 'title-info')
-    xml_description = _get_first_child_or_create_and_insert_first(xml_doc, xml_fiction_book, 'description')
+
-    xml_titleinfo = _get_first_child_or_create_and_insert_first(xml_doc, xml_description, 'title-info')
+    indent = ti.text
+
+    _set_comments(ti, mi)
+    _set_series(ti, mi)
+    _set_tags(ti, mi)
+    _set_authors(ti, mi)
+    _set_title(ti, mi)
+
+    for child in ti:
+        child.tail = indent
+
+    stream.seek(0)
+    stream.truncate()
+    stream.write(etree.tostring(root, method='xml', encoding='utf-8',
+        xml_declaration=True))
 
-    _set_title(xml_doc, xml_titleinfo, mi)
-    _set_authors(xml_doc, xml_titleinfo, mi)
-    _set_series(xml_doc, xml_titleinfo, mi)
-    
-    stream.truncate(0)
-    stream.write(xml_doc.toxml(xml_doc.encoding))

src/calibre/ebooks/mobi/reader/headers.py

@@ -45,6 +45,10 @@ class EXTHHeader(object): # {{{
             elif idx == 202:
                 self.thumbnail_offset, = struct.unpack('>L', content)
             elif idx == 501:
+                try:
+                    self.cdetype = content.decode('ascii')
+                except UnicodeDecodeError:
+                    self.cdetype = None
                 # cdetype
                 if content == b'EBSP':
                     if not self.mi.tags:
@@ -109,8 +113,11 @@ class EXTHHeader(object): # {{{
                         self.mi.isbn = raw
             except:
                 pass
-        elif idx == 113:
+        elif idx == 113: # ASIN or other id
-            pass # ASIN or UUID
+            try:
+                self.uuid = content.decode('ascii')
+            except:
+                self.uuid = None
         elif idx == 116:
             self.start_offset, = struct.unpack(b'>L', content)
         elif idx == 121: