Merge from trunk

src/calibre/ebooks/__init__.py

@@ -8,6 +8,7 @@ from various formats.
 '''
 
 import traceback, os, re
+from cStringIO import StringIO
 from calibre import CurrentDir
 
 class ConversionError(Exception):
@@ -209,4 +210,45 @@ def unit_convert(value, base, font, dpi):
             result = value * 0.40
     return result
 
+def generate_masthead(title, output_path=None, width=600, height=60):
+    from calibre.ebooks.conversion.config import load_defaults
+    from calibre.utils.fonts import fontconfig
+    font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf')
+    recs = load_defaults('mobi_output')
+    masthead_font_family = recs.get('masthead_font', 'Default')
+
+    if masthead_font_family != 'Default':
+        masthead_font = fontconfig.files_for_family(masthead_font_family)
+        # Assume 'normal' always in dict, else use default
+        # {'normal': (path_to_font, friendly name)}
+        if 'normal' in masthead_font:
+            font_path = masthead_font['normal'][0]
+
+    if not font_path or not os.access(font_path, os.R_OK):
+        font_path = default_font
+
+    try:
+        from PIL import Image, ImageDraw, ImageFont
+        Image, ImageDraw, ImageFont
+    except ImportError:
+        import Image, ImageDraw, ImageFont
+
+    img = Image.new('RGB', (width, height), 'white')
+    draw = ImageDraw.Draw(img)
+    try:
+        font = ImageFont.truetype(font_path, 48)
+    except:
+        font = ImageFont.truetype(default_font, 48)
+    text = title.encode('utf-8')
+    width, height = draw.textsize(text, font=font)
+    left = max(int((width - width)/2.), 0)
+    top = max(int((height - height)/2.), 0)
+    draw.text((left, top), text, fill=(0,0,0), font=font)
+    if output_path is None:
+        f = StringIO()
+        img.save(f, 'JPEG')
+        return f.getvalue()
+    else:
+        with open(output_path, 'wb') as f:
+            img.save(f, 'JPEG')
 

src/calibre/ebooks/mobi/debug.py

@@ -1089,6 +1089,11 @@ class TextRecord(object): # {{{
             self.trailing_data['uncrossable_breaks'] = self.trailing_data.pop(2)
         self.trailing_data['raw_bytes'] = raw_trailing_bytes
 
+        for typ, val in self.trailing_data.iteritems():
+            if isinstance(typ, int):
+                print ('Record %d has unknown trailing data of type: %d : %r'%
+                        (idx, typ, val))
+
         self.idx = idx
 
     def dump(self, folder):
@@ -1192,8 +1197,7 @@ class TBSIndexing(object): # {{{
             '(%d ends, %d complete, %d starts)')%tuple(map(len, (s+e+c, e,
                 c, s))))
         byts = bytearray(r.trailing_data.get('indexing', b''))
-        sbyts = tuple(hex(b)[2:] for b in byts)
+        ans.append('TBS bytes: %s'%format_bytes(byts))
-        ans.append('TBS bytes: %s'%(' '.join(sbyts)))
         for typ, entries in (('Ends', e), ('Complete', c), ('Starts', s)):
             if entries:
                 ans.append('\t%s:'%typ)
@@ -1220,8 +1224,14 @@ class TBSIndexing(object): # {{{
             ans.append('Outermost index: %d'%outermost_index)
             ans.append('Unknown extra start bytes: %s'%repr_extra(extra))
             if is_periodical: # Hierarchical periodical
+                try:
                     byts, a = self.interpret_periodical(tbs_type, byts,
                         dat['geom'][0])
+                except:
+                    import traceback
+                    traceback.print_exc()
+                    a = []
+                    print ('Failed to decode TBS bytes for record: %d'%r.idx)
                 ans += a
             if byts:
                 sbyts = tuple(hex(b)[2:] for b in byts)
@@ -1372,7 +1382,7 @@ class MOBIFile(object): # {{{
                     self.index_header, self.cncx)
             self.indexing_record_nums = set(xrange(pir,
                 pir+2+self.index_header.num_of_cncx_blocks))
-        self.secondary_index_record = self.secondary_index_record = None
+        self.secondary_index_record = self.secondary_index_header = None
         sir = self.mobi_header.secondary_index_record
         if sir != 0xffffffff:
             self.secondary_index_header = SecondaryIndexHeader(self.records[sir])

src/calibre/ebooks/mobi/utils.py

@@ -169,18 +169,25 @@ def get_trailing_data(record, extra_data_flags):
     :return: Trailing data, record - trailing data
     '''
     data = OrderedDict()
-    for i in xrange(16, -1, -1):
+    flags = extra_data_flags >> 1
-        flag = 1 << i # 2**i
+
-        if flag & extra_data_flags:
+    num = 0
-            if i == 0:
+    while flags:
+        num += 1
+        if flags & 0b1:
+            sz, consumed = decint(record, forward=False)
+            if sz > consumed:
+                data[num] = record[-sz:-consumed]
+            record = record[:-sz]
+        flags >>= 1
+    # Read multibyte chars if any
+    if extra_data_flags & 0b1:
         # Only the first two bits are used for the size since there can
         # never be more than 3 trailing multibyte chars
         sz = (ord(record[-1]) & 0b11) + 1
         consumed = 1
-            else:
-                sz, consumed = decint(record, forward=False)
         if sz > consumed:
-                data[i] = record[-sz:-consumed]
+            data[0] = record[-sz:-consumed]
         record = record[:-sz]
     return data, record
 

src/calibre/ebooks/mobi/writer.py

@@ -430,6 +430,7 @@ class MobiWriter(object):
         text.seek(npos)
         return data, overlap
 
+    # TBS {{{
     def _generate_flat_indexed_navpoints(self):
         # Assemble a HTMLRecordData instance for each HTML record
         # Return True if valid, False if invalid
@@ -1174,6 +1175,8 @@ class MobiWriter(object):
 
         self._tbSequence = tbSequence
 
+    # }}}
+
     def _evaluate_periodical_toc(self):
         '''
         Periodical:

src/calibre/ebooks/mobi/writer2/indexer.py

@@ -14,7 +14,7 @@ from collections import OrderedDict, defaultdict
 
 from calibre.ebooks.mobi.writer2 import RECORD_SIZE
 from calibre.ebooks.mobi.utils import (encint, encode_number_as_hex,
-        encode_tbs, align_block, utf8_text, detect_periodical)
+        encode_tbs, align_block, utf8_text)
 
 
 class CNCX(object): # {{{
@@ -323,16 +323,22 @@ class TBS(object): # {{{
 class Indexer(object): # {{{
 
     def __init__(self, serializer, number_of_text_records,
-            size_of_last_text_record, opts, oeb):
+            size_of_last_text_record, masthead_offset, is_periodical,
+            opts, oeb):
         self.serializer = serializer
         self.number_of_text_records = number_of_text_records
         self.text_size = (RECORD_SIZE * (self.number_of_text_records-1) +
                             size_of_last_text_record)
+        self.masthead_offset = masthead_offset
+
         self.oeb = oeb
         self.log = oeb.log
         self.opts = opts
 
-        self.is_periodical = detect_periodical(self.oeb.toc, self.log)
+        self.is_periodical = is_periodical
+        if self.is_periodical and self.masthead_offset is None:
+            raise ValueError('Periodicals must have a masthead')
+
         self.log('Generating MOBI index for a %s'%('periodical' if
             self.is_periodical else 'book'))
         self.is_flat_periodical = False

src/calibre/ebooks/mobi/writer2/main.py

@@ -11,7 +11,7 @@ import re, random, time
 from cStringIO import StringIO
 from struct import pack
 
-from calibre.ebooks import normalize
+from calibre.ebooks import normalize, generate_masthead
 from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES
 from calibre.ebooks.mobi.writer2.serializer import Serializer
 from calibre.ebooks.compression.palmdoc import compress_doc
@@ -19,7 +19,7 @@ from calibre.ebooks.mobi.langcodes import iana2mobi
 from calibre.utils.filenames import ascii_filename
 from calibre.ebooks.mobi.writer2 import (PALMDOC, UNCOMPRESSED, RECORD_SIZE)
 from calibre.ebooks.mobi.utils import (rescale_image, encint,
-        encode_trailing_data, align_block)
+        encode_trailing_data, align_block, detect_periodical)
 from calibre.ebooks.mobi.writer2.indexer import Indexer
 
 EXTH_CODES = {
@@ -35,6 +35,9 @@ EXTH_CODES = {
     'type': 111,
     'source': 112,
     'versionnumber': 114,
+    'coveroffset': 201,
+    'thumboffset': 202,
+    'hasfakecover': 203,
     'lastupdatetime': 502,
     'title': 503,
     }
@@ -79,13 +82,14 @@ class MobiWriter(object):
         self.write_content()
 
     def generate_content(self):
-        self.map_image_names()
+        self.is_periodical = detect_periodical(self.oeb.toc, self.oeb.log)
+        self.generate_images()
         self.generate_text()
+        # The uncrossable breaks trailing entries come before the indexing
+        # trailing entries
+        self.write_uncrossable_breaks()
         # Index records come after text records
         self.generate_index()
-        self.write_uncrossable_breaks()
-        # Image records come after index records
-        self.generate_images()
 
     # Indexing {{{
     def generate_index(self):
@@ -93,6 +97,7 @@ class MobiWriter(object):
         try:
             self.indexer = Indexer(self.serializer, self.last_text_record_idx,
                     len(self.records[self.last_text_record_idx]),
+                    self.masthead_offset, self.is_periodical,
                     self.opts, self.oeb)
         except:
             self.log.exception('Failed to generate MOBI index:')
@@ -104,11 +109,6 @@ class MobiWriter(object):
                 self.records[i] += encode_trailing_data(tbs)
             self.records.extend(self.indexer.records)
 
-    @property
-    def is_periodical(self):
-        return (self.primary_index_record_idx is None or not
-                self.indexer.is_periodical)
-
     # }}}
 
     def write_uncrossable_breaks(self): # {{{
@@ -138,58 +138,51 @@ class MobiWriter(object):
     # }}}
 
     # Images {{{
-    def map_image_names(self):
-        '''
-        Map image names to record indices, ensuring that the masthead image if
-        present has index number 1.
-        '''
-        index = 1
-        self.images = images = {}
-        mh_href = None
-
-        if 'masthead' in self.oeb.guide:
-            mh_href = self.oeb.guide['masthead'].href
-            images[mh_href] = 1
-            index += 1
-
-        for item in self.oeb.manifest.values():
-            if item.media_type in OEB_RASTER_IMAGES:
-                if item.href == mh_href: continue
-                images[item.href] = index
-                index += 1
 
     def generate_images(self):
-        self.oeb.logger.info('Serializing images...')
+        oeb = self.oeb
-        images = [(index, href) for href, index in self.images.iteritems()]
+        oeb.logger.info('Serializing images...')
-        images.sort()
+        self.image_records = []
-        self.first_image_record = None
+
-        for _, href in images:
+        mh_href = self.masthead_offset = None
-            item = self.oeb.manifest.hrefs[href]
+        if 'masthead' in oeb.guide:
+            mh_href = oeb.guide['masthead'].href
+        elif self.is_periodical:
+            # Generate a default masthead
+            data = generate_masthead(unicode(self.oeb.metadata('title')[0]))
+            self.image_records.append(data)
+            self.masthead_offset = 0
+
+        cover_href = self.cover_offset = self.thumbnail_offset = None
+        if (oeb.metadata.cover and
+                unicode(oeb.metadata.cover[0]) in oeb.manifest.ids):
+            cover_id = unicode(oeb.metadata.cover[0])
+            item = oeb.manifest.ids[cover_id]
+            cover_href = item.href
+
+        for item in self.oeb.manifest.values():
+            if item.media_type not in OEB_RASTER_IMAGES: continue
             try:
                 data = rescale_image(item.data)
             except:
-                self.oeb.logger.warn('Bad image file %r' % item.href)
+                oeb.logger.warn('Bad image file %r' % item.href)
                 continue
-            finally:
+            else:
-                item.unload_data_from_memory()
+                if item.href == mh_href:
-            self.records.append(data)
+                    self.masthead_offset = len(self.image_records) - 1
-            if self.first_image_record is None:
+                elif item.href == cover_href:
-                self.first_image_record = len(self.records) - 1
+                    self.image_records.append(data)
-
+                    self.cover_offset = len(self.image_records) - 1
-    def add_thumbnail(self, item):
                     try:
                         data = rescale_image(item.data, dimen=MAX_THUMB_DIMEN,
                             maxsizeb=MAX_THUMB_SIZE)
-        except IOError:
+                    except:
-            self.oeb.logger.warn('Bad image file %r' % item.href)
+                        oeb.logger.warn('Failed to generate thumbnail')
-            return None
+                    else:
-        manifest = self.oeb.manifest
+                        self.image_records.append(data)
-        id, href = manifest.generate('thumbnail', 'thumbnail.jpeg')
+                        self.thumbnail_offset = len(self.image_records) - 1
-        manifest.add(id, href, 'image/jpeg', data=data)
+            finally:
-        index = len(self.images) + 1
+                item.unload_data_from_memory()
-        self.images[href] = index
-        self.records.append(data)
-        return index
 
     # }}}
 
@@ -282,9 +275,13 @@ class MobiWriter(object):
     def generate_record0(self): #  MOBI header {{{
         metadata = self.oeb.metadata
         exth = self.build_exth()
+        first_image_record = None
+        if self.image_records:
+            first_image_record  = len(self.records)
+            self.records.extend(self.image_records)
         last_content_record = len(self.records) - 1
 
-        # FCIS/FLIS (Seem to server no purpose)
+        # FCIS/FLIS (Seems to serve no purpose)
         flis_number = len(self.records)
         self.records.append(
             b'FLIS\0\0\0\x08\0\x41\0\0\0\0\0\0\xff\xff\xff\xff\0\x01\0\x03\0\0\0\x03\0\0\0\x01'+
@@ -363,8 +360,7 @@ class MobiWriter(object):
         # 0x58 - 0x5b : Format version
         # 0x5c - 0x5f : First image record number
         record0.write(pack(b'>II',
-            6, self.first_image_record if self.first_image_record else
+            6, first_image_record if first_image_record else len(self.records)))
-            len(self.records)-1))
 
         # 0x60 - 0x63 : First HUFF/CDIC record number
         # 0x64 - 0x67 : Number of HUFF/CDIC records
@@ -539,19 +535,14 @@ class MobiWriter(object):
             exth.write(pack(b'>III', code, 12, val))
             nrecs += 1
 
-        if (oeb.metadata.cover and
+        if self.cover_offset is not None:
-                unicode(oeb.metadata.cover[0]) in oeb.manifest.ids):
+            exth.write(pack(b'>III', EXTH_CODES['coveroffset'], 12,
-            id = unicode(oeb.metadata.cover[0])
+                self.cover_offset))
-            item = oeb.manifest.ids[id]
+            exth.write(pack(b'>III', EXTH_CODES['hasfakecover'], 12, 0))
-            href = item.href
-            if href in self.images:
-                index = self.images[href] - 1
-                exth.write(pack(b'>III', 0xc9, 0x0c, index))
-                exth.write(pack(b'>III', 0xcb, 0x0c, 0))
             nrecs += 2
-                index = self.add_thumbnail(item)
+        if self.thumbnail_offset is not None:
-                if index is not None:
+            exth.write(pack(b'>III', EXTH_CODES['thumboffset'], 12,
-                    exth.write(pack(b'>III', 0xca, 0x0c, index - 1))
+                self.thumbnail_offset))
             nrecs += 1
 
         exth = exth.getvalue()

src/calibre/gui2/dialogs/tag_list_editor.py

@@ -1,7 +1,8 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-from PyQt4.Qt import (Qt, QDialog, QTableWidgetItem, QIcon, QByteArray, QString)
+from PyQt4.Qt import (Qt, QDialog, QTableWidgetItem, QIcon, QByteArray,
+        QString, QSize)
 
 from calibre.gui2.dialogs.tag_list_editor_ui import Ui_TagListEditor
 from calibre.gui2 import question_dialog, error_dialog, gprefs
@@ -84,8 +85,6 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         try:
             self.table_column_widths = \
                         gprefs.get('tag_list_editor_table_widths', None)
-            geom = gprefs.get('tag_list_editor_dialog_geometry', bytearray(''))
-            self.restoreGeometry(QByteArray(geom))
         except:
             pass
 
@@ -147,6 +146,15 @@ class TagListEditor(QDialog, Ui_TagListEditor):
         self.table.itemChanged.connect(self.finish_editing)
         self.buttonBox.accepted.connect(self.accepted)
 
+        try:
+            geom = gprefs.get('tag_list_editor_dialog_geometry', None)
+            if geom is not None:
+                self.restoreGeometry(QByteArray(geom))
+            else:
+                self.resize(self.sizeHint()+QSize(150, 100))
+        except:
+            pass
+
     def table_column_resized(self, col, old, new):
         self.table_column_widths = []
         for c in range(0, self.table.columnCount()):

src/calibre/translations/eu.po

@@ -951,7 +951,7 @@ msgstr "Araztu saioa"
 
 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:13
 msgid "Communicate with Android phones."
-msgstr "Adroid telefonoekin komunikatu."
+msgstr "Android telefonoekin komunikatu."
 
 #: /home/kovid/work/calibre/src/calibre/devices/android/driver.py:113
 msgid ""

src/calibre/web/feeds/news.py

@@ -1083,40 +1083,9 @@ class BasicNewsRecipe(Recipe):
     MI_HEIGHT = 60
 
     def default_masthead_image(self, out_path):
-        from calibre.ebooks.conversion.config import load_defaults
+        from calibre.ebooks import generate_masthead
-        from calibre.utils.fonts import fontconfig
+        generate_masthead(self.get_masthead_title(), output_path=out_path,
-        font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf')
+                width=self.MI_WIDTH, height=self.MI_HEIGHT)
-        recs = load_defaults('mobi_output')
-        masthead_font_family = recs.get('masthead_font', 'Default')
-
-        if masthead_font_family != 'Default':
-            masthead_font = fontconfig.files_for_family(masthead_font_family)
-            # Assume 'normal' always in dict, else use default
-            # {'normal': (path_to_font, friendly name)}
-            if 'normal' in masthead_font:
-                font_path = masthead_font['normal'][0]
-
-        if not font_path or not os.access(font_path, os.R_OK):
-            font_path = default_font
-
-        try:
-            from PIL import Image, ImageDraw, ImageFont
-            Image, ImageDraw, ImageFont
-        except ImportError:
-            import Image, ImageDraw, ImageFont
-
-        img = Image.new('RGB', (self.MI_WIDTH, self.MI_HEIGHT), 'white')
-        draw = ImageDraw.Draw(img)
-        try:
-            font = ImageFont.truetype(font_path, 48)
-        except:
-            font = ImageFont.truetype(default_font, 48)
-        text = self.get_masthead_title().encode('utf-8')
-        width, height = draw.textsize(text, font=font)
-        left = max(int((self.MI_WIDTH - width)/2.), 0)
-        top = max(int((self.MI_HEIGHT - height)/2.), 0)
-        draw.text((left, top), text, fill=(0,0,0), font=font)
-        img.save(open(out_path, 'wb'), 'JPEG')
 
     def prepare_masthead_image(self, path_to_image, out_path):
         from calibre import fit_image