Merge from trunk

src/calibre/ebooks/mobi/debug.py

@@ -650,8 +650,6 @@ class Tag(object): # {{{
             'article'   : {
                     5  : ('Class offset in cncx', 'class_offset'),
                     21 : ('Parent section index', 'parent_index'),
-                    22 : ('Description offset in cncx', 'desc_offset'),
-                    23 : ('Author offset in cncx', 'author_offset'),
                     69 : ('Offset from first image record num to the'
                         ' image record associated with this article',
                         'image_index'),

src/calibre/ebooks/mobi/writer2/indexer.py

@@ -2,7 +2,7 @@
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 from __future__ import (unicode_literals, division, absolute_import,
                         print_function)
-from future_builtins import filter
+from future_builtins import filter, map
 
 __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
@@ -16,7 +16,6 @@ from calibre.ebooks.mobi.writer2 import RECORD_SIZE
 from calibre.ebooks.mobi.utils import (encint, encode_number_as_hex,
         encode_tbs, align_block, utf8_text)
 
-
 class CNCX(object): # {{{
 
     '''
@@ -61,7 +60,63 @@ class CNCX(object): # {{{
         return self.strings[string]
 # }}}
 
-class IndexEntry(object): # {{{
+class TAGX(object): # {{{
+
+    BITMASKS = {11:0b1}
+    BITMASKS.update({x:i+1 for i, x in enumerate([1, 2, 3, 4, 5, 21, 22, 23])})
+    BITMASKS.update({x:i+1 for i, x in enumerate([69, 70, 71, 72, 73])})
+
+    NUM_VALUES = defaultdict(lambda x:1)
+    NUM_VALUES[11] = 3
+    NUM_VALUES[0] = 0
+
+    def __init__(self):
+        self.byts = bytearray()
+
+    def add_tag(self, tag):
+        buf = self.byts
+        buf.append(tag)
+        buf.append(self.NUM_VALUES[tag])
+        # bitmask
+        buf.append((1 << (self.BITMASKS[tag])) if tag else 0)
+        # eof
+        buf.append(0 if tag else 1)
+
+    def header(self, control_byte_count):
+        header = b'TAGX'
+        # table length, control byte count
+        header += pack(b'>II', 12+len(self.byts), control_byte_count)
+        return header
+
+    @property
+    def periodical(self):
+        '''
+        TAGX block for the Primary index header of a periodical
+        '''
+        map(self.add_tag, (1, 2, 3, 4, 5, 21, 22, 23, 0, 69, 70, 71, 72, 73, 0))
+        return self.header(2) + bytes(self.byts)
+
+    @property
+    def secondary(self):
+        '''
+        TAGX block for the secondary index header of a periodical
+        '''
+        map(self.add_tag, (11, 0))
+        return self.header(1) + bytes(self.byts)
+
+    @property
+    def flat_book(self):
+        '''
+        TAGX block for the primary index header of a flat book
+        '''
+        map(self.add_tag, (1, 2, 3, 4, 0))
+        return self.header(1) + bytes(self.byts)
+
+# }}}
+
+# Index Entries {{{
+
+class IndexEntry(object):
 
     TAG_VALUES = {
             'offset': 1,
@@ -69,17 +124,22 @@ class IndexEntry(object): # {{{
             'label_offset': 3,
             'depth': 4,
             'class_offset': 5,
+            'secondary': 11,
             'parent_index': 21,
             'first_child_index': 22,
             'last_child_index': 23,
+            'image_index': 69,
+            'desc_offset': 70,
+            'author_offset': 73,
     }
     RTAG_MAP = {v:k for k, v in TAG_VALUES.iteritems()}
 
-    BITMASKS = [1, 2, 3, 4, 5, 21, 22, 23,]
 
-    def __init__(self, offset, label_offset, depth=0, class_offset=None):
+    def __init__(self, offset, label_offset, depth=0, class_offset=None,
+            control_byte_count=1):
         self.offset, self.label_offset = offset, label_offset
         self.depth, self.class_offset = depth, class_offset
+        self.control_byte_count = control_byte_count
 
         self.length = 0
         self.index = 0
@@ -88,6 +148,10 @@ class IndexEntry(object): # {{{
         self.first_child_index = None
         self.last_child_index = None
 
+        self.image_index = None
+        self.author_offset = None
+        self.desc_offset = None
+
     def __repr__(self):
         return ('IndexEntry(offset=%r, depth=%r, length=%r, index=%r,'
                 ' parent_index=%r)')%(self.offset, self.depth, self.length,
@@ -99,35 +163,6 @@ class IndexEntry(object): # {{{
         def fset(self, val): self.length = val
         return property(fget=fget, fset=fset, doc='Alias for length')
 
-    @classmethod
-    def tagx_block(cls, for_periodical=True):
-        buf = bytearray()
-
-        def add_tag(tag, num_values=1):
-            buf.append(tag)
-            buf.append(num_values)
-            # bitmask
-            buf.append(1 << (cls.BITMASKS.index(tag)))
-            # eof
-            buf.append(0)
-
-        for tag in xrange(1, 5):
-            add_tag(tag)
-
-        if for_periodical:
-            for tag in (5, 21, 22, 23):
-                add_tag(tag)
-
-        # End of TAGX record
-        for i in xrange(3): buf.append(0)
-        buf.append(1)
-
-        header = b'TAGX'
-        header += pack(b'>I', 12+len(buf)) # table length
-        header += pack(b'>I', 1) # control byte count
-
-        return header + bytes(buf)
-
     @property
     def next_offset(self):
         return self.offset + self.length
@@ -145,24 +180,58 @@ class IndexEntry(object): # {{{
     def entry_type(self):
         ans = 0
         for tag in self.tag_nums:
-            ans |= (1 << self.BITMASKS.index(tag)) # 1 << x == 2**x
+            ans |= (1 << (TAGX.BITMASKS[tag])) # 1 << x == 2**x
         return ans
 
     @property
     def bytestring(self):
         buf = StringIO()
+        if isinstance(self.index, int):
             buf.write(encode_number_as_hex(self.index))
+        else:
+            raw = bytearray(self.index.encode('ascii'))
+            raw.insert(0, len(raw))
+            buf.write(bytes(raw))
         et = self.entry_type
         buf.write(bytes(bytearray([et])))
 
         for tag in self.tag_nums:
             attr = self.RTAG_MAP[tag]
             val = getattr(self, attr)
-            buf.write(encint(val))
+            if isinstance(val, int):
+                val = [val]
+            for x in val:
+                buf.write(encint(x))
 
         ans = buf.getvalue()
         return ans
 
+class SecondaryIndexEntry(IndexEntry):
+
+    INDEX_MAP = {'author':73, 'caption':72, 'credit':71, 'description':70,
+                'mastheadImage':69}
+
+    def __init__(self, index):
+        IndexEntry.__init__(self, index, 0, 0)
+
+        tag = self.INDEX_MAP[index]
+        self.secondary = [len(self.INDEX_MAP) if tag == min(
+            self.INDEX_MAP.itervalues()) else 0, 0, tag]
+
+    @property
+    def tag_nums(self):
+        yield 11
+
+    @property
+    def entry_type(self):
+        return 1
+
+    @classmethod
+    def entries(cls):
+        rmap = {v:k for k,v in cls.INDEX_MAP.iteritems()}
+        for tag in sorted(rmap, reverse=True):
+            yield cls(rmap[tag])
+
 # }}}
 
 class TBS(object): # {{{
@@ -407,7 +476,8 @@ class Indexer(object): # {{{
 
     def create_header(self): # {{{
         buf = StringIO()
-        tagx_block = IndexEntry.tagx_block(self.is_periodical)
+        tagx_block = (TAGX().periodical if self.is_periodical else
+                            TAGX().flat_book)
         header_length = 192
 
         # Ident 0 - 4

src/calibre/ebooks/mobi/writer2/main.py

@@ -35,6 +35,7 @@ EXTH_CODES = {
     'type': 111,
     'source': 112,
     'versionnumber': 114,
+    'startreading': 116,
     'coveroffset': 201,
     'thumboffset': 202,
     'hasfakecover': 203,
@@ -83,6 +84,8 @@ class MobiWriter(object):
 
     def generate_content(self):
         self.is_periodical = detect_periodical(self.oeb.toc, self.oeb.log)
+        # Image records are stored in their own list, they are merged into the
+        # main record list at the end
         self.generate_images()
         self.generate_text()
         # The uncrossable breaks trailing entries come before the indexing
@@ -545,6 +548,11 @@ class MobiWriter(object):
                 self.thumbnail_offset))
             nrecs += 1
 
+        if self.serializer.start_offset is not None:
+            exth.write(pack(b'>III', EXTH_CODES['startreading'], 12,
+                self.serializer.start_offset))
+            nrecs += 1
+
         exth = exth.getvalue()
         trail = len(exth) % 4
         pad = b'\0' * (4 - trail) # Always pad w/ at least 1 byte

src/calibre/ebooks/mobi/writer2/serializer.py

@@ -39,6 +39,10 @@ class Serializer(object):
         self.logger = oeb.logger
         self.write_page_breaks_after_item = write_page_breaks_after_item
 
+        # If not None, this is a number pointing to the location at which to
+        # open the MOBI file on the Kindle
+        self.start_offset = None
+
         # Mapping of hrefs (urlnormalized) to the offset in the buffer where
         # the resource pointed to by the href lives. Used at the end to fill in
         # the correct values into all filepos="..." links.
@@ -144,6 +148,8 @@ class Serializer(object):
                 buf.write(b'title="')
                 self.serialize_text(ref.title, quot=True)
                 buf.write(b'" ')
+                if ref.title == 'start':
+                    self._start_href = ref.href
             self.serialize_href(ref.href)
             # Space required or won't work, I kid you not
             buf.write(b' />')
@@ -283,6 +289,7 @@ class Serializer(object):
         buf = self.buf
         id_offsets = self.id_offsets
         for href, hoffs in self.href_offsets.items():
+            is_start = (href and href == getattr(self, '_start_href', None))
             # Iterate over all filepos items
             if href not in id_offsets:
                 self.logger.warn('Hyperlink target %r not found' % href)
@@ -290,6 +297,8 @@ class Serializer(object):
                 href, _ = urldefrag(href)
             if href in self.id_offsets:
                 ioff = self.id_offsets[href]
+                if is_start:
+                    self.start_offset = ioff
                 for hoff in hoffs:
                     buf.seek(hoff)
                     buf.write(b'%010d' % ioff)

src/calibre/gui2/book_details.py

@@ -23,6 +23,7 @@ from calibre.gui2 import (config, open_local_file, open_url, pixmap_to_data,
         gprefs)
 from calibre.utils.icu import sort_key
 from calibre.utils.formatter import EvalFormatter
+from calibre.utils.date import is_date_undefined
 
 def render_html(mi, css, vertical, widget, all_fields=False): # {{{
     table = render_data(mi, all_fields=all_fields,
@@ -163,6 +164,10 @@ def render_data(mi, use_roman_numbers=True, all_fields=False):
                 val = _('Book %(sidx)s of <span class="series_name">%(series)s</span>')%dict(
                         sidx=fmt_sidx(sidx, use_roman=use_roman_numbers),
                         series=prepare_string_for_xml(getattr(mi, field)))
+            elif metadata['datatype'] == 'datetime':
+                aval = getattr(mi, field)
+                if is_date_undefined(aval):
+                    val = ''
 
             ans.append((field, u'<td class="title">%s</td><td>%s</td>'%(name, val)))