From 097123968126290d14f56c2d712be7c8f394a679 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 1 Aug 2011 05:51:45 -0600 Subject: [PATCH 1/2] ... --- src/calibre/ebooks/mobi/debug.py | 4 +- src/calibre/ebooks/mobi/writer2/indexer.py | 146 +++++++++++++----- src/calibre/ebooks/mobi/writer2/main.py | 8 + src/calibre/ebooks/mobi/writer2/serializer.py | 9 ++ 4 files changed, 126 insertions(+), 41 deletions(-) diff --git a/src/calibre/ebooks/mobi/debug.py b/src/calibre/ebooks/mobi/debug.py index aff8543624..8034117f9b 100644 --- a/src/calibre/ebooks/mobi/debug.py +++ b/src/calibre/ebooks/mobi/debug.py @@ -650,8 +650,6 @@ class Tag(object): # {{{ 'article' : { 5 : ('Class offset in cncx', 'class_offset'), 21 : ('Parent section index', 'parent_index'), - 22 : ('Description offset in cncx', 'desc_offset'), - 23 : ('Author offset in cncx', 'author_offset'), 69 : ('Offset from first image record num to the' ' image record associated with this article', 'image_index'), @@ -1033,7 +1031,7 @@ class IndexRecord(object): # {{{ # }}} -class CNCX(object) : # {{{ +class CNCX(object): # {{{ ''' Parses the records that contain the compiled NCX (all strings from the diff --git a/src/calibre/ebooks/mobi/writer2/indexer.py b/src/calibre/ebooks/mobi/writer2/indexer.py index 54f74c1664..f1ec1fcec1 100644 --- a/src/calibre/ebooks/mobi/writer2/indexer.py +++ b/src/calibre/ebooks/mobi/writer2/indexer.py @@ -2,7 +2,7 @@ # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai from __future__ import (unicode_literals, division, absolute_import, print_function) -from future_builtins import filter +from future_builtins import filter, map __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' @@ -16,7 +16,6 @@ from calibre.ebooks.mobi.writer2 import RECORD_SIZE from calibre.ebooks.mobi.utils import (encint, encode_number_as_hex, encode_tbs, align_block, utf8_text) - class CNCX(object): # {{{ ''' @@ -61,7 +60,63 @@ class CNCX(object): # {{{ return self.strings[string] # }}} -class IndexEntry(object): # {{{ +class TAGX(object): # {{{ + + BITMASKS = {11:0b1} + BITMASKS.update({x:i+1 for i, x in enumerate([1, 2, 3, 4, 5, 21, 22, 23])}) + BITMASKS.update({x:i+1 for i, x in enumerate([69, 70, 71, 72, 73])}) + + NUM_VALUES = defaultdict(lambda x:1) + NUM_VALUES[11] = 3 + NUM_VALUES[0] = 0 + + def __init__(self): + self.byts = bytearray() + + def add_tag(self, tag): + buf = self.byts + buf.append(tag) + buf.append(self.NUM_VALUES[tag]) + # bitmask + buf.append((1 << (self.BITMASKS[tag])) if tag else 0) + # eof + buf.append(0 if tag else 1) + + def header(self, control_byte_count): + header = b'TAGX' + # table length, control byte count + header += pack(b'>II', 12+len(self.byts), control_byte_count) + return header + + @property + def periodical(self): + ''' + TAGX block for the Primary index header of a periodical + ''' + map(self.add_tag, (1, 2, 3, 4, 5, 21, 22, 23, 0, 69, 70, 71, 72, 73, 0)) + return self.header(2) + bytes(self.byts) + + @property + def secondary(self): + ''' + TAGX block for the secondary index header of a periodical + ''' + map(self.add_tag, (11, 0)) + return self.header(1) + bytes(self.byts) + + @property + def flat_book(self): + ''' + TAGX block for the primary index header of a flat book + ''' + map(self.add_tag, (1, 2, 3, 4, 0)) + return self.header(1) + bytes(self.byts) + +# }}} + +# Index Entries {{{ + +class IndexEntry(object): TAG_VALUES = { 'offset': 1, @@ -69,17 +124,22 @@ class IndexEntry(object): # {{{ 'label_offset': 3, 'depth': 4, 'class_offset': 5, + 'secondary': 11, 'parent_index': 21, 'first_child_index': 22, 'last_child_index': 23, + 'image_index': 69, + 'desc_offset': 70, + 'author_offset': 73, } RTAG_MAP = {v:k for k, v in TAG_VALUES.iteritems()} - BITMASKS = [1, 2, 3, 4, 5, 21, 22, 23,] - def __init__(self, offset, label_offset, depth=0, class_offset=None): + def __init__(self, offset, label_offset, depth=0, class_offset=None, + control_byte_count=1): self.offset, self.label_offset = offset, label_offset self.depth, self.class_offset = depth, class_offset + self.control_byte_count = control_byte_count self.length = 0 self.index = 0 @@ -88,6 +148,10 @@ class IndexEntry(object): # {{{ self.first_child_index = None self.last_child_index = None + self.image_index = None + self.author_offset = None + self.desc_offset = None + def __repr__(self): return ('IndexEntry(offset=%r, depth=%r, length=%r, index=%r,' ' parent_index=%r)')%(self.offset, self.depth, self.length, @@ -99,35 +163,6 @@ class IndexEntry(object): # {{{ def fset(self, val): self.length = val return property(fget=fget, fset=fset, doc='Alias for length') - @classmethod - def tagx_block(cls, for_periodical=True): - buf = bytearray() - - def add_tag(tag, num_values=1): - buf.append(tag) - buf.append(num_values) - # bitmask - buf.append(1 << (cls.BITMASKS.index(tag))) - # eof - buf.append(0) - - for tag in xrange(1, 5): - add_tag(tag) - - if for_periodical: - for tag in (5, 21, 22, 23): - add_tag(tag) - - # End of TAGX record - for i in xrange(3): buf.append(0) - buf.append(1) - - header = b'TAGX' - header += pack(b'>I', 12+len(buf)) # table length - header += pack(b'>I', 1) # control byte count - - return header + bytes(buf) - @property def next_offset(self): return self.offset + self.length @@ -145,24 +180,58 @@ class IndexEntry(object): # {{{ def entry_type(self): ans = 0 for tag in self.tag_nums: - ans |= (1 << self.BITMASKS.index(tag)) # 1 << x == 2**x + ans |= (1 << (TAGX.BITMASKS[tag])) # 1 << x == 2**x return ans @property def bytestring(self): buf = StringIO() - buf.write(encode_number_as_hex(self.index)) + if isinstance(self.index, int): + buf.write(encode_number_as_hex(self.index)) + else: + raw = bytearray(self.index.encode('ascii')) + raw.insert(0, len(raw)) + buf.write(bytes(raw)) et = self.entry_type buf.write(bytes(bytearray([et]))) for tag in self.tag_nums: attr = self.RTAG_MAP[tag] val = getattr(self, attr) - buf.write(encint(val)) + if isinstance(val, int): + val = [val] + for x in val: + buf.write(encint(x)) ans = buf.getvalue() return ans +class SecondaryIndexEntry(IndexEntry): + + INDEX_MAP = {'author':73, 'caption':72, 'credit':71, 'description':70, + 'mastheadImage':69} + + def __init__(self, index): + IndexEntry.__init__(self, index, 0, 0) + + tag = self.INDEX_MAP[index] + self.secondary = [len(self.INDEX_MAP) if tag == min( + self.INDEX_MAP.itervalues()) else 0, 0, tag] + + @property + def tag_nums(self): + yield 11 + + @property + def entry_type(self): + return 1 + + @classmethod + def entries(cls): + rmap = {v:k for k,v in cls.INDEX_MAP.iteritems()} + for tag in sorted(rmap, reverse=True): + yield cls(rmap[tag]) + # }}} class TBS(object): # {{{ @@ -407,7 +476,8 @@ class Indexer(object): # {{{ def create_header(self): # {{{ buf = StringIO() - tagx_block = IndexEntry.tagx_block(self.is_periodical) + tagx_block = (TAGX().periodical if self.is_periodical else + TAGX().flat_book) header_length = 192 # Ident 0 - 4 diff --git a/src/calibre/ebooks/mobi/writer2/main.py b/src/calibre/ebooks/mobi/writer2/main.py index ab24b197d3..b4f781e5b0 100644 --- a/src/calibre/ebooks/mobi/writer2/main.py +++ b/src/calibre/ebooks/mobi/writer2/main.py @@ -35,6 +35,7 @@ EXTH_CODES = { 'type': 111, 'source': 112, 'versionnumber': 114, + 'startreading': 116, 'coveroffset': 201, 'thumboffset': 202, 'hasfakecover': 203, @@ -83,6 +84,8 @@ class MobiWriter(object): def generate_content(self): self.is_periodical = detect_periodical(self.oeb.toc, self.oeb.log) + # Image records are stored in their own list, they are merged into the + # main record list at the end self.generate_images() self.generate_text() # The uncrossable breaks trailing entries come before the indexing @@ -545,6 +548,11 @@ class MobiWriter(object): self.thumbnail_offset)) nrecs += 1 + if self.serializer.start_offset is not None: + exth.write(pack(b'>III', EXTH_CODES['startreading'], 12, + self.serializer.start_offset)) + nrecs += 1 + exth = exth.getvalue() trail = len(exth) % 4 pad = b'\0' * (4 - trail) # Always pad w/ at least 1 byte diff --git a/src/calibre/ebooks/mobi/writer2/serializer.py b/src/calibre/ebooks/mobi/writer2/serializer.py index 0bd93def51..594f8bfd78 100644 --- a/src/calibre/ebooks/mobi/writer2/serializer.py +++ b/src/calibre/ebooks/mobi/writer2/serializer.py @@ -39,6 +39,10 @@ class Serializer(object): self.logger = oeb.logger self.write_page_breaks_after_item = write_page_breaks_after_item + # If not None, this is a number pointing to the location at which to + # open the MOBI file on the Kindle + self.start_offset = None + # Mapping of hrefs (urlnormalized) to the offset in the buffer where # the resource pointed to by the href lives. Used at the end to fill in # the correct values into all filepos="..." links. @@ -144,6 +148,8 @@ class Serializer(object): buf.write(b'title="') self.serialize_text(ref.title, quot=True) buf.write(b'" ') + if ref.title == 'start': + self._start_href = ref.href self.serialize_href(ref.href) # Space required or won't work, I kid you not buf.write(b' />') @@ -283,6 +289,7 @@ class Serializer(object): buf = self.buf id_offsets = self.id_offsets for href, hoffs in self.href_offsets.items(): + is_start = (href and href == getattr(self, '_start_href', None)) # Iterate over all filepos items if href not in id_offsets: self.logger.warn('Hyperlink target %r not found' % href) @@ -290,6 +297,8 @@ class Serializer(object): href, _ = urldefrag(href) if href in self.id_offsets: ioff = self.id_offsets[href] + if is_start: + self.start_offset = ioff for hoff in hoffs: buf.seek(hoff) buf.write(b'%010d' % ioff) From 5571c742878b71c76c2f17bc2e20bb15aa2c621e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 1 Aug 2011 06:17:23 -0600 Subject: [PATCH 2/2] Display undefined dates properly in the Book details panel. Fixes #819222 (Publishing date) --- src/calibre/gui2/book_details.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index c65b6b5d14..802535b4e2 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -23,6 +23,7 @@ from calibre.gui2 import (config, open_local_file, open_url, pixmap_to_data, gprefs) from calibre.utils.icu import sort_key from calibre.utils.formatter import EvalFormatter +from calibre.utils.date import is_date_undefined def render_html(mi, css, vertical, widget, all_fields=False): # {{{ table = render_data(mi, all_fields=all_fields, @@ -163,6 +164,10 @@ def render_data(mi, use_roman_numbers=True, all_fields=False): val = _('Book %(sidx)s of %(series)s')%dict( sidx=fmt_sidx(sidx, use_roman=use_roman_numbers), series=prepare_string_for_xml(getattr(mi, field))) + elif metadata['datatype'] == 'datetime': + aval = getattr(mi, field) + if is_date_undefined(aval): + val = '' ans.append((field, u'%s%s'%(name, val)))