From ca37c601ec6bb272b4c6c391b0b8dd866abcbba5 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Sun, 7 Dec 2008 21:30:28 -0500
Subject: [PATCH 01/22] Refactor the LitReader:   - Protect whitespace-only
 nodes with <span/> tags so they can be safely     pretty-printed.   -
 Separate the LitReader class into a low-level LitFile class and a    
 high-level LitReader class.

---
 src/calibre/ebooks/lit/reader.py | 373 +++++++++++++++++--------------
 1 file changed, 206 insertions(+), 167 deletions(-)

diff --git a/src/calibre/ebooks/lit/reader.py b/src/calibre/ebooks/lit/reader.py
index 671e48ab76..1ab11548e5 100644
--- a/src/calibre/ebooks/lit/reader.py
+++ b/src/calibre/ebooks/lit/reader.py
@@ -7,9 +7,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net> ' \
     'and Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys, struct, cStringIO, os
+import sys, struct, os
 import functools
 import re
+from cStringIO import StringIO
 from lxml import etree
 from calibre.ebooks.lit import LitError
 from calibre.ebooks.lit.maps import OPF_MAP, HTML_MAP
@@ -19,6 +20,8 @@ from calibre import plugins
 lzx, lxzerror = plugins['lzx']
 msdes, msdeserror = plugins['msdes']
 
+__all__ = ["LitReader"]
+
 XML_DECL = """<?xml version="1.0" encoding="UTF-8" ?>
 """
 OPF_DECL = """<?xml version="1.0" encoding="UTF-8" ?>
@@ -106,31 +109,54 @@ def consume_sized_utf8_string(bytes, zpad=False):
         pos += 1
     return u''.join(result), bytes[pos:]
 
+
 class UnBinary(object):
     AMPERSAND_RE = re.compile(
         r'&(?!(?:#[0-9]+|#x[0-9a-fA-F]+|[a-zA-Z_:][a-zA-Z0-9.-_:]+);)')
     OPEN_ANGLE_RE = re.compile(r'<<(?![!]--)')
-    CLOSE_ANGLE_RE = re.compile(r'(?<!--)>>')
+    CLOSE_ANGLE_RE = re.compile(r'(?<!--)>>(?=>>|[^>])')
     DOUBLE_ANGLE_RE = re.compile(r'([<>])\1')
     
     def __init__(self, bin, path, manifest={}, map=HTML_MAP):
         self.manifest = manifest
         self.tag_map, self.attr_map, self.tag_to_attr_map = map
-        self.opf = map is OPF_MAP
-        self.bin = bin
+        self.is_html = map is HTML_MAP
         self.dir = os.path.dirname(path)
-        self.buf = cStringIO.StringIO()
-        self.binary_to_text()
-        self.raw = self.buf.getvalue().lstrip().decode('utf-8')
-        self.escape_reserved()
+        buf = StringIO()
+        self.binary_to_text(bin, buf)
+        raw = buf.getvalue().lstrip().decode('utf-8')
+        raw = self.escape_reserved(raw)
+        self.tree = self.fixup_tree(raw)
 
-    def escape_reserved(self):
-        raw = self.raw
+    def fixup_node(self, node, in_head=False):
+        in_head = in_head or (node.tag == 'head')
+        if self.is_html and not in_head:
+            text = node.text
+            if text and text.isspace() and len(node) > 0:
+                node.text = None
+                span = etree.SubElement(node, 'span')
+                span.text = text
+        text = node.tail
+        if text and text.isspace():
+            node.tail = None
+            if self.is_html and not in_head:
+                span = etree.Element('span')
+                span.text = text
+                node.addnext(span)
+        for child in node.iterchildren():
+            if isinstance(child.tag, basestring):
+                self.fixup_node(child, in_head)
+        return node
+        
+    def fixup_tree(self, raw):
+        return self.fixup_node(etree.fromstring(raw))
+        
+    def escape_reserved(self, raw):
         raw = self.AMPERSAND_RE.sub(r'&amp;', raw)
         raw = self.OPEN_ANGLE_RE.sub(r'&lt;', raw)
         raw = self.CLOSE_ANGLE_RE.sub(r'&gt;', raw)
         raw = self.DOUBLE_ANGLE_RE.sub(r'\1', raw)
-        self.raw = raw
+        return raw
     
     def item_path(self, internal_id):
         try:
@@ -148,19 +174,15 @@ class UnBinary(object):
         relpath = (['..'] * (len(base) - index)) + target[index:]
         return '/'.join(relpath)
     
-    def __unicode__(self):
-        return self.raw
-    
-    def binary_to_text(self, base=0, depth=0):
+    def binary_to_text(self, bin, buf, index=0, depth=0):
         tag_name = current_map = None
         dynamic_tag = errors = 0
         in_censorship = is_goingdown = False
         state = 'text'
-        index = base
         flags = 0
         
-        while index < len(self.bin):
-            c, index = read_utf8_char(self.bin, index)
+        while index < len(bin):
+            c, index = read_utf8_char(bin, index)
             oc = ord(c)
             
             if state == 'text':
@@ -173,7 +195,7 @@ class UnBinary(object):
                     c = '>>'
                 elif c == '<':
                     c = '<<'
-                self.buf.write(c.encode('ascii', 'xmlcharrefreplace'))
+                buf.write(c.encode('ascii', 'xmlcharrefreplace'))
             
             elif state == 'get flags':
                 if oc == 0:
@@ -186,7 +208,7 @@ class UnBinary(object):
                 state = 'text' if oc == 0 else 'get attr'
                 if flags & FLAG_OPENING:
                     tag = oc
-                    self.buf.write('<')
+                    buf.write('<')
                     if not (flags & FLAG_CLOSING):
                         is_goingdown = True
                     if tag == 0x8000:
@@ -203,7 +225,7 @@ class UnBinary(object):
                         tag_name = '?'+unichr(tag)+'?'
                         current_map = self.tag_to_attr_map[tag]
                         print 'WARNING: tag %s unknown' % unichr(tag)
-                    self.buf.write(unicode(tag_name).encode('utf-8'))
+                    buf.write(unicode(tag_name).encode('utf-8'))
                 elif flags & FLAG_CLOSING:
                     if depth == 0:
                         raise LitError('Extra closing tag')
@@ -215,14 +237,14 @@ class UnBinary(object):
                     if not is_goingdown:
                         tag_name = None
                         dynamic_tag = 0
-                        self.buf.write(' />')
+                        buf.write(' />')
                     else:
-                        self.buf.write('>')
-                        index = self.binary_to_text(base=index, depth=depth+1)
+                        buf.write('>')
+                        index = self.binary_to_text(bin, buf, index, depth+1)
                         is_goingdown = False
                         if not tag_name:
                             raise LitError('Tag ends before it begins.')
-                        self.buf.write(u''.join(
+                        buf.write(u''.join(
                                 ('</', tag_name, '>')).encode('utf-8'))
                         dynamic_tag = 0
                         tag_name = None
@@ -243,7 +265,7 @@ class UnBinary(object):
                         in_censorship = True
                         state = 'get value length'
                         continue
-                    self.buf.write(' ' + unicode(attr).encode('utf-8') + '=')
+                    buf.write(' ' + unicode(attr).encode('utf-8') + '=')
                     if attr in ['href', 'src']:
                         state = 'get href length'
                     else:
@@ -251,40 +273,40 @@ class UnBinary(object):
             
             elif state == 'get value length':
                 if not in_censorship:
-                    self.buf.write('"')
+                    buf.write('"')
                 count = oc - 1
                 if count == 0:
                     if not in_censorship:
-                        self.buf.write('"')
+                        buf.write('"')
                     in_censorship = False
                     state = 'get attr'
                     continue
                 state = 'get value'
                 if oc == 0xffff:
                     continue
-                if count < 0 or count > (len(self.bin) - index):
+                if count < 0 or count > (len(bin) - index):
                     raise LitError('Invalid character count %d' % count)
             
             elif state == 'get value':
                 if count == 0xfffe:
                     if not in_censorship:
-                        self.buf.write('%s"' % (oc - 1))
+                        buf.write('%s"' % (oc - 1))
                     in_censorship = False
                     state = 'get attr'
                 elif count > 0:
                     if not in_censorship:
-                        self.buf.write(c.encode(
+                        buf.write(c.encode(
                             'ascii', 'xmlcharrefreplace'))
                     count -= 1
                 if count == 0:
                     if not in_censorship:
-                        self.buf.write('"')
+                        buf.write('"')
                     in_censorship = False
                     state = 'get attr'
             
             elif state == 'get custom length':
                 count = oc - 1
-                if count <= 0 or count > len(self.bin)-index:
+                if count <= 0 or count > len(bin)-index:
                     raise LitError('Invalid character count %d' % count)
                 dynamic_tag += 1
                 state = 'get custom'
@@ -294,26 +316,26 @@ class UnBinary(object):
                 tag_name += c
                 count -= 1
                 if count == 0:
-                    self.buf.write(unicode(tag_name).encode('utf-8'))
+                    buf.write(unicode(tag_name).encode('utf-8'))
                     state = 'get attr'
             
             elif state == 'get attr length':
                 count = oc - 1
-                if count <= 0 or count > (len(self.bin) - index):
+                if count <= 0 or count > (len(bin) - index):
                     raise LitError('Invalid character count %d' % count)
-                self.buf.write(' ')
+                buf.write(' ')
                 state = 'get custom attr'
             
             elif state == 'get custom attr':
-                self.buf.write(unicode(c).encode('utf-8'))
+                buf.write(unicode(c).encode('utf-8'))
                 count -= 1
                 if count == 0:
-                    self.buf.write('=')
+                    buf.write('=')
                     state = 'get value length'
 
             elif state == 'get href length':
                 count = oc - 1
-                if count <= 0 or count > (len(self.bin) - index):
+                if count <= 0 or count > (len(bin) - index):
                     raise LitError('Invalid character count %d' % count)
                 href = ''
                 state = 'get href'
@@ -326,11 +348,12 @@ class UnBinary(object):
                     path = self.item_path(doc)
                     if m and frag:
                         path += m + frag
-                    self.buf.write((u'"%s"' % path).encode(
+                    buf.write((u'"%s"' % path).encode(
                         'ascii', 'xmlcharrefreplace'))
                     state = 'get attr'
         return index
     
+
 class DirectoryEntry(object):
     def __init__(self, name, section, offset, size):
         self.name = name
@@ -345,6 +368,7 @@ class DirectoryEntry(object):
     def __str__(self):
         return repr(self)
 
+
 class ManifestItem(object):
     def __init__(self, original, internal, mime_type, offset, root, state):
         self.original = original
@@ -372,65 +396,87 @@ class ManifestItem(object):
             % (self.internal, self.path, self.mime_type, self.offset,
                self.root, self.state)
 
+
 def preserve(function):
     def wrapper(self, *args, **kwargs):
-        opos = self._stream.tell()
+        opos = self.stream.tell()
         try:
             return function(self, *args, **kwargs)
         finally:
-            self._stream.seek(opos)
+            self.stream.seek(opos)
     functools.update_wrapper(wrapper, function)
     return wrapper
     
-class LitReader(object):
+class LitFile(object):
     PIECE_SIZE = 16
-    XML_PARSER = etree.XMLParser(
-        remove_blank_text=True, resolve_entities=False)
+
+    def __init__(self, filename_or_stream):
+        if hasattr(filename_or_stream, 'read'):
+            self.stream = filename_or_stream
+        else:
+            self.stream = open(filename_or_stream, 'rb')
+        try:
+            self.opf_path = os.path.splitext(
+                os.path.basename(self.stream.name))[0] + '.opf'
+        except AttributeError:
+            self.opf_path = 'content.opf'
+        if self.magic != 'ITOLITLS':
+            raise LitError('Not a valid LIT file')
+        if self.version != 1:
+            raise LitError('Unknown LIT version %d' % (self.version,))
+        self.read_secondary_header()
+        self.read_header_pieces()
+        self.read_section_names()
+        self.read_manifest()
+        self.read_drm()
+
+    def warn(self, msg):
+        print "WARNING: %s" % (msg,)
 
     def magic():
         @preserve
         def fget(self):
-            self._stream.seek(0)
-            return self._stream.read(8)
+            self.stream.seek(0)
+            return self.stream.read(8)
         return property(fget=fget)
     magic = magic()
     
     def version():
         def fget(self):
-            self._stream.seek(8)
-            return u32(self._stream.read(4))
+            self.stream.seek(8)
+            return u32(self.stream.read(4))
         return property(fget=fget)
     version = version()
     
     def hdr_len():
         @preserve
         def fget(self):
-            self._stream.seek(12)
-            return int32(self._stream.read(4))
+            self.stream.seek(12)
+            return int32(self.stream.read(4))
         return property(fget=fget)
     hdr_len = hdr_len()
     
     def num_pieces():
         @preserve
         def fget(self):
-            self._stream.seek(16)
-            return int32(self._stream.read(4))
+            self.stream.seek(16)
+            return int32(self.stream.read(4))
         return property(fget=fget)
     num_pieces = num_pieces()
     
     def sec_hdr_len():
         @preserve
         def fget(self):
-            self._stream.seek(20)
-            return int32(self._stream.read(4))
+            self.stream.seek(20)
+            return int32(self.stream.read(4))
         return property(fget=fget)
     sec_hdr_len = sec_hdr_len()
     
     def guid():
         @preserve
         def fget(self):
-            self._stream.seek(24)
-            return self._stream.read(16)
+            self.stream.seek(24)
+            return self.stream.read(16)
         return property(fget=fget)
     guid = guid()
     
@@ -440,44 +486,27 @@ class LitReader(object):
             size = self.hdr_len \
                 + (self.num_pieces * self.PIECE_SIZE) \
                 + self.sec_hdr_len
-            self._stream.seek(0)
-            return self._stream.read(size)
+            self.stream.seek(0)
+            return self.stream.read(size)
         return property(fget=fget)
     header = header()
     
-    def __init__(self, filename_or_stream):
-        if hasattr(filename_or_stream, 'read'):
-            self._stream = filename_or_stream
-        else:
-            self._stream = open(filename_or_stream, 'rb')
-        if self.magic != 'ITOLITLS':
-            raise LitError('Not a valid LIT file')
-        if self.version != 1:
-            raise LitError('Unknown LIT version %d' % (self.version,))
-        self.entries = {}
-        self._read_secondary_header()
-        self._read_header_pieces()
-        self._read_section_names()
-        self._read_manifest()
-        self._read_meta()
-        self._read_drm()
-
     @preserve
     def __len__(self):
-        self._stream.seek(0, 2)
-        return self._stream.tell()
+        self.stream.seek(0, 2)
+        return self.stream.tell()
 
     @preserve
-    def _read_raw(self, offset, size):
-        self._stream.seek(offset)
-        return self._stream.read(size)
+    def read_raw(self, offset, size):
+        self.stream.seek(offset)
+        return self.stream.read(size)
 
-    def _read_content(self, offset, size):
-        return self._read_raw(self.content_offset + offset, size)
+    def read_content(self, offset, size):
+        return self.read_raw(self.content_offset + offset, size)
     
-    def _read_secondary_header(self):
+    def read_secondary_header(self):
         offset = self.hdr_len + (self.num_pieces * self.PIECE_SIZE)
-        bytes = self._read_raw(offset, self.sec_hdr_len)
+        bytes = self.read_raw(offset, self.sec_hdr_len)
         offset = int32(bytes[4:])
         while offset < len(bytes):
             blocktype = bytes[offset:offset+4]
@@ -505,21 +534,21 @@ class LitReader(object):
         if not hasattr(self, 'content_offset'):
             raise LitError('Could not figure out the content offset')
     
-    def _read_header_pieces(self):
+    def read_header_pieces(self):
         src = self.header[self.hdr_len:]
         for i in xrange(self.num_pieces):
             piece = src[i * self.PIECE_SIZE:(i + 1) * self.PIECE_SIZE]
             if u32(piece[4:]) != 0 or u32(piece[12:]) != 0:
                 raise LitError('Piece %s has 64bit value' % repr(piece))
             offset, size = u32(piece), int32(piece[8:])
-            piece = self._read_raw(offset, size)
+            piece = self.read_raw(offset, size)
             if i == 0:
                 continue # Dont need this piece
             elif i == 1:
                 if u32(piece[8:])  != self.entry_chunklen or \
                    u32(piece[12:]) != self.entry_unknown:
                     raise LitError('Secondary header does not match piece')
-                self._read_directory(piece)
+                self.read_directory(piece)
             elif i == 2:
                 if u32(piece[8:])  != self.count_chunklen or \
                    u32(piece[12:]) != self.count_unknown:
@@ -530,12 +559,13 @@ class LitReader(object):
             elif i == 4:
                 self.piece4_guid = piece
                 
-    def _read_directory(self, piece):
+    def read_directory(self, piece):
         if not piece.startswith('IFCM'):
             raise LitError('Header piece #1 is not main directory.')
         chunk_size, num_chunks = int32(piece[8:12]), int32(piece[24:28])
         if (32 + (num_chunks * chunk_size)) != len(piece):
-            raise LitError('IFCM HEADER has incorrect length')
+            raise LitError('IFCM header has incorrect length')
+        self.entries = {}
         for i in xrange(num_chunks):
             offset = 32 + (i * chunk_size)
             chunk = piece[offset:offset + chunk_size]
@@ -569,17 +599,17 @@ class LitReader(object):
                 entry = DirectoryEntry(name, section, offset, size)
                 self.entries[name] = entry
 
-    def _read_section_names(self):
+    def read_section_names(self):
         if '::DataSpace/NameList' not in self.entries:
             raise LitError('Lit file does not have a valid NameList')
         raw = self.get_file('::DataSpace/NameList')
         if len(raw) < 4:
             raise LitError('Invalid Namelist section')
         pos = 4
-        self.num_sections = u16(raw[2:pos])
-        self.section_names = [""]*self.num_sections
-        self.section_data = [None]*self.num_sections
-        for section in xrange(self.num_sections):
+        num_sections = u16(raw[2:pos])
+        self.section_names = [""] * num_sections
+        self.section_data = [None] * num_sections
+        for section in xrange(num_sections):
             size = u16(raw[pos:pos+2])
             pos += 2
             size = size*2 + 2
@@ -589,11 +619,12 @@ class LitReader(object):
                 raw[pos:pos+size].decode('utf-16-le').rstrip('\000')
             pos += size
 
-    def _read_manifest(self):
+    def read_manifest(self):
         if '/manifest' not in self.entries:
             raise LitError('Lit file does not have a valid manifest')
         raw = self.get_file('/manifest')
         self.manifest = {}
+        self.paths = {self.opf_path: None}
         while raw:
             slen, raw = ord(raw[0]), raw[1:]
             if slen == 0: break
@@ -632,28 +663,9 @@ class LitReader(object):
         for item in mlist:
             if item.path[0] == '/':
                 item.path = os.path.basename(item.path)
+            self.paths[item.path] = item
 
-    def _pretty_print(self, xml):
-        f = cStringIO.StringIO(xml.encode('utf-8'))
-        doc = etree.parse(f, parser=self.XML_PARSER)
-        pretty = etree.tostring(doc, encoding='ascii', pretty_print=True)
-        return XML_DECL + unicode(pretty)
-                
-    def _read_meta(self):
-        path = 'content.opf'
-        raw = self.get_file('/meta')
-        xml = OPF_DECL
-        try:
-            xml += unicode(UnBinary(raw, path, self.manifest, OPF_MAP))
-        except LitError:
-            if 'PENGUIN group' not in raw: raise
-            print "WARNING: attempting PENGUIN malformed OPF fix"
-            raw = raw.replace(
-                'PENGUIN group', '\x00\x01\x18\x00PENGUIN group', 1)
-            xml += unicode(UnBinary(raw, path, self.manifest, OPF_MAP))
-        self.meta = xml
-
-    def _read_drm(self):
+    def read_drm(self):
         self.drmlevel = 0
         if '/DRMStorage/Licenses/EUL' in self.entries:
             self.drmlevel = 5
@@ -664,7 +676,7 @@ class LitReader(object):
         else:
             return
         if self.drmlevel < 5:
-            msdes.deskey(self._calculate_deskey(), msdes.DE1)
+            msdes.deskey(self.calculate_deskey(), msdes.DE1)
             bookkey = msdes.des(self.get_file('/DRMStorage/DRMSealed'))
             if bookkey[0] != '\000':
                 raise LitError('Unable to decrypt title key!')
@@ -672,7 +684,7 @@ class LitReader(object):
         else:
             raise DRMError("Cannot access DRM-protected book")
 
-    def _calculate_deskey(self):
+    def calculate_deskey(self):
         hashfiles = ['/meta', '/DRMStorage/DRMSource']
         if self.drmlevel == 3:
             hashfiles.append('/DRMStorage/DRMBookplate')
@@ -696,18 +708,18 @@ class LitReader(object):
     def get_file(self, name):
         entry = self.entries[name]
         if entry.section == 0:
-            return self._read_content(entry.offset, entry.size)
+            return self.read_content(entry.offset, entry.size)
         section = self.get_section(entry.section)
         return section[entry.offset:entry.offset+entry.size]
 
     def get_section(self, section):
         data = self.section_data[section]
         if not data:
-            data = self._get_section(section)
+            data = self.get_section_uncached(section)
             self.section_data[section] = data
         return data
 
-    def _get_section(self, section):
+    def get_section_uncached(self, section):
         name = self.section_names[section]
         path = '::DataSpace/Storage/' + name
         transform = self.get_file(path + '/Transform/List')
@@ -719,29 +731,29 @@ class LitReader(object):
                 raise LitError("ControlData is too short")
             guid = msguid(transform)
             if guid == DESENCRYPT_GUID:
-                content = self._decrypt(content)
+                content = self.decrypt(content)
                 control = control[csize:]
             elif guid == LZXCOMPRESS_GUID:
                 reset_table = self.get_file(
                     '/'.join(('::DataSpace/Storage', name, 'Transform',
                               LZXCOMPRESS_GUID, 'InstanceData/ResetTable')))
-                content = self._decompress(content, control, reset_table)
+                content = self.decompress(content, control, reset_table)
                 control = control[csize:]
             else:
                 raise LitError("Unrecognized transform: %s." % repr(guid))
             transform = transform[16:]
         return content
 
-    def _decrypt(self, content):
+    def decrypt(self, content):
         length = len(content)
         extra = length & 0x7
         if extra > 0:
-            self._warn("content length not a multiple of block size")
+            self.warn("content length not a multiple of block size")
             content += "\0" * (8 - extra)
         msdes.deskey(self.bookkey, msdes.DE1)
         return msdes.des(content)
 
-    def _decompress(self, content, control, reset_table):
+    def decompress(self, content, control, reset_table):
         if len(control) < 32 or control[CONTROL_TAG:CONTROL_TAG+4] != "LZXC":
             raise LitError("Invalid ControlData tag value")
         if len(reset_table) < (RESET_INTERVAL + 8):
@@ -782,7 +794,7 @@ class LitReader(object):
                         result.append(
                             lzx.decompress(content[base:size], window_bytes))
                     except lzx.LzxError:
-                        self._warn("LZX decompression error; skipping chunk")
+                        self.warn("LZX decompression error; skipping chunk")
                     bytes_remaining -= window_bytes
                     base = size
             accum += int32(reset_table[RESET_INTERVAL:])
@@ -792,55 +804,81 @@ class LitReader(object):
             try:
                 result.append(lzx.decompress(content[base:], bytes_remaining))
             except lzx.LzxError:
-                self._warn("LZX decompression error; skipping chunk")
+                self.warn("LZX decompression error; skipping chunk")
             bytes_remaining = 0
         if bytes_remaining > 0:
             raise LitError("Failed to completely decompress section")
         return ''.join(result)
 
-    def get_entry_content(self, entry, pretty_print=False):
-        if 'spine' in entry.state:
-            name = '/'.join(('/data', entry.internal, 'content'))
-            path = entry.path
-            raw = self.get_file(name)
-            decl, map = (OPF_DECL, OPF_MAP) \
-                if name == '/meta' else (HTML_DECL, HTML_MAP)
-            content = decl + unicode(UnBinary(raw, path, self.manifest, map))
-            if pretty_print:
-                content = self._pretty_print(content)
-            content = content.encode('utf-8')
-        else:
-            name = '/'.join(('/data', entry.internal))
-            content = self.get_file(name)
-        return content
-                    
-    def extract_content(self, output_dir=os.getcwdu(), pretty_print=False):
-        output_dir = os.path.abspath(output_dir)
-        try:
-            opf_path = os.path.splitext(
-                os.path.basename(self._stream.name))[0] + '.opf'
-        except AttributeError:
-            opf_path = 'content.opf'
-        opf_path = os.path.join(output_dir, opf_path)
-        self._ensure_dir(opf_path)
-        with open(opf_path, 'wb') as f:
-            xml = self.meta
-            if pretty_print:
-                xml = self._pretty_print(xml)
-            f.write(xml.encode('utf-8'))
-        for entry in self.manifest.values():
-            path = os.path.join(output_dir, entry.path)
-            self._ensure_dir(path)
-            with open(path, 'wb') as f:
-                f.write(self.get_entry_content(entry, pretty_print))
 
+class LitReader(object):
+    def __init__(self, filename_or_stream):
+        self._litfile = LitFile(filename_or_stream)
+
+    def namelist(self):
+        return self._litfile.paths.keys()
+    
+    def _read_meta(self):
+        path = 'content.opf'
+        raw = self._litfile.get_file('/meta')
+        try:
+            unbin = UnBinary(raw, path, self._litfile.manifest, OPF_MAP)
+        except LitError:
+            if 'PENGUIN group' not in raw: raise
+            print "WARNING: attempting PENGUIN malformed OPF fix"
+            raw = raw.replace(
+                'PENGUIN group', '\x00\x01\x18\x00PENGUIN group', 1)
+            unbin = UnBinary(raw, path, self._litfile.manifest, OPF_MAP)
+        return unbin.tree
+
+    def read_xml(self, name):
+        entry = self._litfile.paths[name] if name else None
+        if entry is None:
+            content = self._read_meta()
+        elif 'spine' in entry.state:
+            internal = '/'.join(('/data', entry.internal, 'content'))
+            raw = self._litfile.get_file(internal)
+            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
+            content = unbin.tree
+        else:
+            raise LitError('Requested non-XML content as XML')
+        return content
+
+    def read(self, name, pretty_print=False):
+        entry = self._litfile.paths[name] if name else None
+        if entry is None:
+            meta = self._read_meta()
+            content = OPF_DECL + etree.tostring(
+                meta, encoding='ascii', pretty_print=pretty_print)
+        elif 'spine' in entry.state:
+            internal = '/'.join(('/data', entry.internal, 'content'))
+            raw = self._litfile.get_file(internal)
+            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
+            content = HTML_DECL + etree.tostring(
+                unbin.tree, encoding='ascii', pretty_print=pretty_print)
+        else:
+            internal = '/'.join(('/data', entry.internal))
+            content = self._litfile.get_file(internal)
+        return content
+
+    def meta():
+        def fget(self):
+            return self.read(self._litfile.opf_path)
+        return property(fget=fget)
+    meta = meta()
+    
     def _ensure_dir(self, path):
         dir = os.path.dirname(path)
         if not os.path.isdir(dir):
             os.makedirs(dir)
 
-    def _warn(self, msg):
-        print "WARNING: %s" % (msg,)
+    def extract_content(self, output_dir=os.getcwdu(), pretty_print=False):
+        for name in self.namelist():
+            path = os.path.join(output_dir, name)
+            self._ensure_dir(path)
+            with open(path, 'wb') as f:
+                f.write(self.read(name, pretty_print=pretty_print))
+
 
 def option_parser():
     from calibre.utils.config import OptionParser
@@ -850,7 +888,8 @@ def option_parser():
         help=_('Output directory. Defaults to current directory.'))
     parser.add_option(
         '-p', '--pretty-print', default=False, action='store_true',
-        help=_('Legibly format extracted markup. May modify meaningful whitespace.'))
+        help=_('Legibly format extracted markup.' \
+                   ' May modify meaningful whitespace.'))
     parser.add_option(
         '--verbose', default=False, action='store_true',
         help=_('Useful for debugging.'))

From 22a672ab4b1148b7477393c313e2242b95031dce Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Mon, 15 Dec 2008 22:38:56 -0500
Subject: [PATCH 02/22] Changes to support OEBBook-based transformations.

---
 src/calibre/ebooks/lit/oeb.py      | 87 ++++++++++++++++++++++--------
 src/calibre/ebooks/lit/stylizer.py |  7 ++-
 2 files changed, 70 insertions(+), 24 deletions(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index 6378c99219..85299b0c78 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -36,12 +36,14 @@ def OPF(name): return '{%s}%s' % (OPF2_NS, name)
 def DC(name): return '{%s}%s' % (DC11_NS, name)
 def NCX(name): return '{%s}%s' % (NCX_NS, name)
 
+EPUB_MIME = 'application/epub+zip'
 XHTML_MIME = 'application/xhtml+xml'
 CSS_MIME = 'text/css'
 NCX_MIME = 'application/x-dtbncx+xml'
 OPF_MIME = 'application/oebps-package+xml'
 OEB_DOC_MIME = 'text/x-oeb1-document'
 OEB_CSS_MIME = 'text/x-oeb1-css'
+OPENTYPE_MIME = 'font/opentype'
 
 OEB_STYLES = set([CSS_MIME, OEB_CSS_MIME, 'text/x-oeb-css'])
 OEB_DOCS = set([XHTML_MIME, 'text/html', OEB_DOC_MIME, 'text/x-oeb-document'])
@@ -65,7 +67,11 @@ def barename(name):
 def xpath(elem, expr):
     return elem.xpath(expr, namespaces=XPNSMAP)
 
-URL_UNSAFE = r"""`!@#$%^&*[](){}?+=;:'",<>\| """
+ASCII_CHARS = set(chr(x) for x in xrange(128))
+URL_SAFE = set(u'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+               u'abcdefghijklmnopqrstuvwxyz'
+               u'0123456789' u'_.-/~')
+URL_UNSAFE = ASCII_CHARS - URL_SAFE
 def urlquote(href):
     result = []
     for char in href:
@@ -212,7 +218,8 @@ class Metadata(object):
 
 class Manifest(object):
     class Item(object):
-        def __init__(self, id, href, media_type, fallback=None, loader=str):
+        def __init__(self, id, href, media_type,
+                     fallback=None, loader=str, data=None):
             self.id = id
             self.href = self.path = urlnormalize(href)
             self.media_type = media_type
@@ -220,7 +227,7 @@ class Manifest(object):
             self.spine_position = None
             self.linear = True
             self._loader = loader
-            self._data = None
+            self._data = data
 
         def __repr__(self):
             return 'Item(id=%r, href=%r, media_type=%r)' \
@@ -228,10 +235,10 @@ class Manifest(object):
 
         def data():
             def fget(self):
-                if self._data:
+                if self._data is not None:
                     return self._data
                 data = self._loader(self.href)
-                if self.media_type == XHTML_MIME:
+                if self.media_type in OEB_DOCS:
                     data = etree.fromstring(data, parser=XML_PARSER)
                     if namespace(data.tag) != XHTML_NS:
                         data.attrib['xmlns'] = XHTML_NS
@@ -256,42 +263,59 @@ class Manifest(object):
     
     def __init__(self, oeb):
         self.oeb = oeb
-        self.items = {}
+        self.ids = {}
         self.hrefs = {}
 
-    def add(self, id, href, media_type, fallback=None):
+    def add(self, id, href, media_type, fallback=None, loader=None, data=None):
+        loader = loader or self.oeb.container.read
         item = self.Item(
-            id, href, media_type, fallback, self.oeb.container.read)
-        self.items[item.id] = item
+            id, href, media_type, fallback, loader, data)
+        self.ids[item.id] = item
         self.hrefs[item.href] = item
         return item
 
-    def remove(self, id):
-        href = self.items[id].href
-        del self.items[id]
-        del self.hrefs[href]
+    def remove(self, item):
+        if item in self.ids:
+            item = self.ids[item]
+        del self.ids[item.id]
+        del self.hrefs[item.href]
+        if item in self.oeb.spine:
+            self.oeb.spine.remove(item)
+
+    def generate(self, id, href):
+        base = id
+        index = 1
+        while id in self.ids:
+            id = base + str(index)
+            index += 1
+        base, ext = os.path.splitext(href)
+        index = 1
+        while href in self.hrefs:
+            href = base + str(index) + ext
+            index += 1
+        return id, href
 
     def __iter__(self):
-        for id in self.items:
+        for id in self.ids:
             yield id
 
     def __getitem__(self, id):
-        return self.items[id]
+        return self.ids[id]
 
     def values(self):
-        for item in self.items.values():
+        for item in self.ids.values():
             yield item
 
     def items(self):
-        for id, item in self.refs.items():
-            yield id, items
+        for id, item in self.ids.items():
+            yield id, item
     
     def __contains__(self, key):
-        return id in self.items
+        return id in self.ids
 
     def to_opf1(self, parent=None):
         elem = element(parent, 'manifest')
-        for item in self.items.values():
+        for item in self.ids.values():
             media_type = item.media_type
             if media_type == XHTML_MIME:
                 media_type = OEB_DOC_MIME
@@ -306,7 +330,7 @@ class Manifest(object):
     
     def to_opf2(self, parent=None):
         elem = element(parent, OPF('manifest'))
-        for item in self.items.values():
+        for item in self.ids.values():
             attrib = {'id': item.id, 'href': item.href,
                       'media-type': item.media_type}
             if item.fallback:
@@ -320,18 +344,35 @@ class Spine(object):
         self.oeb = oeb
         self.items = []
 
-    def add(self, item, linear):
+    def _linear(self, linear):
         if isinstance(linear, StringTypes):
             linear = linear.lower()
         if linear is None or linear in ('yes', 'true'):
             linear = True
         elif linear in ('no', 'false'):
             linear = False
-        item.linear = linear
+        return linear
+        
+    def add(self, item, linear=None):
+        item.linear = self._linear(linear)
         item.spine_position = len(self.items)
         self.items.append(item)
         return item
     
+    def insert(self, index, item, linear):
+        item.linear = self._linear(linear)
+        item.spine_position = index
+        self.items.insert(index, item)
+        for i in xrange(index, len(self.items)):
+            self.items[i].spine_position = i
+        return item
+    
+    def remove(self, item):
+        index = item.spine_position
+        self.items.pop(index)
+        for i in xrange(index, len(self.items)):
+            self.items[i].spine_position = i
+    
     def __iter__(self):
         for item in self.items:
             yield item
diff --git a/src/calibre/ebooks/lit/stylizer.py b/src/calibre/ebooks/lit/stylizer.py
index 7a89474d89..dade67adbb 100644
--- a/src/calibre/ebooks/lit/stylizer.py
+++ b/src/calibre/ebooks/lit/stylizer.py
@@ -148,6 +148,7 @@ class Stylizer(object):
         rules = []
         index = 0
         self.stylesheets = set()
+        self.page_rule = {}
         for stylesheet in stylesheets:
             href = stylesheet.href
             self.stylesheets.add(href)
@@ -169,7 +170,7 @@ class Stylizer(object):
                 results.append((specificity, selector, style, text, href))
         elif isinstance(rule, CSSPageRule):
             style = self.flatten_style(rule.style)
-            results.append(((0, 0, 0, 0), [], style, '@page', href))
+            self.page_rule.update(style)
         return results
 
     def flatten_style(self, cssstyle):
@@ -441,4 +442,8 @@ class Style(object):
     
     def __str__(self):
         items = self._style.items()
+        items.sort()
         return '; '.join("%s: %s" % (key, val) for key, val in items)
+
+    def cssdict(self):
+        return dict(self._style)

From 83470f3d6bd49f794437897d9587ce41e337e57a Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Tue, 16 Dec 2008 12:21:58 -0500
Subject: [PATCH 03/22] Remove some of the crazier aspects of the LitReader
 refactoring.

---
 src/calibre/ebooks/lit/oeb.py    |   4 +-
 src/calibre/ebooks/lit/reader.py | 169 +++++++++++++++----------------
 2 files changed, 84 insertions(+), 89 deletions(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index 85299b0c78..35497dfb25 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -516,7 +516,9 @@ class TOC(object):
 
 
 class OEBBook(object):
-    def __init__(self, opfpath, container=None):
+    def __init__(self, opfpath=None, container=None):
+        if not opfpath:
+            opfpath = 'content.opf'
         if not container:
             container = DirContainer(os.path.dirname(opfpath))
             opfpath = os.path.basename(opfpath)
diff --git a/src/calibre/ebooks/lit/reader.py b/src/calibre/ebooks/lit/reader.py
index e50436ee4f..e872cad85b 100644
--- a/src/calibre/ebooks/lit/reader.py
+++ b/src/calibre/ebooks/lit/reader.py
@@ -16,7 +16,7 @@ from lxml import etree
 from calibre.ebooks.lit import LitError
 from calibre.ebooks.lit.maps import OPF_MAP, HTML_MAP
 import calibre.ebooks.lit.mssha1 as mssha1
-from calibre.ebooks.lit.oeb import urlnormalize
+from calibre.ebooks.lit.oeb import XML_PARSER, urlnormalize
 from calibre.ebooks import DRMError
 from calibre import plugins
 lzx, lxzerror = plugins['lzx']
@@ -111,6 +111,8 @@ def consume_sized_utf8_string(bytes, zpad=False):
         pos += 1
     return u''.join(result), bytes[pos:]
 
+def encode(string):
+    return unicode(string).encode('ascii', 'xmlcharrefreplace')
 
 class UnBinary(object):
     AMPERSAND_RE = re.compile(
@@ -126,39 +128,17 @@ class UnBinary(object):
         self.dir = os.path.dirname(path)
         buf = StringIO()
         self.binary_to_text(bin, buf)
-        raw = buf.getvalue().lstrip().decode('utf-8')
-        raw = self.escape_reserved(raw)
-        self.tree = self.fixup_tree(raw)
+        self.raw = buf.getvalue().lstrip().decode('utf-8')
+        self.escape_reserved()
+        self._tree = None
 
-    def fixup_node(self, node, in_head=False):
-        in_head = in_head or (node.tag == 'head')
-        if self.is_html and not in_head:
-            text = node.text
-            if text and text.isspace() and len(node) > 0:
-                node.text = None
-                span = etree.SubElement(node, 'span')
-                span.text = text
-        text = node.tail
-        if text and text.isspace():
-            node.tail = None
-            if self.is_html and not in_head:
-                span = etree.Element('span')
-                span.text = text
-                node.addnext(span)
-        for child in node.iterchildren():
-            if isinstance(child.tag, basestring):
-                self.fixup_node(child, in_head)
-        return node
-        
-    def fixup_tree(self, raw):
-        return self.fixup_node(etree.fromstring(raw))
-        
-    def escape_reserved(self, raw):
+    def escape_reserved(self):
+        raw = self.raw
         raw = self.AMPERSAND_RE.sub(r'&amp;', raw)
         raw = self.OPEN_ANGLE_RE.sub(r'&lt;', raw)
         raw = self.CLOSE_ANGLE_RE.sub(r'&gt;', raw)
         raw = self.DOUBLE_ANGLE_RE.sub(r'\1', raw)
-        return raw
+        self.raw = raw
     
     def item_path(self, internal_id):
         try:
@@ -176,6 +156,17 @@ class UnBinary(object):
         relpath = (['..'] * (len(base) - index)) + target[index:]
         return '/'.join(relpath)
     
+    def __unicode__(self):
+        return self.raw
+
+    def tree():
+        def fget(self):
+            if not self._tree:
+                self._tree = etree.fromstring(self.raw, parser=XML_PARSER)
+            return self._tree
+        return property(fget=fget)
+    tree = tree()
+    
     def binary_to_text(self, bin, buf, index=0, depth=0):
         tag_name = current_map = None
         dynamic_tag = errors = 0
@@ -197,7 +188,7 @@ class UnBinary(object):
                     c = '>>'
                 elif c == '<':
                     c = '<<'
-                buf.write(c.encode('ascii', 'xmlcharrefreplace'))
+                buf.write(encode(c))
             
             elif state == 'get flags':
                 if oc == 0:
@@ -227,7 +218,7 @@ class UnBinary(object):
                         tag_name = '?'+unichr(tag)+'?'
                         current_map = self.tag_to_attr_map[tag]
                         print 'WARNING: tag %s unknown' % unichr(tag)
-                    buf.write(unicode(tag_name).encode('utf-8'))
+                    buf.write(encode(tag_name))
                 elif flags & FLAG_CLOSING:
                     if depth == 0:
                         raise LitError('Extra closing tag')
@@ -246,8 +237,7 @@ class UnBinary(object):
                         is_goingdown = False
                         if not tag_name:
                             raise LitError('Tag ends before it begins.')
-                        buf.write(u''.join(
-                                ('</', tag_name, '>')).encode('utf-8'))
+                        buf.write(encode(u''.join(('</', tag_name, '>'))))
                         dynamic_tag = 0
                         tag_name = None
                     state = 'text'
@@ -267,7 +257,7 @@ class UnBinary(object):
                         in_censorship = True
                         state = 'get value length'
                         continue
-                    buf.write(' ' + unicode(attr).encode('utf-8') + '=')
+                    buf.write(' ' + encode(attr) + '=')
                     if attr in ['href', 'src']:
                         state = 'get href length'
                     else:
@@ -297,8 +287,7 @@ class UnBinary(object):
                     state = 'get attr'
                 elif count > 0:
                     if not in_censorship:
-                        buf.write(c.encode(
-                            'ascii', 'xmlcharrefreplace'))
+                        buf.write(encode(c))
                     count -= 1
                 if count == 0:
                     if not in_censorship:
@@ -318,7 +307,7 @@ class UnBinary(object):
                 tag_name += c
                 count -= 1
                 if count == 0:
-                    buf.write(unicode(tag_name).encode('utf-8'))
+                    buf.write(encode(tag_name))
                     state = 'get attr'
             
             elif state == 'get attr length':
@@ -329,7 +318,7 @@ class UnBinary(object):
                 state = 'get custom attr'
             
             elif state == 'get custom attr':
-                buf.write(unicode(c).encode('utf-8'))
+                buf.write(encode(c))
                 count -= 1
                 if count == 0:
                     buf.write('=')
@@ -351,7 +340,7 @@ class UnBinary(object):
                     if frag:
                         path = '#'.join((path, frag))
                     path = urlnormalize(path)
-                    self.buf.write((u'"%s"' % path).encode('utf-8'))
+                    buf.write(encode(u'"%s"' % path))
                     state = 'get attr'
         return index
     
@@ -816,10 +805,62 @@ class LitFile(object):
 class LitReader(object):
     def __init__(self, filename_or_stream):
         self._litfile = LitFile(filename_or_stream)
-
+    
     def namelist(self):
         return self._litfile.paths.keys()
     
+    def read_xml(self, name):
+        entry = self._litfile.paths[name] if name else None
+        if entry is None:
+            content = self._read_meta()
+        elif 'spine' in entry.state:
+            internal = '/'.join(('/data', entry.internal, 'content'))
+            raw = self._litfile.get_file(internal)
+            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
+            content = unbin.tree
+        else:
+            raise LitError('Requested non-XML content as XML')
+        return content
+    
+    def read(self, name, pretty_print=False):
+        entry = self._litfile.paths[name] if name else None
+        if entry is None:
+            meta = self._read_meta()
+            content = OPF_DECL + etree.tostring(
+                meta, encoding='ascii', pretty_print=pretty_print)
+        elif 'spine' in entry.state:
+            internal = '/'.join(('/data', entry.internal, 'content'))
+            raw = self._litfile.get_file(internal)
+            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
+            content = HTML_DECL
+            if pretty_print:
+                content += etree.tostring(unbin.tree,
+                    encoding='ascii', pretty_print=True)
+            else:
+                content += unicode(unbin)
+        else:
+            internal = '/'.join(('/data', entry.internal))
+            content = self._litfile.get_file(internal)
+        return content
+    
+    def meta():
+        def fget(self):
+            return self.read(self._litfile.opf_path)
+        return property(fget=fget)
+    meta = meta()
+    
+    def _ensure_dir(self, path):
+        dir = os.path.dirname(path)
+        if not os.path.isdir(dir):
+            os.makedirs(dir)
+    
+    def extract_content(self, output_dir=os.getcwdu(), pretty_print=False):
+        for name in self.namelist():
+            path = os.path.join(output_dir, name)
+            self._ensure_dir(path)
+            with open(path, 'wb') as f:
+                f.write(self.read(name, pretty_print=pretty_print))
+    
     def _read_meta(self):
         path = 'content.opf'
         raw = self._litfile.get_file('/meta')
@@ -833,54 +874,6 @@ class LitReader(object):
             unbin = UnBinary(raw, path, self._litfile.manifest, OPF_MAP)
         return unbin.tree
 
-    def read_xml(self, name):
-        entry = self._litfile.paths[name] if name else None
-        if entry is None:
-            content = self._read_meta()
-        elif 'spine' in entry.state:
-            internal = '/'.join(('/data', entry.internal, 'content'))
-            raw = self._litfile.get_file(internal)
-            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
-            content = unbin.tree
-        else:
-            raise LitError('Requested non-XML content as XML')
-        return content
-
-    def read(self, name, pretty_print=False):
-        entry = self._litfile.paths[name] if name else None
-        if entry is None:
-            meta = self._read_meta()
-            content = OPF_DECL + etree.tostring(
-                meta, encoding='ascii', pretty_print=pretty_print)
-        elif 'spine' in entry.state:
-            internal = '/'.join(('/data', entry.internal, 'content'))
-            raw = self._litfile.get_file(internal)
-            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
-            content = HTML_DECL + etree.tostring(
-                unbin.tree, encoding='ascii', pretty_print=pretty_print)
-        else:
-            internal = '/'.join(('/data', entry.internal))
-            content = self._litfile.get_file(internal)
-        return content
-
-    def meta():
-        def fget(self):
-            return self.read(self._litfile.opf_path)
-        return property(fget=fget)
-    meta = meta()
-    
-    def _ensure_dir(self, path):
-        dir = os.path.dirname(path)
-        if not os.path.isdir(dir):
-            os.makedirs(dir)
-
-    def extract_content(self, output_dir=os.getcwdu(), pretty_print=False):
-        for name in self.namelist():
-            path = os.path.join(output_dir, name)
-            self._ensure_dir(path)
-            with open(path, 'wb') as f:
-                f.write(self.read(name, pretty_print=pretty_print))
-
 
 def option_parser():
     from calibre.utils.config import OptionParser

From 0c7a0b831da57cf565654e40d535cce780bf2519 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Tue, 16 Dec 2008 12:33:27 -0500
Subject: [PATCH 04/22] Fix some str vs. unicode issues.  Re-add dropped
 TOC.iterdescendents() method.

---
 src/calibre/ebooks/lit/oeb.py    | 7 +++++++
 src/calibre/ebooks/lit/reader.py | 7 +++++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index 35497dfb25..a085478b5d 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -247,6 +247,7 @@ class Manifest(object):
                 elif self.media_type.startswith('application/') \
                      and self.media_type.endswith('+xml'):
                     data = etree.fromstring(data, parser=XML_PARSER)
+                self._data = data
                 return data
             def fset(self, value):
                 self._data = value
@@ -476,6 +477,12 @@ class TOC(object):
         node = TOC(title, href, klass, id)
         self.nodes.append(node)
         return node
+
+    def iterdescendents(self):
+        for node in self.nodes:
+            yield node
+            for child in node.iterdescendents():
+                yield child
     
     def __iter__(self):
         for node in self.nodes:
diff --git a/src/calibre/ebooks/lit/reader.py b/src/calibre/ebooks/lit/reader.py
index e872cad85b..1abb4704e3 100644
--- a/src/calibre/ebooks/lit/reader.py
+++ b/src/calibre/ebooks/lit/reader.py
@@ -128,7 +128,7 @@ class UnBinary(object):
         self.dir = os.path.dirname(path)
         buf = StringIO()
         self.binary_to_text(bin, buf)
-        self.raw = buf.getvalue().lstrip().decode('utf-8')
+        self.raw = buf.getvalue().lstrip()
         self.escape_reserved()
         self._tree = None
 
@@ -157,6 +157,9 @@ class UnBinary(object):
         return '/'.join(relpath)
     
     def __unicode__(self):
+        return self.raw.decode('utf-8')
+
+    def __str__(self):
         return self.raw
 
     def tree():
@@ -837,7 +840,7 @@ class LitReader(object):
                 content += etree.tostring(unbin.tree,
                     encoding='ascii', pretty_print=True)
             else:
-                content += unicode(unbin)
+                content += str(unbin)
         else:
             internal = '/'.join(('/data', entry.internal))
             content = self._litfile.get_file(internal)

From e1eb5450051cb854e91afba18e7594d3e8cf0e8b Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Tue, 16 Dec 2008 12:34:34 -0500
Subject: [PATCH 05/22] Correct iterdescendants() method name.

---
 src/calibre/ebooks/lit/oeb.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index a085478b5d..8ef7f41add 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -478,10 +478,10 @@ class TOC(object):
         self.nodes.append(node)
         return node
 
-    def iterdescendents(self):
+    def iterdescendants(self):
         for node in self.nodes:
             yield node
-            for child in node.iterdescendents():
+            for child in node.iterdescendants():
                 yield child
     
     def __iter__(self):

From a6717b1ec36799849f31e80223242f46c48d9168 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Tue, 16 Dec 2008 12:48:09 -0500
Subject: [PATCH 06/22] Try protocol where Containers see "None" as metadata
 request.

---
 src/calibre/ebooks/lit/oeb.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index 8ef7f41add..02a4996dd0 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -524,8 +524,6 @@ class TOC(object):
 
 class OEBBook(object):
     def __init__(self, opfpath=None, container=None):
-        if not opfpath:
-            opfpath = 'content.opf'
         if not container:
             container = DirContainer(os.path.dirname(opfpath))
             opfpath = os.path.basename(opfpath)

From 3eaede190cd30639000370e5c78edd4ffc3939f1 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Tue, 16 Dec 2008 14:25:11 -0500
Subject: [PATCH 07/22] Switch to LIT-optimized font size list.

---
 src/calibre/ebooks/lit/stylizer.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/calibre/ebooks/lit/stylizer.py b/src/calibre/ebooks/lit/stylizer.py
index dade67adbb..f958e7c98a 100644
--- a/src/calibre/ebooks/lit/stylizer.py
+++ b/src/calibre/ebooks/lit/stylizer.py
@@ -82,14 +82,14 @@ DEFAULTS = {'azimuth': 'center', 'background-attachment': 'scroll',
 FONT_SIZE_NAMES = set(['xx-small', 'x-small', 'small', 'medium', 'large',
                        'x-large', 'xx-large'])
 
-FONT_SIZE_LIST = [('xx-small', 1,     6.),
-                  ('x-small',  None,  7.),
-                  ('small',    2,     8.),
-                  ('medium',   3,     9.),
-                  ('large',    4,    11.),
-                  ('x-large',  5,    13.),
-                  ('xx-large', 6,    15.),
-                  (None,       7,    17.)]
+FONT_SIZE_LIST = [('xx-small', 1,    10.),
+                  ('x-small',  None, 11.),
+                  ('small',    2,    13.),
+                  ('medium',   3,    16.),
+                  ('large',    4,    18.),
+                  ('x-large',  5,    20.),
+                  ('xx-large', 6,    22.),
+                  (None,       7,    24.)]
 
 FONT_SIZE_BY_NAME = {}
 FONT_SIZE_BY_NUM = {}

From d46abf8c5011b17713baa35d77f31cca3ba106c8 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Tue, 16 Dec 2008 17:42:11 -0500
Subject: [PATCH 08/22] Convert page dimensions to points.  Add auto-layering
 method to TOC.

---
 src/calibre/ebooks/lit/oeb.py      | 9 +++++++++
 src/calibre/ebooks/lit/stylizer.py | 5 +++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index 02a4996dd0..d9fc73f249 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -490,6 +490,15 @@ class TOC(object):
     
     def __getitem__(self, index):
         return self.nodes[index]
+
+    def autolayer(self):
+        prev = None
+        for node in list(self.nodes):
+            if prev and urldefrag(prev.href)[0] == urldefrag(node.href)[0]:
+                self.nodes.remove(node)
+                prev.nodes.append(node)
+            else:
+                prev = node
     
     def depth(self, level=0):
         if self.nodes:
diff --git a/src/calibre/ebooks/lit/stylizer.py b/src/calibre/ebooks/lit/stylizer.py
index f958e7c98a..d50d1957be 100644
--- a/src/calibre/ebooks/lit/stylizer.py
+++ b/src/calibre/ebooks/lit/stylizer.py
@@ -104,8 +104,8 @@ def xpath(elem, expr):
 
 class Page(object):
     def __init__(self, width, height, dpi):
-        self.width = float(width)
-        self.height = float(height)
+        self.width = (float(width) / dpi) * 72.
+        self.height = (float(height) / dpi) * 72.
         self.dpi = float(dpi)
 
 class Profiles(object):
@@ -251,6 +251,7 @@ class Stylizer(object):
             rules.append('%s {\n    %s;\n}' % (selector, style))
         return '\n'.join(rules)
 
+
 class Style(object):
     def __init__(self, element, stylizer):
         self._element = element

From a52c0b5c217914067b1786286652b4c149d34888 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Tue, 16 Dec 2008 19:11:19 -0500
Subject: [PATCH 09/22] Fix bug in Manifest __contains__() method.

---
 src/calibre/ebooks/lit/oeb.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index d9fc73f249..fb1aa14cd6 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -312,7 +312,7 @@ class Manifest(object):
             yield id, item
     
     def __contains__(self, key):
-        return id in self.ids
+        return key in self.ids
 
     def to_opf1(self, parent=None):
         elem = element(parent, 'manifest')

From 156d923b8c3675ab3997f7a7df1c043956a38d17 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Tue, 16 Dec 2008 23:55:10 -0500
Subject: [PATCH 10/22] Accurately namespace in-head stylesheets.  Be stricter
 with element matching rules.

---
 src/calibre/ebooks/lit/stylizer.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/calibre/ebooks/lit/stylizer.py b/src/calibre/ebooks/lit/stylizer.py
index d50d1957be..486e505f1d 100644
--- a/src/calibre/ebooks/lit/stylizer.py
+++ b/src/calibre/ebooks/lit/stylizer.py
@@ -20,7 +20,7 @@ import cssutils
 from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \
     CSSValueList, cssproperties
 from lxml import etree
-from calibre.ebooks.lit.oeb import XHTML_NS, CSS_MIME, OEB_STYLES
+from calibre.ebooks.lit.oeb import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
 from calibre.ebooks.lit.oeb import barename, urlnormalize
 from calibre.resources import html_css
 
@@ -126,12 +126,12 @@ class Stylizer(object):
         parser = cssutils.CSSParser()
         parser.setFetcher(lambda path: ('utf-8', oeb.container.read(path)))
         for elem in head:
-            tag = barename(elem.tag)
-            if tag == 'style':
-                text = ''.join(elem.text)
+            if elem.tag == XHTML('style') and elem.text \
+               and elem.get('type', CSS_MIME) in OEB_STYLES:
+                text = XHTML_CSS_NAMESPACE + elem.text
                 stylesheet = parser.parseString(text, href=cssname)
                 stylesheets.append(stylesheet)
-            elif tag == 'link' \
+            elif elem.tag == XHTML('link') and elem.get('href') \
                  and elem.get('rel', 'stylesheet') == 'stylesheet' \
                  and elem.get('type', CSS_MIME) in OEB_STYLES:
                 href = urlnormalize(elem.attrib['href'])

From 46c52b3bb56aae896d899445edfaa4e1e061833f Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Thu, 18 Dec 2008 17:19:23 -0500
Subject: [PATCH 11/22] Fix bug in parsing OPF <guide/> elements. Add relhref
 method for building hrefs relative to a manifest item.

---
 src/calibre/ebooks/lit/oeb.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index 5fb34d4665..e52b5b1ecb 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -119,7 +119,7 @@ class DirContainer(AbstractContainer):
 
     def exists(self, path):
         path = os.path.join(self.rootdir, path)
-        return os.path.isfile(path)
+        return os.path.isfile(urlunquote(path))
 
 
 class Metadata(object):
@@ -274,6 +274,22 @@ class Manifest(object):
             if result != 0:
                 return result
             return cmp(self.id, other.id)
+        
+        def relhref(self, href):
+            if '/' not in self.href:
+                return href
+            base = os.path.dirname(self.href).split('/')
+            target, frag = urldefrag(href)
+            target = target.split('/')
+            for index in xrange(min(len(base), len(target))):
+                if base[index] != target[index]: break
+            else:
+                index += 1
+            relhref = (['..'] * (len(base) - index)) + target[index:]
+            relhref = '/'.join(relhref)
+            if frag:
+                relhref = '#'.join((relhref, frag))
+            return relhref
     
     def __init__(self, oeb):
         self.oeb = oeb
@@ -297,6 +313,7 @@ class Manifest(object):
             self.oeb.spine.remove(item)
 
     def generate(self, id, href):
+        href = urlnormalize(href)
         base = id
         index = 1
         while id in self.ids:
@@ -640,7 +657,8 @@ class OEBBook(object):
         self.guide = guide = Guide(self)
         for elem in xpath(opf, '/o2:package/o2:guide/o2:reference'):
             href = elem.get('href')
-            if href not in self.manifest.hrefs:
+            path, frag = urldefrag(href)
+            if path not in self.manifest.hrefs:
                 self.logger.log_warn(u'Guide reference %r not found' % href)
                 continue
             guide.add(elem.get('type'), elem.get('title'), href)

From 1f6eda3ff18926f44247090b83807b7817ed9f84 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Thu, 18 Dec 2008 17:20:15 -0500
Subject: [PATCH 12/22] Add exists() method to the LitReader (qua OEB-archive).

---
 src/calibre/ebooks/lit/reader.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/lit/reader.py b/src/calibre/ebooks/lit/reader.py
index 1abb4704e3..8b8d37cd4d 100644
--- a/src/calibre/ebooks/lit/reader.py
+++ b/src/calibre/ebooks/lit/reader.py
@@ -12,6 +12,7 @@ import functools
 import re
 from urlparse import urldefrag
 from cStringIO import StringIO
+from urllib import unquote as urlunquote
 from lxml import etree
 from calibre.ebooks.lit import LitError
 from calibre.ebooks.lit.maps import OPF_MAP, HTML_MAP
@@ -811,9 +812,12 @@ class LitReader(object):
     
     def namelist(self):
         return self._litfile.paths.keys()
+
+    def exists(self, name):
+        return urlunquote(name) in self._litfile.paths
     
     def read_xml(self, name):
-        entry = self._litfile.paths[name] if name else None
+        entry = self._litfile.paths[urlunquote(name)] if name else None
         if entry is None:
             content = self._read_meta()
         elif 'spine' in entry.state:
@@ -826,7 +830,7 @@ class LitReader(object):
         return content
     
     def read(self, name, pretty_print=False):
-        entry = self._litfile.paths[name] if name else None
+        entry = self._litfile.paths[urlunquote(name)] if name else None
         if entry is None:
             meta = self._read_meta()
             content = OPF_DECL + etree.tostring(

From 4ab6b412c4d11273469742c5f78f748c2efc5344 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Thu, 18 Dec 2008 22:29:43 -0500
Subject: [PATCH 13/22] Find more NCX ToCs. Have profile-specific logical CSS
 font-sizes.

---
 src/calibre/ebooks/lit/oeb.py      | 12 +++++---
 src/calibre/ebooks/lit/stylizer.py | 49 ++++++++++++++++--------------
 2 files changed, 34 insertions(+), 27 deletions(-)

diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/lit/oeb.py
index e52b5b1ecb..6e9e11a483 100644
--- a/src/calibre/ebooks/lit/oeb.py
+++ b/src/calibre/ebooks/lit/oeb.py
@@ -563,13 +563,14 @@ class TOC(object):
     
 class OEBBook(object):
     def __init__(self, opfpath=None, container=None, logger=FauxLogger()):
-        if not container:
+        if opfpath and not container:
             container = DirContainer(os.path.dirname(opfpath))
             opfpath = os.path.basename(opfpath)
         self.container = container
         self.logger = logger
-        opf = self._read_opf(opfpath)
-        self._all_from_opf(opf)
+        if opfpath or container:
+            opf = self._read_opf(opfpath)
+            self._all_from_opf(opf)
     
     def _convert_opf1(self, opf):
         nroot = etree.Element(OPF('package'),
@@ -676,7 +677,10 @@ class OEBBook(object):
     def _toc_from_ncx(self, opf):
         result = xpath(opf, '/o2:package/o2:spine/@toc')
         if not result:
-            return False
+            expr = '/o2:package/o2:manifest[@media-type="%s"]/@id'
+            result = xpath(opf, expr % NCX_MIME)
+            if len(result) != 1:
+                return False
         id = result[0]
         ncx = self.manifest[id].data
         self.manifest.remove(id)
diff --git a/src/calibre/ebooks/lit/stylizer.py b/src/calibre/ebooks/lit/stylizer.py
index 486e505f1d..dbd10a320e 100644
--- a/src/calibre/ebooks/lit/stylizer.py
+++ b/src/calibre/ebooks/lit/stylizer.py
@@ -16,6 +16,7 @@ import itertools
 import types
 import re
 import copy
+from itertools import izip
 import cssutils
 from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \
     CSSValueList, cssproperties
@@ -25,7 +26,7 @@ from calibre.ebooks.lit.oeb import barename, urlnormalize
 from calibre.resources import html_css
 
 HTML_CSS_STYLESHEET = cssutils.parseString(html_css)
-XHTML_CSS_NAMESPACE = "@namespace url(http://www.w3.org/1999/xhtml);\n"
+XHTML_CSS_NAMESPACE = "@namespace url(%s);\n" % XHTML_NS
 
 INHERITED = set(['azimuth', 'border-collapse', 'border-spacing',
                  'caption-side', 'color', 'cursor', 'direction', 'elevation',
@@ -82,20 +83,15 @@ DEFAULTS = {'azimuth': 'center', 'background-attachment': 'scroll',
 FONT_SIZE_NAMES = set(['xx-small', 'x-small', 'small', 'medium', 'large',
                        'x-large', 'xx-large'])
 
-FONT_SIZE_LIST = [('xx-small', 1,    10.),
-                  ('x-small',  None, 11.),
-                  ('small',    2,    13.),
-                  ('medium',   3,    16.),
-                  ('large',    4,    18.),
-                  ('x-large',  5,    20.),
-                  ('xx-large', 6,    22.),
-                  (None,       7,    24.)]
+FONT_SIZES = [('xx-small', 1),
+              ('x-small',  None),
+              ('small',    2),
+              ('medium',   3),
+              ('large',    4),
+              ('x-large',  5),
+              ('xx-large', 6),
+              (None,       7)]
 
-FONT_SIZE_BY_NAME = {}
-FONT_SIZE_BY_NUM = {}
-for name, num, size in FONT_SIZE_LIST:
-    FONT_SIZE_BY_NAME[name] = size
-    FONT_SIZE_BY_NUM[num] = size
 
 XPNSMAP = {'h': XHTML_NS,}
 def xpath(elem, expr):
@@ -103,14 +99,20 @@ def xpath(elem, expr):
 
 
 class Page(object):
-    def __init__(self, width, height, dpi):
+    def __init__(self, width, height, dpi, fbase, fsizes):
         self.width = (float(width) / dpi) * 72.
         self.height = (float(height) / dpi) * 72.
         self.dpi = float(dpi)
+        self.fbase = float(fbase)
+        self.fsizes = []
+        for (name, num), size in izip(FONT_SIZES, fsizes):
+            self.fsizes.append((name, num, float(size)))
+        self.fnames = dict((name, sz) for name, _, sz in self.fsizes if name)
+        self.fnums = dict((num, sz) for _, num, sz in self.fsizes if num)
 
 class Profiles(object):
-    PRS500 = Page(584, 754, 168.451)
-    PRS505 = PRS500
+    PRS505 = Page(584, 754, 168.451, 12, [7.5, 9, 10, 12, 15.5, 20, 22, 24])
+    MSLIT = Page(652, 480, 168.451, 13, [10, 11, 13, 16, 18, 20, 22, 24])
 
     
 class Stylizer(object):    
@@ -187,7 +189,7 @@ class Stylizer(object):
             size = style['font-size']
             if size == 'normal': size = 'medium'
             if size in FONT_SIZE_NAMES:
-                style['font-size'] = "%dpt" % FONT_SIZE_BY_NAME[size]
+                style['font-size'] = "%dpt" % self.page.fnames[size]
         return style
     
     def _normalize_edge(self, cssvalue, name):
@@ -385,18 +387,19 @@ class Style(object):
             result = None
             factor = None
             if value == 'inherit':
-                value = 'medium'
+                # We should only see this if the root element
+                value = self._stylizer.page.fbase
             if value in FONT_SIZE_NAMES:
-                result = FONT_SIZE_BY_NAME[value]
+                result = self._stylizer.page.fnames[value]
             elif value == 'smaller':
                 factor = 1.0/1.2
-                for _, _, size in FONT_SIZE_LIST:
+                for _, _, size in self._stylizer.page.fsizes:
                     if base <= size: break
                     factor = None
                     result = size
             elif value == 'larger':
                 factor = 1.2
-                for _, _, size in reversed(FONT_SIZE_LIST):
+                for _, _, size in reversed(self._stylizer.page.fsizes):
                     if base >= size: break
                     factor = None
                     result = size
@@ -412,7 +415,7 @@ class Style(object):
             styles = self._stylizer._styles
             base = styles[self._element.getparent()].fontSize
         else:
-            base = normalize_fontsize(DEFAULTS['font-size'])
+            base = self._styles.page.fbase
         if 'font-size' in self._style:
             size = self._style['font-size']
             result = normalize_fontsize(size, base)

From 921f4296f0f80b7b61f5723a82879b0c761a2ff1 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Thu, 18 Dec 2008 22:29:50 -0500
Subject: [PATCH 14/22] Fix Windows LZX compression endianness bug

---
 src/calibre/utils/lzx/lzxc.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/calibre/utils/lzx/lzxc.c b/src/calibre/utils/lzx/lzxc.c
index 1617e544d2..626c829480 100644
--- a/src/calibre/utils/lzx/lzxc.c
+++ b/src/calibre/utils/lzx/lzxc.c
@@ -32,9 +32,11 @@
 #include <assert.h>
 #include <math.h>
 
-#if BYTE_ORDER == BIG_ENDIAN
-#define LZX_BIG_ENDIAN
-#endif
+#ifdef BYTE_ORDER
+# if BYTE_ORDER == BIG_ENDIAN
+#  define LZX_BIG_ENDIAN
+# endif /* BYTE_ORDER == BIG_ENDIAN */
+#endif /* BYTE_ORDER */
 
 #ifdef NONSLIDE
 #include "lzc.h"

From 4f4a0e0c524afcf0e82ff417620ed4dac9d83aab Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Fri, 19 Dec 2008 00:46:37 -0500
Subject: [PATCH 15/22] The great file re-shuffling.

---
 setup.py                                       | 2 +-
 src/calibre/ebooks/lit/reader.py               | 2 +-
 src/calibre/ebooks/lit/writer.py               | 8 ++++----
 src/calibre/ebooks/{lit/oeb.py => oeb/base.py} | 0
 src/calibre/ebooks/{lit => oeb}/html.css       | 0
 src/calibre/ebooks/{lit => oeb}/stylizer.py    | 4 ++--
 6 files changed, 8 insertions(+), 8 deletions(-)
 rename src/calibre/ebooks/{lit/oeb.py => oeb/base.py} (100%)
 rename src/calibre/ebooks/{lit => oeb}/html.css (100%)
 rename src/calibre/ebooks/{lit => oeb}/stylizer.py (99%)

diff --git a/setup.py b/setup.py
index 457c51342d..5438dffd7a 100644
--- a/setup.py
+++ b/setup.py
@@ -146,7 +146,7 @@ if __name__ == '__main__':
             metadata_sqlite = 'library/metadata_sqlite.sql',
             jquery          = 'gui2/viewer/jquery.js',
             jquery_scrollTo = 'gui2/viewer/jquery_scrollTo.js',
-            html_css        = 'ebooks/lit/html.css',
+            html_css        = 'ebooks/oeb/html.css',
         )
         
         DEST = os.path.join('src', APPNAME, 'resources.py')
diff --git a/src/calibre/ebooks/lit/reader.py b/src/calibre/ebooks/lit/reader.py
index 8b8d37cd4d..0e7f9a1ccf 100644
--- a/src/calibre/ebooks/lit/reader.py
+++ b/src/calibre/ebooks/lit/reader.py
@@ -17,7 +17,7 @@ from lxml import etree
 from calibre.ebooks.lit import LitError
 from calibre.ebooks.lit.maps import OPF_MAP, HTML_MAP
 import calibre.ebooks.lit.mssha1 as mssha1
-from calibre.ebooks.lit.oeb import XML_PARSER, urlnormalize
+from calibre.ebooks.oeb.base import XML_PARSER, urlnormalize
 from calibre.ebooks import DRMError
 from calibre import plugins
 lzx, lxzerror = plugins['lzx']
diff --git a/src/calibre/ebooks/lit/writer.py b/src/calibre/ebooks/lit/writer.py
index f824e6aac3..6e2d079e1c 100644
--- a/src/calibre/ebooks/lit/writer.py
+++ b/src/calibre/ebooks/lit/writer.py
@@ -23,11 +23,11 @@ from urllib import unquote as urlunquote
 from lxml import etree
 from calibre.ebooks.lit.reader import DirectoryEntry
 import calibre.ebooks.lit.maps as maps
-from calibre.ebooks.lit.oeb import OEB_DOCS, OEB_STYLES, OEB_CSS_MIME, \
+from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, OEB_CSS_MIME, \
     CSS_MIME, OPF_MIME, XML_NS, XML
-from calibre.ebooks.lit.oeb import namespace, barename, urlnormalize, xpath
-from calibre.ebooks.lit.oeb import FauxLogger, OEBBook
-from calibre.ebooks.lit.stylizer import Stylizer
+from calibre.ebooks.oeb.base import namespace, barename, urlnormalize, xpath
+from calibre.ebooks.oeb.base import FauxLogger, OEBBook
+from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.lit.lzx import Compressor
 import calibre
 from calibre import LoggingInterface
diff --git a/src/calibre/ebooks/lit/oeb.py b/src/calibre/ebooks/oeb/base.py
similarity index 100%
rename from src/calibre/ebooks/lit/oeb.py
rename to src/calibre/ebooks/oeb/base.py
diff --git a/src/calibre/ebooks/lit/html.css b/src/calibre/ebooks/oeb/html.css
similarity index 100%
rename from src/calibre/ebooks/lit/html.css
rename to src/calibre/ebooks/oeb/html.css
diff --git a/src/calibre/ebooks/lit/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py
similarity index 99%
rename from src/calibre/ebooks/lit/stylizer.py
rename to src/calibre/ebooks/oeb/stylizer.py
index dbd10a320e..4bd828e47b 100644
--- a/src/calibre/ebooks/lit/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@@ -21,8 +21,8 @@ import cssutils
 from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \
     CSSValueList, cssproperties
 from lxml import etree
-from calibre.ebooks.lit.oeb import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
-from calibre.ebooks.lit.oeb import barename, urlnormalize
+from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
+from calibre.ebooks.oeb.base import barename, urlnormalize
 from calibre.resources import html_css
 
 HTML_CSS_STYLESHEET = cssutils.parseString(html_css)

From 124eb4cba94f375db965949e071dc4afa6dc4fca Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Fri, 19 Dec 2008 00:54:54 -0500
Subject: [PATCH 16/22] Fine-tune the MSLIT profile. Fix a bug in the moving of
 font-sizes to a "page" property.

---
 src/calibre/ebooks/oeb/stylizer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py
index 4bd828e47b..5891509fdb 100644
--- a/src/calibre/ebooks/oeb/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@@ -112,7 +112,7 @@ class Page(object):
 
 class Profiles(object):
     PRS505 = Page(584, 754, 168.451, 12, [7.5, 9, 10, 12, 15.5, 20, 22, 24])
-    MSLIT = Page(652, 480, 168.451, 13, [10, 11, 13, 16, 18, 20, 22, 24])
+    MSLIT = Page(652, 480, 168.451, 13, [10, 11, 13, 16, 18, 20, 22, 26])
 
     
 class Stylizer(object):    
@@ -415,7 +415,7 @@ class Style(object):
             styles = self._stylizer._styles
             base = styles[self._element.getparent()].fontSize
         else:
-            base = self._styles.page.fbase
+            base = self._stylizer.page.fbase
         if 'font-size' in self._style:
             size = self._style['font-size']
             result = normalize_fontsize(size, base)

From f6c1a684c31ee91f78af444d8d257ba24f7fd345 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Fri, 19 Dec 2008 08:57:04 -0500
Subject: [PATCH 17/22] Let "oeb" be an actual module.

---
 src/calibre/ebooks/oeb/__init__.py | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 src/calibre/ebooks/oeb/__init__.py

diff --git a/src/calibre/ebooks/oeb/__init__.py b/src/calibre/ebooks/oeb/__init__.py
new file mode 100644
index 0000000000..4f8588535f
--- /dev/null
+++ b/src/calibre/ebooks/oeb/__init__.py
@@ -0,0 +1,2 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'

From 30c232591336df10d923038b0e27a21476ece26a Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Fri, 19 Dec 2008 08:57:52 -0500
Subject: [PATCH 18/22] Switch to using lxml.cssselect instead of my
 home-rolled selector.

---
 src/calibre/ebooks/oeb/html.css    |  16 -----
 src/calibre/ebooks/oeb/stylizer.py | 105 +++++++++++------------------
 2 files changed, 40 insertions(+), 81 deletions(-)

diff --git a/src/calibre/ebooks/oeb/html.css b/src/calibre/ebooks/oeb/html.css
index 9401b19cf2..38e4f314a4 100644
--- a/src/calibre/ebooks/oeb/html.css
+++ b/src/calibre/ebooks/oeb/html.css
@@ -392,22 +392,6 @@ spacer {
   float: none ! important;
 }
 
-/* focusable content: anything w/ tabindex >=0 is focusable */
-abbr:focus, acronym:focus, address:focus, applet:focus, b:focus, 
-base:focus, big:focus, blockquote:focus, br:focus, canvas:focus, caption:focus,
-center:focus, cite:focus, code:focus, col:focus, colgroup:focus, dd:focus,
-del:focus, dfn:focus, dir:focus, div:focus, dl:focus, dt:focus, em:focus,
-fieldset:focus, font:focus, form:focus, h1:focus, h2:focus, h3:focus, h4:focus,
-h5:focus, h6:focus, hr:focus, i:focus, img:focus, ins:focus, 
-kbd:focus, label:focus, legend:focus, li:focus, link:focus, menu:focus, 
-object:focus, ol:focus, p:focus, pre:focus, q:focus, s:focus, samp:focus, 
-small:focus, span:focus, strike:focus, strong:focus, sub:focus, sup:focus, 
-table:focus, tbody:focus, td:focus, tfoot:focus, th:focus, thead:focus, 
-tr:focus, tt:focus, u:focus, ul:focus, var:focus {
-  /* Don't specify the outline-color, we should always use initial value. */
-   outline: 1px dotted;
-}
-
 /* hidden elements */
 area, base, basefont, head, meta, script, style, title,
 noembed, param, link {
diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py
index 5891509fdb..9a4776c6b9 100644
--- a/src/calibre/ebooks/oeb/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@@ -21,12 +21,14 @@ import cssutils
 from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \
     CSSValueList, cssproperties
 from lxml import etree
+from lxml.cssselect import css_to_xpath, ExpressionError
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
 from calibre.ebooks.oeb.base import barename, urlnormalize
 from calibre.resources import html_css
 
-HTML_CSS_STYLESHEET = cssutils.parseString(html_css)
 XHTML_CSS_NAMESPACE = "@namespace url(%s);\n" % XHTML_NS
+HTML_CSS_STYLESHEET = cssutils.parseString(html_css)
+HTML_CSS_STYLESHEET.namespaces['h'] = XHTML_NS
 
 INHERITED = set(['azimuth', 'border-collapse', 'border-spacing',
                  'caption-side', 'color', 'cursor', 'direction', 'elevation',
@@ -97,6 +99,18 @@ XPNSMAP = {'h': XHTML_NS,}
 def xpath(elem, expr):
     return elem.xpath(expr, namespaces=XPNSMAP)
 
+class CSSSelector(etree.XPath):
+    def __init__(self, css, namespaces=XPNSMAP):
+        path = css_to_xpath(css)
+        etree.XPath.__init__(self, path, namespaces=namespaces)
+        self.css = css
+
+    def __repr__(self):
+        return '<%s %s for %r>' % (
+            self.__class__.__name__,
+            hex(abs(id(self)))[2:],
+            self.css)
+
 
 class Page(object):
     def __init__(self, width, height, dpi, fbase, fsizes):
@@ -132,6 +146,7 @@ class Stylizer(object):
                and elem.get('type', CSS_MIME) in OEB_STYLES:
                 text = XHTML_CSS_NAMESPACE + elem.text
                 stylesheet = parser.parseString(text, href=cssname)
+                stylesheet.namespaces['h'] = XHTML_NS
                 stylesheets.append(stylesheet)
             elif elem.tag == XHTML('link') and elem.get('href') \
                  and elem.get('rel', 'stylesheet') == 'stylesheet' \
@@ -145,6 +160,7 @@ class Stylizer(object):
                     data = XHTML_CSS_NAMESPACE
                     data += oeb.manifest.hrefs[path].data
                     stylesheet = parser.parseString(data, href=path)
+                    stylesheet.namespaces['h'] = XHTML_NS
                     self.STYLESHEETS[path] = stylesheet
                 stylesheets.append(stylesheet)
         rules = []
@@ -160,6 +176,16 @@ class Stylizer(object):
         rules.sort()
         self.rules = rules
         self._styles = {}
+        for _, _, cssdict, text, _ in rules:
+            try:
+                selector = CSSSelector(text)
+            except ExpressionError:
+                continue
+            for elem in selector(tree):
+                self.style(elem)._update_cssdict(cssdict)
+        for elem in tree.xpath('//*[@style]'):
+            self.style(elem)._apply_style_tag()
+        
 
     def flatten_rule(self, rule, href, index):
         results = []
@@ -236,9 +262,10 @@ class Stylizer(object):
         return style
 
     def style(self, element):
-        try: return self._styles[element]
-        except: pass
-        return Style(element, self)
+        try:
+            return self._styles[element]
+        except KeyError:
+            return Style(element, self)
 
     def stylesheet(self, name, font_scale=None):
         rules = []
@@ -259,69 +286,17 @@ class Style(object):
         self._element = element
         self._page = stylizer.page
         self._stylizer = stylizer
-        self._style = self._assemble_style(element, stylizer)
+        self._style = {}
         stylizer._styles[element] = self
+
+    def _update_cssdict(self, cssdict):
+        self._style.update(cssdict)
         
-    def _assemble_style(self, element, stylizer):
-        result = {}
-        rules = stylizer.rules
-        for _, selector, style, _, _ in rules:
-            if self._selects_element(element, selector):
-                result.update(style)
-        try:
-            style = CSSStyleDeclaration(element.attrib['style'])
-            result.update(stylizer.flatten_style(style))
-        except KeyError:
-            pass
-        return result
-        
-    def _selects_element(self, element, selector):
-        def _selects_element(element, items, index):
-            if index == -1:
-                return True
-            item = items[index]
-            if item.type == 'universal':
-                pass
-            elif item.type == 'type-selector':
-                name1 = ("{%s}%s" % item.value).lower()
-                name2 = element.tag.lower()
-                if name1 != name2:
-                    return False
-            elif item.type == 'id':
-                name1 = item.value[1:]
-                name2 = element.get('id', '')
-                if name1 != name2:
-                    return False
-            elif item.type == 'class':
-                name = item.value[1:].lower()
-                classes = element.get('class', '').lower().split()
-                if name not in classes:
-                    return False
-            elif item.type == 'child':
-                parent = element.getparent()
-                if parent is None:
-                    return False
-                element = parent
-            elif item.type == 'descendant':
-                element = element.getparent()
-                while element is not None:
-                    if _selects_element(element, items, index - 1):
-                        return True
-                    element = element.getparent()
-                return False
-            elif item.type == 'pseudo-class':
-                if item.value == ':first-child':
-                    e = element.getprevious()
-                    if e is not None:
-                        return False
-                else:
-                    return False
-            elif item.type == 'pseudo-element':
-                return False
-            else:
-                return False
-            return _selects_element(element, items, index - 1)
-        return _selects_element(element, selector, len(selector) - 1)
+    def _apply_style_tag(self):
+        attrib = self._element.attrib
+        if 'style' in attrib:
+            style = CSSStyleDeclaration(attrib['style'])
+            self._style.update(self._stylizer.flatten_style(style))
 
     def _has_parent(self):
         parent = self._element.getparent()

From 479385ea34dac4814609a8a0c48ebbc3afdd4e16 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Fri, 19 Dec 2008 08:58:37 -0500
Subject: [PATCH 19/22] Remove unnecessary level of attribute-resolution.

---
 src/calibre/ebooks/oeb/stylizer.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py
index 9a4776c6b9..6f1fc5d774 100644
--- a/src/calibre/ebooks/oeb/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@@ -363,18 +363,18 @@ class Style(object):
             factor = None
             if value == 'inherit':
                 # We should only see this if the root element
-                value = self._stylizer.page.fbase
+                value = self._page.fbase
             if value in FONT_SIZE_NAMES:
-                result = self._stylizer.page.fnames[value]
+                result = self._page.fnames[value]
             elif value == 'smaller':
                 factor = 1.0/1.2
-                for _, _, size in self._stylizer.page.fsizes:
+                for _, _, size in self._page.fsizes:
                     if base <= size: break
                     factor = None
                     result = size
             elif value == 'larger':
                 factor = 1.2
-                for _, _, size in reversed(self._stylizer.page.fsizes):
+                for _, _, size in reversed(self._page.fsizes):
                     if base >= size: break
                     factor = None
                     result = size
@@ -390,7 +390,7 @@ class Style(object):
             styles = self._stylizer._styles
             base = styles[self._element.getparent()].fontSize
         else:
-            base = self._stylizer.page.fbase
+            base = self._page.fbase
         if 'font-size' in self._style:
             size = self._style['font-size']
             result = normalize_fontsize(size, base)

From 03cb13e7ab0f8c8141172d5c94093751d3275dd1 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Fri, 19 Dec 2008 23:18:26 -0500
Subject: [PATCH 20/22] Make CSS namespace additions compliant with the CSS3
 namespace rec. Set the MSLIT profile to its actual apparent resolution of
 100dpi.

---
 src/calibre/ebooks/oeb/stylizer.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py
index 6f1fc5d774..b6e4ffc184 100644
--- a/src/calibre/ebooks/oeb/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@@ -26,7 +26,7 @@ from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
 from calibre.ebooks.oeb.base import barename, urlnormalize
 from calibre.resources import html_css
 
-XHTML_CSS_NAMESPACE = "@namespace url(%s);\n" % XHTML_NS
+XHTML_CSS_NAMESPACE = '@namespace "%s";\n' % XHTML_NS
 HTML_CSS_STYLESHEET = cssutils.parseString(html_css)
 HTML_CSS_STYLESHEET.namespaces['h'] = XHTML_NS
 
@@ -126,7 +126,7 @@ class Page(object):
 
 class Profiles(object):
     PRS505 = Page(584, 754, 168.451, 12, [7.5, 9, 10, 12, 15.5, 20, 22, 24])
-    MSLIT = Page(652, 480, 168.451, 13, [10, 11, 13, 16, 18, 20, 22, 26])
+    MSLIT = Page(652, 480, 100.0, 13, [10, 11, 13, 16, 18, 20, 22, 26])
 
     
 class Stylizer(object):    

From dc1bfe4d204085d7c29b437237e45366287f5c2b Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Sun, 28 Dec 2008 15:46:52 -0500
Subject: [PATCH 21/22] Add DirWriter.  Add str() for manifest items.

---
 src/calibre/ebooks/oeb/base.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py
index 9b67bc6ad6..c7824a6ab3 100644
--- a/src/calibre/ebooks/oeb/base.py
+++ b/src/calibre/ebooks/oeb/base.py
@@ -69,6 +69,9 @@ def barename(name):
 def xpath(elem, expr):
     return elem.xpath(expr, namespaces=XPNSMAP)
 
+def xml2str(root):
+    return etree.tostring(root, encoding='utf-8', xml_declaration=True)
+
 ASCII_CHARS = set(chr(x) for x in xrange(128))
 URL_SAFE = set(u'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
                u'abcdefghijklmnopqrstuvwxyz'
@@ -114,6 +117,9 @@ class DirContainer(AbstractContainer):
 
     def write(self, path, data):
         path = os.path.join(self.rootdir, path)
+        dir = os.path.dirname(path)
+        if not os.path.isdir(dir):
+            os.makedirs(dir)
         with open(urlunquote(path), 'wb') as f:
             return f.write(data)
 
@@ -121,6 +127,21 @@ class DirContainer(AbstractContainer):
         path = os.path.join(self.rootdir, path)
         return os.path.isfile(urlunquote(path))
 
+class DirWriter(object):
+    def __init__(self, version=2.0):
+        self.version = version
+
+    def dump(self, oeb, path):
+        if not os.path.isdir(path):
+            os.mkdir(path)
+        output = DirContainer(path)
+        for item in oeb.manifest.values():
+            output.write(item.href, str(item))
+        metadata = oeb.to_opf2() if self.version == 2 else oeb.to_opf1()
+        for href, data in metadata.values():
+            output.write(href, xml2str(data))
+        return
+
 
 class Metadata(object):
     TERMS = set(['contributor', 'coverage', 'creator', 'date', 'description',
@@ -269,6 +290,12 @@ class Manifest(object):
             return property(fget, fset, fdel)
         data = data()
 
+        def __str__(self):
+            data = self.data
+            if isinstance(data, etree._Element):
+                return xml2str(data)
+            return str(data)
+
         def __cmp__(self, other):
             result = cmp(self.spine_position, other.spine_position)
             if result != 0:

From d0e1f3d26d6eeace7872bb3b61c79d7069d313d1 Mon Sep 17 00:00:00 2001
From: "Marshall T. Vandegrift" <llasram@gmail.com>
Date: Sun, 28 Dec 2008 15:47:39 -0500
Subject: [PATCH 22/22] Clean up unnecessary CSS rules.

---
 src/calibre/ebooks/oeb/html.css    | 13 ++-----------
 src/calibre/ebooks/oeb/stylizer.py |  2 +-
 2 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/src/calibre/ebooks/oeb/html.css b/src/calibre/ebooks/oeb/html.css
index 38e4f314a4..a454b9b716 100644
--- a/src/calibre/ebooks/oeb/html.css
+++ b/src/calibre/ebooks/oeb/html.css
@@ -45,7 +45,6 @@ html, div, map, dt, isindex, form {
 
 body {
   display: block;
-  margin: 8px;
 }
 
 p, dl, multicol {
@@ -59,7 +58,7 @@ dd {
 
 blockquote {
   display: block;
-  margin: 1em 40px;
+  margin: 1em;
 }
 
 address {
@@ -74,7 +73,7 @@ center {
 
 blockquote[type=cite] {
   display: block;
-  margin: 1em 0px;
+  margin: 1em 0em;
   border-color: blue;
   border-width: thin;
 }
@@ -234,14 +233,6 @@ th {
 
 /* inlines */
 
-q:before {
-  content: open-quote;
-}
-
-q:after {
-  content: close-quote;
-}
-
 b, strong {
   font-weight: bolder;
 }
diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py
index b6e4ffc184..28f3218fb6 100644
--- a/src/calibre/ebooks/oeb/stylizer.py
+++ b/src/calibre/ebooks/oeb/stylizer.py
@@ -179,7 +179,7 @@ class Stylizer(object):
         for _, _, cssdict, text, _ in rules:
             try:
                 selector = CSSSelector(text)
-            except ExpressionError:
+            except ExpressionError, e:
                 continue
             for elem in selector(tree):
                 self.style(elem)._update_cssdict(cssdict)