Add test framework for book container

Also add test for container cloning

src/calibre/ebooks/oeb/polish/container.py

@@ -71,7 +71,7 @@ def clone_container(container, dest_dir):
         return cls(None, None, container.log, clone_data=clone_data)
     return cls(None, container.log, clone_data=clone_data)
 
-class Container(object):
+class Container(object):  # {{{
 
     '''
     A container represents an Open EBook as a directory full of files and an
@@ -107,7 +107,9 @@ class Container(object):
 
         if clone_data is not None:
             self.cloned = True
-            self.name_path_map = clone_data['name_path_map']
+            for x in ('name_path_map', 'opf_name', 'mime_map', 'pretty_print', 'encoding_map'):
+                setattr(self, x, clone_data[x])
+            self.opf_dir = os.path.dirname(self.name_path_map[self.opf_name])
             return
 
         # Map of relative paths with '/' separators from root of unzipped ePub
@@ -135,6 +137,21 @@ class Container(object):
             if name in self.mime_map:
                 self.mime_map[name] = item.get('media-type')
 
+    def clone_data(self, dest_dir):
+        Container.commit(self, keep_parsed=True)
+        self.cloned = True
+        clone_dir(self.root, dest_dir)
+        return {
+            'root': dest_dir,
+            'opf_name': self.opf_name,
+            'mime_map': self.mime_map.copy(),
+            'pretty_print': set(self.pretty_print),
+            'encoding_map': self.encoding_map.copy(),
+            'name_path_map': {
+                name:os.path.join(dest_dir, os.path.relpath(path, self.root))
+                for name, path in self.name_path_map.iteritems()}
+        }
+
     def abspath_to_name(self, fullpath):
         return self.relpath(os.path.abspath(fullpath)).replace(os.sep, '/')
 
@@ -292,21 +309,26 @@ class Container(object):
             for item in self.opf_xpath('//opf:guide/opf:reference[@href and @type]')}
 
     @property
-    def spine_items(self):
+    def spine_names(self):
         manifest_id_map = self.manifest_id_map
 
-        linear, non_linear = [], []
+        non_linear = []
         for item in self.opf_xpath('//opf:spine/opf:itemref[@idref]'):
             idref = item.get('idref')
             name = manifest_id_map.get(idref, None)
             path = self.name_path_map.get(name, None)
             if path:
                 if item.get('linear', 'yes') == 'yes':
-                    yield path
+                    yield name
                 else:
-                    non_linear.append(path)
+                    non_linear.append(name)
-        for path in non_linear:
+        for name in non_linear:
-            yield path
+            yield name
+
+    @property
+    def spine_items(self):
+        for name in self.spine_names:
+            yield self.name_path_map[name]
 
     def remove_item(self, name):
         '''
@@ -498,17 +520,6 @@ class Container(object):
         for name in tuple(self.dirtied):
             self.commit_item(name, keep_parsed=keep_parsed)
 
-    def clone_data(self, dest_dir):
-        self.commit(keep_parsed=True)
-        self.cloned = True
-        clone_dir(self.root, dest_dir)
-        return {
-            'root': dest_dir,
-            'name_path_map': {
-                name:os.path.join(dest_dir, os.path.relpath(path, self.root))
-                for name, path in self.name_path_map.iteritems()}
-        }
-
     def compare_to(self, other):
         if set(self.name_path_map) != set(other.name_path_map):
             return 'Set of files is not the same'
@@ -519,6 +530,7 @@ class Container(object):
                 if f1.read() != f2.read():
                     mismatches.append('The file %s is not the same'%name)
         return '\n'.join(mismatches)
+# }}}
 
 # EPUB {{{
 class InvalidEpub(InvalidBook):
@@ -539,7 +551,7 @@ class EpubContainer(Container):
             'rights.xml': False,
     }
 
-    def __init__(self, pathtoepub, log, clone_data=None):
+    def __init__(self, pathtoepub, log, clone_data=None, tdir=None):
         if clone_data is not None:
             super(EpubContainer, self).__init__(None, None, log, clone_data=clone_data)
             for x in ('pathtoepub', 'container', 'obfuscated_fonts'):
@@ -547,7 +559,9 @@ class EpubContainer(Container):
             return
 
         self.pathtoepub = pathtoepub
-        tdir = self.root = os.path.abspath(os.path.realpath(PersistentTemporaryDirectory('_epub_container')))
+        if tdir is None:
+            tdir = os.path.abspath(os.path.realpath(PersistentTemporaryDirectory('_epub_container')))
+        self.root = tdir
         with open(self.pathtoepub, 'rb') as stream:
             try:
                 zf = ZipFile(stream)
@@ -685,7 +699,7 @@ class AZW3Container(Container):
 
     book_type = 'azw3'
 
-    def __init__(self, pathtoazw3, log, clone_data=None):
+    def __init__(self, pathtoazw3, log, clone_data=None, tdir=None):
         if clone_data is not None:
             super(AZW3Container, self).__init__(None, None, log, clone_data=clone_data)
             for x in ('pathtoazw3', 'obfuscated_fonts'):
@@ -693,7 +707,9 @@ class AZW3Container(Container):
             return
 
         self.pathtoazw3 = pathtoazw3
-        tdir = self.root = os.path.abspath(os.path.realpath(PersistentTemporaryDirectory('_azw3_container')))
+        if tdir is None:
+            tdir = os.path.abspath(os.path.realpath(PersistentTemporaryDirectory('_azw3_container')))
+        self.root = tdir
         with open(pathtoazw3, 'rb') as stream:
             raw = stream.read(3)
             if raw == b'TPZ':
@@ -752,11 +768,11 @@ class AZW3Container(Container):
         outp.convert(oeb, outpath, inp, plumber.opts, default_log)
 # }}}
 
-def get_container(path, log=None):
+def get_container(path, log=None, tdir=None):
     if log is None:
         log = default_log
     ebook = (AZW3Container if path.rpartition('.')[-1].lower() in {'azw3', 'mobi'}
-            else EpubContainer)(path, log)
+            else EpubContainer)(path, log, tdir=tdir)
     return ebook
 
 def test_roundtrip():

src/calibre/ebooks/oeb/polish/tests/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+
+

src/calibre/ebooks/oeb/polish/tests/base.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import os, unittest, shutil
+
+from calibre.ptempfile import PersistentTemporaryDirectory
+from calibre.utils.logging import DevNull
+import calibre.ebooks.oeb.polish.container as pc
+
+def get_cache():
+    from calibre.constants import cache_dir
+    cache = os.path.join(cache_dir(), 'polish-test')
+    if not os.path.exists(cache):
+        os.mkdir(cache)
+    return cache
+
+def needs_recompile(obj, srcs):
+    if isinstance(srcs, type('')):
+        srcs = [srcs]
+    try:
+        obj_mtime = os.stat(obj).st_mtime
+    except OSError:
+        return True
+    for src in srcs:
+        if os.stat(src).st_mtime > obj_mtime:
+            return True
+    return False
+
+def build_book(src, dest, args=()):
+    from calibre.ebooks.conversion.cli import main
+    main(['ebook-convert', src, dest] + list(args))
+
+def get_simple_book(fmt='epub'):
+    cache = get_cache()
+    ans = os.path.join(cache, 'simple.'+fmt)
+    src = os.path.join(os.path.dirname(__file__), 'simple.html')
+    if needs_recompile(ans, src):
+        x = src.replace('simple.html', 'index.html')
+        raw = open(src, 'rb').read().decode('utf-8')
+        raw = raw.replace('LMONOI', P('fonts/liberation/LiberationMono-Italic.ttf'))
+        raw = raw.replace('LMONO', P('fonts/liberation/LiberationMono-Regular.ttf'))
+        raw = raw.replace('IMAGE1', I('marked.png'))
+        try:
+            with open(x, 'wb') as f:
+                f.write(raw.encode('utf-8'))
+            build_book(x, ans, args=['--level1-toc=//h:h2', '--language=en', '--authors=Kovid Goyal',
+                                        '--cover=' + I('lt.png')])
+        finally:
+            try:
+                os.remove('index.html')
+            except:
+                pass
+    return ans
+
+devnull = DevNull()
+
+class BaseTest(unittest.TestCase):
+
+    longMessage = True
+    maxDiff = None
+
+    def setUp(self):
+        pc.default_log = devnull
+        self.tdir = PersistentTemporaryDirectory(suffix='-polish-test')
+
+    def tearDown(self):
+        shutil.rmtree(self.tdir, ignore_errors=True)
+        del self.tdir
+

src/calibre/ebooks/oeb/polish/tests/container.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import os
+
+from calibre.ebooks.oeb.polish.tests.base import BaseTest, get_simple_book
+
+from calibre.ebooks.oeb.polish.container import get_container, clone_container
+from calibre.utils.filenames import nlinks_file
+
+class ContainerTests(BaseTest):
+
+    def test_clone(self):
+        ' Test cloning of containers '
+        for fmt in ('epub', 'azw3'):
+            base = os.path.join(self.tdir, fmt + '-')
+            book = get_simple_book(fmt)
+            tdir = base + 'first'
+            os.mkdir(tdir)
+            c1 = get_container(book, tdir=tdir)
+            tdir = base + 'second'
+            os.mkdir(tdir)
+            c2 = clone_container(c1, tdir)
+
+            for c in (c1, c2):
+                for name, path in c.name_path_map.iteritems():
+                    self.assertEqual(2, nlinks_file(path), 'The file %s is not linked' % name)
+
+            for name in c1.name_path_map:
+                self.assertIn(name, c2.name_path_map)
+                self.assertEqual(c1.open(name).read(), c2.open(name).read(), 'The file %s differs' % name)
+
+            spine_names = tuple(c1.spine_names)
+            text = spine_names[0]
+            root = c2.parsed(text)
+            root.xpath('//*[local-name()="body"]')[0].set('id', 'changed id for test')
+            c2.dirty(text)
+            c2.commit_item(text)
+            for c in (c1, c2):
+                self.assertEqual(1, nlinks_file(c.name_path_map[text]))
+            self.assertNotEqual(c1.open(text).read(), c2.open(text).read())
+
+            name = spine_names[1]
+            with c1.open(name, mode='r+b') as f:
+                f.seek(0, 2)
+                f.write(b'    ')
+            for c in (c1, c2):
+                self.assertEqual(1, nlinks_file(c.name_path_map[name]))
+            self.assertNotEqual(c1.open(name).read(), c2.open(name).read())
+
+            x = base + 'out.' + fmt
+            for c in (c1, c2):
+                c.commit(outpath=x)
+

src/calibre/ebooks/oeb/polish/tests/index.html

@@ -0,0 +1,64 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>A simple test page</title>
+		<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+        <style type="text/css">
+@font-face {
+    font-family: "Liberation Mono";
+    src: url(/home/kovid/work/calibre/resources/fonts/liberation/LiberationMono-Regular.ttf);
+}
+
+@font-face {
+    font-family: "Liberation Mono";
+    src: url(/home/kovid/work/calibre/resources/fonts/liberation/LiberationMono-Italic.ttf);
+    font-style: italic;
+}
+
+h1 {
+    color: DarkCyan;
+    text-align: center;
+}
+
+p { font-family: "Liberation Mono"; }
+        </style>
+	</head>
+	<body>
+    <h2>A simple test page</h2>
+<!--lorem-->
+<p>To pursue pleasure rationally encounter consequences that are extremely
+painful.</p>
+
+<p>Nor again is there anyone who loves or pursues or desires to obtain pain of
+itself, because it is pain, but because occasionally circumstances occur in
+which toil and pain can procure him some great pleasure. To take a trivial
+example, which of us ever undertakes laborious physical exercise, except to
+obtain some advantage from it? But who has any right to find fault with a man
+who chooses to enjoy a pleasure that has no <em>annoying</em> consequences, or one who
+avoids a pain that produces no resultant pleasure?</p>
+
+<div style="text-align:center"><img alt="test" src="/home/kovid/work/calibre/resources/images/marked.png"></div>
+
+<p>On the other hand.</p>
+
+<!--/lorem-->
+
+<h2>Another test page</h2>
+
+<!--lorem-->
+<p>The great explorer of the truth, the master-builder of human happiness. No
+one rejects, dislikes, or avoids pleasure itself, because it is pleasure, but
+because those who do not know how to pursue pleasure rationally encounter
+consequences that are extremely painful.</p>
+
+<p>Nor again is there anyone who loves or pursues or desires to obtain pain of
+itself, because it is pain, but because occasionally circumstances occur in
+which toil and pain can procure him some great pleasure. To take a trivial
+example, which of us ever undertakes laborious physical exercise, except to
+obtain some advantage from it? But who has any right.</p>
+<!--/lorem-->
+
+
+	</body>
+</html>
+

src/calibre/ebooks/oeb/polish/tests/main.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+
+try:
+    import init_calibre  # noqa
+except ImportError:
+    pass
+
+import os, unittest
+
+def find_tests():
+    return unittest.defaultTestLoader.discover(os.path.dirname(os.path.abspath(__file__)), pattern='*.py')
+
+if __name__ == '__main__':
+    from calibre.db.tests.main import run_tests
+    run_tests(find_tests=find_tests)
+
+

src/calibre/ebooks/oeb/polish/tests/simple.html

@@ -0,0 +1,64 @@
+<!DOCTYPE html>
+<html>
+	<head>
+		<title>A simple test page</title>
+		<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
+        <style type="text/css">
+@font-face {
+    font-family: "Liberation Mono";
+    src: url(LMONO);
+}
+
+@font-face {
+    font-family: "Liberation Mono";
+    src: url(LMONOI);
+    font-style: italic;
+}
+
+h1 {
+    color: DarkCyan;
+    text-align: center;
+}
+
+p { font-family: "Liberation Mono"; }
+        </style>
+	</head>
+	<body>
+    <h2>A simple test page</h2>
+<!--lorem-->
+<p>To pursue pleasure rationally encounter consequences that are extremely
+painful.</p>
+
+<p>Nor again is there anyone who loves or pursues or desires to obtain pain of
+itself, because it is pain, but because occasionally circumstances occur in
+which toil and pain can procure him some great pleasure. To take a trivial
+example, which of us ever undertakes laborious physical exercise, except to
+obtain some advantage from it? But who has any right to find fault with a man
+who chooses to enjoy a pleasure that has no <em>annoying</em> consequences, or one who
+avoids a pain that produces no resultant pleasure?</p>
+
+<div style="text-align:center"><img alt="test" src="IMAGE1"></div>
+
+<p>On the other hand.</p>
+
+<!--/lorem-->
+
+<h2>Another test page</h2>
+
+<!--lorem-->
+<p>The great explorer of the truth, the master-builder of human happiness. No
+one rejects, dislikes, or avoids pleasure itself, because it is pleasure, but
+because those who do not know how to pursue pleasure rationally encounter
+consequences that are extremely painful.</p>
+
+<p>Nor again is there anyone who loves or pursues or desires to obtain pain of
+itself, because it is pain, but because occasionally circumstances occur in
+which toil and pain can procure him some great pleasure. To take a trivial
+example, which of us ever undertakes laborious physical exercise, except to
+obtain some advantage from it? But who has any right.</p>
+<!--/lorem-->
+
+
+	</body>
+</html>
+