Merge branch 'csv_module' of https://github.com/jamesbroadhead/calibre into jamesbroadhead-csv_module

.gitignore

@@ -5,6 +5,7 @@
 .bzr
 .bzrignore
 .build-cache
+.cache
 src/calibre/plugins
 resources/images.qrc
 manual/generated

.travis.yml

@@ -25,5 +25,8 @@ before_install:
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then sudo mkdir -p $SWBASE && sudo chown $USER $SWBASE; fi
     - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then curl https://download.calibre-ebook.com/travis/sw-osx.tar.bz2 | tar xj -C $SWBASE; fi
     - npm install --no-optional rapydscript-ng && echo $PATH && which rapydscript && rapydscript --version
+    - python $SW/bin/easy_install --user "pip==9.0.1"
+    - pip install --user -r requirements.txt
     - python setup.py bootstrap --ephemeral
+
 script: python setup.py test

appveyor.yml

@@ -20,6 +20,8 @@ platform:
 
 before_build:
   - C:\Python35-x64\python.exe setup/win-ci.py sw
+  - C:\sw\private\python\easy_install.exe --user "pip==9.0.1"
+  - C:\sw\private\python\pip install --user -r requirements.txt
 
 build_script:
     - C:\sw\private\python\python.exe setup/win-ci.py build

requirements.txt

@@ -0,0 +1,2 @@
+# Note, this file is work-in-progress, and is partial only.
+mock==2.0.0

setup/test.py

@@ -8,7 +8,7 @@ import unittest
 
 from setup import Command
 
-TEST_MODULES = frozenset('srv db polish opf css docx cfi matcher icu smartypants build misc'.split())
+TEST_MODULES = frozenset('srv db polish opf css docx cfi matcher icu smartypants build misc library'.split())
 
 def find_tests(which_tests=None):
     ans = []
@@ -64,10 +64,13 @@ def find_tests(which_tests=None):
         a(find_tests())
         from calibre.utils.shared_file import find_tests
         a(find_tests())
+    if ok('library'):
+        a(unittest.TestLoader().discover('src/calibre/library'))
 
     tests = unittest.TestSuite(ans)
     return tests
 
+
 class Test(Command):
 
     description = 'Run the calibre test suite'

src/calibre/library/cli.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 Command line interface to the calibre database.
 '''
 
-import sys, os, cStringIO, re
+import cStringIO, csv, os, re, sys
 import unicodedata
 from textwrap import TextWrapper
 from optparse import OptionValueError, OptionGroup
@@ -1362,18 +1362,6 @@ def command_check_library(args, dbpath):
     else:
         exts = [f.strip() for f in opts.exts.split(',') if f.strip()]
 
-    def print_one(checker, check):
-        attr = check[0]
-        list = getattr(checker, attr, None)
-        if list is None:
-            return
-        if opts.csv:
-            for i in list:
-                print check[1] + ',' + i[0] + ',' + i[1]
-        else:
-            print check[1]
-            for i in list:
-                print '    %-40.40s - %-40.40s'%(i[0], i[1])
 
     if not LibraryDatabase.exists_at(dbpath):
         prints('No library found at', dbpath, file=sys.stderr)
@@ -1383,7 +1371,25 @@ def command_check_library(args, dbpath):
     checker = CheckLibrary(dbpath, db)
     checker.scan_library(names, exts)
     for check in checks:
-        print_one(checker, check)
+        _print_check_library_results(checker, check, opts)
+
+
+def _print_check_library_results(checker, check, opts):
+    attr = check[0]
+    list = getattr(checker, attr, None)
+    if list is None:
+        return
+
+    if opts.csv:
+        to_output = [(check[1], i[0], i[1]) for i in list]
+        csv_print = csv.writer(sys.stdout)
+        for line in to_output:
+            csv_print.writerow(line)
+
+    else:
+        print check[1]
+        for i in list:
+            print '    %-40.40s - %-40.40s'%(i[0], i[1])
 
 
 def restore_database_option_parser():

src/calibre/library/test.py

@@ -1,105 +0,0 @@
-#!/usr/bin/env  python2
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
-__docformat__ = 'restructuredtext en'
-
-'''
-Unit tests for database layer.
-'''
-
-import sys, unittest, os, cStringIO
-from itertools import repeat
-
-from calibre.ptempfile import PersistentTemporaryDirectory
-from calibre.library.database2 import LibraryDatabase2
-from calibre.ebooks.metadata import MetaInformation
-
-
-class DBTest(unittest.TestCase):
-
-    img = '\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00d\x00d\x00\x00\xff\xdb\x00C\x00\x05\x03\x04\x04\x04\x03\x05\x04\x04\x04\x05\x05\x05\x06\x07\x0c\x08\x07\x07\x07\x07\x0f\x0b\x0b\t\x0c\x11\x0f\x12\x12\x11\x0f\x11\x11\x13\x16\x1c\x17\x13\x14\x1a\x15\x11\x11\x18!\x18\x1a\x1d\x1d\x1f\x1f\x1f\x13\x17"$"\x1e$\x1c\x1e\x1f\x1e\xff\xdb\x00C\x01\x05\x05\x05\x07\x06\x07\x0e\x08\x08\x0e\x1e\x14\x11\x14\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\x1e\xff\xc0\x00\x11\x08\x00\x01\x00\x01\x03\x01\x11\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x14\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\xff\xc4\x00\x14\x10\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x06\xff\xc4\x00\x14\x11\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00p\xf9+\xff\xd9'  # noqa
-
-    def setUp(self):
-        self.tdir    = PersistentTemporaryDirectory('_calibre_dbtest')
-        self.db      = LibraryDatabase2(self.tdir)
-        f = open(os.path.join(self.tdir, 'test.txt'), 'w+b')
-        f.write('test')
-        paths = list(repeat(f, 3))
-        formats = list(repeat('txt', 3))
-        m1 = MetaInformation('Test Ebook 1', ['Test Author 1'])
-        m1.tags = ['tag1', 'tag2']
-        m1.publisher = 'Test Publisher 1'
-        m1.rating = 2
-        m1.series = 'Test Series 1'
-        m1.series_index = 3
-        m1.author_sort = 'as1'
-        m1.isbn = 'isbn1'
-        m1.cover_data = ('jpg', self.img)
-        m2 = MetaInformation('Test Ebook 2', ['Test Author 2'])
-        m2.tags = ['tag3', 'tag4']
-        m2.publisher = 'Test Publisher 2'
-        m2.rating = 3
-        m2.series = 'Test Series 2'
-        m2.series_index = 1
-        m2.author_sort = 'as1'
-        m2.isbn = 'isbn1'
-        self.db.add_books(paths, formats, [m1, m2, m2], add_duplicates=True)
-        self.m1, self.m2 = m1, m2
-
-    def testAdding(self):
-        m1, m2 = self.db.get_metadata(1, True), self.db.get_metadata(2, True)
-        for p in ('title', 'authors', 'publisher', 'rating', 'series',
-                  'series_index', 'author_sort', 'isbn', 'tags'):
-
-            def ga(mi, p):
-                val = getattr(mi, p)
-                if isinstance(val, list):
-                    val = set(val)
-                return val
-
-            self.assertEqual(ga(self.m1, p), ga(m1, p))
-            self.assertEqual(ga(self.m2, p), ga(m2, p))
-
-        self.assertEqual(self.db.format(1, 'txt', index_is_id=True), 'test')
-        self.assertEqual(self.db.formats(1, index_is_id=True), 'TXT')
-        self.db.add_format(1, 'html', cStringIO.StringIO('<html/>'), index_is_id=True)
-        self.assertEqual(self.db.formats(1, index_is_id=True), 'HTML,TXT')
-        self.db.remove_format(1, 'html', index_is_id=True)
-        self.assertEqual(self.db.formats(1, index_is_id=True), 'TXT')
-        self.assertNotEqual(self.db.cover(1, index_is_id=True), None)
-        self.assertEqual(self.db.cover(2, index_is_id=True), None)
-
-    def testMetadata(self):
-        self.db.refresh('timestamp', True)
-        for x in ('title', 'author_sort', 'series', 'publisher', 'isbn', 'series_index', 'rating'):
-            val = 3 if x in ['rating', 'series_index'] else 'dummy'
-            getattr(self.db, 'set_'+x)(3, val)
-            self.db.refresh_ids([3])
-            self.assertEqual(getattr(self.db, x)(2), val)
-
-        self.db.set_authors(3, ['new auth'])
-        self.db.refresh_ids([3])
-        self.assertEqual('new auth', self.db.authors(2))
-        self.assertEqual(self.db.format(3, 'txt', index_is_id=True), 'test')
-
-    def testSorting(self):
-        self.db.sort('authors', True)
-        self.assertEqual(self.db.authors(0), 'Test Author 1')
-        self.db.sort('rating', False)
-        self.assertEqual(self.db.rating(0), 3)
-
-
-def suite():
-    return unittest.TestLoader().loadTestsFromTestCase(DBTest)
-
-
-def test():
-    unittest.TextTestRunner(verbosity=2).run(suite())
-
-
-def main(args=sys.argv):
-    test()
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/library/test_cli.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python2
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__docformat__ = 'restructuredtext en'
+
+'''
+Test the CLI of the calibre database management tool
+'''
+import csv
+import unittest
+from StringIO import StringIO
+
+from mock import Mock, patch
+
+from calibre.library.check_library import CheckLibrary
+from calibre.library.cli import _print_check_library_results
+
+
+class PrintCheckLibraryResultsTest(unittest.TestCase):
+    """
+    Asserts the format of the output to the CLI to avoid regressions
+    """
+    check_machine_name = 'dummy_check'
+    check_human_name = 'Dummy Check'
+    check = (check_machine_name, check_human_name, True, False)
+
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_prints_nothing_if_no_errors(self, mock_stdout):
+        checker = Mock(name='checker', spec=CheckLibrary)
+        setattr(checker, self.check_machine_name, None)
+        opts = Mock()
+
+        opts.csv = False
+        _print_check_library_results(checker, self.check, opts)
+        self.assertEqual(mock_stdout.getvalue(), '')
+
+        opts.csv = True
+        _print_check_library_results(checker, self.check, opts)
+        self.assertEqual(mock_stdout.getvalue(), '')
+
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_human_readable_output(self, mock_stdout):
+        """
+        Basic check of the human-readable output.
+
+        Does not test: the full line format, truncation
+        """
+        checker = Mock(name='checker', speck=CheckLibrary)
+        data = [['first', 'second']]
+        opts = Mock()
+        opts.csv = False
+        setattr(checker, self.check_machine_name, data)
+        _print_check_library_results(checker, self.check, opts)
+
+        result = mock_stdout.getvalue().split('\n')
+        self.assertEqual(len(result), len(data)+2)
+        self.assertEqual(result[0], self.check_human_name)
+
+        result_first = result[1].split('-')[0].strip()
+        result_second = result[1].split('-')[1].strip()
+
+        self.assertEqual(result_first, 'first')
+        self.assertEqual(result_second, 'second')
+
+        self.assertEqual(result[-1], '')
+
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_basic_csv_output(self, mock_stdout):
+        """
+        Test simple csv output
+        """
+        checker = Mock(name='checker', speck=CheckLibrary)
+        data = [['first', 'second']]
+        opts = Mock()
+        opts.csv = True
+        setattr(checker, self.check_machine_name, data)
+        _print_check_library_results(checker, self.check, opts)
+
+        result = mock_stdout.getvalue().split('\n')
+        parsed_result = [l for l in csv.reader(result) if l]
+        self.assertEqual(parsed_result, [[self.check_human_name, data[0][0], data[0][1]]])
+
+    @patch('sys.stdout', new_callable=StringIO)
+    def test_escaped_csv_output(self, mock_stdout):
+        """
+        Test more complex csv output
+        """
+        checker = Mock(name='checker', speck=CheckLibrary)
+        data = [['I, Caesar', 'second']]
+        opts = Mock()
+        opts.csv = True
+        setattr(checker, self.check_machine_name, data)
+        _print_check_library_results(checker, self.check, opts)
+
+        result = mock_stdout.getvalue().split('\n')
+        parsed_result = [l for l in csv.reader(result) if l]
+        self.assertEqual(parsed_result, [[self.check_human_name, data[0][0], data[0][1]]])