Merge branch 'kovidgoyal/master'

src/calibre/db/adding.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+
+import os
+from calibre.ebooks import BOOK_EXTENSIONS
+
+def find_books_in_directory(dirpath, single_book_per_directory):
+    dirpath = os.path.abspath(dirpath)
+    if single_book_per_directory:
+        formats = []
+        for path in os.listdir(dirpath):
+            path = os.path.abspath(os.path.join(dirpath, path))
+            if os.path.isdir(path) or not os.access(path, os.R_OK):
+                continue
+            ext = os.path.splitext(path)[1]
+            if not ext:
+                continue
+            ext = ext[1:].lower()
+            if ext not in BOOK_EXTENSIONS and ext != 'opf':
+                continue
+            formats.append(path)
+        yield formats
+    else:
+        books = {}
+        for path in os.listdir(dirpath):
+            path = os.path.abspath(os.path.join(dirpath, path))
+            if os.path.isdir(path) or not os.access(path, os.R_OK):
+                continue
+            ext = os.path.splitext(path)[1]
+            if not ext:
+                continue
+            ext = ext[1:].lower()
+            if ext not in BOOK_EXTENSIONS:
+                continue
+
+            key = os.path.splitext(path)[0]
+            if key not in books:
+                books[key] = []
+            books[key].append(path)
+
+        for formats in books.values():
+            yield formats
+
+def import_book_directory_multiple(db, dirpath, callback=None,
+        added_ids=None):
+    from calibre.ebooks.metadata.meta import metadata_from_formats
+
+    duplicates = []
+    for formats in find_books_in_directory(dirpath, False):
+        mi = metadata_from_formats(formats)
+        if mi.title is None:
+            continue
+        if db.has_book(mi):
+            duplicates.append((mi, formats))
+            continue
+        book_id = db.import_book(mi, formats)
+        if added_ids is not None:
+            added_ids.add(book_id)
+        if callable(callback):
+            if callback(mi.title):
+                break
+    return duplicates
+
+def import_book_directory(db, dirpath, callback=None, added_ids=None):
+    from calibre.ebooks.metadata.meta import metadata_from_formats
+    dirpath = os.path.abspath(dirpath)
+    formats = find_books_in_directory(dirpath, True)
+    formats = list(formats)[0]
+    if not formats:
+        return
+    mi = metadata_from_formats(formats)
+    if mi.title is None:
+        return
+    if db.has_book(mi):
+        return [(mi, formats)]
+    book_id = db.import_book(mi, formats)
+    if added_ids is not None:
+        added_ids.add(book_id)
+    if callable(callback):
+        callback(mi.title)
+
+def recursive_import(db, root, single_book_per_directory=True,
+        callback=None, added_ids=None):
+    root = os.path.abspath(root)
+    duplicates  = []
+    for dirpath in os.walk(root):
+        res = (import_book_directory(db, dirpath[0], callback=callback,
+            added_ids=added_ids) if single_book_per_directory else
+            import_book_directory_multiple(db, dirpath[0],
+                callback=callback, added_ids=added_ids))
+        if res is not None:
+            duplicates.extend(res)
+        if callable(callback):
+            if callback(''):
+                break
+    return duplicates
+

src/calibre/db/cache.py

@@ -1117,7 +1117,7 @@ class Cache(object):
         return book_id
 
     @write_api
-    def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, dbapi=None):
+    def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, run_hooks=True, dbapi=None):
         duplicates, ids = [], []
         for mi, format_map in books:
             book_id = self._create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid)
@@ -1126,7 +1126,7 @@ class Cache(object):
             else:
                 ids.append(book_id)
                 for fmt, stream_or_path in format_map.iteritems():
-                    self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi)
+                    self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi, run_hooks=run_hooks)
         return ids, duplicates
 
     @write_api

src/calibre/db/legacy.py

@@ -8,8 +8,10 @@ __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import os, traceback
 from functools import partial
+from future_builtins import zip
 
 from calibre.db import _get_next_series_num_for_list, _get_series_values
+from calibre.db.adding import find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import
 from calibre.db.backend import DB
 from calibre.db.cache import Cache
 from calibre.db.categories import CATEGORY_SORTS
@@ -36,7 +38,7 @@ class LibraryDatabase(object):
             progress_callback=lambda x, y:True, restore_all_prefs=False):
 
         self.is_second_db = is_second_db  # TODO: Use is_second_db
-        self.listeners = set([])
+        self.listeners = set()
 
         backend = self.backend = DB(library_path, default_prefs=default_prefs,
                      read_only=read_only, restore_all_prefs=restore_all_prefs,
@@ -150,7 +152,7 @@ class LibraryDatabase(object):
     def path(self, index, index_is_id=False):
         'Return the relative path to the directory containing this books files as a unicode string.'
         book_id = index if index_is_id else self.data.index_to_id(index)
-        return self.data.cache.field_for('path', book_id).replace('/', os.sep)
+        return self.new_api.field_for('path', book_id).replace('/', os.sep)
 
     def abspath(self, index, index_is_id=False, create_dirs=True):
         'Return the absolute path to the directory containing this books files as a unicode string.'
@@ -159,6 +161,52 @@ class LibraryDatabase(object):
             os.makedirs(path)
         return path
 
+    # Adding books {{{
+    def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None):
+        return self.new_api.create_book_entry(mi, cover=cover, add_duplicates=add_duplicates, force_id=force_id)
+
+    def add_books(self, paths, formats, metadata, add_duplicates=True, return_ids=False):
+        books = [(mi, {fmt:path}) for mi, path, fmt in zip(metadata, paths, formats)]
+        book_ids, duplicates = self.new_api.add_books(books, add_duplicates=add_duplicates, dbapi=self)
+        if duplicates:
+            paths, formats, metadata = [], [], []
+            for mi, format_map in duplicates:
+                metadata.append(mi)
+                for fmt, path in format_map.iteritems():
+                    formats.append(fmt)
+                    paths.append(path)
+            duplicates = (paths, formats, metadata)
+        ids = book_ids if return_ids else len(book_ids)
+        return duplicates or None, ids
+
+    def import_book(self, mi, formats, notify=True, import_hooks=True, apply_import_tags=True, preserve_uuid=False):
+        format_map = {}
+        for path in formats:
+            ext = os.path.splitext(path)[1][1:].upper()
+            if ext == 'OPF':
+                continue
+            format_map[ext] = path
+        book_ids, duplicates = self.new_api.add_books(
+            [(mi, format_map)], add_duplicates=True, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid, dbapi=self, run_hooks=import_hooks)
+        if notify:
+            self.notify('add', book_ids)
+        return book_ids[0]
+
+    def find_books_in_directory(self, dirpath, single_book_per_directory):
+        return find_books_in_directory(dirpath, single_book_per_directory)
+
+    def import_book_directory_multiple(self, dirpath, callback=None,
+            added_ids=None):
+        return import_book_directory_multiple(self, dirpath, callback=callback, added_ids=added_ids)
+
+    def import_book_directory(self, dirpath, callback=None, added_ids=None):
+        return import_book_directory(self, dirpath, callback=callback, added_ids=added_ids)
+
+    def recursive_import(self, root, single_book_per_directory=True,
+            callback=None, added_ids=None):
+        return recursive_import(self, root, single_book_per_directory=single_book_per_directory, callback=callback, added_ids=added_ids)
+    # }}}
+
     # Private interface {{{
 
     def __iter__(self):

src/calibre/db/tests/legacy.py

@@ -7,8 +7,34 @@ __license__ = 'GPL v3'
 __copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import inspect
+from repr import repr
+from functools import partial
+from tempfile import NamedTemporaryFile
+
 from calibre.db.tests.base import BaseTest
 
+class ET(object):
+
+    def __init__(self, func_name, args, kwargs={}, old=None, legacy=None):
+        self.func_name = func_name
+        self.args, self.kwargs = args, kwargs
+        self.old, self.legacy = old, legacy
+
+    def __call__(self, test):
+        old = self.old or test.init_old(test.cloned_library)
+        legacy = self.legacy or test.init_legacy(test.cloned_library)
+        oldres = getattr(old, self.func_name)(*self.args, **self.kwargs)
+        newres = getattr(legacy, self.func_name)(*self.args, **self.kwargs)
+        test.assertEqual(oldres, newres, 'Equivalence test for %s with args: %s and kwargs: %s failed' % (
+            self.func_name, repr(self.args), repr(self.kwargs)))
+        self.retval = newres
+        return newres
+
+def compare_argspecs(old, new, attr):
+    ok = len(old.args) == len(new.args) and len(old.defaults or ()) == len(new.defaults or ()) and old.args[-len(old.defaults or ()):] == new.args[-len(new.defaults or ()):]  # noqa
+    if not ok:
+        raise AssertionError('The argspec for %s does not match. %r != %r' % (attr, old, new))
+
 class LegacyTest(BaseTest):
 
     ''' Test the emulation of the legacy interface. '''
@@ -119,6 +145,41 @@ class LegacyTest(BaseTest):
         db.close()
         # }}}
 
+    def test_legacy_adding_books(self):  # {{{
+        'Test various adding books methods'
+        from calibre.ebooks.metadata.book.base import Metadata
+        legacy, old = self.init_legacy(self.cloned_library), self.init_old(self.cloned_library)
+        mi = Metadata('Added Book0', authors=('Added Author',))
+        with NamedTemporaryFile(suffix='.aff') as f:
+            f.write(b'xxx')
+            f.flush()
+            T = partial(ET, 'add_books', ([f.name], ['AFF'], [mi]), old=old, legacy=legacy)
+            T()(self)
+            book_id = T(kwargs={'return_ids':True})(self)[1][0]
+            self.assertEqual(legacy.new_api.formats(book_id), ('AFF',))
+            T(kwargs={'add_duplicates':False})(self)
+            mi.title = 'Added Book1'
+            mi.uuid = 'uuu'
+            T = partial(ET, 'import_book', (mi,[f.name]), old=old, legacy=legacy)
+            book_id = T()(self)
+            self.assertNotEqual(legacy.uuid(book_id, index_is_id=True), old.uuid(book_id, index_is_id=True))
+            book_id = T(kwargs={'preserve_uuid':True})(self)
+            self.assertEqual(legacy.uuid(book_id, index_is_id=True), old.uuid(book_id, index_is_id=True))
+            self.assertEqual(legacy.new_api.formats(book_id), ('AFF',))
+        with NamedTemporaryFile(suffix='.opf') as f:
+            f.write(b'zzzz')
+            f.flush()
+            T = partial(ET, 'import_book', (mi,[f.name]), old=old, legacy=legacy)
+            book_id = T()(self)
+            self.assertFalse(legacy.new_api.formats(book_id))
+
+        mi.title = 'Added Book2'
+        T = partial(ET, 'create_book_entry', (mi,), old=old, legacy=legacy)
+        T()
+        T({'add_duplicates':False})
+        T({'force_id':1000})
+    # }}}
+
     def test_legacy_coverage(self):  # {{{
         ' Check that the emulation of the legacy interface is (almost) total '
         cl = self.cloned_library
@@ -130,15 +191,20 @@ class LegacyTest(BaseTest):
             '_set_title', '_set_custom', '_update_author_in_cache',
         }
         SKIP_ARGSPEC = {
-            '__init__',
+            '__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors',
         }
 
+        missing = []
+
         try:
+            total = 0
             for attr in dir(db):
                 if attr in SKIP_ATTRS:
                     continue
+                total += 1
                 if not hasattr(ndb, attr):
-                    raise AssertionError('The attribute %s is missing' % attr)
+                    missing.append(attr)
+                    continue
                 obj, nobj  = getattr(db, attr), getattr(ndb, attr)
                 if attr not in SKIP_ARGSPEC:
                     try:
@@ -146,11 +212,15 @@ class LegacyTest(BaseTest):
                     except TypeError:
                         pass
                     else:
-                        self.assertEqual(argspec, inspect.getargspec(nobj), 'argspec for %s not the same' % attr)
+                        compare_argspecs(argspec, inspect.getargspec(nobj), attr)
         finally:
             for db in (ndb, db):
                 db.close()
                 db.break_cycles()
 
+        if missing:
+            pc = len(missing)/total
+            raise AssertionError('{0:.1%} of API ({2} attrs) are missing. For example: {1}'.format(pc, missing[0], len(missing)))
+
     # }}}
 

src/calibre/gui2/__init__.py

@@ -110,6 +110,7 @@ defs['bd_overlay_cover_size'] = False
 defs['tags_browser_category_icons'] = {}
 defs['cover_browser_reflections'] = True
 defs['extra_row_spacing'] = 0
+defs['refresh_book_list_on_bulk_edit'] = True
 del defs
 # }}}
 

src/calibre/gui2/actions/edit_metadata.py

@@ -10,7 +10,7 @@ from functools import partial
 
 from PyQt4.Qt import QMenu, QModelIndex, QTimer, QIcon
 
-from calibre.gui2 import error_dialog, Dispatcher, question_dialog
+from calibre.gui2 import error_dialog, Dispatcher, question_dialog, gprefs
 from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 from calibre.gui2.dialogs.device_category_editor import DeviceCategoryEditor
@@ -366,8 +366,11 @@ class EditMetadataAction(InterfaceAction):
             self.gui.tags_view.blockSignals(False)
         if changed:
             m = self.gui.library_view.model()
-            m.refresh(reset=False)
+            if gprefs['refresh_book_list_on_bulk_edit']:
-            m.research()
+                m.refresh(reset=False)
+                m.research()
+            else:
+                m.refresh_ids(book_ids)
             self.gui.tags_view.recount()
             if self.gui.cover_flow:
                 self.gui.cover_flow.dataChanged()

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -317,6 +317,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         Ui_MetadataBulkDialog.__init__(self)
         self.model = model
         self.db = model.db
+        self.refresh_book_list.setChecked(gprefs['refresh_book_list_on_bulk_edit'])
+        self.refresh_book_list.toggled.connect(self.save_refresh_booklist)
         self.ids = [self.db.id(r) for r in rows]
         self.box_title.setText('<p>' +
                 _('Editing meta information for <b>%d books</b>') %
@@ -380,6 +382,9 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog):
         self.authors.setFocus(Qt.OtherFocusReason)
         self.exec_()
 
+    def save_refresh_booklist(self, *args):
+        gprefs['refresh_book_list_on_bulk_edit'] = bool(self.refresh_book_list.isChecked())
+
     def save_state(self, *args):
         gprefs['bulk_metadata_window_geometry'] = \
             bytearray(self.saveGeometry())

src/calibre/gui2/dialogs/metadata_bulk.ui

@@ -45,7 +45,7 @@
         <x>0</x>
         <y>0</y>
         <width>950</width>
-        <height>576</height>
+        <height>577</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_2">
@@ -1113,7 +1113,7 @@ not multiple and the destination field is multiple</string>
                 <x>0</x>
                 <y>0</y>
                 <width>934</width>
-                <height>213</height>
+                <height>256</height>
                </rect>
               </property>
               <layout class="QGridLayout" name="testgrid">
@@ -1170,14 +1170,30 @@ not multiple and the destination field is multiple</string>
     </widget>
    </item>
    <item row="2" column="0">
-    <widget class="QDialogButtonBox" name="button_box">
+    <layout class="QHBoxLayout" name="horizontalLayout_8">
-     <property name="orientation">
+     <item>
-      <enum>Qt::Horizontal</enum>
+      <widget class="QCheckBox" name="refresh_book_list">
-     </property>
+       <property name="toolTip">
-     <property name="standardButtons">
+        <string>If enabled, the book list will be re-sorted and any existing
-      <set>QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+search or Virtual LIbrary will be refreshed after the edit
-     </property>
+is completed. This can be slow on large libraries.</string>
-    </widget>
+       </property>
+       <property name="text">
+        <string>&amp;Refresh book list after edit</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QDialogButtonBox" name="button_box">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="standardButtons">
+        <set>QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+       </property>
+      </widget>
+     </item>
+    </layout>
    </item>
   </layout>
  </widget>
@@ -1241,7 +1257,6 @@ not multiple and the destination field is multiple</string>
   <tabstop>scrollArea</tabstop>
   <tabstop>central_widget</tabstop>
   <tabstop>query_field</tabstop>
-  <tabstop>button_box</tabstop>
   <tabstop>save_button</tabstop>
   <tabstop>remove_button</tabstop>
   <tabstop>search_field</tabstop>

src/calibre/library/database2.py

@@ -37,11 +37,12 @@ from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp,
 from calibre.utils.config import prefs, tweaks, from_json, to_json
 from calibre.utils.icu import sort_key, strcmp, lower
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
-from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
+from calibre.ebooks import check_ebook_format
 from calibre.utils.magick.draw import save_cover_data_to
 from calibre.utils.recycle_bin import delete_file, delete_tree
 from calibre.utils.formatter_functions import load_user_template_functions
 from calibre.db import _get_next_series_num_for_list, _get_series_values
+from calibre.db.adding import find_books_in_directory, import_book_directory_multiple, import_book_directory, recursive_import
 from calibre.db.errors import NoSuchFormat
 from calibre.db.lazy import FormatMetadata, FormatsList
 from calibre.db.categories import Tag, CATEGORY_SORTS
@@ -3728,95 +3729,18 @@ books_series_link      feeds
         return len(books)
 
     def find_books_in_directory(self, dirpath, single_book_per_directory):
-        dirpath = os.path.abspath(dirpath)
+        return find_books_in_directory(dirpath, single_book_per_directory)
-        if single_book_per_directory:
-            formats = []
-            for path in os.listdir(dirpath):
-                path = os.path.abspath(os.path.join(dirpath, path))
-                if os.path.isdir(path) or not os.access(path, os.R_OK):
-                    continue
-                ext = os.path.splitext(path)[1]
-                if not ext:
-                    continue
-                ext = ext[1:].lower()
-                if ext not in BOOK_EXTENSIONS and ext != 'opf':
-                    continue
-                formats.append(path)
-            yield formats
-        else:
-            books = {}
-            for path in os.listdir(dirpath):
-                path = os.path.abspath(os.path.join(dirpath, path))
-                if os.path.isdir(path) or not os.access(path, os.R_OK):
-                    continue
-                ext = os.path.splitext(path)[1]
-                if not ext:
-                    continue
-                ext = ext[1:].lower()
-                if ext not in BOOK_EXTENSIONS:
-                    continue
-
-                key = os.path.splitext(path)[0]
-                if key not in books:
-                    books[key] = []
-                books[key].append(path)
-
-            for formats in books.values():
-                yield formats
 
     def import_book_directory_multiple(self, dirpath, callback=None,
             added_ids=None):
-        from calibre.ebooks.metadata.meta import metadata_from_formats
+        return import_book_directory_multiple(self, dirpath, callback=callback, added_ids=added_ids)
-
-        duplicates = []
-        for formats in self.find_books_in_directory(dirpath, False):
-            mi = metadata_from_formats(formats)
-            if mi.title is None:
-                continue
-            if self.has_book(mi):
-                duplicates.append((mi, formats))
-                continue
-            book_id = self.import_book(mi, formats)
-            if added_ids is not None:
-                added_ids.add(book_id)
-            if callable(callback):
-                if callback(mi.title):
-                    break
-        return duplicates
 
     def import_book_directory(self, dirpath, callback=None, added_ids=None):
-        from calibre.ebooks.metadata.meta import metadata_from_formats
+        return import_book_directory(self, dirpath, callback=callback, added_ids=added_ids)
-        dirpath = os.path.abspath(dirpath)
-        formats = self.find_books_in_directory(dirpath, True)
-        formats = list(formats)[0]
-        if not formats:
-            return
-        mi = metadata_from_formats(formats)
-        if mi.title is None:
-            return
-        if self.has_book(mi):
-            return [(mi, formats)]
-        book_id = self.import_book(mi, formats)
-        if added_ids is not None:
-            added_ids.add(book_id)
-        if callable(callback):
-            callback(mi.title)
 
     def recursive_import(self, root, single_book_per_directory=True,
             callback=None, added_ids=None):
-        root = os.path.abspath(root)
+        return recursive_import(self, root, single_book_per_directory=single_book_per_directory, callback=callback, added_ids=added_ids)
-        duplicates  = []
-        for dirpath in os.walk(root):
-            res = (self.import_book_directory(dirpath[0], callback=callback,
-                added_ids=added_ids) if single_book_per_directory else
-                self.import_book_directory_multiple(dirpath[0],
-                    callback=callback, added_ids=added_ids))
-            if res is not None:
-                duplicates.extend(res)
-            if callable(callback):
-                if callback(''):
-                    break
-        return duplicates
 
     def add_custom_book_data(self, book_id, name, val):
         x = self.conn.get('SELECT id FROM books WHERE ID=?', (book_id,), all=False)