From 27b36ebefc4e3490fdc658bd9779c3a818734b18 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 5 Aug 2013 00:00:34 +0530 Subject: [PATCH] Fix write_lock being help on db when postimport plugins are run --- src/calibre/db/adding.py | 43 +++++++++++++------------- src/calibre/db/cache.py | 67 ++++++++++++++++++++-------------------- 2 files changed, 56 insertions(+), 54 deletions(-) diff --git a/src/calibre/db/adding.py b/src/calibre/db/adding.py index d777e7065e..0ca8c611d8 100644 --- a/src/calibre/db/adding.py +++ b/src/calibre/db/adding.py @@ -106,26 +106,27 @@ def add_catalog(cache, path, title): from calibre.utils.date import utcnow fmt = os.path.splitext(path)[1][1:].lower() - with lopen(path, 'rb') as stream, cache.write_lock: - matches = cache._search('title:="%s" and tags:="%s"' % (title.replace('"', '\\"'), _('Catalog')), None) - db_id = None - if matches: - db_id = list(matches)[0] - try: - mi = get_metadata(stream, fmt) - mi.authors = ['calibre'] - except: - mi = Metadata(title, ['calibre']) - mi.title, mi.authors = title, ['calibre'] - mi.tags = [_('Catalog')] - mi.pubdate = mi.timestamp = utcnow() - if fmt == 'mobi': - mi.cover, mi.cover_data = None, (None, None) - if db_id is None: - db_id = cache._create_book_entry(mi, apply_import_tags=False) - else: - cache._set_metadata(db_id, mi) - cache._add_format(db_id, fmt, stream) + with lopen(path, 'rb') as stream: + with cache.write_lock: + matches = cache._search('title:="%s" and tags:="%s"' % (title.replace('"', '\\"'), _('Catalog')), None) + db_id = None + if matches: + db_id = list(matches)[0] + try: + mi = get_metadata(stream, fmt) + mi.authors = ['calibre'] + except: + mi = Metadata(title, ['calibre']) + mi.title, mi.authors = title, ['calibre'] + mi.tags = [_('Catalog')] + mi.pubdate = mi.timestamp = utcnow() + if fmt == 'mobi': + mi.cover, mi.cover_data = None, (None, None) + if db_id is None: + db_id = cache._create_book_entry(mi, apply_import_tags=False) + else: + cache._set_metadata(db_id, mi) + cache.add_format(db_id, fmt, stream) # Cant keep write lock since post-import hooks might run return db_id @@ -156,7 +157,7 @@ def add_news(cache, path, arg): mi.timestamp = utcnow() db_id = cache._create_book_entry(mi, apply_import_tags=False) - cache._add_format(db_id, fmt, stream) + cache.add_format(db_id, fmt, stream) # Cant keep write lock since post-import hooks might run if not hasattr(path, 'read'): stream.close() diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 28058d7e8a..7db3a8b279 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -679,11 +679,10 @@ class Cache(object): fmtfile = self.format(book_id, original_fmt, as_file=True) if fmtfile is not None: fmt = original_fmt.partition('_')[2] - with self.write_lock: - with fmtfile: - self._add_format(book_id, fmt, fmtfile, run_hooks=False) - self._remove_formats({book_id:(original_fmt,)}) - return True + with fmtfile: + self.add_format(book_id, fmt, fmtfile, run_hooks=False) + self.remove_formats({book_id:(original_fmt,)}) + return True return False @read_api @@ -1150,38 +1149,40 @@ class Cache(object): self._reload_from_db() raise - @write_api + @api def add_format(self, book_id, fmt, stream_or_path, replace=True, run_hooks=True, dbapi=None): - if run_hooks: - # Run import plugins - npath = run_import_plugins(stream_or_path, fmt) - fmt = os.path.splitext(npath)[-1].lower().replace('.', '').upper() - stream_or_path = lopen(npath, 'rb') - fmt = check_ebook_format(stream_or_path, fmt) + with self.write_lock: + if run_hooks: + # Run import plugins + npath = run_import_plugins(stream_or_path, fmt) + fmt = os.path.splitext(npath)[-1].lower().replace('.', '').upper() + stream_or_path = lopen(npath, 'rb') + fmt = check_ebook_format(stream_or_path, fmt) - fmt = (fmt or '').upper() - self.format_metadata_cache[book_id].pop(fmt, None) - try: - name = self.fields['formats'].format_fname(book_id, fmt) - except: - name = None + fmt = (fmt or '').upper() + self.format_metadata_cache[book_id].pop(fmt, None) + try: + name = self.fields['formats'].format_fname(book_id, fmt) + except: + name = None - if name and not replace: - return False + if name and not replace: + return False - path = self._field_for('path', book_id).replace('/', os.sep) - title = self._field_for('title', book_id, default_value=_('Unknown')) - author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] - stream = stream_or_path if hasattr(stream_or_path, 'read') else lopen(stream_or_path, 'rb') - size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path) - del stream + path = self._field_for('path', book_id).replace('/', os.sep) + title = self._field_for('title', book_id, default_value=_('Unknown')) + author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] + stream = stream_or_path if hasattr(stream_or_path, 'read') else lopen(stream_or_path, 'rb') + size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path) + del stream - max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) - self.fields['size'].table.update_sizes({book_id: max_size}) - self._update_last_modified((book_id,)) + max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) + self.fields['size'].table.update_sizes({book_id: max_size}) + self._update_last_modified((book_id,)) if run_hooks: - # Run post import plugins + # Run post import plugins, the write lock is released so the plugin + # can call api without a locking violation. run_plugins_on_postimport(dbapi or self, book_id, fmt) stream_or_path.close() @@ -1305,17 +1306,17 @@ class Cache(object): return book_id - @write_api + @api def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, run_hooks=True, dbapi=None): duplicates, ids = [], [] for mi, format_map in books: - book_id = self._create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid) + book_id = self.create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid) if book_id is None: duplicates.append((mi, format_map)) else: ids.append(book_id) for fmt, stream_or_path in format_map.iteritems(): - self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi, run_hooks=run_hooks) + self.add_format(book_id, fmt, stream_or_path, dbapi=dbapi, run_hooks=run_hooks) return ids, duplicates @write_api