Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Initial implementation of new adding logic

Browse Source
This commit is contained in:
Kovid Goyal 2014-11-12 12:19:26 +05:30
parent 5d1adac683
commit bba5cbf11e
5 changed files with 513 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

200
src/calibre/ebooks/metadata/worker.py
View File

@ -6,178 +6,62 @@ __license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
from threading import Thread
from Queue import Empty
import os, time, sys, shutil
import os, shutil, errno
from calibre.utils.ipc.job import ParallelJob
from calibre.utils.ipc.server import Server
from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory
from calibre import prints
from calibre.constants import filesystem_encoding
from calibre.customize.ui import run_plugins_on_import
from calibre.db.utils import find_identical_books
from calibre.ebooks.metadata.meta import metadata_from_formats
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.utils.filenames import samefile
def debug(*args):
prints(*args)
sys.stdout.flush()
def serialize_metadata_for(formats, tdir, id_):
from calibre.ebooks.metadata.meta import metadata_from_formats
from calibre.ebooks.metadata.opf2 import metadata_to_opf
mi = metadata_from_formats(formats)
def serialize_metadata_for(paths, tdir, group_id):
mi = metadata_from_formats(paths)
mi.cover = None
cdata = None
if mi.cover_data:
cdata = mi.cover_data[-1]
mi.cover_data = None
mi.cover_data = (None, None)
if not mi.application_id:
mi.application_id = '__calibre_dummy__'
with open(os.path.join(tdir, '%s.opf'%id_), 'wb') as f:
f.write(metadata_to_opf(mi, default_lang='und'))
opf = metadata_to_opf(mi, default_lang='und')
has_cover = False
if cdata:
with open(os.path.join(tdir, str(id_)), 'wb') as f:
with open(os.path.join(tdir, '%s.cdata' % group_id), 'wb') as f:
f.write(cdata)
has_cover = True
return mi, opf, has_cover
def read_metadata_(task, tdir, notification=lambda x,y:x):
with TemporaryDirectory() as mdir:
do_read_metadata(task, tdir, mdir, notification)
def do_read_metadata(task, tdir, mdir, notification):
from calibre.customize.ui import run_plugins_on_import
for x in task:
try:
id_, formats = x
except:
def run_import_plugins(paths, group_id, tdir):
final_paths = []
for path in paths:
if not os.access(path, os.R_OK):
continue
try:
if isinstance(formats, basestring):
formats = [formats]
import_map = {}
fmts, metadata_fmts = [], []
for format in formats:
mfmt = format
name, ext = os.path.splitext(os.path.basename(format))
nfp = run_plugins_on_import(format)
if not nfp or nfp == format or not os.access(nfp, os.R_OK):
nfp = None
else:
nfp = run_plugins_on_import(path)
if nfp and os.access(nfp, os.R_OK) and not samefile(nfp, path):
# Ensure that the filename is preserved so that
# reading metadata from filename is not broken
nfp = os.path.abspath(nfp)
nfext = os.path.splitext(nfp)[1]
mfmt = os.path.join(mdir, name + nfext)
shutil.copyfile(nfp, mfmt)
metadata_fmts.append(mfmt)
fmts.append(nfp)
serialize_metadata_for(metadata_fmts, tdir, id_)
for format, nfp in zip(formats, fmts):
if not nfp:
continue
if isinstance(nfp, unicode):
nfp.encode(filesystem_encoding)
x = lambda j : os.path.abspath(os.path.normpath(os.path.normcase(j)))
if x(nfp) != x(format) and os.access(nfp, os.R_OK|os.W_OK):
fmt = os.path.splitext(format)[1].replace('.', '').lower()
nfmt = os.path.splitext(nfp)[1].replace('.', '').lower()
dest = os.path.join(tdir, '%s.%s'%(id_, nfmt))
shutil.copyfile(nfp, dest)
import_map[fmt] = dest
if import_map:
with open(os.path.join(tdir, str(id_)+'.import'), 'wb') as f:
for fmt, nfp in import_map.items():
f.write(fmt+':'+nfp+'\n')
notification(0.5, id_)
except:
import traceback
with open(os.path.join(tdir, '%s.error'%id_), 'wb') as f:
f.write(traceback.format_exc())
class Progress(object):
def __init__(self, result_queue, tdir):
self.result_queue = result_queue
self.tdir = tdir
def __call__(self, id):
cover = os.path.join(self.tdir, str(id))
if not os.path.exists(cover):
cover = None
res = os.path.join(self.tdir, '%s.error'%id)
if not os.path.exists(res):
res = res.replace('.error', '.opf')
self.result_queue.put((id, res, cover))
class ReadMetadata(Thread):
def __init__(self, tasks, result_queue, spare_server=None):
self.tasks, self.result_queue = tasks, result_queue
self.spare_server = spare_server
self.canceled = False
Thread.__init__(self)
self.daemon = True
self.failure_details = {}
self.tdir = PersistentTemporaryDirectory('_rm_worker')
def run(self):
jobs, ids = set([]), set([])
for t in self.tasks:
for b in t:
ids.add(b[0])
progress = Progress(self.result_queue, self.tdir)
server = Server() if self.spare_server is None else self.spare_server
name = os.path.splitext(os.path.basename(path))[0]
ext = os.path.splitext(nfp)[1]
path = os.path.join(tdir, '%s' % group_id, name + ext)
try:
for i, task in enumerate(self.tasks):
job = ParallelJob('read_metadata',
'Read metadata (%d of %d)'%(i, len(self.tasks)),
lambda x,y:x, args=[task, self.tdir])
jobs.add(job)
server.add_job(job)
while not self.canceled:
time.sleep(0.2)
running = False
for job in jobs:
while True:
os.mkdir(os.path.dirname(path))
except EnvironmentError as err:
if err.errno != errno.EEXIST:
raise
try:
id = job.notifications.get_nowait()[-1]
if id in ids:
progress(id)
ids.remove(id)
except Empty:
break
job.update(consume_notifications=False)
if not job.is_finished:
running = True
os.rename(nfp, path)
except EnvironmentError:
shutil.copyfile(nfp, path)
final_paths.append(path)
return final_paths
if not running:
break
finally:
server.close()
time.sleep(1)
if self.canceled:
return
for id in ids:
progress(id)
for job in jobs:
if job.failed:
prints(job.details)
if os.path.exists(job.log_path):
try:
os.remove(job.log_path)
except:
pass
def read_metadata(paths, result_queue, chunk=50, spare_server=None):
tasks = []
pos = 0
while pos < len(paths):
tasks.append(paths[pos:pos+chunk])
pos += chunk
t = ReadMetadata(tasks, result_queue, spare_server=spare_server)
t.start()
return t
def read_metadata(paths, group_id, tdir, common_data=None):
paths = run_import_plugins(paths, group_id, tdir)
mi, opf, has_cover = serialize_metadata_for(paths, tdir, group_id)
duplicate_info = None
if common_data is not None:
if isinstance(common_data, (set, frozenset)):
duplicate_info = mi.title and icu_lower(mi.title) in common_data
else:
duplicate_info = find_identical_books(mi, common_data)
return paths, opf, has_cover, duplicate_info

128
src/calibre/gui2/actions/add.py
View File

@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en'
import os
from functools import partial
from collections import defaultdict
from PyQt5.Qt import QPixmap, QTimer
@ -20,8 +19,6 @@ from calibre.gui2.dialogs.progress import ProgressDialog
from calibre.gui2.widgets import IMAGE_EXTENSIONS
from calibre.ebooks import BOOK_EXTENSIONS
from calibre.utils.filenames import ascii_filename
from calibre.utils.icu import sort_key
from calibre.constants import filesystem_encoding
from calibre.gui2.actions import InterfaceAction
from calibre.gui2 import question_dialog
from calibre.ebooks.metadata import MetaInformation
@ -154,15 +151,15 @@ class AddAction(InterfaceAction):
_('Select root folder'))
if not root:
return
lp = os.path.normcase(os.path.abspath(self.gui.current_db.library_path))
if lp.startswith(os.path.normcase(os.path.abspath(root)) + os.pathsep):
return error_dialog(self.gui, _('Cannot add'), _(
'Cannot add books from the folder: %s as it contains the currently opened calibre library') % root, show=True)
self.do_add_recursive(root, single)
def do_add_recursive(self, root, single):
from calibre.gui2.add import Adder
self._adder = Adder(self.gui,
self.gui.library_view.model().db,
self.Dispatcher(self._files_added), spare_server=self.gui.spare_server)
self.gui.tags_view.disable_recounting = True
self._adder.add_recursive(root, single)
from calibre.gui2.add2 import Adder
Adder(root, single_book_per_directory=single, db=self.gui.current_db, callback=self._files_added, parent=self.gui)
def add_recursive_single(self, *args):
'''
@ -364,75 +361,58 @@ class AddAction(InterfaceAction):
'cardb' if self.gui.stack.currentIndex() == 3 else None
if not paths:
return
from calibre.gui2.add import Adder
self.__adder_func = partial(self._files_added, on_card=on_card)
self._adder = Adder(self.gui,
None if to_device else self.gui.library_view.model().db,
self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server)
self.gui.tags_view.disable_recounting = True
self._adder.add(paths)
from calibre.gui2.add2 import Adder
Adder(paths, db=None if to_device else self.gui.current_db, parent=self.gui, callback=partial(self._files_added, on_card=on_card))
def _files_added(self, paths=[], names=[], infos=[], on_card=None):
self.gui.tags_view.disable_recounting = False
if paths:
self.gui.upload_books(paths,
list(map(ascii_filename, names)),
infos, on_card=on_card)
def _files_added(self, adder, on_card=None):
if adder.items:
paths, infos, names = [], [], []
for mi, cover_path, format_paths in adder.items:
mi.cover = cover_path
paths.append(format_paths[0]), infos.append(mi)
names.append(ascii_filename(os.path.basename(paths[-1])))
self.gui.upload_books(paths, names, infos, on_card=on_card)
self.gui.status_bar.show_message(
_('Uploading books to device.'), 2000)
if getattr(self._adder, 'number_of_books_added', 0) > 0:
self.gui.library_view.model().books_added(self._adder.number_of_books_added)
return
if adder.number_of_books_added > 0:
self.gui.library_view.model().books_added(adder.number_of_books_added)
self.gui.library_view.set_current_row(0)
if hasattr(self.gui, 'db_images'):
self.gui.db_images.beginResetModel(), self.gui.db_images.endResetModel()
self.gui.tags_view.recount()
if getattr(self._adder, 'merged_books', False):
merged = defaultdict(list)
for title, author in self._adder.merged_books:
merged[author].append(title)
lines = []
for author in sorted(merged, key=sort_key):
lines.append(author)
for title in sorted(merged[author], key=sort_key):
lines.append('\t' + title)
lines.append('')
info_dialog(self.gui, _('Merged some books'),
_('The following %d duplicate books were found and incoming '
'book formats were processed and merged into your '
'Calibre database according to your automerge '
'settings:')%len(self._adder.merged_books),
det_msg='\n'.join(lines), show=True)
# if getattr(self._adder, 'merged_books', False):
# merged = defaultdict(list)
# for title, author in self._adder.merged_books:
# merged[author].append(title)
# lines = []
# for author in sorted(merged, key=sort_key):
# lines.append(author)
# for title in sorted(merged[author], key=sort_key):
# lines.append('\t' + title)
# lines.append('')
# info_dialog(self.gui, _('Merged some books'),
# _('The following %d duplicate books were found and incoming '
# 'book formats were processed and merged into your '
# 'Calibre database according to your automerge '
# 'settings:')%len(self._adder.merged_books),
# det_msg='\n'.join(lines), show=True)
#
if getattr(self._adder, 'number_of_books_added', 0) > 0 or \
getattr(self._adder, 'merged_books', False):
# The formats of the current book could have changed if
# automerge is enabled
current_idx = self.gui.library_view.currentIndex()
if current_idx.isValid():
self.gui.library_view.model().current_changed(current_idx,
current_idx)
# if getattr(self._adder, 'number_of_books_added', 0) > 0 or \
# getattr(self._adder, 'merged_books', False):
# # The formats of the current book could have changed if
# # automerge is enabled
# current_idx = self.gui.library_view.currentIndex()
# if current_idx.isValid():
# self.gui.library_view.model().current_changed(current_idx,
# current_idx)
#
if getattr(self._adder, 'critical', None):
det_msg = []
for name, log in self._adder.critical.items():
if isinstance(name, str):
name = name.decode(filesystem_encoding, 'replace')
det_msg.append(name+'\n'+log)
warning_dialog(self.gui, _('Failed to read metadata'),
_('Failed to read metadata from the following')+':',
det_msg='\n\n'.join(det_msg), show=True)
if hasattr(self._adder, 'cleanup'):
self._adder.cleanup()
self._adder.setParent(None)
del self._adder
self._adder = None
def _add_from_device_adder(self, paths=[], names=[], infos=[],
on_card=None, model=None):
self._files_added(paths, names, infos, on_card=on_card)
def _add_from_device_adder(self, adder, on_card=None, model=None):
self._files_added(adder, on_card=on_card)
# set the in-library flags, and as a consequence send the library's
# metadata for this book to the device. This sets the uuid to the
# correct value. Note that set_books_in_library might sync_booklists
@ -503,12 +483,6 @@ class AddAction(InterfaceAction):
show=True)
if ok_paths:
from calibre.gui2.add import Adder
self.__adder_func = partial(self._add_from_device_adder, on_card=None,
model=view.model())
self._adder = Adder(self.gui, self.gui.library_view.model().db,
self.Dispatcher(self.__adder_func), spare_server=self.gui.spare_server)
self._adder.add(ok_paths)
from calibre.gui2.add2 import Adder
callback = partial(self._add_from_device_adder, on_card=None, model=view.model())
Adder(ok_paths, db=None, parent=self.gui, callback=callback)

388
src/calibre/gui2/add2.py Normal file
View File

@ -0,0 +1,388 @@
#!/usr/bin/env python
# vim:fileencoding=utf-8
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
import shutil, os, weakref, traceback
from threading import Thread
from collections import OrderedDict
from Queue import Empty
from io import BytesIO
from PyQt5.Qt import QObject, Qt, pyqtSignal
from calibre import prints
from calibre.customize.ui import run_plugins_on_postimport
from calibre.db.adding import find_books_in_directory
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.metadata.opf2 import OPF
from calibre.gui2 import error_dialog, warning_dialog
from calibre.gui2.dialogs.duplicates import DuplicatesQuestion
from calibre.gui2.dialogs.progress import ProgressDialog
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre.utils import join_with_timeout
from calibre.utils.config import prefs
from calibre.utils.ipc.pool import Pool, Failure
def validate_source(source, parent=None): # {{{
if isinstance(source, basestring):
if not os.path.exists(source):
error_dialog(parent, _('Cannot add books'), _(
'The path %s does not exist') % source, show=True)
return False
if os.path.isdir(source):
if not os.access(source, os.X_OK|os.R_OK):
error_dialog(parent, _('Cannot add books'), _(
'You do not have permission to read %s') % source, show=True)
return False
else:
if not os.access(source, os.R_OK):
error_dialog(parent, _('Cannot add books'), _(
'You do not have permission to read %s') % source, show=True)
return False
if not source.lower().rpartition(os.extsep) in {'zip', 'rar'}:
error_dialog(parent, _('Cannot add books'), _(
'The file %s is not a recognized archive format') % source, show=True)
return False
return True
# }}}
class Adder(QObject):
do_one_signal = pyqtSignal()
def __init__(self, source, single_book_per_directory=True, db=None, parent=None, callback=None):
if not validate_source(source, parent):
raise ValueError('Bad source')
QObject.__init__(self, parent)
self.single_book_per_directory = single_book_per_directory
self.callback = callback
self.add_formats_to_existing = prefs['add_formats_to_existing']
self.do_one_signal.connect(self.tick, type=Qt.QueuedConnection)
self.tdir = PersistentTemporaryDirectory('_add_books')
self.pool = None
self.pd = ProgressDialog(_('Adding books...'), _('Scanning for files...'), min=0, max=0, parent=parent, icon='add_book.png')
self.db = getattr(db, 'new_api', None)
if self.db is not None:
self.dbref = weakref.ref(db)
self.source = source
self.tdir = PersistentTemporaryDirectory('_add_books')
self.scan_error = None
self.file_groups = OrderedDict()
self.abort_scan = False
self.duplicates = []
self.report = []
self.items = []
self.added_book_ids = set()
self.added_duplicate_info = ({}, {}, {}) if self.add_formats_to_existing else set()
self.pd.show()
self.scan_thread = Thread(target=self.scan, name='ScanBooks')
self.scan_thread.daemon = True
self.scan_thread.start()
self.do_one = self.monitor_scan
self.do_one_signal.emit()
def break_cycles(self):
self.abort_scan = True
self.pd.close()
self.pd.deleteLater()
shutil.rmtree(self.tdir, ignore_errors=True)
if self.pool is not None:
self.pool.shutdown()
if not self.items:
shutil.rmtree(self.tdir, ignore_errors=True)
self.setParent(None)
self.added_duplicate_info = self.pool = self.items = self.duplicates = self.pd = self.db = self.dbref = self.tdir = self.file_groups = self.scan_thread = None # noqa
self.deleteLater()
def tick(self):
if self.pd.canceled:
try:
if callable(self.callback):
self.callback(self)
finally:
self.break_cycles()
return
self.do_one()
# Filesystem scan {{{
def scan(self):
try:
if isinstance(self.source, basestring):
if os.path.isdir(self.source):
root = self.source
else:
root = self.extract()
for dirpath, dirnames, filenames in os.walk(root):
for files in find_books_in_directory(dirpath, self.single_book_per_directory):
if self.abort_scan:
return
if files:
self.file_groups[len(self.file_groups)] = files
else:
unreadable_files = []
for path in self.source:
if self.abort_scan:
return
if os.access(path, os.R_OK):
self.file_groups[len(self.file_groups)] = [path]
else:
unreadable_files.append(path)
if unreadable_files:
if not self.file_groups:
self.scan_error = _('You do not have permission to read the selected file(s).') + '\n'
self.scan_error += '\n'.join(unreadable_files)
else:
a = self.report.append
for f in unreadable_files:
a(_('Could not add %s as you do not have permission to read the file' % f))
a('')
except Exception:
self.scan_error = traceback.format_exc()
def extract(self):
tdir = os.path.join(self.tdir, 'archive')
if self.source.lower().endswith('.zip'):
from calibre.utils.zipfile import ZipFile
try:
with ZipFile(self.source) as zf:
zf.extractall(tdir)
except Exception:
prints('Corrupt ZIP file, trying to use local headers')
from calibre.utils.localunzip import extractall
extractall(self.source, tdir)
elif self.path.lower().endswith('.rar'):
from calibre.utils.unrar import extract
extract(self.source, tdir)
return tdir
def monitor_scan(self):
self.scan_thread.join(0.05)
if self.scan_thread.is_alive():
self.do_one_signal.emit()
return
if self.scan_error is not None:
error_dialog(self.pd, _('Cannot add books'), _(
'Failed to add any books, click "Show details" for more information.'),
det_msg=self.scan_error, show=True)
self.break_cycles()
return
if not self.file_groups:
error_dialog(self.pd, _('Could not add'), _(
'No ebook files were found in %s') % self.source, show=True)
self.break_cycles()
return
self.pd.msg = _('Reading metadata and adding to library...')
self.pd.max = len(self.file_groups)
self.pd.value = 0
self.pool = Pool(name='AddBooks') if self.pool is None else self.pool
if self.db is not None:
data = self.db.data_for_find_identical_books() if self.add_formats_to_existing else self.db.data_for_has_book()
try:
self.pool.set_common_data(data)
except Failure as err:
error_dialog(self.pd, _('Cannot add books'), _(
'Failed to add any books, click "Show details" for more information.'),
det_msg=unicode(err.failure_message) + '\n' + unicode(err.details), show=True)
self.pd.canceled = True
self.groups_to_add = iter(self.file_groups)
self.do_one = self.do_one_group
self.do_one_signal.emit()
# }}}
def do_one_group(self):
try:
group_id = next(self.groups_to_add)
except StopIteration:
self.do_one = self.monitor_pool
self.do_one_signal.emit()
return
try:
self.pool(group_id, 'calibre.ebooks.metadata.worker', 'read_metadata',
self.file_groups[group_id], group_id, self.tdir)
except Failure as err:
error_dialog(self.pd, _('Cannot add books'), _(
'Failed to add any books, click "Show details" for more information.'),
det_msg=unicode(err.failure_message) + '\n' + unicode(err.details), show=True)
self.pd.canceled = True
self.do_one_signal.emit()
def monitor_pool(self):
try:
worker_result = self.pool.results.get(True, 0.05)
self.pool.results.task_done()
except Empty:
try:
self.pool.wait_for_tasks(timeout=0.01)
except RuntimeError:
pass # Tasks still remaining
except Failure as err:
error_dialog(self.pd, _('Cannot add books'), _(
'Failed to add some books, click "Show details" for more information.'),
det_msg=unicode(err.failure_message) + '\n' + unicode(err.details), show=True)
self.pd.canceled = True
else:
# All tasks completed
try:
join_with_timeout(self.pool.results, 0.01)
except RuntimeError:
pass # There are results remaining
else:
# No results left
self.process_duplicates()
return
else:
group_id = worker_result.id
if worker_result.is_terminal_failure:
error_dialog(self.pd, _('Critical failure'), _(
'The read metadata worker process crashed while processing'
' some files. Adding of books is aborted. Click "Show details"'
' to see which files caused the problem.'), show=True,
det_msg='\n'.join(self.file_groups[group_id]))
self.pd.canceled = True
else:
try:
self.process_result(group_id, worker_result.result)
except Exception:
self.report_metadata_failure(group_id, traceback.format_exc())
self.pd.value += 1
self.do_one_signal.emit()
def report_metadata_failure(self, group_id, details):
a = self.report.append
paths = self.file_groups[group_id]
a(''), a('-' * 70)
a(_('Failed to read metadata from the file(s):'))
[a('\t' + f) for f in paths]
a(_('With error:')), a(details)
mi = Metadata(_('Unknown'))
mi.read_metadata_failed = False
return mi
def process_result(self, group_id, result):
if result.err:
mi = self.report_metadata_failure(group_id, result.traceback)
paths = self.file_groups[group_id]
has_cover = False
duplicate_info = set() if self.add_formats_to_existing else False
else:
paths, opf, has_cover, duplicate_info = result.value
try:
mi = OPF(BytesIO(opf), basedir=self.tdir, populate_spine=False, try_to_guess_cover=False).to_book_metadata()
mi.read_metadata_failed = False
except Exception:
mi = self.report_metadata_failure(group_id, traceback.format_exc())
if mi.is_null('title'):
for path in paths:
mi.title = os.path.splitext(os.path.basename(path))[0]
break
if mi.application_id == '__calibre_dummy__':
mi.application_id = None
self.pd.msg = mi.title
cover_path = os.path.join(self.tdir, '%s.cdata' % group_id) if has_cover else None
if self.db is None:
if paths:
self.items.append((mi, cover_path, paths))
return
if self.add_formats_to_existing:
pass # TODO: Implement this
else:
if duplicate_info or icu_lower(mi.title or _('Unknown')) in self.added_duplicate_info:
self.duplicates.append((mi, cover_path, paths))
else:
self.add_book(mi, cover_path, paths)
def add_book(self, mi, cover_path, paths):
try:
cdata = None
if cover_path:
with open(cover_path, 'rb') as f:
cdata = f.read()
book_id = self.dbref().create_book_entry(mi, cover=cdata)
self.added_book_ids.add(book_id)
except Exception:
a = self.report.append
a(''), a('-' * 70)
a(_('Failed to add the book: ') + mi.title)
[a('\t' + f) for f in paths]
a(_('With error:')), a(traceback.format_exc())
return
else:
self.add_formats(book_id, paths, mi)
if self.add_formats_to_existing:
pass # TODO: Implement this
else:
self.added_duplicate_info.add(icu_lower(mi.title or _('Unknown')))
def add_formats(self, book_id, paths, mi):
fmap = {p.rpartition(os.path.extsep)[-1].lower():p for p in paths}
for fmt, path in fmap.iteritems():
# The onimport plugins have already been run by the read metadata
# worker
try:
if self.db.add_format(book_id, fmt, path, run_hooks=False):
run_plugins_on_postimport(self.dbref(), book_id, fmt)
except Exception:
a = self.report.append
a(''), a('-' * 70)
a(_('Failed to add the file {0} to the book: {1}').format(path, mi.title))
a(_('With error:')), a(traceback.format_exc())
def process_duplicates(self):
if self.duplicates:
d = DuplicatesQuestion(self.dbref(), self.duplicates, self.pd)
duplicates = tuple(d.duplicates)
d.deleteLater()
if duplicates:
self.do_one = self.process_duplicate
self.duplicates_to_process = iter(duplicates)
self.do_one_signal.emit()
return
self.finish()
def process_duplicate(self):
try:
mi, cover_path, paths = next(self.duplicates_to_process)
except StopIteration:
self.finish()
return
self.add_book(mi, cover_path, paths)
self.do_one_signal.emit()
def finish(self):
if self.report:
added_some = self.items or self.added_book_ids
d = warning_dialog if added_some else error_dialog
msg = _('There were problems adding some files, click "Show details" for more information') if added_some else _(
'Failed to add any books, click "Show details" for more information')
d(self.pd, _('Errors while adding'), msg, det_msg='\n'.join(self.report), show=True)
try:
if callable(self.callback):
self.callback(self)
finally:
self.break_cycles()
@property
def number_of_books_added(self):
return len(self.added_book_ids)
# TODO: Duplicates and auto-merge (in particular adding duplicated files as well as adding files already in the db)
# TODO: Test importing with filetype plugin (archive, de-obfuscate)
# TODO: Test recursive adding when no books are found
# TODO: Test handling of exception in metadata read function
# TODO: Report terminal erros where some books have been added better
# TODO: Test direct add of books to device
# TODO: Test adding form device to library
# TODO: Check aborting after a few books have been added

6
src/calibre/gui2/main_window.py
View File

@ -109,6 +109,12 @@ class MainWindow(QMainWindow):
if disable_automatic_gc:
self._gc = GarbageCollector(self, debug=False)
def enable_garbage_collection(self, enabled=True):
if hasattr(self, '_gc'):
self._gc.timer.blockSignals(not enabled)
else:
gc.enable() if enabled else gc.disable()
def unhandled_exception(self, type, value, tb):
if type == KeyboardInterrupt:
self.keyboard_interrupt.emit()

3
src/calibre/utils/ipc/worker.py
View File

@ -46,9 +46,6 @@ PARALLEL_FUNCS = {
'move_library' :
('calibre.library.move', 'move_library', 'notification'),
'read_metadata' :
('calibre.ebooks.metadata.worker', 'read_metadata_', 'notification'),
'arbitrary' :
('calibre.utils.ipc.worker', 'arbitrary', None),