Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Adding books: Run in the main thread to prevent unfortunate interactions with the metadata backup. Also fix regression that broke the Abort button.

Browse Source
This commit is contained in:
Kovid Goyal 2010-12-07 10:52:36 -07:00
parent 03c6b10d95
commit 6e43796c9d
1 changed files with 98 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

172
src/calibre/gui2/add.py
View File

@ -3,41 +3,55 @@ UI for adding books to the database and saving books to disk
''' '''
import os, shutil, time import os, shutil, time
from Queue import Queue, Empty from Queue import Queue, Empty
from threading import Thread from functools import partial
from PyQt4.Qt import QThread, SIGNAL, QObject, QTimer, Qt, \ from PyQt4.Qt import QThread, QObject, Qt, QProgressDialog, pyqtSignal, QTimer
QProgressDialog
from calibre.gui2.dialogs.progress import ProgressDialog from calibre.gui2.dialogs.progress import ProgressDialog
from calibre.gui2 import question_dialog, error_dialog, info_dialog from calibre.gui2 import question_dialog, error_dialog, info_dialog
from calibre.ebooks.metadata.opf2 import OPF from calibre.ebooks.metadata.opf2 import OPF
from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.metadata import MetaInformation
from calibre.constants import preferred_encoding, filesystem_encoding from calibre.constants import preferred_encoding, filesystem_encoding, DEBUG
from calibre.utils.config import prefs from calibre.utils.config import prefs
from calibre import prints
single_shot = partial(QTimer.singleShot, 75)
class DuplicatesAdder(QObject): # {{{
added = pyqtSignal(object)
adding_done = pyqtSignal()
class DuplicatesAdder(QThread): # {{{
# Add duplicate books
def __init__(self, parent, db, duplicates, db_adder): def __init__(self, parent, db, duplicates, db_adder):
QThread.__init__(self, parent) QObject.__init__(self, parent)
self.db, self.db_adder = db, db_adder self.db, self.db_adder = db, db_adder
self.duplicates = duplicates self.duplicates = list(duplicates)
self.count = 0
single_shot(self.add_one)
def add_one(self):
if not self.duplicates:
self.adding_done.emit()
return
mi, cover, formats = self.duplicates.pop()
formats = [f for f in formats if not f.lower().endswith('.opf')]
id = self.db.create_book_entry(mi, cover=cover,
add_duplicates=True)
# here we add all the formats for dupe book record created above
self.db_adder.add_formats(id, formats)
self.db_adder.number_of_books_added += 1
self.count += 1
self.added.emit(self.count)
single_shot(self.add_one)
def run(self):
count = 1
for mi, cover, formats in self.duplicates:
formats = [f for f in formats if not f.lower().endswith('.opf')]
id = self.db.create_book_entry(mi, cover=cover,
add_duplicates=True)
# here we add all the formats for dupe book record created above
self.db_adder.add_formats(id, formats)
self.db_adder.number_of_books_added += 1
self.emit(SIGNAL('added(PyQt_PyObject)'), count)
count += 1
self.emit(SIGNAL('adding_done()'))
# }}} # }}}
class RecursiveFind(QThread): # {{{ class RecursiveFind(QThread): # {{{
update = pyqtSignal(object)
found = pyqtSignal(object)
def __init__(self, parent, db, root, single): def __init__(self, parent, db, root, single):
QThread.__init__(self, parent) QThread.__init__(self, parent)
self.db = db self.db = db
@ -50,8 +64,8 @@ class RecursiveFind(QThread): # {{{
for dirpath in os.walk(root): for dirpath in os.walk(root):
if self.canceled: if self.canceled:
return return
self.emit(SIGNAL('update(PyQt_PyObject)'), self.update.emit(
_('Searching in')+' '+dirpath[0]) _('Searching in')+' '+dirpath[0])
self.books += list(self.db.find_books_in_directory(dirpath[0], self.books += list(self.db.find_books_in_directory(dirpath[0],
self.single_book_per_directory)) self.single_book_per_directory))
@ -71,46 +85,55 @@ class RecursiveFind(QThread): # {{{
msg = unicode(err) msg = unicode(err)
except: except:
msg = repr(err) msg = repr(err)
self.emit(SIGNAL('found(PyQt_PyObject)'), msg) self.found.emit(msg)
return return
self.books = [formats for formats in self.books if formats] self.books = [formats for formats in self.books if formats]
if not self.canceled: if not self.canceled:
self.emit(SIGNAL('found(PyQt_PyObject)'), self.books) self.found.emit(self.books)
# }}} # }}}
class DBAdder(Thread): # {{{ class DBAdder(QObject): # {{{
def __init__(self, parent, db, ids, nmap):
QObject.__init__(self, parent)
def __init__(self, db, ids, nmap):
self.db, self.ids, self.nmap = db, dict(**ids), dict(**nmap) self.db, self.ids, self.nmap = db, dict(**ids), dict(**nmap)
self.end = False
self.critical = {} self.critical = {}
self.number_of_books_added = 0 self.number_of_books_added = 0
self.duplicates = [] self.duplicates = []
self.names, self.paths, self.infos = [], [], [] self.names, self.paths, self.infos = [], [], []
Thread.__init__(self)
self.daemon = True
self.input_queue = Queue() self.input_queue = Queue()
self.output_queue = Queue() self.output_queue = Queue()
self.merged_books = set([]) self.merged_books = set([])
def run(self): def end(self):
while not self.end: self.input_queue.put((None, None, None))
try:
id, opf, cover = self.input_queue.get(True, 0.2) def start(self):
except Empty: try:
continue id, opf, cover = self.input_queue.get_nowait()
name = self.nmap.pop(id) except Empty:
title = None single_shot(self.start)
try: return
title = self.add(id, opf, cover, name) if id is None and opf is None and cover is None:
except: return
import traceback name = self.nmap.pop(id)
self.critical[name] = traceback.format_exc() title = None
title = name if DEBUG:
self.output_queue.put(title) st = time.time()
try:
title = self.add(id, opf, cover, name)
except:
import traceback
self.critical[name] = traceback.format_exc()
title = name
self.output_queue.put(title)
if DEBUG:
prints('Added', title, 'to db in:', time.time() - st, 'seconds')
single_shot(self.start)
def process_formats(self, opf, formats): def process_formats(self, opf, formats):
imp = opf[:-4]+'.import' imp = opf[:-4]+'.import'
@ -201,10 +224,10 @@ class Adder(QObject): # {{{
self.pd.setModal(True) self.pd.setModal(True)
self.pd.show() self.pd.show()
self._parent = parent self._parent = parent
self.rfind = self.worker = self.timer = None self.rfind = self.worker = None
self.callback = callback self.callback = callback
self.callback_called = False self.callback_called = False
self.connect(self.pd, SIGNAL('canceled()'), self.canceled) self.pd.canceled_signal.connect(self.canceled)
def add_recursive(self, root, single=True): def add_recursive(self, root, single=True):
self.path = root self.path = root
@ -213,10 +236,8 @@ class Adder(QObject): # {{{
self.pd.set_max(0) self.pd.set_max(0)
self.pd.value = 0 self.pd.value = 0
self.rfind = RecursiveFind(self, self.db, root, single) self.rfind = RecursiveFind(self, self.db, root, single)
self.connect(self.rfind, SIGNAL('update(PyQt_PyObject)'), self.rfind.update.connect(self.pd.set_msg, type=Qt.QueuedConnection)
self.pd.set_msg, Qt.QueuedConnection) self.rfind.found.connect(self.add, type=Qt.QueuedConnection)
self.connect(self.rfind, SIGNAL('found(PyQt_PyObject)'),
self.add, Qt.QueuedConnection)
self.rfind.start() self.rfind.start()
def add(self, books): def add(self, books):
@ -246,12 +267,12 @@ class Adder(QObject): # {{{
self.pd.set_min(0) self.pd.set_min(0)
self.pd.set_max(len(self.ids)) self.pd.set_max(len(self.ids))
self.pd.value = 0 self.pd.value = 0
self.db_adder = DBAdder(self.db, self.ids, self.nmap) self.db_adder = DBAdder(self, self.db, self.ids, self.nmap)
self.db_adder.start() self.db_adder.start()
self.last_added_at = time.time() self.last_added_at = time.time()
self.entry_count = len(self.ids) self.entry_count = len(self.ids)
self.continue_updating = True self.continue_updating = True
QTimer.singleShot(200, self.update) single_shot(self.update)
def canceled(self): def canceled(self):
self.continue_updating = False self.continue_updating = False
@ -260,14 +281,14 @@ class Adder(QObject): # {{{
if self.worker is not None: if self.worker is not None:
self.worker.canceled = True self.worker.canceled = True
if hasattr(self, 'db_adder'): if hasattr(self, 'db_adder'):
self.db_adder.end = True self.db_adder.end()
self.pd.hide() self.pd.hide()
if not self.callback_called: if not self.callback_called:
self.callback(self.paths, self.names, self.infos) self.callback(self.paths, self.names, self.infos)
self.callback_called = True self.callback_called = True
def duplicates_processed(self): def duplicates_processed(self):
self.db_adder.end = True self.db_adder.end()
if not self.callback_called: if not self.callback_called:
self.callback(self.paths, self.names, self.infos) self.callback(self.paths, self.names, self.infos)
self.callback_called = True self.callback_called = True
@ -300,7 +321,7 @@ class Adder(QObject): # {{{
if (time.time() - self.last_added_at) > self.ADD_TIMEOUT: if (time.time() - self.last_added_at) > self.ADD_TIMEOUT:
self.continue_updating = False self.continue_updating = False
self.pd.hide() self.pd.hide()
self.db_adder.end = True self.db_adder.end()
if not self.callback_called: if not self.callback_called:
self.callback([], [], []) self.callback([], [], [])
self.callback_called = True self.callback_called = True
@ -311,7 +332,7 @@ class Adder(QObject): # {{{
'find the problem book.'), show=True) 'find the problem book.'), show=True)
if self.continue_updating: if self.continue_updating:
QTimer.singleShot(200, self.update) single_shot(self.update)
def process_duplicates(self): def process_duplicates(self):
@ -332,11 +353,8 @@ class Adder(QObject): # {{{
self.__p_d = pd self.__p_d = pd
self.__d_a = DuplicatesAdder(self._parent, self.db, duplicates, self.__d_a = DuplicatesAdder(self._parent, self.db, duplicates,
self.db_adder) self.db_adder)
self.connect(self.__d_a, SIGNAL('added(PyQt_PyObject)'), self.__d_a.added.connect(pd.setValue)
pd.setValue) self.__d_a.adding_done.connect(self.duplicates_processed)
self.connect(self.__d_a, SIGNAL('adding_done()'),
self.duplicates_processed)
self.__d_a.start()
else: else:
return self.duplicates_processed() return self.duplicates_processed()
@ -407,14 +425,12 @@ class Saver(QObject): # {{{
self.worker = SaveWorker(self.rq, db, self.ids, path, self.opts, self.worker = SaveWorker(self.rq, db, self.ids, path, self.opts,
spare_server=self.spare_server) spare_server=self.spare_server)
self.pd.canceled_signal.connect(self.canceled) self.pd.canceled_signal.connect(self.canceled)
self.timer = QTimer(self) self.continue_updating = True
self.connect(self.timer, SIGNAL('timeout()'), self.update) single_shot(self.update)
self.timer.start(200)
def canceled(self): def canceled(self):
if self.timer is not None: self.continue_updating = False
self.timer.stop()
if self.worker is not None: if self.worker is not None:
self.worker.canceled = True self.worker.canceled = True
self.pd.hide() self.pd.hide()
@ -424,27 +440,35 @@ class Saver(QObject): # {{{
def update(self): def update(self):
if not self.ids or not self.worker.is_alive(): if not self.continue_updating:
self.timer.stop() return
self.pd.hide() if not self.worker.is_alive():
# Check that all ids were processed
while self.ids: while self.ids:
# Get all queued results since worker is dead
before = len(self.ids) before = len(self.ids)
self.get_result() self.get_result()
if before == len(self.ids): if before == len(self.ids):
# No results available => worker died unexpectedly
for i in list(self.ids): for i in list(self.ids):
self.failures.add(('id:%d'%i, 'Unknown error')) self.failures.add(('id:%d'%i, 'Unknown error'))
self.ids.remove(i) self.ids.remove(i)
break
if not self.ids:
self.continue_updating = False
self.pd.hide()
if not self.callback_called: if not self.callback_called:
try: try:
self.worker.join(1.5) # Give the worker time to clean up and set worker.error
self.worker.join(2)
except: except:
pass # The worker was not yet started pass # The worker was not yet started
self.callback(self.worker.path, self.failures, self.worker.error)
self.callback_called = True self.callback_called = True
return self.callback(self.worker.path, self.failures, self.worker.error)
self.get_result() if self.continue_updating:
self.get_result()
single_shot(self.update)
def get_result(self): def get_result(self):