Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add a check database integrity button to the Advanced Preferences. Also make adding books more robust

Browse Source
This commit is contained in:
Kovid Goyal 2009-08-09 23:09:00 -06:00
parent 56acf8fb56
commit aefd2d55a0
13 changed files with 366 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/calibre/constants.py
View File

@ -38,6 +38,11 @@ fcntl = None if iswindows else __import__('fcntl')
filesystem_encoding = sys.getfilesystemencoding()
if filesystem_encoding is None: filesystem_encoding = 'utf-8'
DEBUG = False
def debug():
global DEBUG
DEBUG = True
################################################################################
plugins = None

2
src/calibre/debug.py
View File

@ -164,6 +164,8 @@ def add_simple_plugin(path_to_plugin):
def main(args=sys.argv):
from calibre.constants import debug
debug()
opts, args = option_parser().parse_args(args)
if opts.gui:
from calibre.gui2.main import main

6
src/calibre/devices/blackberry/driver.py
View File

@ -17,15 +17,15 @@ class BLACKBERRY(USBMS):
FORMATS = ['mobi', 'prc']
VENDOR_ID = [0x0fca]
PRODUCT_ID = [0x8004]
BCD = [0x0200]
PRODUCT_ID = [0x8004, 0x0004]
BCD = [0x0200, 0x0107]
VENDOR_NAME = 'RIM'
WINDOWS_MAIN_MEM = 'BLACKBERRY_SD'
#OSX_MAIN_MEM = 'Kindle Internal Storage Media'
MAIN_MEMORY_VOLUME_LABEL = 'Blackberry Main Memory'
MAIN_MEMORY_VOLUME_LABEL = 'Blackberry SD Card'
EBOOK_DIR_MAIN = 'ebooks'
SUPPORTS_SUB_DIRS = True

5
src/calibre/devices/usbms/books.py
View File

@ -17,7 +17,10 @@ class Book(object):
self.authors = authors
self.mime = mime
self.size = os.path.getsize(path)
self.datetime = time.gmtime(os.path.getctime(path))
try:
self.datetime = time.gmtime(os.path.getctime(path))
except ValueError:
self.datetime = time.gmtime()
self.path = path
self.thumbnail = None
self.tags = []

74
src/calibre/ebooks/metadata/worker.py
View File

@ -8,12 +8,13 @@ __docformat__ = 'restructuredtext en'
from threading import Thread
from Queue import Empty
import os, time, sys
import os, time, sys, shutil
from calibre.utils.ipc.job import ParallelJob
from calibre.utils.ipc.server import Server
from calibre.ptempfile import PersistentTemporaryDirectory
from calibre import prints
from calibre.constants import filesystem_encoding
def debug(*args):
@ -23,6 +24,7 @@ def debug(*args):
def read_metadata_(task, tdir, notification=lambda x,y:x):
from calibre.ebooks.metadata.meta import metadata_from_formats
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.customize.ui import run_plugins_on_import
for x in task:
try:
id, formats = x
@ -38,6 +40,24 @@ def read_metadata_(task, tdir, notification=lambda x,y:x):
if cdata:
with open(os.path.join(tdir, str(id)), 'wb') as f:
f.write(cdata)
import_map = {}
for format in formats:
nfp = run_plugins_on_import(format)
nfp = os.path.abspath(nfp)
if isinstance(nfp, unicode):
nfp.encode(filesystem_encoding)
x = lambda j : os.path.abspath(os.path.normpath(os.path.normcase(j)))
if x(nfp) != x(format) and os.access(nfp, os.R_OK|os.W_OK):
fmt = os.path.splitext(format)[1].replace('.', '').lower()
nfmt = os.path.splitext(nfp)[1].replace('.', '').lower()
dest = os.path.join(tdir, '%s.%s'%(id, nfmt))
shutil.copyfile(nfp, dest)
import_map[fmt] = dest
os.remove(nfp)
if import_map:
with open(os.path.join(tdir, str(id)+'.import'), 'wb') as f:
for fmt, nfp in import_map.items():
f.write(fmt+':'+nfp+'\n')
notification(0.5, id)
except:
import traceback
@ -66,6 +86,7 @@ class ReadMetadata(Thread):
self.canceled = False
Thread.__init__(self)
self.daemon = True
self.failure_details = {}
self.tdir = PersistentTemporaryDirectory('_rm_worker')
@ -76,33 +97,34 @@ class ReadMetadata(Thread):
ids.add(b[0])
progress = Progress(self.result_queue, self.tdir)
server = Server() if self.spare_server is None else self.spare_server
for i, task in enumerate(self.tasks):
job = ParallelJob('read_metadata',
'Read metadata (%d of %d)'%(i, len(self.tasks)),
lambda x,y:x, args=[task, self.tdir])
jobs.add(job)
server.add_job(job)
try:
for i, task in enumerate(self.tasks):
job = ParallelJob('read_metadata',
'Read metadata (%d of %d)'%(i, len(self.tasks)),
lambda x,y:x, args=[task, self.tdir])
jobs.add(job)
server.add_job(job)
while not self.canceled:
time.sleep(0.2)
running = False
for job in jobs:
while True:
try:
id = job.notifications.get_nowait()[-1]
if id in ids:
progress(id)
ids.remove(id)
except Empty:
break
job.update(consume_notifications=False)
if not job.is_finished:
running = True
while not self.canceled:
time.sleep(0.2)
running = False
for job in jobs:
while True:
try:
id = job.notifications.get_nowait()[-1]
if id in ids:
progress(id)
ids.remove(id)
except Empty:
break
job.update(consume_notifications=False)
if not job.is_finished:
running = True
if not running:
break
server.close()
if not running:
break
finally:
server.close()
time.sleep(1)
if self.canceled:

4
src/calibre/ebooks/oeb/transforms/rasterize.py
View File

@ -187,6 +187,10 @@ class SVGRasterizer(object):
covers = self.oeb.metadata.cover
if not covers:
return
if unicode(covers[0]) not in self.oeb.manifest.ids:
self.oeb.logger.warn('Cover not in manifest, skipping.')
self.oeb.metadata.clear('cover')
return
cover = self.oeb.manifest.ids[unicode(covers[0])]
if not cover.media_type == SVG_MIME:
return

40
src/calibre/gui2/__init__.py
View File

@ -103,6 +103,33 @@ def available_width():
def extension(path):
return os.path.splitext(path)[1][1:].lower()
class CopyButton(QPushButton):
ACTION_KEYS = [Qt.Key_Enter, Qt.Key_Return, Qt.Key_Space]
def copied(self):
self.emit(SIGNAL('copy()'))
self.setDisabled(True)
self.setText(_('Copied to clipboard'))
def keyPressEvent(self, ev):
if ev.key() in self.ACTION_KEYS:
self.copied()
else:
QPushButton.event(self, ev)
def keyReleaseEvent(self, ev):
if ev.key() in self.ACTION_KEYS:
pass
else:
QPushButton.event(self, ev)
def mouseReleaseEvent(self, ev):
ev.accept()
self.copied()
class MessageBox(QMessageBox):
def __init__(self, type_, title, msg, buttons, parent, det_msg=''):
@ -111,9 +138,16 @@ class MessageBox(QMessageBox):
self.msg = msg
self.det_msg = det_msg
self.setDetailedText(det_msg)
self.cb = QPushButton(_('Copy to Clipboard'))
self.layout().addWidget(self.cb)
self.connect(self.cb, SIGNAL('clicked()'), self.copy_to_clipboard)
# Cannot set keyboard shortcut as the event is not easy to filter
self.cb = CopyButton(_('Copy to Clipboard'))
self.connect(self.cb, SIGNAL('copy()'), self.copy_to_clipboard)
self.addButton(self.cb, QMessageBox.ActionRole)
default_button = self.button(self.Ok)
if default_button is None:
default_button = self.button(self.Yes)
if default_button is not None:
self.setDefaultButton(default_button)
def copy_to_clipboard(self):
QApplication.clipboard().setText('%s: %s\n\n%s' %

216
src/calibre/gui2/add.py
View File

@ -1,13 +1,14 @@
'''
UI for adding books to the database and saving books to disk
'''
import os, shutil
import os, shutil, time
from Queue import Queue, Empty
from threading import Thread
from PyQt4.Qt import QThread, SIGNAL, QObject, QTimer, Qt
from calibre.gui2.dialogs.progress import ProgressDialog
from calibre.gui2 import question_dialog
from calibre.gui2 import question_dialog, error_dialog
from calibre.ebooks.metadata.opf2 import OPF
from calibre.ebooks.metadata import MetaInformation
from calibre.constants import preferred_encoding
@ -36,6 +37,96 @@ class RecursiveFind(QThread):
if not self.canceled:
self.emit(SIGNAL('found(PyQt_PyObject)'), self.books)
class DBAdder(Thread):
def __init__(self, db, ids, nmap):
self.db, self.ids, self.nmap = db, dict(**ids), dict(**nmap)
self.end = False
self.critical = {}
self.number_of_books_added = 0
self.duplicates = []
self.names, self.path, self.infos = [], [], []
Thread.__init__(self)
self.daemon = True
self.input_queue = Queue()
self.output_queue = Queue()
def run(self):
while not self.end:
try:
id, opf, cover = self.input_queue.get(True, 0.2)
except Empty:
continue
name = self.nmap.pop(id)
title = None
try:
title = self.add(id, opf, cover, name)
except:
import traceback
self.critical[name] = traceback.format_exc()
title = name
self.output_queue.put(title)
def process_formats(self, opf, formats):
imp = opf[:-4]+'.import'
if not os.access(imp, os.R_OK):
return formats
fmt_map = {}
for line in open(imp, 'rb').readlines():
if ':' not in line:
continue
f, _, p = line.partition(':')
fmt_map[f] = p.rstrip()
fmts = []
for fmt in formats:
e = os.path.splitext(fmt)[1].replace('.', '').lower()
fmts.append(fmt_map.get(e, fmt))
if not os.access(fmts[-1], os.R_OK):
fmts[-1] = fmt
return fmts
def add(self, id, opf, cover, name):
formats = self.ids.pop(id)
if opf.endswith('.error'):
mi = MetaInformation('', [_('Unknown')])
self.critical[name] = open(opf, 'rb').read().decode('utf-8', 'replace')
else:
try:
mi = MetaInformation(OPF(opf))
except:
import traceback
mi = MetaInformation('', [_('Unknown')])
self.critical[name] = traceback.format_exc()
formats = self.process_formats(opf, formats)
if not mi.title:
mi.title = os.path.splitext(name)[0]
mi.title = mi.title if isinstance(mi.title, unicode) else \
mi.title.decode(preferred_encoding, 'replace')
if self.db is not None:
if cover:
cover = open(cover, 'rb').read()
id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
self.number_of_books_added += 1
if id is None:
self.duplicates.append((mi, cover, formats))
else:
self.add_formats(id, formats)
else:
self.names.append(name)
self.paths.append(formats[0])
self.infos.append({'title':mi.title,
'authors':', '.join(mi.authors),
'cover':None,
'tags':mi.tags if mi.tags else []})
return mi.title
def add_formats(self, id, formats):
for path in formats:
fmt = os.path.splitext(path)[-1].replace('.', '').upper()
with open(path, 'rb') as f:
self.db.add_format(id, fmt, f, index_is_id=True,
notify=False)
class Adder(QObject):
@ -44,15 +135,12 @@ class Adder(QObject):
self.pd = ProgressDialog(_('Adding...'), parent=parent)
self.spare_server = spare_server
self.db = db
self.critical = {}
self.pd.setModal(True)
self.pd.show()
self._parent = parent
self.number_of_books_added = 0
self.rfind = self.worker = self.timer = None
self.callback = callback
self.callback_called = False
self.infos, self.paths, self.names = [], [], []
self.connect(self.pd, SIGNAL('canceled()'), self.canceled)
def add_recursive(self, root, single=True):
@ -87,32 +175,33 @@ class Adder(QObject):
self.pd.set_max(len(self.ids))
self.pd.value = 0
self.timer = QTimer(self)
self.db_adder = DBAdder(self.db, self.ids, self.nmap)
self.db_adder.start()
self.connect(self.timer, SIGNAL('timeout()'), self.update)
self.last_added_at = time.time()
self.entry_count = len(self.ids)
self.timer.start(200)
def add_formats(self, id, formats):
for path in formats:
fmt = os.path.splitext(path)[-1].replace('.', '').upper()
self.db.add_format_with_hooks(id, fmt, path, index_is_id=True,
notify=False)
def canceled(self):
if self.rfind is not None:
self.rfind.cenceled = True
self.rfind.canceled = True
if self.timer is not None:
self.timer.stop()
if self.worker is not None:
self.worker.canceled = True
if hasattr(self, 'db_adder'):
self.db_adder.end = True
self.pd.hide()
if not self.callback_called:
self.callback(self.paths, self.names, self.infos)
self.callback_called = True
def update(self):
if not self.ids:
if self.entry_count <= 0:
self.timer.stop()
self.process_duplicates()
self.pd.hide()
self.db_adder.end = True
if not self.callback_called:
self.callback(self.paths, self.names, self.infos)
self.callback_called = True
@ -120,61 +209,53 @@ class Adder(QObject):
try:
id, opf, cover = self.rq.get_nowait()
self.db_adder.input_queue.put((id, opf, cover))
self.last_added_at = time.time()
except Empty:
return
self.pd.value += 1
formats = self.ids.pop(id)
name = self.nmap.pop(id)
if opf.endswith('.error'):
mi = MetaInformation('', [_('Unknown')])
self.critical[name] = open(opf, 'rb').read().decode('utf-8', 'replace')
else:
try:
mi = MetaInformation(OPF(opf))
except:
import traceback
mi = MetaInformation('', [_('Unknown')])
self.critical[name] = traceback.format_exc()
if not mi.title:
mi.title = os.path.splitext(name)[0]
mi.title = mi.title if isinstance(mi.title, unicode) else \
mi.title.decode(preferred_encoding, 'replace')
pass
if self.db is not None:
if cover:
cover = open(cover, 'rb').read()
id = self.db.create_book_entry(mi, cover=cover, add_duplicates=False)
self.number_of_books_added += 1
if id is None:
self.duplicates.append((mi, cover, formats))
else:
self.add_formats(id, formats)
else:
self.names.append(name)
self.paths.append(formats[0])
self.infos.append({'title':mi.title,
'authors':', '.join(mi.authors),
'cover':None,
'tags':mi.tags if mi.tags else []})
try:
title = self.db_adder.output_queue.get_nowait()
self.pd.value += 1
self.pd.set_msg(_('Added')+' '+title)
self.last_added_at = time.time()
self.entry_count -= 1
except Empty:
pass
self.pd.set_msg(_('Added')+' '+mi.title)
if (time.time() - self.last_added_at) > 300:
self.timer.stop()
self.pd.hide()
self.db_adder.end = True
if not self.callback_called:
self.callback([], [], [])
self.callback_called = True
error_dialog(self._parent, _('Adding failed'),
_('The add books process seems to have hung.'
' Try restarting calibre and adding the '
'books in smaller increments, until you '
'find the problem book.'), show=True)
def process_duplicates(self):
if not self.duplicates:
duplicates = self.db_adder.duplicates
if not duplicates:
return
files = [x[0].title for x in self.duplicates]
self.pd.hide()
files = [x[0].title for x in duplicates]
if question_dialog(self._parent, _('Duplicates found!'),
_('Books with the same title as the following already '
'exist in the database. Add them anyway?'),
'\n'.join(files)):
for mi, cover, formats in self.duplicates:
for mi, cover, formats in duplicates:
id = self.db.create_book_entry(mi, cover=cover,
add_duplicates=True)
self.add_formats(id, formats)
self.number_of_books_added += 1
self.db_adder.add_formats(id, formats)
self.db_adder.number_of_books_added += 1
def cleanup(self):
if hasattr(self, 'pd'):
self.pd.hide()
if hasattr(self, 'worker') and hasattr(self.worker, 'tdir') and \
self.worker.tdir is not None:
if os.path.exists(self.worker.tdir):
@ -183,6 +264,35 @@ class Adder(QObject):
except:
pass
@property
def number_of_books_added(self):
return getattr(getattr(self, 'db_adder', None), 'number_of_books_added',
0)
@property
def critical(self):
return getattr(getattr(self, 'db_adder', None), 'critical',
{})
@property
def paths(self):
return getattr(getattr(self, 'db_adder', None), 'paths',
[])
@property
def names(self):
return getattr(getattr(self, 'db_adder', None), 'names',
[])
@property
def infos(self):
return getattr(getattr(self, 'db_adder', None), 'infos',
[])
###############################################################################
############################## END ADDER ######################################
###############################################################################
class Saver(QObject):
def __init__(self, parent, db, callback, rows, path,

42
src/calibre/gui2/dialogs/config.py
View File

@ -5,14 +5,15 @@ import os, re, time, textwrap
from PyQt4.Qt import QDialog, QMessageBox, QListWidgetItem, QIcon, \
QDesktopServices, QVBoxLayout, QLabel, QPlainTextEdit, \
QStringListModel, QAbstractItemModel, QFont, \
SIGNAL, QTimer, Qt, QSize, QVariant, QUrl, \
SIGNAL, QThread, Qt, QSize, QVariant, QUrl, \
QModelIndex, QInputDialog, QAbstractTableModel, \
QDialogButtonBox, QTabWidget, QBrush, QLineEdit
from calibre.constants import islinux, iswindows
from calibre.gui2.dialogs.config_ui import Ui_Dialog
from calibre.gui2 import qstring_to_unicode, choose_dir, error_dialog, config, \
ALL_COLUMNS, NONE, info_dialog, choose_files
ALL_COLUMNS, NONE, info_dialog, choose_files, \
warning_dialog
from calibre.utils.config import prefs
from calibre.gui2.widgets import FilenamePattern
from calibre.gui2.library import BooksModel
@ -736,19 +737,46 @@ class ConfigDialog(QDialog, Ui_Dialog):
config['frequently_used_directories'] = self.directories
QDialog.accept(self)
class VacThread(QThread):
def __init__(self, parent, db):
QThread.__init__(self, parent)
self.db = db
self._parent = parent
def run(self):
bad = self.db.check_integrity()
self.emit(SIGNAL('check_done(PyQt_PyObject)'), bad)
class Vacuum(QMessageBox):
def __init__(self, parent, db):
self.db = db
QMessageBox.__init__(self, QMessageBox.Information, _('Compacting...'),
_('Compacting database. This may take a while.'),
QMessageBox.__init__(self, QMessageBox.Information, _('Checking...'),
_('Checking database integrity. This may take a while.'),
QMessageBox.NoButton, parent)
QTimer.singleShot(200, self.vacuum)
self.vthread = VacThread(self, db)
self.connect(self.vthread, SIGNAL('check_done(PyQt_PyObject)'),
self.check_done,
Qt.QueuedConnection)
self.vthread.start()
def vacuum(self):
self.db.vacuum()
def check_done(self, bad):
if bad:
titles = [self.db.title(x, index_is_id=True) for x in bad]
det_msg = '\n'.join(titles)
warning_dialog(self, _('Some inconsistencies found'),
_('The following books had formats listed in the '
'database that are not actually available. '
'The entries for the formats have been removed. '
'You should check them manually. This can '
'happen if you manipulate the files in the '
'library folder directly.'), det_msg=det_msg, show=True)
self.accept()
if __name__ == '__main__':
from calibre.library.database2 import LibraryDatabase2
from PyQt4.Qt import QApplication

2
src/calibre/gui2/dialogs/config.ui
View File

@ -710,7 +710,7 @@
<string>Free unused diskspace from the database</string>
</property>
<property name="text">
<string>&amp;Compact database</string>
<string>&amp;Check database integrity</string>
</property>
</widget>
</item>

34
src/calibre/library/database2.py
View File

@ -1787,4 +1787,38 @@ books_series_link feeds
return duplicates
def check_integrity(self):
bad = {}
for id in self.data.universal_set():
formats = self.data.get(id, FIELD_MAP['formats'], row_is_id=True)
if not formats:
formats = []
else:
formats = [x.lower() for x in formats.split(',')]
actual_formats = self.formats(id, index_is_id=True)
if not actual_formats:
actual_formats = []
else:
actual_formats = [x.lower() for x in actual_formats.split(',')]
mismatch = False
for fmt in formats:
if fmt in actual_formats:
continue
mismatch = True
if id not in bad:
bad[id] = []
bad[id].append(fmt)
for id in bad:
for fmt in bad[id]:
self.conn.execute('DELETE FROM data WHERE book=? AND format=?', (id, fmt.upper()))
self.conn.commit()
self.refresh_ids(list(bad.keys()))
self.vacuum()
return bad

6
src/calibre/utils/ipc/job.py
View File

@ -11,6 +11,9 @@ _count = 0
import time, cStringIO
from Queue import Queue, Empty
from calibre import prints
from calibre.constants import DEBUG
class BaseJob(object):
WAITING = 0
@ -47,6 +50,9 @@ class BaseJob(object):
self._status_text = _('Stopped')
else:
self._status_text = _('Error') if self.failed else _('Finished')
if DEBUG:
prints('Job:', self.id, self.description, 'finished')
prints('\t'.join(self.details.splitlines(True)))
if not self._done_called:
self._done_called = True
try:

28
src/calibre/utils/ipc/server.py
View File

@ -119,13 +119,33 @@ class Server(Thread):
'CALIBRE_WORKER_KEY' : hexlify(self.auth_key),
'CALIBRE_WORKER_RESULT' : hexlify(rfile),
}
for i in range(2):
# Try launch twice as occasionally on OS X
# Listener.accept fails with EINTR
cw = self.do_launch(env, gui, redirect_output, rfile)
if isinstance(cw, ConnectedWorker):
break
if isinstance(cw, basestring):
raise Exception('Failed to launch worker process:\n'+cw)
return cw
def do_launch(self, env, gui, redirect_output, rfile):
w = Worker(env, gui=gui)
if redirect_output is None:
redirect_output = not gui
w(redirect_output=redirect_output)
conn = self.listener.accept()
if conn is None:
raise Exception('Failed to launch worker process')
try:
w(redirect_output=redirect_output)
conn = self.listener.accept()
if conn is None:
raise Exception('Failed to launch worker process')
except BaseException:
try:
w.kill()
except:
pass
import traceback
return traceback.format_exc()
return ConnectedWorker(w, conn, rfile)
def add_job(self, job):