Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Allow using a tweak to switch db backends

Browse Source
This commit is contained in:
Kovid Goyal 2013-07-19 21:11:08 +05:30
parent 8d4d404b5d
commit d0b826c39a
11 changed files with 80 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
src/calibre/db/__init__.py
View File

@ -118,65 +118,26 @@ def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None):
return data
def get_db_loader():
from calibre.utils.config_base import tweaks
if tweaks.get('use_new_db', False):
from calibre.db.legacy import LibraryDatabase as cls
import apsw
errs = (apsw.Error,)
else:
from calibre.library.database2 import LibraryDatabase2 as cls
from calibre.library.sqlite import sqlite, DatabaseException
errs = (sqlite.Error, DatabaseException)
return cls, errs
'''
Rewrite of the calibre database backend.
Broad Objectives:
* Use the sqlite db only as a datastore. i.e. do not do
sorting/searching/concatenation or anything else in sqlite. Instead
mirror the sqlite tables in memory, create caches and lookup maps from
them and create a set_* API that updates the memory caches and the sqlite
correctly.
* Move from keeping a list of books in memory as a cache to a per table
cache. This allows much faster search and sort operations at the expense
of slightly slower lookup operations. That slowdown can be mitigated by
keeping lots of maps and updating them in the set_* API. Also
get_categories becomes blazingly fast.
* Separate the database layer from the cache layer more cleanly. Rather
than having the db layer refer to the cache layer and vice versa, the
cache layer will refer to the db layer only and the new API will be
defined on the cache layer.
* Get rid of index_is_id and other poor design decisions
* Minimize the API as much as possible and define it cleanly
* Do not change the on disk format of metadata.db at all (this is for
backwards compatibility)
* Get rid of the need for a separate db access thread by switching to apsw
to access sqlite, which is thread safe
* The new API will have methods to efficiently do bulk operations and will
use shared/exclusive/pending locks to serialize access to the in-mem data
structures. Use the same locking scheme as sqlite itself does.
How this will proceed:
1. Create the new API
2. Create a test suite for it
3. Write a replacement for LibraryDatabase2 that uses the new API
internally
4. Lots of testing of calibre with the new LibraryDatabase2
5. Gradually migrate code to use the (much faster) new api wherever possible (the new api
will be exposed via db.new_api)
I plan to work on this slowly, in parallel to normal calibre development
work.
Various things that require other things before they can be migrated:
1. From initialize_dynamic(): Also add custom
columns/categories/searches info into
self.field_metadata.
2. Catching DatabaseException and sqlite.Error when creating new
libraries/switching/on calibre startup.
3. Port library/restore.py
4. Replace the metadatabackup thread with the new implementation when using the new backend.
5. grep the sources for TODO
6. Check that content server reloading on metadata,db change, metadata
2. Port library/restore.py
3. Replace the metadatabackup thread with the new implementation when using the new backend.
4. Check that content server reloading on metadata,db change, metadata
backup, refresh gui on calibredb add and moving libraries all work (check
them on windows as well for file locking issues)
'''

17
src/calibre/gui2/actions/choose_library.py
View File

@ -19,9 +19,12 @@ from calibre.utils.config import prefs, tweaks
from calibre.utils.icu import sort_key
from calibre.gui2 import (gprefs, warning_dialog, Dispatcher, error_dialog,
question_dialog, info_dialog, open_local_file, choose_dir)
from calibre.library.database2 import LibraryDatabase2
from calibre.gui2.actions import InterfaceAction
def db_class():
from calibre.db import get_db_loader
return get_db_loader()[0]
class LibraryUsageStats(object): # {{{
def __init__(self):
@ -139,7 +142,7 @@ class MovedDialog(QDialog): # {{{
def accept(self):
newloc = unicode(self.loc.text())
if not LibraryDatabase2.exists_at(newloc):
if not db_class.exists_at(newloc):
error_dialog(self, _('No library found'),
_('No existing calibre library found at %s')%newloc,
show=True)
@ -313,6 +316,7 @@ class ChooseLibraryAction(InterfaceAction):
self.qaction.setEnabled(enabled)
def rename_requested(self, name, location):
LibraryDatabase = db_class()
loc = location.replace('/', os.sep)
base = os.path.dirname(loc)
newname, ok = QInputDialog.getText(self.gui, _('Rename') + ' ' + name,
@ -328,10 +332,10 @@ class ChooseLibraryAction(InterfaceAction):
_('The folder %s already exists. Delete it first.') %
newloc, show=True)
if (iswindows and len(newloc) >
LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
return error_dialog(self.gui, _('Too long'),
_('Path to library too long. Must be less than'
' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
show=True)
if not os.path.exists(loc):
error_dialog(self.gui, _('Not found'),
@ -387,16 +391,17 @@ class ChooseLibraryAction(InterfaceAction):
'rate of approximately 1 book every three seconds.'), show=True)
def restore_database(self):
LibraryDatabase = db_class()
m = self.gui.library_view.model()
db = m.db
if (iswindows and len(db.library_path) >
LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
return error_dialog(self.gui, _('Too long'),
_('Path to library too long. Must be less than'
' %d characters. Move your library to a location with'
' a shorter path using Windows Explorer, then point'
' calibre to the new location and try again.')%
LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
show=True)
from calibre.gui2.dialogs.restore_library import restore_database

4
src/calibre/gui2/actions/copy_to_library.py
View File

@ -55,8 +55,8 @@ class Worker(Thread): # {{{
notify=False, replace=replace)
def doit(self):
from calibre.library.database2 import LibraryDatabase2
newdb = LibraryDatabase2(self.loc, is_second_db=True)
from calibre.db import get_db_loader
newdb = get_db_loader()[0](self.loc, is_second_db=True)
with closing(newdb):
self._doit(newdb)
newdb.break_cycles()

7
src/calibre/gui2/dialogs/choose_library.py
View File

@ -15,7 +15,6 @@ from calibre.constants import (filesystem_encoding, iswindows,
get_portable_base)
from calibre import isbytestring, patheq, force_unicode
from calibre.gui2.wizard import move_library
from calibre.library.database2 import LibraryDatabase2
class ChooseLibrary(QDialog, Ui_Dialog):
@ -86,6 +85,8 @@ class ChooseLibrary(QDialog, Ui_Dialog):
show=True)
return False
if ac in ('new', 'move'):
from calibre.db import get_db_loader
LibraryDatabase = get_db_loader()[0]
if not empty:
error_dialog(self, _('Not empty'),
_('The folder %s is not empty. Please choose an empty'
@ -93,10 +94,10 @@ class ChooseLibrary(QDialog, Ui_Dialog):
show=True)
return False
if (iswindows and len(loc) >
LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
error_dialog(self, _('Too long'),
_('Path to library too long. Must be less than'
' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
show=True)
return False

27
src/calibre/gui2/main.py
View File

@ -15,8 +15,6 @@ from calibre.gui2 import (ORG_NAME, APP_UID, initialize_file_icon_provider,
Application, choose_dir, error_dialog, question_dialog, gprefs)
from calibre.gui2.main_window import option_parser as _option_parser
from calibre.utils.config import prefs, dynamic
from calibre.library.database2 import LibraryDatabase2
from calibre.library.sqlite import sqlite, DatabaseException
if iswindows:
winutil = plugins['winutil'][0]
@ -51,7 +49,8 @@ path_to_ebook to the database.
def find_portable_library():
base = get_portable_base()
if base is None: return
if base is None:
return
import glob
candidates = [os.path.basename(os.path.dirname(x)) for x in glob.glob(
os.path.join(base, u'*%smetadata.db'%os.sep))]
@ -123,7 +122,7 @@ def get_default_library_path():
def get_library_path(parent=None):
library_path = prefs['library_path']
if library_path is None: # Need to migrate to new database layout
if library_path is None: # Need to migrate to new database layout
base = os.path.expanduser('~')
if iswindows:
base = winutil.special_folder_path(winutil.CSIDL_PERSONAL)
@ -181,7 +180,7 @@ class GuiRunner(QObject):
main = Main(self.opts, gui_debug=self.gui_debug)
if self.splash_screen is not None:
self.splash_screen.showMessage(_('Initializing user interface...'))
with gprefs: # Only write gui.json after initialization is complete
with gprefs: # Only write gui.json after initialization is complete
main.initialize(self.library_path, db, self.listener, self.actions)
if self.splash_screen is not None:
self.splash_screen.finish(main)
@ -224,7 +223,7 @@ class GuiRunner(QObject):
try:
self.library_path = candidate
db = LibraryDatabase2(candidate)
db = self.db_class(candidate)
except:
error_dialog(self.splash_screen, _('Bad database location'),
_('Bad database location %r. calibre will now quit.'
@ -235,10 +234,12 @@ class GuiRunner(QObject):
self.start_gui(db)
def initialize_db(self):
from calibre.db import get_db_loader
db = None
self.db_class, errs = get_db_loader()
try:
db = LibraryDatabase2(self.library_path)
except (sqlite.Error, DatabaseException):
db = self.db_class(self.library_path)
except errs:
repair = question_dialog(self.splash_screen, _('Corrupted database'),
_('The library database at %s appears to be corrupted. Do '
'you want calibre to try and rebuild it automatically? '
@ -249,7 +250,7 @@ class GuiRunner(QObject):
)
if repair:
if repair_library(self.library_path):
db = LibraryDatabase2(self.library_path)
db = self.db_class(self.library_path)
except:
error_dialog(self.splash_screen, _('Bad database location'),
_('Bad database location %r. Will start with '
@ -383,7 +384,7 @@ def shutdown_other(rc=None):
rc = build_pipe(print_error=False)
if rc.conn is None:
prints(_('No running calibre found'))
return # No running instance found
return # No running instance found
from calibre.utils.lock import singleinstance
rc.conn.send('shutdown:')
prints(_('Shutdown command sent, waiting for shutdown...'))
@ -441,7 +442,7 @@ def main(args=sys.argv):
otherinstance = False
try:
listener = Listener(address=gui_socket_address())
except socket.error: # Good si is correct (on UNIX)
except socket.error: # Good si is correct (on UNIX)
otherinstance = True
else:
# On windows only singleinstance can be trusted
@ -458,7 +459,8 @@ if __name__ == '__main__':
try:
sys.exit(main())
except Exception as err:
if not iswindows: raise
if not iswindows:
raise
tb = traceback.format_exc()
from PyQt4.QtGui import QErrorMessage
logfile = os.path.join(os.path.expanduser('~'), 'calibre.log')
@ -470,3 +472,4 @@ if __name__ == '__main__':
unicode(tb).replace('\n', '<br>'),
log.replace('\n', '<br>')))

12
src/calibre/gui2/ui.py
View File

@ -22,7 +22,7 @@ from calibre import prints, force_unicode
from calibre.constants import __appname__, isosx, filesystem_encoding
from calibre.utils.config import prefs, dynamic
from calibre.utils.ipc.server import Server
from calibre.library.database2 import LibraryDatabase2
from calibre.db import get_db_loader
from calibre.customize.ui import interface_actions, available_store_plugins
from calibre.gui2 import (error_dialog, GetMetadata, open_url,
gprefs, max_available_height, config, info_dialog, Dispatcher,
@ -42,7 +42,6 @@ from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin
from calibre.gui2.tag_browser.ui import TagBrowserMixin
from calibre.gui2.keyboard import Manager
from calibre.gui2.auto_add import AutoAdder
from calibre.library.sqlite import sqlite, DatabaseException
from calibre.gui2.proceed import ProceedQuestion
from calibre.gui2.dialogs.message_box import JobError
from calibre.gui2.job_indicator import Pointer
@ -572,12 +571,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
default_prefs = olddb.prefs
from calibre.utils.formatter_functions import unload_user_template_functions
unload_user_template_functions(olddb.library_id )
unload_user_template_functions(olddb.library_id)
except:
olddb = None
db_class, errs = get_db_loader()
try:
db = LibraryDatabase2(newloc, default_prefs=default_prefs)
except (DatabaseException, sqlite.Error):
db = db_class(newloc, default_prefs=default_prefs)
except errs:
if not allow_rebuild:
raise
import traceback
@ -591,7 +591,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
if repair:
from calibre.gui2.dialogs.restore_library import repair_library_at
if repair_library_at(newloc, parent=self):
db = LibraryDatabase2(newloc, default_prefs=default_prefs)
db = db_class(newloc, default_prefs=default_prefs)
else:
return
else:

17
src/calibre/gui2/wizard/__init__.py
View File

@ -14,7 +14,6 @@ from contextlib import closing
from PyQt4.Qt import (QWizard, QWizardPage, QPixmap, Qt, QAbstractListModel,
QVariant, QItemSelectionModel, SIGNAL, QObject, QTimer, pyqtSignal)
from calibre import __appname__, patheq
from calibre.library.database2 import LibraryDatabase2
from calibre.library.move import MoveLibrary
from calibre.constants import (filesystem_encoding, iswindows, plugins,
isportable)
@ -34,6 +33,10 @@ from calibre.customize.ui import device_plugins
if iswindows:
winutil = plugins['winutil'][0]
def db_class():
from calibre.db import get_db_loader
return get_db_loader()[0]
# Devices {{{
class Device(object):
@ -623,7 +626,7 @@ def move_library(oldloc, newloc, parent, callback_on_complete):
if oldloc and os.access(os.path.join(oldloc, 'metadata.db'), os.R_OK):
# Move old library to new location
try:
db = LibraryDatabase2(oldloc)
db = db_class()(oldloc)
except:
return move_library(None, newloc, parent,
callback)
@ -636,13 +639,13 @@ def move_library(oldloc, newloc, parent, callback_on_complete):
return
else:
# Create new library at new location
db = LibraryDatabase2(newloc)
db = db_class()(newloc)
callback(newloc)
return
# Try to load existing library at new location
try:
LibraryDatabase2(newloc)
db_class()(newloc)
except Exception as err:
det = traceback.format_exc()
error_dialog(parent, _('Invalid database'),
@ -729,7 +732,7 @@ class LibraryPage(QWizardPage, LibraryUI):
def is_library_dir_suitable(self, x):
try:
return LibraryDatabase2.exists_at(x) or not os.listdir(x)
return db_class().exists_at(x) or not os.listdir(x)
except:
return False
@ -745,10 +748,10 @@ class LibraryPage(QWizardPage, LibraryUI):
_('Select location for books'))
if x:
if (iswindows and len(x) >
LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
db_class().WINDOWS_LIBRARY_PATH_LIMIT):
return error_dialog(self, _('Too long'),
_('Path to library too long. Must be less than'
' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
' %d characters.')%(db_class().WINDOWS_LIBRARY_PATH_LIMIT),
show=True)
if not os.path.exists(x):
try:

7
src/calibre/library/__init__.py
View File

@ -3,13 +3,13 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
''' Code to manage ebook library'''
def db(path=None, read_only=False):
from calibre.library.database2 import LibraryDatabase2
from calibre.db import get_db_loader
from calibre.utils.config import prefs
return LibraryDatabase2(path if path else prefs['library_path'],
return get_db_loader()[0](path if path else prefs['library_path'],
read_only=read_only)
def generate_test_db(library_path, # {{{
def generate_test_db(library_path, # {{{
num_of_records=20000,
num_of_authors=6000,
num_of_tags=10000,
@ -76,3 +76,4 @@ def current_library_name():
if path:
return posixpath.basename(path)

13
src/calibre/library/cli.py
View File

@ -15,15 +15,18 @@ from calibre import preferred_encoding, prints, isbytestring
from calibre.utils.config import OptionParser, prefs, tweaks
from calibre.ebooks.metadata.meta import get_metadata
from calibre.ebooks.metadata.book.base import field_from_string
from calibre.library.database2 import LibraryDatabase2
from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
from calibre.utils.date import isoformat
from calibre.db import get_db_loader
FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
'formats', 'isbn', 'uuid', 'pubdate', 'cover', 'last_modified',
'identifiers'])
def db_class():
return get_db_loader()[0]
do_notify = True
def send_message(msg=''):
global do_notify
@ -62,7 +65,7 @@ def get_db(dbpath, options):
dbpath = os.path.abspath(dbpath)
if options.dont_notify_gui:
do_notify = False
return LibraryDatabase2(dbpath)
return db_class()(dbpath)
def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator,
prefix, limit, subtitle='Books in the calibre database'):
@ -1101,7 +1104,7 @@ def command_backup_metadata(args, dbpath):
dbpath = opts.library_path
if isbytestring(dbpath):
dbpath = dbpath.decode(preferred_encoding)
db = LibraryDatabase2(dbpath)
db = db_class()(dbpath)
book_ids = None
if opts.all:
book_ids = db.all_ids()
@ -1183,7 +1186,7 @@ def command_check_library(args, dbpath):
for i in list:
print ' %-40.40s - %-40.40s'%(i[0], i[1])
db = LibraryDatabase2(dbpath)
db = db_class()(dbpath)
checker = CheckLibrary(dbpath, db)
checker.scan_library(names, exts)
for check in checks:
@ -1296,7 +1299,7 @@ def command_list_categories(args, dbpath):
if isbytestring(dbpath):
dbpath = dbpath.decode(preferred_encoding)
db = LibraryDatabase2(dbpath)
db = db_class()(dbpath)
category_data = db.get_categories()
data = []
report_on = [c.strip() for c in opts.report.split(',') if c.strip()]

6
src/calibre/library/move.py
View File

@ -10,14 +10,14 @@ import time, os
from threading import Thread
from Queue import Empty
from calibre.library.database2 import LibraryDatabase2
from calibre.utils.ipc.server import Server
from calibre.utils.ipc.job import ParallelJob
def move_library(from_, to, notification = lambda x:x):
def move_library(from_, to, notification=lambda x:x):
from calibre.db import get_db_loader
time.sleep(1)
old = LibraryDatabase2(from_)
old = get_db_loader()[0](from_)
old.move_library_to(to, notification)
return True

4
src/calibre/library/server/main.py
View File

@ -100,7 +100,7 @@ def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
def main(args=sys.argv):
from calibre.library.database2 import LibraryDatabase2
from calibre.db import get_db_loader
parser = option_parser()
opts, args = parser.parse_args(args)
if opts.daemonize and not iswindows:
@ -116,7 +116,7 @@ def main(args=sys.argv):
print('No saved library path. Use the --with-library option'
' to specify the path to the library you want to use.')
return 1
db = LibraryDatabase2(opts.with_library)
db = get_db_loader()[0](opts.with_library)
server = LibraryServer(db, opts, show_tracebacks=opts.develop)
server.start()
return 0