diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py
index 5bc0437fb5..b8e46b4ed2 100644
--- a/src/calibre/db/__init__.py
+++ b/src/calibre/db/__init__.py
@@ -118,65 +118,26 @@ def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None):
return data
+def get_db_loader():
+ from calibre.utils.config_base import tweaks
+ if tweaks.get('use_new_db', False):
+ from calibre.db.legacy import LibraryDatabase as cls
+ import apsw
+ errs = (apsw.Error,)
+ else:
+ from calibre.library.database2 import LibraryDatabase2 as cls
+ from calibre.library.sqlite import sqlite, DatabaseException
+ errs = (sqlite.Error, DatabaseException)
+ return cls, errs
+
'''
-Rewrite of the calibre database backend.
-
-Broad Objectives:
-
- * Use the sqlite db only as a datastore. i.e. do not do
- sorting/searching/concatenation or anything else in sqlite. Instead
- mirror the sqlite tables in memory, create caches and lookup maps from
- them and create a set_* API that updates the memory caches and the sqlite
- correctly.
-
- * Move from keeping a list of books in memory as a cache to a per table
- cache. This allows much faster search and sort operations at the expense
- of slightly slower lookup operations. That slowdown can be mitigated by
- keeping lots of maps and updating them in the set_* API. Also
- get_categories becomes blazingly fast.
-
- * Separate the database layer from the cache layer more cleanly. Rather
- than having the db layer refer to the cache layer and vice versa, the
- cache layer will refer to the db layer only and the new API will be
- defined on the cache layer.
-
- * Get rid of index_is_id and other poor design decisions
-
- * Minimize the API as much as possible and define it cleanly
-
- * Do not change the on disk format of metadata.db at all (this is for
- backwards compatibility)
-
- * Get rid of the need for a separate db access thread by switching to apsw
- to access sqlite, which is thread safe
-
- * The new API will have methods to efficiently do bulk operations and will
- use shared/exclusive/pending locks to serialize access to the in-mem data
- structures. Use the same locking scheme as sqlite itself does.
-
-How this will proceed:
-
- 1. Create the new API
- 2. Create a test suite for it
- 3. Write a replacement for LibraryDatabase2 that uses the new API
- internally
- 4. Lots of testing of calibre with the new LibraryDatabase2
- 5. Gradually migrate code to use the (much faster) new api wherever possible (the new api
- will be exposed via db.new_api)
-
- I plan to work on this slowly, in parallel to normal calibre development
- work.
-
Various things that require other things before they can be migrated:
1. From initialize_dynamic(): Also add custom
columns/categories/searches info into
self.field_metadata.
- 2. Catching DatabaseException and sqlite.Error when creating new
- libraries/switching/on calibre startup.
- 3. Port library/restore.py
- 4. Replace the metadatabackup thread with the new implementation when using the new backend.
- 5. grep the sources for TODO
- 6. Check that content server reloading on metadata,db change, metadata
+ 2. Port library/restore.py
+ 3. Replace the metadatabackup thread with the new implementation when using the new backend.
+ 4. Check that content server reloading on metadata,db change, metadata
backup, refresh gui on calibredb add and moving libraries all work (check
them on windows as well for file locking issues)
'''
diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py
index 347b6862a0..6857454ae4 100644
--- a/src/calibre/gui2/actions/choose_library.py
+++ b/src/calibre/gui2/actions/choose_library.py
@@ -19,9 +19,12 @@ from calibre.utils.config import prefs, tweaks
from calibre.utils.icu import sort_key
from calibre.gui2 import (gprefs, warning_dialog, Dispatcher, error_dialog,
question_dialog, info_dialog, open_local_file, choose_dir)
-from calibre.library.database2 import LibraryDatabase2
from calibre.gui2.actions import InterfaceAction
+def db_class():
+ from calibre.db import get_db_loader
+ return get_db_loader()[0]
+
class LibraryUsageStats(object): # {{{
def __init__(self):
@@ -139,7 +142,7 @@ class MovedDialog(QDialog): # {{{
def accept(self):
newloc = unicode(self.loc.text())
- if not LibraryDatabase2.exists_at(newloc):
+ if not db_class.exists_at(newloc):
error_dialog(self, _('No library found'),
_('No existing calibre library found at %s')%newloc,
show=True)
@@ -313,6 +316,7 @@ class ChooseLibraryAction(InterfaceAction):
self.qaction.setEnabled(enabled)
def rename_requested(self, name, location):
+ LibraryDatabase = db_class()
loc = location.replace('/', os.sep)
base = os.path.dirname(loc)
newname, ok = QInputDialog.getText(self.gui, _('Rename') + ' ' + name,
@@ -328,10 +332,10 @@ class ChooseLibraryAction(InterfaceAction):
_('The folder %s already exists. Delete it first.') %
newloc, show=True)
if (iswindows and len(newloc) >
- LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+ LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
return error_dialog(self.gui, _('Too long'),
_('Path to library too long. Must be less than'
- ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+ ' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
show=True)
if not os.path.exists(loc):
error_dialog(self.gui, _('Not found'),
@@ -387,16 +391,17 @@ class ChooseLibraryAction(InterfaceAction):
'rate of approximately 1 book every three seconds.'), show=True)
def restore_database(self):
+ LibraryDatabase = db_class()
m = self.gui.library_view.model()
db = m.db
if (iswindows and len(db.library_path) >
- LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+ LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
return error_dialog(self.gui, _('Too long'),
_('Path to library too long. Must be less than'
' %d characters. Move your library to a location with'
' a shorter path using Windows Explorer, then point'
' calibre to the new location and try again.')%
- LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+ LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
show=True)
from calibre.gui2.dialogs.restore_library import restore_database
diff --git a/src/calibre/gui2/actions/copy_to_library.py b/src/calibre/gui2/actions/copy_to_library.py
index 8465cb98f0..f32f060de7 100644
--- a/src/calibre/gui2/actions/copy_to_library.py
+++ b/src/calibre/gui2/actions/copy_to_library.py
@@ -55,8 +55,8 @@ class Worker(Thread): # {{{
notify=False, replace=replace)
def doit(self):
- from calibre.library.database2 import LibraryDatabase2
- newdb = LibraryDatabase2(self.loc, is_second_db=True)
+ from calibre.db import get_db_loader
+ newdb = get_db_loader()[0](self.loc, is_second_db=True)
with closing(newdb):
self._doit(newdb)
newdb.break_cycles()
diff --git a/src/calibre/gui2/dialogs/choose_library.py b/src/calibre/gui2/dialogs/choose_library.py
index 91048e8ff1..52e8ef644a 100644
--- a/src/calibre/gui2/dialogs/choose_library.py
+++ b/src/calibre/gui2/dialogs/choose_library.py
@@ -15,7 +15,6 @@ from calibre.constants import (filesystem_encoding, iswindows,
get_portable_base)
from calibre import isbytestring, patheq, force_unicode
from calibre.gui2.wizard import move_library
-from calibre.library.database2 import LibraryDatabase2
class ChooseLibrary(QDialog, Ui_Dialog):
@@ -86,6 +85,8 @@ class ChooseLibrary(QDialog, Ui_Dialog):
show=True)
return False
if ac in ('new', 'move'):
+ from calibre.db import get_db_loader
+ LibraryDatabase = get_db_loader()[0]
if not empty:
error_dialog(self, _('Not empty'),
_('The folder %s is not empty. Please choose an empty'
@@ -93,10 +94,10 @@ class ChooseLibrary(QDialog, Ui_Dialog):
show=True)
return False
if (iswindows and len(loc) >
- LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+ LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
error_dialog(self, _('Too long'),
_('Path to library too long. Must be less than'
- ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+ ' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
show=True)
return False
diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py
index f35a9ca083..bcfc88d239 100644
--- a/src/calibre/gui2/main.py
+++ b/src/calibre/gui2/main.py
@@ -15,8 +15,6 @@ from calibre.gui2 import (ORG_NAME, APP_UID, initialize_file_icon_provider,
Application, choose_dir, error_dialog, question_dialog, gprefs)
from calibre.gui2.main_window import option_parser as _option_parser
from calibre.utils.config import prefs, dynamic
-from calibre.library.database2 import LibraryDatabase2
-from calibre.library.sqlite import sqlite, DatabaseException
if iswindows:
winutil = plugins['winutil'][0]
@@ -51,7 +49,8 @@ path_to_ebook to the database.
def find_portable_library():
base = get_portable_base()
- if base is None: return
+ if base is None:
+ return
import glob
candidates = [os.path.basename(os.path.dirname(x)) for x in glob.glob(
os.path.join(base, u'*%smetadata.db'%os.sep))]
@@ -123,7 +122,7 @@ def get_default_library_path():
def get_library_path(parent=None):
library_path = prefs['library_path']
- if library_path is None: # Need to migrate to new database layout
+ if library_path is None: # Need to migrate to new database layout
base = os.path.expanduser('~')
if iswindows:
base = winutil.special_folder_path(winutil.CSIDL_PERSONAL)
@@ -181,7 +180,7 @@ class GuiRunner(QObject):
main = Main(self.opts, gui_debug=self.gui_debug)
if self.splash_screen is not None:
self.splash_screen.showMessage(_('Initializing user interface...'))
- with gprefs: # Only write gui.json after initialization is complete
+ with gprefs: # Only write gui.json after initialization is complete
main.initialize(self.library_path, db, self.listener, self.actions)
if self.splash_screen is not None:
self.splash_screen.finish(main)
@@ -224,7 +223,7 @@ class GuiRunner(QObject):
try:
self.library_path = candidate
- db = LibraryDatabase2(candidate)
+ db = self.db_class(candidate)
except:
error_dialog(self.splash_screen, _('Bad database location'),
_('Bad database location %r. calibre will now quit.'
@@ -235,10 +234,12 @@ class GuiRunner(QObject):
self.start_gui(db)
def initialize_db(self):
+ from calibre.db import get_db_loader
db = None
+ self.db_class, errs = get_db_loader()
try:
- db = LibraryDatabase2(self.library_path)
- except (sqlite.Error, DatabaseException):
+ db = self.db_class(self.library_path)
+ except errs:
repair = question_dialog(self.splash_screen, _('Corrupted database'),
_('The library database at %s appears to be corrupted. Do '
'you want calibre to try and rebuild it automatically? '
@@ -249,7 +250,7 @@ class GuiRunner(QObject):
)
if repair:
if repair_library(self.library_path):
- db = LibraryDatabase2(self.library_path)
+ db = self.db_class(self.library_path)
except:
error_dialog(self.splash_screen, _('Bad database location'),
_('Bad database location %r. Will start with '
@@ -383,7 +384,7 @@ def shutdown_other(rc=None):
rc = build_pipe(print_error=False)
if rc.conn is None:
prints(_('No running calibre found'))
- return # No running instance found
+ return # No running instance found
from calibre.utils.lock import singleinstance
rc.conn.send('shutdown:')
prints(_('Shutdown command sent, waiting for shutdown...'))
@@ -441,7 +442,7 @@ def main(args=sys.argv):
otherinstance = False
try:
listener = Listener(address=gui_socket_address())
- except socket.error: # Good si is correct (on UNIX)
+ except socket.error: # Good si is correct (on UNIX)
otherinstance = True
else:
# On windows only singleinstance can be trusted
@@ -458,7 +459,8 @@ if __name__ == '__main__':
try:
sys.exit(main())
except Exception as err:
- if not iswindows: raise
+ if not iswindows:
+ raise
tb = traceback.format_exc()
from PyQt4.QtGui import QErrorMessage
logfile = os.path.join(os.path.expanduser('~'), 'calibre.log')
@@ -470,3 +472,4 @@ if __name__ == '__main__':
unicode(tb).replace('\n', '
'),
log.replace('\n', '
')))
+
diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py
index 80aa66601b..5e4f75895a 100644
--- a/src/calibre/gui2/ui.py
+++ b/src/calibre/gui2/ui.py
@@ -22,7 +22,7 @@ from calibre import prints, force_unicode
from calibre.constants import __appname__, isosx, filesystem_encoding
from calibre.utils.config import prefs, dynamic
from calibre.utils.ipc.server import Server
-from calibre.library.database2 import LibraryDatabase2
+from calibre.db import get_db_loader
from calibre.customize.ui import interface_actions, available_store_plugins
from calibre.gui2 import (error_dialog, GetMetadata, open_url,
gprefs, max_available_height, config, info_dialog, Dispatcher,
@@ -42,7 +42,6 @@ from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin
from calibre.gui2.tag_browser.ui import TagBrowserMixin
from calibre.gui2.keyboard import Manager
from calibre.gui2.auto_add import AutoAdder
-from calibre.library.sqlite import sqlite, DatabaseException
from calibre.gui2.proceed import ProceedQuestion
from calibre.gui2.dialogs.message_box import JobError
from calibre.gui2.job_indicator import Pointer
@@ -572,12 +571,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
default_prefs = olddb.prefs
from calibre.utils.formatter_functions import unload_user_template_functions
- unload_user_template_functions(olddb.library_id )
+ unload_user_template_functions(olddb.library_id)
except:
olddb = None
+ db_class, errs = get_db_loader()
try:
- db = LibraryDatabase2(newloc, default_prefs=default_prefs)
- except (DatabaseException, sqlite.Error):
+ db = db_class(newloc, default_prefs=default_prefs)
+ except errs:
if not allow_rebuild:
raise
import traceback
@@ -591,7 +591,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
if repair:
from calibre.gui2.dialogs.restore_library import repair_library_at
if repair_library_at(newloc, parent=self):
- db = LibraryDatabase2(newloc, default_prefs=default_prefs)
+ db = db_class(newloc, default_prefs=default_prefs)
else:
return
else:
diff --git a/src/calibre/gui2/wizard/__init__.py b/src/calibre/gui2/wizard/__init__.py
index f813eed892..00fd3bdb29 100644
--- a/src/calibre/gui2/wizard/__init__.py
+++ b/src/calibre/gui2/wizard/__init__.py
@@ -14,7 +14,6 @@ from contextlib import closing
from PyQt4.Qt import (QWizard, QWizardPage, QPixmap, Qt, QAbstractListModel,
QVariant, QItemSelectionModel, SIGNAL, QObject, QTimer, pyqtSignal)
from calibre import __appname__, patheq
-from calibre.library.database2 import LibraryDatabase2
from calibre.library.move import MoveLibrary
from calibre.constants import (filesystem_encoding, iswindows, plugins,
isportable)
@@ -34,6 +33,10 @@ from calibre.customize.ui import device_plugins
if iswindows:
winutil = plugins['winutil'][0]
+def db_class():
+ from calibre.db import get_db_loader
+ return get_db_loader()[0]
+
# Devices {{{
class Device(object):
@@ -623,7 +626,7 @@ def move_library(oldloc, newloc, parent, callback_on_complete):
if oldloc and os.access(os.path.join(oldloc, 'metadata.db'), os.R_OK):
# Move old library to new location
try:
- db = LibraryDatabase2(oldloc)
+ db = db_class()(oldloc)
except:
return move_library(None, newloc, parent,
callback)
@@ -636,13 +639,13 @@ def move_library(oldloc, newloc, parent, callback_on_complete):
return
else:
# Create new library at new location
- db = LibraryDatabase2(newloc)
+ db = db_class()(newloc)
callback(newloc)
return
# Try to load existing library at new location
try:
- LibraryDatabase2(newloc)
+ db_class()(newloc)
except Exception as err:
det = traceback.format_exc()
error_dialog(parent, _('Invalid database'),
@@ -729,7 +732,7 @@ class LibraryPage(QWizardPage, LibraryUI):
def is_library_dir_suitable(self, x):
try:
- return LibraryDatabase2.exists_at(x) or not os.listdir(x)
+ return db_class().exists_at(x) or not os.listdir(x)
except:
return False
@@ -745,10 +748,10 @@ class LibraryPage(QWizardPage, LibraryUI):
_('Select location for books'))
if x:
if (iswindows and len(x) >
- LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+ db_class().WINDOWS_LIBRARY_PATH_LIMIT):
return error_dialog(self, _('Too long'),
_('Path to library too long. Must be less than'
- ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+ ' %d characters.')%(db_class().WINDOWS_LIBRARY_PATH_LIMIT),
show=True)
if not os.path.exists(x):
try:
diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py
index 3ae237c919..e219d764d5 100644
--- a/src/calibre/library/__init__.py
+++ b/src/calibre/library/__init__.py
@@ -3,13 +3,13 @@ __copyright__ = '2008, Kovid Goyal '
''' Code to manage ebook library'''
def db(path=None, read_only=False):
- from calibre.library.database2 import LibraryDatabase2
+ from calibre.db import get_db_loader
from calibre.utils.config import prefs
- return LibraryDatabase2(path if path else prefs['library_path'],
+ return get_db_loader()[0](path if path else prefs['library_path'],
read_only=read_only)
-def generate_test_db(library_path, # {{{
+def generate_test_db(library_path, # {{{
num_of_records=20000,
num_of_authors=6000,
num_of_tags=10000,
@@ -76,3 +76,4 @@ def current_library_name():
if path:
return posixpath.basename(path)
+
diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py
index 2978a4e169..2c6e7cd777 100644
--- a/src/calibre/library/cli.py
+++ b/src/calibre/library/cli.py
@@ -15,15 +15,18 @@ from calibre import preferred_encoding, prints, isbytestring
from calibre.utils.config import OptionParser, prefs, tweaks
from calibre.ebooks.metadata.meta import get_metadata
from calibre.ebooks.metadata.book.base import field_from_string
-from calibre.library.database2 import LibraryDatabase2
from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
from calibre.utils.date import isoformat
+from calibre.db import get_db_loader
FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
'formats', 'isbn', 'uuid', 'pubdate', 'cover', 'last_modified',
'identifiers'])
+def db_class():
+ return get_db_loader()[0]
+
do_notify = True
def send_message(msg=''):
global do_notify
@@ -62,7 +65,7 @@ def get_db(dbpath, options):
dbpath = os.path.abspath(dbpath)
if options.dont_notify_gui:
do_notify = False
- return LibraryDatabase2(dbpath)
+ return db_class()(dbpath)
def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator,
prefix, limit, subtitle='Books in the calibre database'):
@@ -1101,7 +1104,7 @@ def command_backup_metadata(args, dbpath):
dbpath = opts.library_path
if isbytestring(dbpath):
dbpath = dbpath.decode(preferred_encoding)
- db = LibraryDatabase2(dbpath)
+ db = db_class()(dbpath)
book_ids = None
if opts.all:
book_ids = db.all_ids()
@@ -1183,7 +1186,7 @@ def command_check_library(args, dbpath):
for i in list:
print ' %-40.40s - %-40.40s'%(i[0], i[1])
- db = LibraryDatabase2(dbpath)
+ db = db_class()(dbpath)
checker = CheckLibrary(dbpath, db)
checker.scan_library(names, exts)
for check in checks:
@@ -1296,7 +1299,7 @@ def command_list_categories(args, dbpath):
if isbytestring(dbpath):
dbpath = dbpath.decode(preferred_encoding)
- db = LibraryDatabase2(dbpath)
+ db = db_class()(dbpath)
category_data = db.get_categories()
data = []
report_on = [c.strip() for c in opts.report.split(',') if c.strip()]
diff --git a/src/calibre/library/move.py b/src/calibre/library/move.py
index d162d962fe..ccebaa0302 100644
--- a/src/calibre/library/move.py
+++ b/src/calibre/library/move.py
@@ -10,14 +10,14 @@ import time, os
from threading import Thread
from Queue import Empty
-from calibre.library.database2 import LibraryDatabase2
from calibre.utils.ipc.server import Server
from calibre.utils.ipc.job import ParallelJob
-def move_library(from_, to, notification = lambda x:x):
+def move_library(from_, to, notification=lambda x:x):
+ from calibre.db import get_db_loader
time.sleep(1)
- old = LibraryDatabase2(from_)
+ old = get_db_loader()[0](from_)
old.move_library_to(to, notification)
return True
diff --git a/src/calibre/library/server/main.py b/src/calibre/library/server/main.py
index 486c0dbb68..a97eb53618 100644
--- a/src/calibre/library/server/main.py
+++ b/src/calibre/library/server/main.py
@@ -100,7 +100,7 @@ def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
def main(args=sys.argv):
- from calibre.library.database2 import LibraryDatabase2
+ from calibre.db import get_db_loader
parser = option_parser()
opts, args = parser.parse_args(args)
if opts.daemonize and not iswindows:
@@ -116,7 +116,7 @@ def main(args=sys.argv):
print('No saved library path. Use the --with-library option'
' to specify the path to the library you want to use.')
return 1
- db = LibraryDatabase2(opts.with_library)
+ db = get_db_loader()[0](opts.with_library)
server = LibraryServer(db, opts, show_tracebacks=opts.develop)
server.start()
return 0