Allow using a tweak to switch db backends

src/calibre/db/__init__.py

@@ -118,65 +118,26 @@ def get_data_as_dict(self, prefix=None, authors_as_string=False, ids=None):
 
     return data
 
+def get_db_loader():
+    from calibre.utils.config_base import tweaks
+    if tweaks.get('use_new_db', False):
+        from calibre.db.legacy import LibraryDatabase as cls
+        import apsw
+        errs = (apsw.Error,)
+    else:
+        from calibre.library.database2 import LibraryDatabase2 as cls
+        from calibre.library.sqlite import sqlite, DatabaseException
+        errs = (sqlite.Error, DatabaseException)
+    return cls, errs
+
 '''
-Rewrite of the calibre database backend.
-
-Broad Objectives:
-
-    * Use the sqlite db only as a datastore. i.e. do not do
-      sorting/searching/concatenation or anything else in sqlite. Instead
-      mirror the sqlite tables in memory, create caches and lookup maps from
-      them and create a set_* API that updates the memory caches and the sqlite
-      correctly.
-
-    * Move from keeping a list of books in memory as a cache to a per table
-      cache. This allows much faster search and sort operations at the expense
-      of slightly slower lookup operations. That slowdown can be mitigated by
-      keeping lots of maps and updating them in the set_* API. Also
-      get_categories becomes blazingly fast.
-
-    * Separate the database layer from the cache layer more cleanly. Rather
-      than having the db layer refer to the cache layer and vice versa, the
-      cache layer will refer to the db layer only and the new API will be
-      defined on the cache layer.
-
-    * Get rid of index_is_id and other poor design decisions
-
-    * Minimize the API as much as possible and define it cleanly
-
-    * Do not change the on disk format of metadata.db at all (this is for
-      backwards compatibility)
-
-    * Get rid of the need for a separate db access thread by switching to apsw
-      to access sqlite, which is thread safe
-
-    * The new API will have methods to efficiently do bulk operations and will
-      use shared/exclusive/pending locks to serialize access to the in-mem data
-      structures. Use the same locking scheme as sqlite itself does.
-
-How this will proceed:
-
-    1. Create the new API
-    2. Create a test suite for it
-    3. Write a replacement for LibraryDatabase2 that uses the new API
-       internally
-    4. Lots of testing of calibre with the new LibraryDatabase2
-    5. Gradually migrate code to use the (much faster) new api wherever possible (the new api
-       will be exposed via db.new_api)
-
-    I plan to work on this slowly, in parallel to normal calibre development
-    work.
-
 Various things that require other things before they can be migrated:
     1. From initialize_dynamic(): Also add custom
                     columns/categories/searches info into
                     self.field_metadata.
-    2. Catching DatabaseException and sqlite.Error when creating new
+    2. Port library/restore.py
-    libraries/switching/on calibre startup.
+    3. Replace the metadatabackup thread with the new implementation when using the new backend.
-    3. Port library/restore.py
+    4. Check that content server reloading on metadata,db change, metadata
-    4. Replace the metadatabackup thread with the new implementation when using the new backend.
-    5. grep the sources for TODO
-    6. Check that content server reloading on metadata,db change, metadata
     backup, refresh gui on calibredb add and moving libraries all work (check
     them on windows as well for file locking issues)
 '''

src/calibre/gui2/actions/choose_library.py

@@ -19,9 +19,12 @@ from calibre.utils.config import prefs, tweaks
 from calibre.utils.icu import sort_key
 from calibre.gui2 import (gprefs, warning_dialog, Dispatcher, error_dialog,
     question_dialog, info_dialog, open_local_file, choose_dir)
-from calibre.library.database2 import LibraryDatabase2
 from calibre.gui2.actions import InterfaceAction
 
+def db_class():
+    from calibre.db import get_db_loader
+    return get_db_loader()[0]
+
 class LibraryUsageStats(object):  # {{{
 
     def __init__(self):
@@ -139,7 +142,7 @@ class MovedDialog(QDialog):  # {{{
 
     def accept(self):
         newloc = unicode(self.loc.text())
-        if not LibraryDatabase2.exists_at(newloc):
+        if not db_class.exists_at(newloc):
             error_dialog(self, _('No library found'),
                     _('No existing calibre library found at %s')%newloc,
                     show=True)
@@ -313,6 +316,7 @@ class ChooseLibraryAction(InterfaceAction):
         self.qaction.setEnabled(enabled)
 
     def rename_requested(self, name, location):
+        LibraryDatabase = db_class()
         loc = location.replace('/', os.sep)
         base = os.path.dirname(loc)
         newname, ok = QInputDialog.getText(self.gui, _('Rename') + ' ' + name,
@@ -328,10 +332,10 @@ class ChooseLibraryAction(InterfaceAction):
                     _('The folder %s already exists. Delete it first.') %
                     newloc, show=True)
         if (iswindows and len(newloc) >
-                LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+                LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
             return error_dialog(self.gui, _('Too long'),
                     _('Path to library too long. Must be less than'
-                    ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+                    ' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
                     show=True)
         if not os.path.exists(loc):
             error_dialog(self.gui, _('Not found'),
@@ -387,16 +391,17 @@ class ChooseLibraryAction(InterfaceAction):
               'rate of approximately 1 book every three seconds.'), show=True)
 
     def restore_database(self):
+        LibraryDatabase = db_class()
         m = self.gui.library_view.model()
         db = m.db
         if (iswindows and len(db.library_path) >
-                LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+                LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
             return error_dialog(self.gui, _('Too long'),
                     _('Path to library too long. Must be less than'
                     ' %d characters. Move your library to a location with'
                     ' a shorter path using Windows Explorer, then point'
                     ' calibre to the new location and try again.')%
-                    LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+                    LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
                     show=True)
 
         from calibre.gui2.dialogs.restore_library import restore_database

src/calibre/gui2/actions/copy_to_library.py

@@ -55,8 +55,8 @@ class Worker(Thread):  # {{{
                         notify=False, replace=replace)
 
     def doit(self):
-        from calibre.library.database2 import LibraryDatabase2
+        from calibre.db import get_db_loader
-        newdb = LibraryDatabase2(self.loc, is_second_db=True)
+        newdb = get_db_loader()[0](self.loc, is_second_db=True)
         with closing(newdb):
             self._doit(newdb)
         newdb.break_cycles()

src/calibre/gui2/dialogs/choose_library.py

@@ -15,7 +15,6 @@ from calibre.constants import (filesystem_encoding, iswindows,
         get_portable_base)
 from calibre import isbytestring, patheq, force_unicode
 from calibre.gui2.wizard import move_library
-from calibre.library.database2 import LibraryDatabase2
 
 class ChooseLibrary(QDialog, Ui_Dialog):
 
@@ -86,6 +85,8 @@ class ChooseLibrary(QDialog, Ui_Dialog):
                     show=True)
             return False
         if ac in ('new', 'move'):
+            from calibre.db import get_db_loader
+            LibraryDatabase = get_db_loader()[0]
             if not empty:
                 error_dialog(self, _('Not empty'),
                     _('The folder %s is not empty. Please choose an empty'
@@ -93,10 +94,10 @@ class ChooseLibrary(QDialog, Ui_Dialog):
                     show=True)
                 return False
             if (iswindows and len(loc) >
-                    LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+                    LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT):
                 error_dialog(self, _('Too long'),
                     _('Path to library too long. Must be less than'
-                    ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+                    ' %d characters.')%LibraryDatabase.WINDOWS_LIBRARY_PATH_LIMIT,
                     show=True)
                 return False
 

src/calibre/gui2/main.py

@@ -15,8 +15,6 @@ from calibre.gui2 import (ORG_NAME, APP_UID, initialize_file_icon_provider,
     Application, choose_dir, error_dialog, question_dialog, gprefs)
 from calibre.gui2.main_window import option_parser as _option_parser
 from calibre.utils.config import prefs, dynamic
-from calibre.library.database2 import LibraryDatabase2
-from calibre.library.sqlite import sqlite, DatabaseException
 
 if iswindows:
     winutil = plugins['winutil'][0]
@@ -51,7 +49,8 @@ path_to_ebook to the database.
 
 def find_portable_library():
     base = get_portable_base()
-    if base is None: return
+    if base is None:
+        return
     import glob
     candidates = [os.path.basename(os.path.dirname(x)) for x in glob.glob(
         os.path.join(base, u'*%smetadata.db'%os.sep))]
@@ -123,7 +122,7 @@ def get_default_library_path():
 
 def get_library_path(parent=None):
     library_path = prefs['library_path']
-    if library_path is None: # Need to migrate to new database layout
+    if library_path is None:  # Need to migrate to new database layout
         base = os.path.expanduser('~')
         if iswindows:
             base = winutil.special_folder_path(winutil.CSIDL_PERSONAL)
@@ -181,7 +180,7 @@ class GuiRunner(QObject):
         main = Main(self.opts, gui_debug=self.gui_debug)
         if self.splash_screen is not None:
             self.splash_screen.showMessage(_('Initializing user interface...'))
-        with gprefs: # Only write gui.json after initialization is complete
+        with gprefs:  # Only write gui.json after initialization is complete
             main.initialize(self.library_path, db, self.listener, self.actions)
         if self.splash_screen is not None:
             self.splash_screen.finish(main)
@@ -224,7 +223,7 @@ class GuiRunner(QObject):
 
             try:
                 self.library_path = candidate
-                db = LibraryDatabase2(candidate)
+                db = self.db_class(candidate)
             except:
                 error_dialog(self.splash_screen, _('Bad database location'),
                     _('Bad database location %r. calibre will now quit.'
@@ -235,10 +234,12 @@ class GuiRunner(QObject):
         self.start_gui(db)
 
     def initialize_db(self):
+        from calibre.db import get_db_loader
         db = None
+        self.db_class, errs = get_db_loader()
         try:
-            db = LibraryDatabase2(self.library_path)
+            db = self.db_class(self.library_path)
-        except (sqlite.Error, DatabaseException):
+        except errs:
             repair = question_dialog(self.splash_screen, _('Corrupted database'),
                     _('The library database at %s appears to be corrupted. Do '
                     'you want calibre to try and rebuild it automatically? '
@@ -249,7 +250,7 @@ class GuiRunner(QObject):
                     )
             if repair:
                 if repair_library(self.library_path):
-                    db = LibraryDatabase2(self.library_path)
+                    db = self.db_class(self.library_path)
         except:
             error_dialog(self.splash_screen, _('Bad database location'),
                     _('Bad database location %r. Will start with '
@@ -383,7 +384,7 @@ def shutdown_other(rc=None):
         rc = build_pipe(print_error=False)
         if rc.conn is None:
             prints(_('No running calibre found'))
-            return # No running instance found
+            return  # No running instance found
     from calibre.utils.lock import singleinstance
     rc.conn.send('shutdown:')
     prints(_('Shutdown command sent, waiting for shutdown...'))
@@ -441,7 +442,7 @@ def main(args=sys.argv):
     otherinstance = False
     try:
         listener = Listener(address=gui_socket_address())
-    except socket.error: # Good si is correct (on UNIX)
+    except socket.error:  # Good si is correct (on UNIX)
         otherinstance = True
     else:
         # On windows only singleinstance can be trusted
@@ -458,7 +459,8 @@ if __name__ == '__main__':
     try:
         sys.exit(main())
     except Exception as err:
-        if not iswindows: raise
+        if not iswindows:
+            raise
         tb = traceback.format_exc()
         from PyQt4.QtGui import QErrorMessage
         logfile = os.path.join(os.path.expanduser('~'), 'calibre.log')
@@ -470,3 +472,4 @@ if __name__ == '__main__':
                     unicode(tb).replace('\n', '<br>'),
                     log.replace('\n', '<br>')))
 
+

src/calibre/gui2/ui.py

@@ -22,7 +22,7 @@ from calibre import prints, force_unicode
 from calibre.constants import __appname__, isosx, filesystem_encoding
 from calibre.utils.config import prefs, dynamic
 from calibre.utils.ipc.server import Server
-from calibre.library.database2 import LibraryDatabase2
+from calibre.db import get_db_loader
 from calibre.customize.ui import interface_actions, available_store_plugins
 from calibre.gui2 import (error_dialog, GetMetadata, open_url,
         gprefs, max_available_height, config, info_dialog, Dispatcher,
@@ -42,7 +42,6 @@ from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin
 from calibre.gui2.tag_browser.ui import TagBrowserMixin
 from calibre.gui2.keyboard import Manager
 from calibre.gui2.auto_add import AutoAdder
-from calibre.library.sqlite import sqlite, DatabaseException
 from calibre.gui2.proceed import ProceedQuestion
 from calibre.gui2.dialogs.message_box import JobError
 from calibre.gui2.job_indicator import Pointer
@@ -572,12 +571,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin,  # {{{
                 default_prefs = olddb.prefs
 
             from calibre.utils.formatter_functions import unload_user_template_functions
-            unload_user_template_functions(olddb.library_id )
+            unload_user_template_functions(olddb.library_id)
         except:
             olddb = None
+        db_class, errs = get_db_loader()
         try:
-            db = LibraryDatabase2(newloc, default_prefs=default_prefs)
+            db = db_class(newloc, default_prefs=default_prefs)
-        except (DatabaseException, sqlite.Error):
+        except errs:
             if not allow_rebuild:
                 raise
             import traceback
@@ -591,7 +591,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin,  # {{{
             if repair:
                 from calibre.gui2.dialogs.restore_library import repair_library_at
                 if repair_library_at(newloc, parent=self):
-                    db = LibraryDatabase2(newloc, default_prefs=default_prefs)
+                    db = db_class(newloc, default_prefs=default_prefs)
                 else:
                     return
             else:

src/calibre/gui2/wizard/__init__.py

@@ -14,7 +14,6 @@ from contextlib import closing
 from PyQt4.Qt import (QWizard, QWizardPage, QPixmap, Qt, QAbstractListModel,
     QVariant, QItemSelectionModel, SIGNAL, QObject, QTimer, pyqtSignal)
 from calibre import __appname__, patheq
-from calibre.library.database2 import LibraryDatabase2
 from calibre.library.move import MoveLibrary
 from calibre.constants import (filesystem_encoding, iswindows, plugins,
         isportable)
@@ -34,6 +33,10 @@ from calibre.customize.ui import device_plugins
 if iswindows:
     winutil = plugins['winutil'][0]
 
+def db_class():
+    from calibre.db import get_db_loader
+    return get_db_loader()[0]
+
 # Devices {{{
 
 class Device(object):
@@ -623,7 +626,7 @@ def move_library(oldloc, newloc, parent, callback_on_complete):
             if oldloc and os.access(os.path.join(oldloc, 'metadata.db'), os.R_OK):
                 # Move old library to new location
                 try:
-                    db = LibraryDatabase2(oldloc)
+                    db = db_class()(oldloc)
                 except:
                     return move_library(None, newloc, parent,
                         callback)
@@ -636,13 +639,13 @@ def move_library(oldloc, newloc, parent, callback_on_complete):
                     return
             else:
                 # Create new library at new location
-                db = LibraryDatabase2(newloc)
+                db = db_class()(newloc)
                 callback(newloc)
                 return
 
         # Try to load existing library at new location
         try:
-            LibraryDatabase2(newloc)
+            db_class()(newloc)
         except Exception as err:
             det = traceback.format_exc()
             error_dialog(parent, _('Invalid database'),
@@ -729,7 +732,7 @@ class LibraryPage(QWizardPage, LibraryUI):
 
     def is_library_dir_suitable(self, x):
         try:
-            return LibraryDatabase2.exists_at(x) or not os.listdir(x)
+            return db_class().exists_at(x) or not os.listdir(x)
         except:
             return False
 
@@ -745,10 +748,10 @@ class LibraryPage(QWizardPage, LibraryUI):
                          _('Select location for books'))
         if x:
             if (iswindows and len(x) >
-                    LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT):
+                    db_class().WINDOWS_LIBRARY_PATH_LIMIT):
                 return error_dialog(self, _('Too long'),
                     _('Path to library too long. Must be less than'
-                    ' %d characters.')%LibraryDatabase2.WINDOWS_LIBRARY_PATH_LIMIT,
+                    ' %d characters.')%(db_class().WINDOWS_LIBRARY_PATH_LIMIT),
                     show=True)
             if not os.path.exists(x):
                 try:

src/calibre/library/__init__.py

@@ -3,13 +3,13 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 ''' Code to manage ebook library'''
 
 def db(path=None, read_only=False):
-    from calibre.library.database2 import LibraryDatabase2
+    from calibre.db import get_db_loader
     from calibre.utils.config import prefs
-    return LibraryDatabase2(path if path else prefs['library_path'],
+    return get_db_loader()[0](path if path else prefs['library_path'],
             read_only=read_only)
 
 
-def generate_test_db(library_path, # {{{
+def generate_test_db(library_path,  # {{{
         num_of_records=20000,
         num_of_authors=6000,
         num_of_tags=10000,
@@ -76,3 +76,4 @@ def current_library_name():
     if path:
         return posixpath.basename(path)
 
+

src/calibre/library/cli.py

@@ -15,15 +15,18 @@ from calibre import preferred_encoding, prints, isbytestring
 from calibre.utils.config import OptionParser, prefs, tweaks
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.ebooks.metadata.book.base import field_from_string
-from calibre.library.database2 import LibraryDatabase2
 from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.utils.date import isoformat
+from calibre.db import get_db_loader
 
 FIELDS = set(['title', 'authors', 'author_sort', 'publisher', 'rating',
     'timestamp', 'size', 'tags', 'comments', 'series', 'series_index',
     'formats', 'isbn', 'uuid', 'pubdate', 'cover', 'last_modified',
     'identifiers'])
 
+def db_class():
+    return get_db_loader()[0]
+
 do_notify = True
 def send_message(msg=''):
     global do_notify
@@ -62,7 +65,7 @@ def get_db(dbpath, options):
     dbpath = os.path.abspath(dbpath)
     if options.dont_notify_gui:
         do_notify = False
-    return LibraryDatabase2(dbpath)
+    return db_class()(dbpath)
 
 def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator,
             prefix, limit, subtitle='Books in the calibre database'):
@@ -1101,7 +1104,7 @@ def command_backup_metadata(args, dbpath):
         dbpath = opts.library_path
     if isbytestring(dbpath):
         dbpath = dbpath.decode(preferred_encoding)
-    db = LibraryDatabase2(dbpath)
+    db = db_class()(dbpath)
     book_ids = None
     if opts.all:
         book_ids = db.all_ids()
@@ -1183,7 +1186,7 @@ def command_check_library(args, dbpath):
             for i in list:
                 print '    %-40.40s - %-40.40s'%(i[0], i[1])
 
-    db = LibraryDatabase2(dbpath)
+    db = db_class()(dbpath)
     checker = CheckLibrary(dbpath, db)
     checker.scan_library(names, exts)
     for check in checks:
@@ -1296,7 +1299,7 @@ def command_list_categories(args, dbpath):
     if isbytestring(dbpath):
         dbpath = dbpath.decode(preferred_encoding)
 
-    db = LibraryDatabase2(dbpath)
+    db = db_class()(dbpath)
     category_data = db.get_categories()
     data = []
     report_on = [c.strip() for c in opts.report.split(',') if c.strip()]

src/calibre/library/move.py

@@ -10,14 +10,14 @@ import time, os
 from threading import Thread
 from Queue import Empty
 
-from calibre.library.database2 import LibraryDatabase2
 from calibre.utils.ipc.server import Server
 from calibre.utils.ipc.job import ParallelJob
 
 
-def move_library(from_, to, notification = lambda x:x):
+def move_library(from_, to, notification=lambda x:x):
+    from calibre.db import get_db_loader
     time.sleep(1)
-    old = LibraryDatabase2(from_)
+    old = get_db_loader()[0](from_)
     old.move_library_to(to, notification)
     return True
 

src/calibre/library/server/main.py

@@ -100,7 +100,7 @@ def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
 
 
 def main(args=sys.argv):
-    from calibre.library.database2 import LibraryDatabase2
+    from calibre.db import get_db_loader
     parser = option_parser()
     opts, args = parser.parse_args(args)
     if opts.daemonize and not iswindows:
@@ -116,7 +116,7 @@ def main(args=sys.argv):
         print('No saved library path. Use the --with-library option'
                 ' to specify the path to the library you want to use.')
         return 1
-    db = LibraryDatabase2(opts.with_library)
+    db = get_db_loader()[0](opts.with_library)
     server = LibraryServer(db, opts, show_tracebacks=opts.develop)
     server.start()
     return 0