Restore custom columns from db prefs instead of trying to aggregate them from the OPFs when doing a library restore

src/calibre/gui2/dialogs/restore_library.py

@@ -100,9 +100,7 @@ def restore_database(db, parent=None):
                 'the database from the individual book '
                 'metadata. This is useful if the '
                 'database has been corrupted and you get a '
-                'blank list of books. Note that restoring only '
+                'blank list of books.'
-                'restores books, not any settings stored in the '
-                'database, or any custom recipes.'
                 '<p>Do you want to restore the database?')):
         return False
     db.conn.close()

src/calibre/library/database2.py

@@ -162,7 +162,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         return path and os.path.exists(os.path.join(path, 'metadata.db'))
 
     def __init__(self, library_path, row_factory=False, default_prefs=None,
-            read_only=False, is_second_db=False):
+            read_only=False, is_second_db=False, progress_callback=None,
+            restore_all_prefs=False):
         self.is_second_db = is_second_db
         try:
             if isbytestring(library_path):
@@ -205,15 +206,21 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         # if we are to copy the prefs and structure from some other DB, then
         # we need to do it before we call initialize_dynamic
         if apply_default_prefs and default_prefs is not None:
+            if progress_callback is None:
+                progress_callback = lambda x, y: True
             dbprefs = DBPrefs(self)
-            for key in default_prefs:
+            progress_callback(None, len(default_prefs))
+            for i, key in enumerate(default_prefs):
                 # be sure that prefs not to be copied are listed below
-                if key in frozenset(['news_to_be_synced']):
+                if not restore_all_prefs and key in frozenset(['news_to_be_synced']):
                     continue
                 dbprefs[key] = default_prefs[key]
+                progress_callback(_('restored preference ') + key, i+1)
             if 'field_metadata' in default_prefs:
                 fmvals = [f for f in default_prefs['field_metadata'].values() if f['is_custom']]
-                for f in fmvals:
+                progress_callback(None, len(fmvals))
+                for i, f in enumerate(fmvals):
+                    progress_callback(_('creating custom column ') + f['label'], i)
                     self.create_custom_column(f['label'], f['name'], f['datatype'],
                             f['is_multiple'] is not None and len(f['is_multiple']) > 0,
                             f['is_editable'], f['display'])

src/calibre/library/prefs.py

@@ -80,21 +80,27 @@ class DBPrefs(dict):
             import traceback
             traceback.print_exc()
 
-    def read_serialized(self, library_path):
+    @classmethod
+    def read_serialized(cls, library_path, recreate_prefs=False):
         try:
             from_filename = os.path.join(library_path,
                     'metadata_db_prefs_backup.json')
             with open(from_filename, "rb") as f:
-                self.clear()
                 d = json.load(f, object_hook=from_json)
-                self.db.conn.execute('DELETE FROM preferences')
+                if not recreate_prefs:
+                    return d
+                cls.clear()
+                cls.db.conn.execute('DELETE FROM preferences')
                 for k,v in d.iteritems():
-                    raw = self.to_raw(v)
+                    raw = cls.to_raw(v)
-                    self.db.conn.execute(
+                    cls.db.conn.execute(
                         'INSERT INTO preferences (key,val) VALUES (?,?)', (k, raw))
-                self.db.conn.commit()
+                cls.db.conn.commit()
-                self.clear()
+                cls.clear()
-                self.update(d)
+                cls.update(d)
+                return d
         except:
             import traceback
             traceback.print_exc()
+            raise
+        return None

src/calibre/library/restore.py

@@ -12,6 +12,7 @@ from operator import itemgetter
 from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.library.database2 import LibraryDatabase2
+from calibre.library.prefs import DBPrefs
 from calibre.constants import filesystem_encoding
 from calibre.utils.date import utcfromtimestamp
 from calibre import isbytestring
@@ -101,8 +102,14 @@ class Restore(Thread):
             with TemporaryDirectory('_library_restore') as tdir:
                 self.library_path = tdir
                 self.scan_library()
-                self.load_preferences()
+                if not self.load_preferences():
-                self.create_cc_metadata()
+                    # Something went wrong with preferences restore. Start over
+                    # with a new database and attempt to rebuild the structure
+                    # from the metadata in the opf
+                    dbpath = os.path.join(self.library_path, 'metadata.db')
+                    if os.path.exists(dbpath):
+                        os.remove(dbpath)
+                    self.create_cc_metadata()
                 self.restore_books()
                 if self.successes == 0 and len(self.dirs) > 0:
                     raise Exception(('Something bad happened'))
@@ -112,21 +119,29 @@ class Restore(Thread):
 
     def load_preferences(self):
         self.progress_callback(None, 1)
-        self.progress_callback('Starting restore preferences', 0)
+        self.progress_callback(_('Starting restoring preferences and column metadata'), 0)
-        dbpath = os.path.join(self.src_library_path, 'metadata_db_prefs_backup.json')
+        prefs_path = os.path.join(self.src_library_path, 'metadata_db_prefs_backup.json')
-        ndbpath = os.path.join(self.library_path, 'metadata_db_prefs_backup.json')
+        if not os.path.exists(prefs_path):
-        if not os.path.exists(dbpath):
+            self.progress_callback(_('Cannot restore preferences. Backup file not found.'), 1)
-            self.progress_callback('Cannot restore preferences. Backup file not found.', 1)
+            return False
-            return
         try:
-            shutil.copyfile(dbpath, ndbpath)
+            prefs = DBPrefs.read_serialized(self.src_library_path, recreate_prefs=False)
-            db = RestoreDatabase(self.library_path)
+            db = RestoreDatabase(self.library_path, default_prefs=prefs,
-            db.prefs.read_serialized(self.library_path)
+                                 restore_all_prefs=True,
+                                 progress_callback=self.progress_callback)
             db.commit()
             db.conn.close()
-            self.progress_callback('Finished restore preferences', 1)
+            self.progress_callback(None, 1)
+            if 'field_metadata' in prefs:
+                self.progress_callback(_('Finished restoring preferences and column metadata'), 1)
+                return True
+            self.progress_callback(_('Finished restoring preferences'), 1)
+            return False
         except:
-            self.progress_callback('Restoring preferences failed', 1)
+            traceback.print_exc()
+            self.progress_callback(None, 1)
+            self.progress_callback(_('Restoring preferences and column metadata failed'), 0)
+        return False
 
     def scan_library(self):
         for dirpath, dirnames, filenames in os.walk(self.src_library_path):