API for dealing with distributed metadata backup

src/calibre/library/custom_columns.py

@@ -382,6 +382,7 @@ class CustomColumns(object):
             )
         # get rid of the temp tables
         self.conn.executescript(drops)
+        self.dirtied(ids, commit=False)
         self.conn.commit()
 
         # set the in-memory copies of the tags
@@ -402,19 +403,21 @@ class CustomColumns(object):
         same length as ids.
         '''
         if extras is not None and len(extras) != len(ids):
-            raise ValueError('Lentgh of ids and extras is not the same')
+            raise ValueError('Length of ids and extras is not the same')
         ev = None
         for idx,id in enumerate(ids):
             if extras is not None:
                 ev = extras[idx]
             self._set_custom(id, val, label=label, num=num, append=append,
                              notify=notify, extra=ev)
+        self.dirtied(ids, commit=False)
         self.conn.commit()
 
     def set_custom(self, id, val, label=None, num=None,
                    append=False, notify=True, extra=None, commit=True):
         self._set_custom(id, val, label=label, num=num, append=append,
                          notify=notify, extra=extra)
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
 

src/calibre/library/database2.py

@@ -13,6 +13,7 @@ from math import floor
 from PyQt4.QtGui import QImage
 
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
+from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.library.database import LibraryDatabase
 from calibre.library.field_metadata import FieldMetadata, TagsIcons
 from calibre.library.schema_upgrades import SchemaUpgrade
@@ -126,6 +127,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def __init__(self, library_path, row_factory=False):
         self.field_metadata = FieldMetadata()
+        self.dirtied_cache = set([])
         if not os.path.exists(library_path):
             os.makedirs(library_path)
         self.listeners = set([])
@@ -337,6 +339,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         setattr(self, 'title_sort', functools.partial(self.get_property,
                 loc=self.FIELD_MAP['sort']))
 
+        d = self.conn.get('SELECT book FROM metadata_dirtied', all=True)
+        self.dirtied_cache.update(set([x[0] for x in d]))
+
         self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
         self.refresh()
         self.last_update_check = self.last_modified()
@@ -550,6 +555,33 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def metadata_for_field(self, key):
         return self.field_metadata[key]
 
+    def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True):
+        for book_id in book_ids:
+            mi = self.get_metadata(book_id, index_is_id=True, get_cover=True)
+            # Always set cover to cover.jpg. Even if cover doesn't exist,
+            # no harm done. This way no need to call dirtied when
+            # cover is set/removed
+            mi.cover = 'cover.jpg'
+            raw = metadata_to_opf(mi)
+            path = self.abspath(book_id, index_is_id=True)
+            with open(os.path.join(path, 'metadata.opf'), 'wb') as f:
+                f.write(raw)
+            if remove_from_dirtied:
+                self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
+                        (book_id,))
+                if book_id in self.dirtied_cache:
+                    self.dirtied_cache.remove(book_id)
+        if commit:
+            self.conn.commit()
+
+    def dirtied(self, book_ids, commit=True):
+        self.conn.executemany(
+            'INSERT OR REPLACE INTO metadata_dirtied VALUES (?)',
+                [(x,) for x in book_ids])
+        if commit:
+            self.conn.commit()
+        self.dirtied.update(set(book_ids))
+
     def get_metadata(self, idx, index_is_id=False, get_cover=False):
         '''
         Convenience method to return metadata as a :class:`Metadata` object.
@@ -1242,6 +1274,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         ss = self.author_sort_from_book(id, index_is_id=True)
         self.conn.execute('UPDATE books SET author_sort=? WHERE id=?',
                           (ss, id))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['authors'],
@@ -1268,6 +1301,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         else:
             self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True)
         self.set_path(id, index_is_id=True)
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         if notify:
@@ -1277,6 +1311,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if dt:
             self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['timestamp'], dt, row_is_id=True)
+            self.dirtied([id], commit=False)
             if commit:
                 self.conn.commit()
             if notify:
@@ -1286,6 +1321,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if dt:
             self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True)
+            self.dirtied([id], commit=False)
             if commit:
                 self.conn.commit()
             if notify:
@@ -1304,6 +1340,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 aid = self.conn.execute('INSERT INTO publishers(name) VALUES (?)', (publisher,)).lastrowid
             self.conn.execute('INSERT INTO books_publishers_link(book, publisher) VALUES (?,?)', (id, aid))
+            self.dirtied([id], commit=False)
             if commit:
                 self.conn.commit()
             self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True)
@@ -1594,6 +1631,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             '''.format(tables[0], tables[1])
             )
         self.conn.executescript(drops)
+        self.dirtied(ids, commit=False)
         self.conn.commit()
 
         for x in ids:
@@ -1639,6 +1677,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         (id, tid), all=False):
                 self.conn.execute('INSERT INTO books_tags_link(book, tag) VALUES (?,?)',
                               (id, tid))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         tags = u','.join(self.get_tags(id))
@@ -1693,6 +1732,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid
             self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['series'], series, row_is_id=True)
@@ -1707,6 +1747,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         except:
             idx = 1.0
         self.conn.execute('UPDATE books SET series_index=? WHERE id=?', (idx, id))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['series_index'], idx, row_is_id=True)
@@ -1719,6 +1760,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         rat = self.conn.get('SELECT id FROM ratings WHERE rating=?', (rating,), all=False)
         rat = rat if rat else self.conn.execute('INSERT INTO ratings(rating) VALUES (?)', (rating,)).lastrowid
         self.conn.execute('INSERT INTO books_ratings_link(book, rating) VALUES (?,?)', (id, rat))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['rating'], rating, row_is_id=True)
@@ -1731,11 +1773,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['comments'], text, row_is_id=True)
+        self.dirtied([id], commit=False)
         if notify:
             self.notify('metadata', [id])
 
     def set_author_sort(self, id, sort, notify=True, commit=True):
         self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (sort, id))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['author_sort'], sort, row_is_id=True)
@@ -1744,6 +1788,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_isbn(self, id, isbn, notify=True, commit=True):
         self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True)

src/calibre/library/schema_upgrades.py

@@ -397,3 +397,15 @@ class SchemaUpgrade(object):
                                  UNIQUE(key));
         '''
         self.conn.executescript(script)
+
+    def upgrade_version_13(self):
+        'Dirtied table for OPF metadata backups'
+        script = '''
+        DROP TABLE IF EXISTS metadata_dirtied;
+        CREATE TABLE metadata_dirtied(id INTEGER PRIMARY KEY,
+                             book INTEGER NOT NULL,
+                             UNIQUE(book));
+        INSERT INTO metadata_dirtied (book) SELECT id FROM books;
+        '''
+        self.conn.executescript(script)
+