EOD on the new db api

resources/metadata_sqlite.sql

@@ -13,8 +13,10 @@ CREATE TABLE books ( id      INTEGER PRIMARY KEY AUTOINCREMENT,
                              isbn TEXT DEFAULT "" COLLATE NOCASE,
                              lccn TEXT DEFAULT "" COLLATE NOCASE,
                              path TEXT NOT NULL DEFAULT "",
-                             flags INTEGER NOT NULL DEFAULT 1
+                             flags INTEGER NOT NULL DEFAULT 1,
-                        , uuid TEXT, has_cover BOOL DEFAULT 0, last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00");
+                             uuid TEXT,
+                             has_cover BOOL DEFAULT 0,
+                             last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00");
 CREATE TABLE books_authors_link ( id INTEGER PRIMARY KEY,
                                           book INTEGER NOT NULL,
                                           author INTEGER NOT NULL,

src/calibre/db/backend.py

@@ -23,6 +23,8 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort
 from calibre.utils.icu import strcmp
 from calibre.utils.config import to_json, from_json, prefs, tweaks
 from calibre.utils.date import utcfromtimestamp
+from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable,
+        SizeTable, FormatsTable, AuthorsTable, IdentifiersTable)
 # }}}
 
 '''
@@ -167,7 +169,7 @@ class Connection(apsw.Connection): # {{{
             return self.cursor().execute(sql)
 # }}}
 
-class DB(SchemaUpgrade):
+class DB(object, SchemaUpgrade):
 
     PATH_LIMIT = 40 if iswindows else 100
     WINDOWS_LIBRARY_PATH_LIMIT = 75
@@ -400,5 +402,40 @@ class DB(SchemaUpgrade):
         ''' Return last modified time as a UTC datetime object '''
         return utcfromtimestamp(os.stat(self.dbpath).st_mtime)
 
+    def read_tables(self):
+        tables = {}
+        for col in ('title', 'sort', 'author_sort', 'series_index', 'comments',
+                'timestamp', 'published', 'uuid', 'path', 'cover',
+                'last_modified'):
+            metadata = self.field_metadata[col].copy()
+            if metadata['table'] is None:
+                metadata['table'], metadata['column'] == 'books', ('has_cover'
+                        if col == 'cover' else col)
+            tables[col] = OneToOneTable(col, metadata)
+
+        for col in ('series', 'publisher', 'rating'):
+            tables[col] = ManyToOneTable(col, self.field_metadata[col].copy())
+
+        for col in ('authors', 'tags', 'formats', 'identifiers'):
+            cls = {
+                    'authors':AuthorsTable,
+                    'formats':FormatsTable,
+                    'identifiers':IdentifiersTable,
+                  }.get(col, ManyToManyTable)
+            tables[col] = cls(col, self.field_metadata[col].copy())
+
+        tables['size'] = SizeTable('size', self.field_metadata['size'].copy())
+
+        with self.conn: # Use a single transaction, to ensure nothing modifies
+                        # the db while we are reading
+            for table in tables.itervalues():
+                try:
+                    table.read()
+                except:
+                    prints('Failed to read table:', table.name)
+                    raise
+
+        return tables
+
    # }}}
 

src/calibre/db/tables.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from datetime import datetime
+
+from dateutil.tz import tzoffset
+
+from calibre.constants import plugins
+from calibre.utils.date import parse_date, local_tz
+from calibre.ebooks.metadata import author_to_author_sort
+
+_c_speedup = plugins['speedup'][0]
+
+def _c_convert_timestamp(val):
+    if not val:
+        return None
+    try:
+        ret = _c_speedup.parse_date(val.strip())
+    except:
+        ret = None
+    if ret is None:
+        return parse_date(val, as_utc=False)
+    year, month, day, hour, minutes, seconds, tzsecs = ret
+    return datetime(year, month, day, hour, minutes, seconds,
+                tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz)
+
+class Table(object):
+
+    def __init__(self, name, metadata):
+        self.name, self.metadata = name, metadata
+
+        # self.adapt() maps values from the db to python objects
+        self.adapt = \
+            {
+                'datetime': _c_convert_timestamp,
+                'bool': bool
+            }.get(
+                metadata['datatype'], lambda x: x)
+        if name == 'authors':
+            # Legacy
+            self.adapt = lambda x: x.replace('|', ',') if x else None
+
+class OneToOneTable(Table):
+
+    def read(self, db):
+        self.book_col_map = {}
+        idcol = 'id' if self.metadata['table'] == 'books' else 'book'
+        for row in db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol,
+            self.metadata['column'], self.metadata['table'])):
+            self.book_col_map[row[0]] = self.adapt(row[1])
+
+class SizeTable(OneToOneTable):
+
+    def read(self, db):
+        self.book_col_map = {}
+        for row in db.conn.execute(
+                'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data '
+                'WHERE data.book=books.id) FROM books'):
+            self.book_col_map[row[0]] = self.adapt(row[1])
+
+class ManyToOneTable(Table):
+
+    def read(self, db):
+        self.id_map = {}
+        self.extra_map = {}
+        self.col_book_map = {}
+        self.book_col_map = {}
+        self.read_id_maps(db)
+        self.read_maps(db)
+
+    def read_id_maps(self, db):
+        for row in db.conn.execute('SELECT id, {0} FROM {1}'.format(
+            self.metadata['name'], self.metadata['table'])):
+            if row[1]:
+                self.id_map[row[0]] = self.adapt(row[1])
+
+    def read_maps(self, db):
+        for row in db.conn.execute(
+                'SELECT book, {0} FROM books_{1}_link'.format(
+                    self.metadata['link_column'], self.metadata['table'])):
+            if row[1] not in self.col_book_map:
+                self.col_book_map[row[1]] = []
+            self.col_book_map.append(row[0])
+            self.book_col_map[row[0]] = row[1]
+
+class ManyToManyTable(ManyToOneTable):
+
+    def read_maps(self, db):
+        for row in db.conn.execute(
+                'SELECT book, {0} FROM books_{1}_link'.format(
+                    self.metadata['link_column'], self.metadata['table'])):
+            if row[1] not in self.col_book_map:
+                self.col_book_map[row[1]] = []
+            self.col_book_map.append(row[0])
+            if row[0] not in self.book_col_map:
+                self.book_col_map[row[0]] = []
+            self.book_col_map[row[0]].append(row[1])
+
+class AuthorsTable(ManyToManyTable):
+
+    def read_id_maps(self, db):
+        for row in db.conn.execute(
+                'SELECT id, name, sort FROM authors'):
+            self.id_map[row[0]] = row[1]
+            self.extra_map[row[0]] = (row[2] if row[2] else
+                    author_to_author_sort(row[1]))
+
+class FormatsTable(ManyToManyTable):
+
+    def read_id_maps(self, db):
+        pass
+
+    def read_maps(self, db):
+        for row in db.conn.execute('SELECT book, format, name FROM data'):
+            if row[1] is not None:
+                if row[1] not in self.col_book_map:
+                    self.col_book_map[row[1]] = []
+                self.col_book_map.append(row[0])
+                if row[0] not in self.book_col_map:
+                    self.book_col_map[row[0]] = []
+                self.book_col_map[row[0]].append((row[1], row[2]))
+
+class IdentifiersTable(ManyToManyTable):
+
+    def read_id_maps(self, db):
+        pass
+
+    def read_maps(self, db):
+        for row in db.conn.execute('SELECT book, type, val FROM identifiers'):
+            if row[1] is not None and row[2] is not None:
+                if row[1] not in self.col_book_map:
+                    self.col_book_map[row[1]] = []
+                self.col_book_map.append(row[0])
+                if row[0] not in self.book_col_map:
+                    self.book_col_map[row[0]] = []
+                self.book_col_map[row[0]].append((row[1], row[2]))
+