From f9dc7d978011f09fc1ef046da316b3ee38871fc2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 20 Jun 2011 21:48:18 -0600 Subject: [PATCH] EOD on the new db api --- resources/metadata_sqlite.sql | 6 +- src/calibre/db/backend.py | 39 +++++++++- src/calibre/db/tables.py | 143 ++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 src/calibre/db/tables.py diff --git a/resources/metadata_sqlite.sql b/resources/metadata_sqlite.sql index 9c4f666449..aa29d4b8de 100644 --- a/resources/metadata_sqlite.sql +++ b/resources/metadata_sqlite.sql @@ -13,8 +13,10 @@ CREATE TABLE books ( id INTEGER PRIMARY KEY AUTOINCREMENT, isbn TEXT DEFAULT "" COLLATE NOCASE, lccn TEXT DEFAULT "" COLLATE NOCASE, path TEXT NOT NULL DEFAULT "", - flags INTEGER NOT NULL DEFAULT 1 - , uuid TEXT, has_cover BOOL DEFAULT 0, last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00"); + flags INTEGER NOT NULL DEFAULT 1, + uuid TEXT, + has_cover BOOL DEFAULT 0, + last_modified TIMESTAMP NOT NULL DEFAULT "2000-01-01 00:00:00+00:00"); CREATE TABLE books_authors_link ( id INTEGER PRIMARY KEY, book INTEGER NOT NULL, author INTEGER NOT NULL, diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 4e6c028b93..159612e52d 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -23,6 +23,8 @@ from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.utils.icu import strcmp from calibre.utils.config import to_json, from_json, prefs, tweaks from calibre.utils.date import utcfromtimestamp +from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, + SizeTable, FormatsTable, AuthorsTable, IdentifiersTable) # }}} ''' @@ -167,7 +169,7 @@ class Connection(apsw.Connection): # {{{ return self.cursor().execute(sql) # }}} -class DB(SchemaUpgrade): +class DB(object, SchemaUpgrade): PATH_LIMIT = 40 if iswindows else 100 WINDOWS_LIBRARY_PATH_LIMIT = 75 @@ -400,5 +402,40 @@ class DB(SchemaUpgrade): ''' Return last modified time as a UTC datetime object ''' return utcfromtimestamp(os.stat(self.dbpath).st_mtime) + def read_tables(self): + tables = {} + for col in ('title', 'sort', 'author_sort', 'series_index', 'comments', + 'timestamp', 'published', 'uuid', 'path', 'cover', + 'last_modified'): + metadata = self.field_metadata[col].copy() + if metadata['table'] is None: + metadata['table'], metadata['column'] == 'books', ('has_cover' + if col == 'cover' else col) + tables[col] = OneToOneTable(col, metadata) + + for col in ('series', 'publisher', 'rating'): + tables[col] = ManyToOneTable(col, self.field_metadata[col].copy()) + + for col in ('authors', 'tags', 'formats', 'identifiers'): + cls = { + 'authors':AuthorsTable, + 'formats':FormatsTable, + 'identifiers':IdentifiersTable, + }.get(col, ManyToManyTable) + tables[col] = cls(col, self.field_metadata[col].copy()) + + tables['size'] = SizeTable('size', self.field_metadata['size'].copy()) + + with self.conn: # Use a single transaction, to ensure nothing modifies + # the db while we are reading + for table in tables.itervalues(): + try: + table.read() + except: + prints('Failed to read table:', table.name) + raise + + return tables + # }}} diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py new file mode 100644 index 0000000000..7240b3ec6e --- /dev/null +++ b/src/calibre/db/tables.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from datetime import datetime + +from dateutil.tz import tzoffset + +from calibre.constants import plugins +from calibre.utils.date import parse_date, local_tz +from calibre.ebooks.metadata import author_to_author_sort + +_c_speedup = plugins['speedup'][0] + +def _c_convert_timestamp(val): + if not val: + return None + try: + ret = _c_speedup.parse_date(val.strip()) + except: + ret = None + if ret is None: + return parse_date(val, as_utc=False) + year, month, day, hour, minutes, seconds, tzsecs = ret + return datetime(year, month, day, hour, minutes, seconds, + tzinfo=tzoffset(None, tzsecs)).astimezone(local_tz) + +class Table(object): + + def __init__(self, name, metadata): + self.name, self.metadata = name, metadata + + # self.adapt() maps values from the db to python objects + self.adapt = \ + { + 'datetime': _c_convert_timestamp, + 'bool': bool + }.get( + metadata['datatype'], lambda x: x) + if name == 'authors': + # Legacy + self.adapt = lambda x: x.replace('|', ',') if x else None + +class OneToOneTable(Table): + + def read(self, db): + self.book_col_map = {} + idcol = 'id' if self.metadata['table'] == 'books' else 'book' + for row in db.conn.execute('SELECT {0}, {1} FROM {2}'.format(idcol, + self.metadata['column'], self.metadata['table'])): + self.book_col_map[row[0]] = self.adapt(row[1]) + +class SizeTable(OneToOneTable): + + def read(self, db): + self.book_col_map = {} + for row in db.conn.execute( + 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' + 'WHERE data.book=books.id) FROM books'): + self.book_col_map[row[0]] = self.adapt(row[1]) + +class ManyToOneTable(Table): + + def read(self, db): + self.id_map = {} + self.extra_map = {} + self.col_book_map = {} + self.book_col_map = {} + self.read_id_maps(db) + self.read_maps(db) + + def read_id_maps(self, db): + for row in db.conn.execute('SELECT id, {0} FROM {1}'.format( + self.metadata['name'], self.metadata['table'])): + if row[1]: + self.id_map[row[0]] = self.adapt(row[1]) + + def read_maps(self, db): + for row in db.conn.execute( + 'SELECT book, {0} FROM books_{1}_link'.format( + self.metadata['link_column'], self.metadata['table'])): + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + self.book_col_map[row[0]] = row[1] + +class ManyToManyTable(ManyToOneTable): + + def read_maps(self, db): + for row in db.conn.execute( + 'SELECT book, {0} FROM books_{1}_link'.format( + self.metadata['link_column'], self.metadata['table'])): + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append(row[1]) + +class AuthorsTable(ManyToManyTable): + + def read_id_maps(self, db): + for row in db.conn.execute( + 'SELECT id, name, sort FROM authors'): + self.id_map[row[0]] = row[1] + self.extra_map[row[0]] = (row[2] if row[2] else + author_to_author_sort(row[1])) + +class FormatsTable(ManyToManyTable): + + def read_id_maps(self, db): + pass + + def read_maps(self, db): + for row in db.conn.execute('SELECT book, format, name FROM data'): + if row[1] is not None: + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append((row[1], row[2])) + +class IdentifiersTable(ManyToManyTable): + + def read_id_maps(self, db): + pass + + def read_maps(self, db): + for row in db.conn.execute('SELECT book, type, val FROM identifiers'): + if row[1] is not None and row[2] is not None: + if row[1] not in self.col_book_map: + self.col_book_map[row[1]] = [] + self.col_book_map.append(row[0]) + if row[0] not in self.book_col_map: + self.book_col_map[row[0]] = [] + self.book_col_map[row[0]].append((row[1], row[2])) +