From e57abb1d283dab55cbeeea8181757e86742abc49 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Jul 2011 20:08:19 -0600 Subject: [PATCH] Start work on cache layer for new db backend --- src/calibre/db/backend.py | 32 ++++++++++- src/calibre/db/cache.py | 19 +++++++ src/calibre/db/fields.py | 114 ++++++++++++++++++++++++++++++++++++++ src/calibre/db/tables.py | 30 +++++++++- src/calibre/db/view.py | 21 +++++++ 5 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 src/calibre/db/fields.py create mode 100644 src/calibre/db/view.py diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 0716cf691c..1b7d3460ef 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -478,7 +478,6 @@ class DB(object): remove.append(data) continue - self.custom_column_label_map[data['label']] = data['num'] self.custom_column_num_map[data['num']] = \ self.custom_column_label_map[data['label']] = data @@ -613,10 +612,31 @@ class DB(object): tables['size'] = SizeTable('size', self.field_metadata['size'].copy()) - for label, data in self.custom_column_label_map.iteritems(): - label = '#' + label + self.FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'timestamp':3, + 'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8, + 'publisher':9, 'series_index':10, 'sort':11, 'author_sort':12, + 'formats':13, 'path':14, 'pubdate':15, 'uuid':16, 'cover':17, + 'au_map':18, 'last_modified':19, 'identifiers':20} + + for k,v in self.FIELD_MAP.iteritems(): + self.field_metadata.set_field_record_index(k, v, prefer_custom=False) + + base = max(self.FIELD_MAP.itervalues()) + + for label_, data in self.custom_column_label_map.iteritems(): + label = '#' + label_ metadata = self.field_metadata[label].copy() link_table = self.custom_table_names(data['num'])[1] + self.FIELD_MAP[data['num']] = base = base+1 + self.field_metadata.set_field_record_index(label_, base, + prefer_custom=True) + if data['datatype'] == 'series': + # account for the series index column. Field_metadata knows that + # the series index is one larger than the series. If you change + # it here, be sure to change it there as well. + self.FIELD_MAP[str(data['num'])+'_index'] = base = base+1 + self.field_metadata.set_field_record_index(label_+'_index', base, + prefer_custom=True) if data['normalized']: if metadata['is_multiple']: @@ -634,6 +654,12 @@ class DB(object): tables[label] = OneToOneTable(label, metadata) else: tables[label] = OneToOneTable(label, metadata) + + self.FIELD_MAP['ondevice'] = base = base+1 + self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False) + self.FIELD_MAP['marked'] = base = base+1 + self.field_metadata.set_field_record_index('marked', base, prefer_custom=False) + # }}} @property diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index cc6da1e995..33487f1f6f 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,5 +7,24 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +from calibre.db.locking import create_locks +from calibre.db.fields import create_field +class Cache(object): + + def __init__(self, backend): + self.backend = backend + self.fields = {} + self.read_lock, self.write_lock = create_locks() + + # Cache Layer API {{{ + + def init(self): + with self.write_lock: + self.backend.read_tables() + + for field, table in self.backend.tables.iteritems(): + self.fields[field] = create_field(field, table) + + # }}} diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py new file mode 100644 index 0000000000..1c36deda2f --- /dev/null +++ b/src/calibre/db/fields.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY + +class Field(object): + + def __init__(self, name, table): + self.name, self.table = name, table + self.has_text_data = self.metadata['datatype'] in ('text', 'comments', + 'series', 'enumeration') + self.table_type = self.table.table_type + + @property + def metadata(self): + return self.table.metadata + + def for_book(self, book_id, default_value=None): + ''' + Return the value of this field for the book identified by book_id. + When no value is found, returns ``default_value``. + ''' + raise NotImplementedError() + + def ids_for_book(self, book_id): + ''' + Return a tuple of items ids for items associated with the book + identified by book_ids. Returns an empty tuple if no such items are + found. + ''' + raise NotImplementedError() + + def books_for(self, item_id): + ''' + Return the ids of all books associated with the item identified by + item_id as a tuple. An empty tuple is returned if no books are found. + ''' + raise NotImplementedError() + + def __iter__(self): + ''' + Iterate over the ids for all values in this field + ''' + raise NotImplementedError() + +class OneToOneField(Field): + + def for_book(self, book_id, default_value=None): + return self.table.book_col_map.get(book_id, default_value) + + def ids_for_book(self, book_id): + return (book_id,) + + def books_for(self, item_id): + return (item_id,) + + def __iter__(self): + return self.table.book_col_map.iterkeys() + +class ManyToOneField(Field): + + def for_book(self, book_id, default_value=None): + ids = self.table.book_col_map.get(book_id, None) + if ids is not None: + ans = self.id_map[ids] + else: + ans = default_value + return ans + + def ids_for_book(self, book_id): + ids = self.table.book_col_map.get(book_id, None) + if ids is None: + return () + return ids + + def books_for(self, item_id): + return self.table.col_book_map.get(item_id, ()) + + def __iter__(self): + return self.table.id_map.iterkeys() + +class ManyToManyField(Field): + + def for_book(self, book_id, default_value=None): + ids = self.table.book_col_map.get(book_id, ()) + if ids: + ans = tuple(self.id_map[i] for i in ids) + else: + ans = default_value + return ans + + def ids_for_book(self, book_id): + return self.table.book_col_map.get(book_id, ()) + + def books_for(self, item_id): + return self.table.col_book_map.get(item_id, ()) + + def __iter__(self): + return self.table.id_map.iterkeys() + +def create_field(name, table): + cls = { + ONE_ONE : OneToOneField, + MANY_ONE : ManyToOneField, + MANY_MANY : ManyToManyField, + }[table.table_type] + return cls(name, table) + diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index cbb3ce0006..29cc1b2bc8 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -17,6 +17,8 @@ from calibre.ebooks.metadata import author_to_author_sort _c_speedup = plugins['speedup'][0] +ONE_ONE, MANY_ONE, MANY_MANY = xrange(3) + def _c_convert_timestamp(val): if not val: return None @@ -57,6 +59,8 @@ class OneToOneTable(Table): timestamp, size, etc. ''' + table_type = ONE_ONE + def read(self, db): self.book_col_map = {} idcol = 'id' if self.metadata['table'] == 'books' else 'book' @@ -82,9 +86,10 @@ class ManyToOneTable(Table): Each book however has only one value for data of this type. ''' + table_type = MANY_ONE + def read(self, db): self.id_map = {} - self.extra_map = {} self.col_book_map = {} self.book_col_map = {} self.read_id_maps(db) @@ -105,6 +110,9 @@ class ManyToOneTable(Table): self.col_book_map[row[1]].append(row[0]) self.book_col_map[row[0]] = row[1] + for key, val in self.col_book_map: + self.col_book_map[key] = tuple(val) + class ManyToManyTable(ManyToOneTable): ''' @@ -113,6 +121,8 @@ class ManyToManyTable(ManyToOneTable): book. For example: tags or authors. ''' + table_type = MANY_MANY + def read_maps(self, db): for row in db.conn.execute( 'SELECT book, {0} FROM {1}'.format( @@ -124,6 +134,12 @@ class ManyToManyTable(ManyToOneTable): self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append(row[1]) + for key, val in self.col_book_map: + self.col_book_map[key] = tuple(val) + + for key, val in self.book_col_map: + self.book_col_map[key] = tuple(val) + class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): @@ -150,6 +166,12 @@ class FormatsTable(ManyToManyTable): self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append((row[1], row[2])) + for key, val in self.col_book_map: + self.col_book_map[key] = tuple(val) + + for key, val in self.book_col_map: + self.book_col_map[key] = tuple(val) + class IdentifiersTable(ManyToManyTable): def read_id_maps(self, db): @@ -165,3 +187,9 @@ class IdentifiersTable(ManyToManyTable): self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append((row[1], row[2])) + for key, val in self.col_book_map: + self.col_book_map[key] = tuple(val) + + for key, val in self.book_col_map: + self.book_col_map[key] = tuple(val) + diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py new file mode 100644 index 0000000000..8f833499aa --- /dev/null +++ b/src/calibre/db/view.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +class View(object): + + def __init__(self, cache): + self.cache = cache + self._field_idx_map = {} + for col, idx in cache.backend.FIELD_MAP.iteritems(): + if isinstance(col, int): + pass # custom column + else: + self._field_idx_map[idx] = col +