From 192e922260e80f32cd385036f0eac175328157f1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 14 Jul 2011 09:25:49 -0600 Subject: [PATCH] More work on cache layer of new db backend --- src/calibre/db/backend.py | 34 +++++++- src/calibre/db/cache.py | 166 ++++++++++++++++++++++++++++++++++++++ src/calibre/db/fields.py | 76 +++++++++++++++++ src/calibre/db/tables.py | 35 +++++--- src/calibre/db/view.py | 68 +++++++++++++++- 5 files changed, 361 insertions(+), 18 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 1b7d3460ef..9158feeb5e 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -8,7 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' # Imports {{{ -import os, shutil, uuid, json +import os, shutil, uuid, json, glob from functools import partial import apsw @@ -25,7 +25,7 @@ from calibre.utils.config import to_json, from_json, prefs, tweaks from calibre.utils.date import utcfromtimestamp, parse_date from calibre.utils.filenames import is_case_sensitive from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, - SizeTable, FormatsTable, AuthorsTable, IdentifiersTable) + SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, CompositeTable) # }}} ''' @@ -624,7 +624,7 @@ class DB(object): base = max(self.FIELD_MAP.itervalues()) for label_, data in self.custom_column_label_map.iteritems(): - label = '#' + label_ + label = self.field_metadata.custom_field_prefix + label_ metadata = self.field_metadata[label].copy() link_table = self.custom_table_names(data['num'])[1] self.FIELD_MAP[data['num']] = base = base+1 @@ -653,7 +653,10 @@ class DB(object): metadata['table'] = link_table tables[label] = OneToOneTable(label, metadata) else: - tables[label] = OneToOneTable(label, metadata) + if data['datatype'] == 'composite': + tables[label] = CompositeTable(label, metadata) + else: + tables[label] = OneToOneTable(label, metadata) self.FIELD_MAP['ondevice'] = base = base+1 self.field_metadata.set_field_record_index('ondevice', base, prefer_custom=False) @@ -758,5 +761,28 @@ class DB(object): pprint.pprint(table.metadata) raise + def format_abspath(self, book_id, fmt, fname, path): + path = os.path.join(self.library_path, path) + fmt = ('.' + fmt.lower()) if fmt else '' + fmt_path = os.path.join(path, fname+fmt) + if os.path.exists(fmt_path): + return fmt_path + try: + candidates = glob.glob(os.path.join(path, '*'+fmt)) + except: # If path contains strange characters this throws an exc + candidates = [] + if fmt and candidates and os.path.exists(candidates[0]): + shutil.copyfile(candidates[0], fmt_path) + return fmt_path + + def format_metadata(self, book_id, fmt, fname, path): + path = self.format_abspath(book_id, fmt, fname, path) + ans = {} + if path is not None: + stat = os.stat(path) + ans['size'] = stat.st_size + ans['mtime'] = utcfromtimestamp(stat.st_mtime) + return ans + # }}} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 6406bba019..ac046143d1 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,10 +7,14 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os +from collections import defaultdict from functools import wraps from calibre.db.locking import create_locks from calibre.db.fields import create_field +from calibre.ebooks.book.base import Metadata +from calibre.utils.date import now def api(f): f.is_cache_api = True @@ -40,6 +44,7 @@ class Cache(object): self.backend = backend self.fields = {} self.read_lock, self.write_lock = create_locks() + self.format_metadata_cache = defaultdict(dict) # Implement locking for all simple read/write API methods # An unlocked version of the method is stored with the name starting @@ -55,6 +60,27 @@ class Cache(object): lock = self.read_lock if ira else self.write_lock setattr(self, name, wrap_simple(lock, func)) + def _format_abspath(self, book_id, fmt): + ''' + Return absolute path to the ebook file of format `format` + + WARNING: This method will return a dummy path for a network backend DB, + so do not rely on it, use format(..., as_path=True) instead. + + Currently used only in calibredb list, the viewer and the catalogs (via + get_data_as_dict()). + + Apart from the viewer, I don't believe any of the others do any file + I/O with the results of this call. + ''' + try: + name = self.fields['formats'].format_fname(book_id, fmt) + path = self._field_for('path', book_id).replace('/', os.sep) + except: + return None + if name and path: + return self.backend.format_abspath(book_id, fmt, name, path) + # Cache Layer API {{{ @api @@ -68,6 +94,8 @@ class Cache(object): for field, table in self.backend.tables.iteritems(): self.fields[field] = create_field(field, table) + self.fields['ondevice'] = create_field('ondevice', None) + @read_api def field_for(self, name, book_id, default_value=None): ''' @@ -82,6 +110,15 @@ class Cache(object): except (KeyError, IndexError): return default_value + @read_api + def composite_for(self, name, book_id, mi, default_value=''): + try: + f = self.fields[name] + except KeyError: + return default_value + + f.render_composite(book_id, mi) + @read_api def field_ids_for(self, name, book_id): ''' @@ -122,6 +159,135 @@ class Cache(object): ''' return frozenset(iter(self.fields[name])) + @read_api + def author_data(self, author_id): + ''' + Return author data as a dictionary with keys: name, sort, link + + If no author with the specified id is found an empty dictionary is + returned. + ''' + try: + return self.fields['authors'].author_data(author_id) + except (KeyError, IndexError): + return {} + + @read_api + def format_metadata(self, book_id, fmt, allow_cache=True): + if not fmt: + return {} + fmt = fmt.upper() + if allow_cache: + x = self.format_metadata_cache[book_id].get(fmt, None) + if x is not None: + return x + try: + name = self.fields['formats'].format_fname(book_id, fmt) + path = self._field_for('path', book_id).replace('/', os.sep) + except: + return {} + + ans = {} + if path and name: + ans = self.backend.format_metadata(book_id, fmt, name, path) + self.format_metadata_cache[book_id][fmt] = ans + return ans + + @read_api + def get_metadata(self, book_id, get_cover=False, + get_user_categories=True, cover_as_data=False): + ''' + Convenience method to return metadata as a :class:`Metadata` object. + Note that the list of formats is not verified. + ''' + mi = Metadata(None) + + author_ids = self._field_ids_for('authors', book_id) + aut_list = [self._author_data(i) for i in author_ids] + aum = [] + aus = {} + aul = {} + for rec in aut_list: + aut = rec['name'] + aum.append(aut) + aus[aut] = rec['sort'] + aul[aut] = rec['link'] + mi.title = self._field_for('title', book_id, + default_value=_('Unknown')) + mi.authors = aum + mi.author_sort = self._field_for('author_sort', book_id, + default_value=_('Unknown')) + mi.author_sort_map = aus + mi.author_link_map = aul + mi.comments = self._field_for('comments', book_id) + mi.publisher = self._field_for('publisher', book_id) + n = now() + mi.timestamp = self._field_for('timestamp', book_id, default_value=n) + mi.pubdate = self._field_for('pubdate', book_id, default_value=n) + mi.uuid = self._field_for('uuid', book_id, + default_value='dummy') + mi.title_sort = self._field_for('sort', book_id, + default_value=_('Unknown')) + mi.book_size = self._field_for('size', book_id, default_value=0) + mi.ondevice_col = self._field_for('ondevice', book_id, default_value='') + mi.last_modified = self._field_for('last_modified', book_id, + default_value=n) + formats = self._field_for('formats', book_id) + mi.format_metadata = {} + if not formats: + formats = None + else: + for f in formats: + mi.format_metadata[f] = self._format_metadata(book_id, f) + formats = ','.join(formats) + mi.formats = formats + mi.has_cover = _('Yes') if self._field_for('cover', book_id, + default_value=False) else '' + mi.tags = list(self._field_for('tags', book_id, default_value=())) + mi.series = self._field_for('series', book_id) + if mi.series: + mi.series_index = self._field_for('series_index', book_id, + default_value=1.0) + mi.rating = self._field_for('rating', book_id) + mi.set_identifiers(self._field_for('identifiers', book_id, + default_value={})) + mi.application_id = book_id + mi.id = book_id + composites = {} + for key, meta in self.field_metadata.custom_iteritems(): + mi.set_user_metadata(key, meta) + if meta['datatype'] == 'composite': + composites.append(key) + else: + mi.set(key, val=self._field_for(meta['label'], book_id), + extra=self._field_for(meta['label']+'_index', book_id)) + for c in composites: + mi.set(key, val=self._composite_for(key, book_id, mi)) + + user_cat_vals = {} + if get_user_categories: + user_cats = self.prefs['user_categories'] + for ucat in user_cats: + res = [] + for name,cat,ign in user_cats[ucat]: + v = mi.get(cat, None) + if isinstance(v, list): + if name in v: + res.append([name,cat]) + elif name == v: + res.append([name,cat]) + user_cat_vals[ucat] = res + mi.user_categories = user_cat_vals + + if get_cover: + if cover_as_data: + cdata = self.cover(id, index_is_id=True) + if cdata: + mi.cover_data = ('jpeg', cdata) + else: + mi.cover = self.cover(id, index_is_id=True, as_path=True) + return mi + # }}} # Testing {{{ diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index 483813d80a..696882c631 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -66,6 +66,60 @@ class OneToOneField(Field): def iter_book_ids(self): return self.table.book_col_map.iterkeys() +class CompositeField(OneToOneField): + + def __init__(self, *args, **kwargs): + OneToOneField.__init__(self, *args, **kwargs) + + self._render_cache = {} + + def render_composite(self, book_id, mi): + ans = self._render_cache.get(book_id, None) + if ans is None: + ans = mi.get(self.metadata['label']) + self._render_cache[book_id] = ans + return ans + + def clear_cache(self): + self._render_cache = {} + + def pop_cache(self, book_id): + self._render_cache.pop(book_id, None) + +class OnDeviceField(OneToOneField): + + def __init__(self, name, table): + self.name = name + self.book_on_device_func = None + + def book_on_device(self, book_id): + if callable(self.book_on_device_func): + return self.book_on_device_func(book_id) + return None + + def set_book_on_device_func(self, func): + self.book_on_device_func = func + + def for_book(self, book_id, default_value=None): + loc = [] + count = 0 + on = self.book_on_device(book_id) + if on is not None: + m, a, b, count = on[:4] + if m is not None: + loc.append(_('Main')) + if a is not None: + loc.append(_('Card A')) + if b is not None: + loc.append(_('Card B')) + return ', '.join(loc) + ((' (%s books)'%count) if count > 1 else '') + + def __iter__(self): + return iter(()) + + def iter_book_ids(self): + return iter(()) + class ManyToOneField(Field): def for_book(self, book_id, default_value=None): @@ -107,11 +161,33 @@ class ManyToManyField(Field): def __iter__(self): return self.table.id_map.iterkeys() +class AuthorsField(ManyToManyField): + + def author_data(self, author_id): + return { + 'name' : self.table.id_map[author_id], + 'sort' : self.table.asort_map[author_id], + 'link' : self.table.alink_map[author_id], + } + +class FormatsField(ManyToManyField): + + def format_fname(self, book_id, fmt): + return self.table.fname_map[book_id][fmt.upper()] + def create_field(name, table): cls = { ONE_ONE : OneToOneField, MANY_ONE : ManyToOneField, MANY_MANY : ManyToManyField, }[table.table_type] + if name == 'authors': + cls = AuthorsField + elif name == 'ondevice': + cls = OnDeviceField + elif name == 'formats': + cls = FormatsField + elif table.metadata['datatype'] == 'composite': + cls = CompositeField return cls(name, table) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index c02c8ed9b7..b75effff4b 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -77,6 +77,17 @@ class SizeTable(OneToOneTable): 'WHERE data.book=books.id) FROM books'): self.book_col_map[row[0]] = self.unserialize(row[1]) +class CompositeTable(OneToOneTable): + + def read(self, db): + self.book_col_map = {} + d = self.metadata['display'] + self.composite_template = ['composite_template'] + self.contains_html = d['contains_html'] + self.make_category = d['make_category'] + self.composite_sort = d['composite_sort'] + self.use_decorations = d['use_decorations'] + class ManyToOneTable(Table): ''' @@ -144,11 +155,11 @@ class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): self.alink_map = {} - self.sort_map = {} + self.asort_map = {} for row in db.conn.execute( 'SELECT id, name, sort, link FROM authors'): self.id_map[row[0]] = row[1] - self.sort_map[row[0]] = (row[2] if row[2] else + self.asort_map[row[0]] = (row[2] if row[2] else author_to_author_sort(row[1])) self.alink_map[row[0]] = row[3] @@ -158,14 +169,19 @@ class FormatsTable(ManyToManyTable): pass def read_maps(self, db): + self.fname_map = {} for row in db.conn.execute('SELECT book, format, name FROM data'): if row[1] is not None: - if row[1] not in self.col_book_map: - self.col_book_map[row[1]] = [] - self.col_book_map[row[1]].append(row[0]) + fmt = row[1].upper() + if fmt not in self.col_book_map: + self.col_book_map[fmt] = [] + self.col_book_map[fmt].append(row[0]) if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] - self.book_col_map[row[0]].append((row[1], row[2])) + self.book_col_map[row[0]].append(fmt) + if row[0] not in self.fname_map: + self.fname_map[row[0]] = {} + self.fname_map[row[0]][fmt] = row[2] for key in tuple(self.col_book_map.iterkeys()): self.col_book_map[key] = tuple(self.col_book_map[key]) @@ -185,12 +201,9 @@ class IdentifiersTable(ManyToManyTable): self.col_book_map[row[1]] = [] self.col_book_map[row[1]].append(row[0]) if row[0] not in self.book_col_map: - self.book_col_map[row[0]] = [] - self.book_col_map[row[0]].append((row[1], row[2])) + self.book_col_map[row[0]] = {} + self.book_col_map[row[0]][row[1]] = row[2] for key in tuple(self.col_book_map.iterkeys()): self.col_book_map[key] = tuple(self.col_book_map[key]) - for key in tuple(self.book_col_map.iterkeys()): - self.book_col_map[key] = tuple(self.book_col_map[key]) - diff --git a/src/calibre/db/view.py b/src/calibre/db/view.py index 8f833499aa..8860f09dc2 100644 --- a/src/calibre/db/view.py +++ b/src/calibre/db/view.py @@ -7,15 +7,77 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' +from functools import partial class View(object): def __init__(self, cache): self.cache = cache - self._field_idx_map = {} + self._field_getters = {} for col, idx in cache.backend.FIELD_MAP.iteritems(): if isinstance(col, int): - pass # custom column + label = self.cache.backend.custom_column_num_map[col]['label'] + label = (self.cache.backend.field_metadata.custom_field_prefix + + label) + self._field_getters[idx] = partial(self.get, label) else: - self._field_idx_map[idx] = col + try: + self._field_getters[idx] = { + 'id' : self._get_id, + 'au_map' : self.get_author_data, + 'ondevice': self.get_ondevice, + 'marked' : self.get_is_marked, + }[col] + except KeyError: + self._field_getters[idx] = partial(self.get, col) + + self._map = list(self.cache.all_book_ids()) + self._map_filtered = list(self._map) + + def _get_id(self, idx, index_is_id=True): + ans = idx if index_is_id else self.index_to_id(idx) + return ans + + def get_field_map_field(self, row, col, index_is_id=True): + ''' + Supports the legacy FIELD_MAP interface for getting metadata. Do not use + in new code. + ''' + getter = self._field_getters[col] + return getter(row, index_is_id=index_is_id) + + def index_to_id(self, idx): + pass + + def get(self, field, idx, index_is_id=True, default_value=None): + id_ = idx if index_is_id else self.index_to_id(idx) + return self.cache.field_for(field, id_) + + def get_ondevice(self, idx, index_is_id=True, default_value=False): + pass + + def get_is_marked(self, idx, index_is_id=True, default_value=False): + pass + + def get_author_data(self, idx, index_is_id=True, default_value=()): + ''' + Return author data for all authors of the book identified by idx as a + tuple of dictionaries. The dictionaries should never be empty, unless + there is a bug somewhere. The list could be empty if idx point to an + non existent book, or book with no authors (though again a book with no + authors should never happen). + + Each dictionary has the keys: name, sort, link. Link can be an empty + string. + + default_value is ignored, this method always returns a tuple + ''' + id_ = idx if index_is_id else self.index_to_id(idx) + with self.cache.read_lock: + ids = self.cache._field_ids_for('authors', id_) + ans = [] + for id_ in ids: + ans.append(self.cache._author_data(id_)) + return tuple(ans) +