From 8355955ef5da8a0e5890534eec2c3c29f6fe1577 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 7 Sep 2011 23:30:21 -0600 Subject: [PATCH] New db backend: get_metadata() tested and working --- src/calibre/db/cache.py | 31 ++++++++++++++---- src/calibre/db/fields.py | 7 ++-- src/calibre/db/tables.py | 2 +- src/calibre/db/tests/base.py | 50 +++++++++++++++++++++++++++++ src/calibre/db/tests/reading.py | 55 +++++++++++++++++++------------- src/calibre/library/database2.py | 2 +- 6 files changed, 114 insertions(+), 33 deletions(-) create mode 100644 src/calibre/db/tests/base.py diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 4e29574077..c7116cc5a7 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -120,8 +120,10 @@ class Cache(object): mi.ondevice_col = self._field_for('ondevice', book_id, default_value='') mi.last_modified = self._field_for('last_modified', book_id, default_value=n) - formats = self._field_for('formats', book_id) + formats = self._field_for('formats', book_id, default_value=()) mi.format_metadata = {} + mi.languages = list(self._field_for('languages', book_id, + default_value=())) if not formats: good_formats = None else: @@ -146,20 +148,26 @@ class Cache(object): default_value={})) mi.application_id = book_id mi.id = book_id - composites = {} + composites = [] for key, meta in self.field_metadata.custom_iteritems(): mi.set_user_metadata(key, meta) if meta['datatype'] == 'composite': composites.append(key) else: - mi.set(key, val=self._field_for(meta['label'], book_id), - extra=self._field_for(meta['label']+'_index', book_id)) - for c in composites: + defval = None + if meta['is_multiple'] and meta['datatype'] == 'text': + defval = [] + val = self._field_for(key, book_id, default_value=defval) + if isinstance(val, tuple): + val = list(val) + extra = self._field_for(key+'_index', book_id) + mi.set(key, val=val, extra=extra) + for key in composites: mi.set(key, val=self._composite_for(key, book_id, mi)) user_cat_vals = {} if get_user_categories: - user_cats = self.prefs['user_categories'] + user_cats = self.backend.prefs['user_categories'] for ucat in user_cats: res = [] for name,cat,ign in user_cats[ucat]: @@ -206,6 +214,17 @@ class Cache(object): The returned value for is_multiple fields are always tuples, unless default_value is returned. + + WARNING: When returning the value for a is_multiple custom field this + method returns None (the default_value) if no value is set. The + get_custom() method from the old interface returned [] + + WARNING: For is_multiple fields this method returns tuples, the old + interface generally returned lists. + + WARNING: For is_multiple fields the order of items is always in link + order (order in which they were entered), whereas the old db had them + in random order for fields other than author. ''' if self.composites and name in self.composites: return self.composite_for(name, book_id, diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index a4a490091e..d425f0d635 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -110,7 +110,7 @@ class CompositeField(OneToOneField): with self._lock: ans = self._render_cache.get(book_id, None) if ans is None: - ans = mi.get(self.metadata['label']) + ans = mi.get('#'+self.metadata['label']) with self._lock: self._render_cache[book_id] = ans return ans @@ -128,7 +128,7 @@ class CompositeField(OneToOneField): ans = self._render_cache.get(book_id, None) if ans is None: mi = get_metadata(book_id) - ans = mi.get(self.metadata['label']) + ans = mi.get('#'+self.metadata['label']) return ans def sort_keys_for_books(self, get_metadata, all_book_ids): @@ -265,6 +265,9 @@ class AuthorsField(ManyToManyField): class FormatsField(ManyToManyField): + def for_book(self, book_id, default_value=None): + return self.table.book_col_map.get(book_id, default_value) + def format_fname(self, book_id, fmt): return self.table.fname_map[book_id][fmt.upper()] diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 6a6730c2b4..c5d7ee216c 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -202,7 +202,7 @@ class FormatsTable(ManyToManyTable): self.col_book_map[key] = tuple(self.col_book_map[key]) for key in tuple(self.book_col_map.iterkeys()): - self.book_col_map[key] = tuple(self.book_col_map[key]) + self.book_col_map[key] = tuple(sorted(self.book_col_map[key])) class IdentifiersTable(ManyToManyTable): diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py new file mode 100644 index 0000000000..3264465050 --- /dev/null +++ b/src/calibre/db/tests/base.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +import unittest, os, shutil + +class BaseTest(unittest.TestCase): + + def create_db(self, library_path): + from calibre.library.database2 import LibraryDatabase2 + if LibraryDatabase2.exists_at(library_path): + raise ValueError('A library already exists at %r'%library_path) + src = os.path.join(os.path.dirname(__file__), 'metadata.db') + db = os.path.join(library_path, 'metadata.db') + shutil.copyfile(src, db) + return db + + def init_cache(self, library_path): + from calibre.db.backend import DB + from calibre.db.cache import Cache + backend = DB(library_path) + cache = Cache(backend) + cache.init() + return cache + + def compare_metadata(self, mi1, mi2): + allfk1 = mi1.all_field_keys() + allfk2 = mi2.all_field_keys() + self.assertEqual(allfk1, allfk2) + + all_keys = {'format_metadata', 'id', 'application_id', + 'author_sort_map', 'author_link_map', 'book_size', + 'ondevice_col', 'last_modified'}.union(allfk1) + for attr in all_keys: + if attr == 'user_metadata': continue + attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr) + self.assertEqual(attr1, attr2, + '%s not the same: %r != %r'%(attr, attr1, attr2)) + if attr.startswith('#'): + attr1, attr2 = mi1.get_extra(attr), mi2.get_extra(attr) + self.assertEqual(attr1, attr2, + '%s {#extra} not the same: %r != %r'%(attr, attr1, attr2)) + + diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 7181d8d645..269bc458c2 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -7,40 +7,24 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' - -import os, shutil, unittest, tempfile, datetime +import shutil, unittest, tempfile, datetime +from cStringIO import StringIO from calibre.utils.date import local_tz +from calibre.db.tests.base import BaseTest -def create_db(library_path): - from calibre.library.database2 import LibraryDatabase2 - if LibraryDatabase2.exists_at(library_path): - raise ValueError('A library already exists at %r'%library_path) - src = os.path.join(os.path.dirname(__file__), 'metadata.db') - db = os.path.join(library_path, 'metadata.db') - shutil.copyfile(src, db) - return db - -def init_cache(library_path): - from calibre.db.backend import DB - from calibre.db.cache import Cache - backend = DB(library_path) - cache = Cache(backend) - cache.init() - return cache - -class ReadingTest(unittest.TestCase): +class ReadingTest(BaseTest): def setUp(self): self.library_path = tempfile.mkdtemp() - create_db(self.library_path) + self.create_db(self.library_path) def tearDown(self): shutil.rmtree(self.library_path) def test_read(self): # {{{ 'Test the reading of data from the database' - cache = init_cache(self.library_path) + cache = self.init_cache(self.library_path) tests = { 3 : { 'title': 'Unknown', @@ -51,6 +35,7 @@ class ReadingTest(unittest.TestCase): 'series_index': 1.0, 'rating': None, 'tags': None, + 'formats':None, 'identifiers': None, 'timestamp': datetime.datetime(2011, 9, 7, 13, 54, 41, tzinfo=local_tz), @@ -81,6 +66,7 @@ class ReadingTest(unittest.TestCase): 'series' : 'Series One', 'series_index': 1.0, 'tags':('Tag Two', 'Tag One'), + 'formats': None, 'rating': 4.0, 'identifiers': {'test':'one'}, 'timestamp': datetime.datetime(2011, 9, 5, 15, 6, @@ -110,6 +96,7 @@ class ReadingTest(unittest.TestCase): 'series_index': 2.0, 'rating': 6.0, 'tags': ('Tag One',), + 'formats':None, 'identifiers': {'test':'two'}, 'timestamp': datetime.datetime(2011, 9, 6, 0, 0, tzinfo=local_tz), @@ -139,7 +126,7 @@ class ReadingTest(unittest.TestCase): def test_sorting(self): # {{{ 'Test sorting' - cache = init_cache(self.library_path) + cache = self.init_cache(self.library_path) for field, order in { 'title' : [2, 1, 3], 'authors': [2, 1, 3], @@ -178,6 +165,28 @@ class ReadingTest(unittest.TestCase): ('title', True)]), 'Subsort failed') # }}} + def test_get_metadata(self): # {{{ + 'Test get_metadata() returns the same data for both backends' + from calibre.library.database2 import LibraryDatabase2 + old = LibraryDatabase2(self.library_path) + for i in xrange(1, 3): + old.add_format(i, 'txt%d'%i, StringIO(b'random%d'%i), + index_is_id=True) + old.add_format(i, 'text%d'%i, StringIO(b'random%d'%i), + index_is_id=True) + + old_metadata = {i:old.get_metadata(i, index_is_id=True) for i in + xrange(1, 4)} + old = None + + cache = self.init_cache(self.library_path) + + new_metadata = {i:cache.get_metadata(i) for i in xrange(1, 4)} + cache = None + for mi2, mi1 in zip(new_metadata.values(), old_metadata.values()): + self.compare_metadata(mi1, mi2) + + # }}} def tests(): return unittest.TestLoader().loadTestsFromTestCase(ReadingTest) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index ebe15ae66c..c65484ff56 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -927,7 +927,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if not formats: good_formats = None else: - formats = formats.split(',') + formats = sorted(formats.split(',')) good_formats = [] for f in formats: try: