New db backend: get_metadata() tested and working

src/calibre/db/cache.py

@@ -120,8 +120,10 @@ class Cache(object):
         mi.ondevice_col = self._field_for('ondevice', book_id, default_value='')
         mi.last_modified = self._field_for('last_modified', book_id,
                 default_value=n)
-        formats = self._field_for('formats', book_id)
+        formats = self._field_for('formats', book_id, default_value=())
         mi.format_metadata = {}
+        mi.languages = list(self._field_for('languages', book_id,
+            default_value=()))
         if not formats:
             good_formats = None
         else:
@@ -146,20 +148,26 @@ class Cache(object):
             default_value={}))
         mi.application_id = book_id
         mi.id = book_id
-        composites = {}
+        composites = []
         for key, meta in self.field_metadata.custom_iteritems():
             mi.set_user_metadata(key, meta)
             if meta['datatype'] == 'composite':
                 composites.append(key)
             else:
-                mi.set(key, val=self._field_for(meta['label'], book_id),
+                defval = None
-                        extra=self._field_for(meta['label']+'_index', book_id))
+                if meta['is_multiple'] and meta['datatype'] == 'text':
-        for c in composites:
+                    defval = []
+                val = self._field_for(key, book_id, default_value=defval)
+                if isinstance(val, tuple):
+                    val = list(val)
+                extra = self._field_for(key+'_index', book_id)
+                mi.set(key, val=val, extra=extra)
+        for key in composites:
             mi.set(key, val=self._composite_for(key, book_id, mi))
 
         user_cat_vals = {}
         if get_user_categories:
-            user_cats = self.prefs['user_categories']
+            user_cats = self.backend.prefs['user_categories']
             for ucat in user_cats:
                 res = []
                 for name,cat,ign in user_cats[ucat]:
@@ -206,6 +214,17 @@ class Cache(object):
 
         The returned value for is_multiple fields are always tuples, unless
         default_value is returned.
+
+        WARNING: When returning the value for a is_multiple custom field this
+        method returns None (the default_value) if no value is set. The
+        get_custom() method from the old interface returned []
+
+        WARNING: For is_multiple fields this method returns tuples, the old
+        interface generally returned lists.
+
+        WARNING: For is_multiple fields the order of items is always in link
+        order (order in which they were entered), whereas the old db had them
+        in random order for fields other than author.
         '''
         if self.composites and name in self.composites:
             return self.composite_for(name, book_id,

src/calibre/db/fields.py

@@ -110,7 +110,7 @@ class CompositeField(OneToOneField):
         with self._lock:
             ans = self._render_cache.get(book_id, None)
         if ans is None:
-            ans = mi.get(self.metadata['label'])
+            ans = mi.get('#'+self.metadata['label'])
             with self._lock:
                 self._render_cache[book_id] = ans
         return ans
@@ -128,7 +128,7 @@ class CompositeField(OneToOneField):
             ans = self._render_cache.get(book_id, None)
         if ans is None:
             mi = get_metadata(book_id)
-            ans = mi.get(self.metadata['label'])
+            ans = mi.get('#'+self.metadata['label'])
         return ans
 
     def sort_keys_for_books(self, get_metadata, all_book_ids):
@@ -265,6 +265,9 @@ class AuthorsField(ManyToManyField):
 
 class FormatsField(ManyToManyField):
 
+    def for_book(self, book_id, default_value=None):
+        return self.table.book_col_map.get(book_id, default_value)
+
     def format_fname(self, book_id, fmt):
         return self.table.fname_map[book_id][fmt.upper()]
 

src/calibre/db/tables.py

@@ -202,7 +202,7 @@ class FormatsTable(ManyToManyTable):
             self.col_book_map[key] = tuple(self.col_book_map[key])
 
         for key in tuple(self.book_col_map.iterkeys()):
-            self.book_col_map[key] = tuple(self.book_col_map[key])
+            self.book_col_map[key] = tuple(sorted(self.book_col_map[key]))
 
 class IdentifiersTable(ManyToManyTable):
 

src/calibre/db/tests/base.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+import unittest, os, shutil
+
+class BaseTest(unittest.TestCase):
+
+    def create_db(self, library_path):
+        from calibre.library.database2 import LibraryDatabase2
+        if LibraryDatabase2.exists_at(library_path):
+            raise ValueError('A library already exists at %r'%library_path)
+        src = os.path.join(os.path.dirname(__file__), 'metadata.db')
+        db = os.path.join(library_path, 'metadata.db')
+        shutil.copyfile(src, db)
+        return db
+
+    def init_cache(self, library_path):
+        from calibre.db.backend import DB
+        from calibre.db.cache import Cache
+        backend = DB(library_path)
+        cache = Cache(backend)
+        cache.init()
+        return cache
+
+    def compare_metadata(self, mi1, mi2):
+        allfk1 = mi1.all_field_keys()
+        allfk2 = mi2.all_field_keys()
+        self.assertEqual(allfk1, allfk2)
+
+        all_keys = {'format_metadata', 'id', 'application_id',
+                'author_sort_map', 'author_link_map', 'book_size',
+                'ondevice_col', 'last_modified'}.union(allfk1)
+        for attr in all_keys:
+            if attr == 'user_metadata': continue
+            attr1, attr2 = getattr(mi1, attr), getattr(mi2, attr)
+            self.assertEqual(attr1, attr2,
+                    '%s not the same: %r != %r'%(attr, attr1, attr2))
+            if attr.startswith('#'):
+                attr1, attr2 = mi1.get_extra(attr), mi2.get_extra(attr)
+                self.assertEqual(attr1, attr2,
+                    '%s {#extra} not the same: %r != %r'%(attr, attr1, attr2))
+
+

src/calibre/db/tests/reading.py

@@ -7,40 +7,24 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-
+import shutil, unittest, tempfile, datetime
-import os, shutil, unittest, tempfile, datetime
+from cStringIO import StringIO
 
 from calibre.utils.date import local_tz
+from calibre.db.tests.base import BaseTest
 
-def create_db(library_path):
+class ReadingTest(BaseTest):
-    from calibre.library.database2 import LibraryDatabase2
-    if LibraryDatabase2.exists_at(library_path):
-        raise ValueError('A library already exists at %r'%library_path)
-    src = os.path.join(os.path.dirname(__file__), 'metadata.db')
-    db = os.path.join(library_path, 'metadata.db')
-    shutil.copyfile(src, db)
-    return db
-
-def init_cache(library_path):
-    from calibre.db.backend import DB
-    from calibre.db.cache import Cache
-    backend = DB(library_path)
-    cache = Cache(backend)
-    cache.init()
-    return cache
-
-class ReadingTest(unittest.TestCase):
 
     def setUp(self):
         self.library_path = tempfile.mkdtemp()
-        create_db(self.library_path)
+        self.create_db(self.library_path)
 
     def tearDown(self):
         shutil.rmtree(self.library_path)
 
     def test_read(self): # {{{
         'Test the reading of data from the database'
-        cache = init_cache(self.library_path)
+        cache = self.init_cache(self.library_path)
         tests = {
                 3  : {
                     'title': 'Unknown',
@@ -51,6 +35,7 @@ class ReadingTest(unittest.TestCase):
                     'series_index': 1.0,
                     'rating': None,
                     'tags': None,
+                    'formats':None,
                     'identifiers': None,
                     'timestamp': datetime.datetime(2011, 9, 7, 13, 54, 41,
                         tzinfo=local_tz),
@@ -81,6 +66,7 @@ class ReadingTest(unittest.TestCase):
                     'series' : 'Series One',
                     'series_index': 1.0,
                     'tags':('Tag Two', 'Tag One'),
+                    'formats': None,
                     'rating': 4.0,
                     'identifiers': {'test':'one'},
                     'timestamp': datetime.datetime(2011, 9, 5, 15, 6,
@@ -110,6 +96,7 @@ class ReadingTest(unittest.TestCase):
                     'series_index': 2.0,
                     'rating': 6.0,
                     'tags': ('Tag One',),
+                    'formats':None,
                     'identifiers': {'test':'two'},
                     'timestamp': datetime.datetime(2011, 9, 6, 0, 0,
                         tzinfo=local_tz),
@@ -139,7 +126,7 @@ class ReadingTest(unittest.TestCase):
 
     def test_sorting(self): # {{{
         'Test sorting'
-        cache = init_cache(self.library_path)
+        cache = self.init_cache(self.library_path)
         for field, order in {
                 'title'  : [2, 1, 3],
                 'authors': [2, 1, 3],
@@ -178,6 +165,28 @@ class ReadingTest(unittest.TestCase):
             ('title', True)]), 'Subsort failed')
     # }}}
 
+    def test_get_metadata(self): # {{{
+        'Test get_metadata() returns the same data for both backends'
+        from calibre.library.database2 import LibraryDatabase2
+        old = LibraryDatabase2(self.library_path)
+        for i in xrange(1, 3):
+            old.add_format(i, 'txt%d'%i, StringIO(b'random%d'%i),
+                    index_is_id=True)
+            old.add_format(i, 'text%d'%i, StringIO(b'random%d'%i),
+                    index_is_id=True)
+
+        old_metadata = {i:old.get_metadata(i, index_is_id=True) for i in
+                xrange(1, 4)}
+        old = None
+
+        cache = self.init_cache(self.library_path)
+
+        new_metadata = {i:cache.get_metadata(i) for i in xrange(1, 4)}
+        cache = None
+        for mi2, mi1 in zip(new_metadata.values(), old_metadata.values()):
+            self.compare_metadata(mi1, mi2)
+
+    # }}}
 
 def tests():
     return unittest.TestLoader().loadTestsFromTestCase(ReadingTest)

src/calibre/library/database2.py

@@ -927,7 +927,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if not formats:
             good_formats = None
         else:
-            formats = formats.split(',')
+            formats = sorted(formats.split(','))
             good_formats = []
             for f in formats:
                 try: