Start work on write API

src/calibre/db/cache.py

@@ -608,6 +608,14 @@ class Cache(object):
         return get_categories(self, sort=sort, book_ids=book_ids,
                               icon_map=icon_map)
 
+    @write_api
+    def set_field(self, name, book_id_to_val_map):
+        # TODO: Specialize title/authors to also update path
+        # TODO: Handle updating caches used by composite fields
+        dirtied = self.fields[name].writer.set_books(
+            book_id_to_val_map, self.backend)
+        return dirtied
+
     # }}}
 
 class SortKey(object):

src/calibre/db/fields.py

@@ -12,6 +12,7 @@ from threading import Lock
 from collections import defaultdict, Counter
 
 from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY
+from calibre.db.write import Writer
 from calibre.ebooks.metadata import title_sort
 from calibre.utils.config_base import tweaks
 from calibre.utils.icu import sort_key
@@ -44,6 +45,7 @@ class Field(object):
             self.category_formatter = lambda x:'\u2605'*int(x/2)
         elif name == 'languages':
             self.category_formatter = calibre_langcode_to_name
+        self.writer = Writer(self)
 
     @property
     def metadata(self):

src/calibre/db/tests/base.py

@@ -7,12 +7,22 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import unittest, os, shutil
+import unittest, os, shutil, tempfile, atexit
+from functools import partial
 from io import BytesIO
 from future_builtins import map
 
+rmtree = partial(shutil.rmtree, ignore_errors=True)
+
 class BaseTest(unittest.TestCase):
 
+    def setUp(self):
+        self.library_path = self.mkdtemp()
+        self.create_db(self.library_path)
+
+    def tearDown(self):
+        shutil.rmtree(self.library_path)
+
     def create_db(self, library_path):
         from calibre.library.database2 import LibraryDatabase2
         if LibraryDatabase2.exists_at(library_path):
@@ -36,6 +46,25 @@ class BaseTest(unittest.TestCase):
         cache.init()
         return cache
 
+    def mkdtemp(self):
+        ans = tempfile.mkdtemp(prefix='db_test_')
+        atexit.register(rmtree, ans)
+        return ans
+
+    def init_old(self, library_path):
+        from calibre.library.database2 import LibraryDatabase2
+        return LibraryDatabase2(library_path)
+
+    def clone_library(self, library_path):
+        if not hasattr(self, 'clone_dir'):
+            self.clone_dir = tempfile.mkdtemp()
+            atexit.register(rmtree, self.clone_dir)
+            self.clone_count = 0
+        self.clone_count += 1
+        dest = os.path.join(self.clone_dir, str(self.clone_count))
+        shutil.copytree(library_path, dest)
+        return dest
+
     def compare_metadata(self, mi1, mi2):
         allfk1 = mi1.all_field_keys()
         allfk2 = mi2.all_field_keys()

src/calibre/db/tests/reading.py

@@ -7,20 +7,13 @@ __license__   = 'GPL v3'
 __copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import shutil, unittest, tempfile, datetime
+import unittest, datetime
 
 from calibre.utils.date import utc_tz
 from calibre.db.tests.base import BaseTest
 
 class ReadingTest(BaseTest):
 
-    def setUp(self):
-        self.library_path = tempfile.mkdtemp()
-        self.create_db(self.library_path)
-
-    def tearDown(self):
-        shutil.rmtree(self.library_path)
-
     def test_read(self): # {{{
         'Test the reading of data from the database'
         cache = self.init_cache(self.library_path)

src/calibre/db/tests/writing.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import unittest
+from collections import namedtuple
+from functools import partial
+
+from calibre.utils.date import UNDEFINED_DATE
+from calibre.db.tests.base import BaseTest
+
+class WritingTest(BaseTest):
+
+    @property
+    def cloned_library(self):
+        return self.clone_library(self.library_path)
+
+    def create_getter(self, name, getter=None):
+        if getter is None:
+            ans = lambda db:partial(db.get_custom, label=name[1:],
+                                       index_is_id=True)
+        else:
+            ans = lambda db:partial(getattr(db, getter), index_is_id=True)
+        return ans
+
+    def create_setter(self, name, setter=None):
+        if setter is None:
+            ans = lambda db:partial(db.set_custom, label=name[1:], commit=True)
+        else:
+            ans = lambda db:partial(getattr(db, setter), commit=True)
+        return ans
+
+    def create_test(self, name, vals, getter=None, setter=None ):
+        T = namedtuple('Test', 'name vals getter setter')
+        return T(name, vals, self.create_getter(name, getter),
+                 self.create_setter(name, setter))
+
+    def run_tests(self, tests):
+        cl = self.cloned_library
+        results = {}
+        for test in tests:
+            results[test] = []
+            for val in test.vals:
+                cache = self.init_cache(cl)
+                cache.set_field(test.name, {1: val})
+                cached_res = cache.field_for(test.name, 1)
+                del cache
+                db = self.init_old(cl)
+                getter = test.getter(db)
+                sqlite_res = getter(1)
+                test.setter(db)(1, val)
+                old_cached_res = getter(1)
+                self.assertEqual(old_cached_res, cached_res,
+                                 'Failed setting for %s with value %r, cached value not the same. Old: %r != New: %r'%(
+                        test.name, val, old_cached_res, cached_res))
+                db.refresh()
+                old_sqlite_res = getter(1)
+                self.assertEqual(old_sqlite_res, sqlite_res,
+                    'Failed setting for %s, sqlite value not the same: %r != %r'%(
+                        test.name, old_sqlite_res, sqlite_res))
+                del db
+
+
+
+    def test_one_one(self):
+        'Test setting of values in one-one fields'
+        tests = []
+        for name, getter, setter in (
+            ('pubdate', 'pubdate', 'set_pubdate'),
+            ('timestamp', 'timestamp', 'set_timestamp'),
+            ('#date', None, None)
+        ):
+            tests.append(self.create_test(
+                name, ('2011-1-12', UNDEFINED_DATE, None), getter, setter))
+
+        self.run_tests(tests)
+
+def tests():
+    return unittest.TestLoader().loadTestsFromTestCase(WritingTest)
+
+def run():
+    unittest.TextTestRunner(verbosity=2).run(tests())
+
+if __name__ == '__main__':
+    run()
+
+

src/calibre/db/write.py

@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from functools import partial
+from datetime import datetime
+
+from calibre.constants import preferred_encoding, ispy3
+from calibre.utils.date import (parse_only_date, parse_date, UNDEFINED_DATE,
+                                isoformat)
+
+# Convert data into values suitable for the db {{{
+
+if ispy3:
+    unicode = str
+
+def single_text(x):
+    if x is None:
+        return x
+    if not isinstance(x, unicode):
+        x = x.decode(preferred_encoding, 'replace')
+    x = x.strip()
+    return x if x else None
+
+def multiple_text(sep, x):
+    if x is None:
+        return ()
+    if isinstance(x, bytes):
+        x = x.decode(preferred_encoding, 'replce')
+    if isinstance(x, unicode):
+        x = x.split(sep)
+    x = (y.strip() for y in x if y.strip())
+    return (' '.join(y.split()) for y in x if y)
+
+def adapt_datetime(x):
+    if isinstance(x, (unicode, bytes)):
+        x = parse_date(x, assume_utc=False, as_utc=False)
+    if x is None:
+        x = UNDEFINED_DATE
+    return x
+
+def adapt_date(x):
+    if isinstance(x, (unicode, bytes)):
+        x = parse_only_date(x)
+    if x is None:
+        x = UNDEFINED_DATE
+    return x
+
+def adapt_number(typ, x):
+    if x is None:
+        return None
+    if isinstance(x, (unicode, bytes)):
+        if x.lower() == 'none':
+            return None
+    return typ(x)
+
+def adapt_bool(x):
+    if isinstance(x, (unicode, bytes)):
+        x = x.lower()
+        if x == 'true':
+            x = True
+        elif x == 'false':
+            x = False
+        elif x == 'none':
+            x = None
+        else:
+            x = bool(int(x))
+    return x if x is None else bool(x)
+
+def get_adapter(name, metadata):
+    dt = metadata['datatype']
+    if dt == 'text':
+        if metadata['is_multiple']:
+            ans = partial(multiple_text, metadata['is_multiple']['ui_to_list'])
+        else:
+            ans = single_text
+    elif dt == 'series':
+        ans = single_text
+    elif dt == 'datetime':
+        ans = adapt_date if name == 'pubdate' else adapt_datetime
+    elif dt == 'int':
+        ans = partial(adapt_number, int)
+    elif dt == 'float':
+        ans = partial(adapt_number, float)
+    elif dt == 'bool':
+        ans = adapt_bool
+    elif dt == 'comments':
+        ans = single_text
+    elif dt == 'rating':
+        ans = lambda x: x if x is None else min(10., max(0., adapt_number(float, x))),
+    elif dt == 'enumeration':
+        ans = single_text
+    elif dt == 'composite':
+        ans = lambda x: x
+
+    if name == 'title':
+        ans = lambda x: ans(x) or _('Unknown')
+    elif name == 'authors':
+        ans = lambda x: ans(x) or (_('Unknown'),)
+
+    return ans
+# }}}
+
+def sqlite_datetime(x):
+    return isoformat(x, sep=' ') if isinstance(x, datetime) else x
+
+def one_one_in_books(book_id_val_map, db, field, *args):
+    'Set a one-one field in the books table'
+    if book_id_val_map:
+        sequence = tuple((sqlite_datetime(v), k) for k, v in book_id_val_map.iteritems())
+        db.conn.executemany(
+            'UPDATE books SET %s=? WHERE id=?'%field.metadata['column'], sequence)
+        field.table.book_col_map.update(book_id_val_map)
+    return set(book_id_val_map)
+
+def one_one_in_other(book_id_val_map, db, field, *args):
+    'Set a one-one field in the non-books table, like comments'
+    deleted = tuple((k,) for k, v in book_id_val_map.iteritems() if v is None)
+    if deleted:
+        db.conn.executemany('DELETE FROM %s WHERE book=?'%field.metadata['table'],
+                        deleted)
+        for book_id in book_id_val_map:
+            field.table.book_col_map.pop(book_id, None)
+    updated = {k:v for k, v in book_id_val_map.iteritems() if v is not None}
+    if updated:
+        db.conn.executemany('INSERT OR REPLACE INTO %s(book,%s) VALUES (?,?)'%(
+            field.metadata['table'], field.metadata['column']),
+            tuple((k, sqlite_datetime(v)) for k, v in updated.iteritems()))
+        field.table.book_col_map.update(updated)
+    return set(book_id_val_map)
+
+def dummy(book_id_val_map, *args):
+    return set()
+
+class Writer(object):
+
+    def __init__(self, field):
+        self.adapter = get_adapter(field.name, field.metadata)
+        self.name = field.name
+        self.field = field
+        dt = field.metadata['datatype']
+        self.filter_vals = lambda x: x
+        if dt == 'composite' or field.name in {'cover', 'size', 'path'}:
+            self.set_books_func = dummy
+        elif field.is_many:
+            # TODO: Implement this
+            pass
+        else:
+            self.set_books_func = (one_one_in_books if field.metadata['table']
+                                   == 'books' else one_one_in_other)
+            if self.name in {'timestamp', 'uuid'}:
+                self.filter_vals = bool
+
+    def set_books(self, book_id_val_map, db):
+        book_id_val_map = {k:self.adapter(v) for k, v in
+                           book_id_val_map.iteritems() if self.filter_vals(v)}
+        if not book_id_val_map:
+            return set()
+        dirtied = self.set_books_func(book_id_val_map, db, self.field)
+        return dirtied
+

src/calibre/library/database2.py

@@ -34,7 +34,7 @@ from calibre import isbytestring
 from calibre.utils.filenames import (ascii_filename, samefile,
         WindowsAtomicFolderMove, hardlink_file)
 from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp,
-        parse_only_date, UNDEFINED_DATE)
+        parse_only_date, UNDEFINED_DATE, parse_date)
 from calibre.utils.config import prefs, tweaks, from_json, to_json
 from calibre.utils.icu import sort_key, strcmp, lower
 from calibre.utils.search_query_parser import saved_searches, set_saved_searches
@@ -2567,6 +2567,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_timestamp(self, id, dt, notify=True, commit=True):
         if dt:
+            if isinstance(dt, (unicode, bytes)):
+                dt = parse_date(dt, as_utc=True, assume_utc=False)
             self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['timestamp'], dt, row_is_id=True)
             self.dirtied([id], commit=False)