From f6a780bf267927e0cee35cb1708622eaf5f627ff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 21 Feb 2013 17:05:32 +0530 Subject: [PATCH] Start work on write API --- src/calibre/db/cache.py | 8 ++ src/calibre/db/fields.py | 2 + src/calibre/db/tests/base.py | 31 +++++- src/calibre/db/tests/reading.py | 9 +- src/calibre/db/tests/writing.py | 92 +++++++++++++++++ src/calibre/db/write.py | 166 +++++++++++++++++++++++++++++++ src/calibre/library/database2.py | 4 +- 7 files changed, 302 insertions(+), 10 deletions(-) create mode 100644 src/calibre/db/tests/writing.py create mode 100644 src/calibre/db/write.py diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 355702d1b9..236d25c3e9 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -608,6 +608,14 @@ class Cache(object): return get_categories(self, sort=sort, book_ids=book_ids, icon_map=icon_map) + @write_api + def set_field(self, name, book_id_to_val_map): + # TODO: Specialize title/authors to also update path + # TODO: Handle updating caches used by composite fields + dirtied = self.fields[name].writer.set_books( + book_id_to_val_map, self.backend) + return dirtied + # }}} class SortKey(object): diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index bd3af5d518..46386b9ba5 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -12,6 +12,7 @@ from threading import Lock from collections import defaultdict, Counter from calibre.db.tables import ONE_ONE, MANY_ONE, MANY_MANY +from calibre.db.write import Writer from calibre.ebooks.metadata import title_sort from calibre.utils.config_base import tweaks from calibre.utils.icu import sort_key @@ -44,6 +45,7 @@ class Field(object): self.category_formatter = lambda x:'\u2605'*int(x/2) elif name == 'languages': self.category_formatter = calibre_langcode_to_name + self.writer = Writer(self) @property def metadata(self): diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index 10fd6c7e0b..ae120ff049 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -7,12 +7,22 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import unittest, os, shutil +import unittest, os, shutil, tempfile, atexit +from functools import partial from io import BytesIO from future_builtins import map +rmtree = partial(shutil.rmtree, ignore_errors=True) + class BaseTest(unittest.TestCase): + def setUp(self): + self.library_path = self.mkdtemp() + self.create_db(self.library_path) + + def tearDown(self): + shutil.rmtree(self.library_path) + def create_db(self, library_path): from calibre.library.database2 import LibraryDatabase2 if LibraryDatabase2.exists_at(library_path): @@ -36,6 +46,25 @@ class BaseTest(unittest.TestCase): cache.init() return cache + def mkdtemp(self): + ans = tempfile.mkdtemp(prefix='db_test_') + atexit.register(rmtree, ans) + return ans + + def init_old(self, library_path): + from calibre.library.database2 import LibraryDatabase2 + return LibraryDatabase2(library_path) + + def clone_library(self, library_path): + if not hasattr(self, 'clone_dir'): + self.clone_dir = tempfile.mkdtemp() + atexit.register(rmtree, self.clone_dir) + self.clone_count = 0 + self.clone_count += 1 + dest = os.path.join(self.clone_dir, str(self.clone_count)) + shutil.copytree(library_path, dest) + return dest + def compare_metadata(self, mi1, mi2): allfk1 = mi1.all_field_keys() allfk2 = mi2.all_field_keys() diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 51689d0f74..c3b458ea58 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -7,20 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import shutil, unittest, tempfile, datetime +import unittest, datetime from calibre.utils.date import utc_tz from calibre.db.tests.base import BaseTest class ReadingTest(BaseTest): - def setUp(self): - self.library_path = tempfile.mkdtemp() - self.create_db(self.library_path) - - def tearDown(self): - shutil.rmtree(self.library_path) - def test_read(self): # {{{ 'Test the reading of data from the database' cache = self.init_cache(self.library_path) diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py new file mode 100644 index 0000000000..4ac1eb7d8d --- /dev/null +++ b/src/calibre/db/tests/writing.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import unittest +from collections import namedtuple +from functools import partial + +from calibre.utils.date import UNDEFINED_DATE +from calibre.db.tests.base import BaseTest + +class WritingTest(BaseTest): + + @property + def cloned_library(self): + return self.clone_library(self.library_path) + + def create_getter(self, name, getter=None): + if getter is None: + ans = lambda db:partial(db.get_custom, label=name[1:], + index_is_id=True) + else: + ans = lambda db:partial(getattr(db, getter), index_is_id=True) + return ans + + def create_setter(self, name, setter=None): + if setter is None: + ans = lambda db:partial(db.set_custom, label=name[1:], commit=True) + else: + ans = lambda db:partial(getattr(db, setter), commit=True) + return ans + + def create_test(self, name, vals, getter=None, setter=None ): + T = namedtuple('Test', 'name vals getter setter') + return T(name, vals, self.create_getter(name, getter), + self.create_setter(name, setter)) + + def run_tests(self, tests): + cl = self.cloned_library + results = {} + for test in tests: + results[test] = [] + for val in test.vals: + cache = self.init_cache(cl) + cache.set_field(test.name, {1: val}) + cached_res = cache.field_for(test.name, 1) + del cache + db = self.init_old(cl) + getter = test.getter(db) + sqlite_res = getter(1) + test.setter(db)(1, val) + old_cached_res = getter(1) + self.assertEqual(old_cached_res, cached_res, + 'Failed setting for %s with value %r, cached value not the same. Old: %r != New: %r'%( + test.name, val, old_cached_res, cached_res)) + db.refresh() + old_sqlite_res = getter(1) + self.assertEqual(old_sqlite_res, sqlite_res, + 'Failed setting for %s, sqlite value not the same: %r != %r'%( + test.name, old_sqlite_res, sqlite_res)) + del db + + + + def test_one_one(self): + 'Test setting of values in one-one fields' + tests = [] + for name, getter, setter in ( + ('pubdate', 'pubdate', 'set_pubdate'), + ('timestamp', 'timestamp', 'set_timestamp'), + ('#date', None, None) + ): + tests.append(self.create_test( + name, ('2011-1-12', UNDEFINED_DATE, None), getter, setter)) + + self.run_tests(tests) + +def tests(): + return unittest.TestLoader().loadTestsFromTestCase(WritingTest) + +def run(): + unittest.TextTestRunner(verbosity=2).run(tests()) + +if __name__ == '__main__': + run() + + diff --git a/src/calibre/db/write.py b/src/calibre/db/write.py new file mode 100644 index 0000000000..32b677eb67 --- /dev/null +++ b/src/calibre/db/write.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from functools import partial +from datetime import datetime + +from calibre.constants import preferred_encoding, ispy3 +from calibre.utils.date import (parse_only_date, parse_date, UNDEFINED_DATE, + isoformat) + +# Convert data into values suitable for the db {{{ + +if ispy3: + unicode = str + +def single_text(x): + if x is None: + return x + if not isinstance(x, unicode): + x = x.decode(preferred_encoding, 'replace') + x = x.strip() + return x if x else None + +def multiple_text(sep, x): + if x is None: + return () + if isinstance(x, bytes): + x = x.decode(preferred_encoding, 'replce') + if isinstance(x, unicode): + x = x.split(sep) + x = (y.strip() for y in x if y.strip()) + return (' '.join(y.split()) for y in x if y) + +def adapt_datetime(x): + if isinstance(x, (unicode, bytes)): + x = parse_date(x, assume_utc=False, as_utc=False) + if x is None: + x = UNDEFINED_DATE + return x + +def adapt_date(x): + if isinstance(x, (unicode, bytes)): + x = parse_only_date(x) + if x is None: + x = UNDEFINED_DATE + return x + +def adapt_number(typ, x): + if x is None: + return None + if isinstance(x, (unicode, bytes)): + if x.lower() == 'none': + return None + return typ(x) + +def adapt_bool(x): + if isinstance(x, (unicode, bytes)): + x = x.lower() + if x == 'true': + x = True + elif x == 'false': + x = False + elif x == 'none': + x = None + else: + x = bool(int(x)) + return x if x is None else bool(x) + +def get_adapter(name, metadata): + dt = metadata['datatype'] + if dt == 'text': + if metadata['is_multiple']: + ans = partial(multiple_text, metadata['is_multiple']['ui_to_list']) + else: + ans = single_text + elif dt == 'series': + ans = single_text + elif dt == 'datetime': + ans = adapt_date if name == 'pubdate' else adapt_datetime + elif dt == 'int': + ans = partial(adapt_number, int) + elif dt == 'float': + ans = partial(adapt_number, float) + elif dt == 'bool': + ans = adapt_bool + elif dt == 'comments': + ans = single_text + elif dt == 'rating': + ans = lambda x: x if x is None else min(10., max(0., adapt_number(float, x))), + elif dt == 'enumeration': + ans = single_text + elif dt == 'composite': + ans = lambda x: x + + if name == 'title': + ans = lambda x: ans(x) or _('Unknown') + elif name == 'authors': + ans = lambda x: ans(x) or (_('Unknown'),) + + return ans +# }}} + +def sqlite_datetime(x): + return isoformat(x, sep=' ') if isinstance(x, datetime) else x + +def one_one_in_books(book_id_val_map, db, field, *args): + 'Set a one-one field in the books table' + if book_id_val_map: + sequence = tuple((sqlite_datetime(v), k) for k, v in book_id_val_map.iteritems()) + db.conn.executemany( + 'UPDATE books SET %s=? WHERE id=?'%field.metadata['column'], sequence) + field.table.book_col_map.update(book_id_val_map) + return set(book_id_val_map) + +def one_one_in_other(book_id_val_map, db, field, *args): + 'Set a one-one field in the non-books table, like comments' + deleted = tuple((k,) for k, v in book_id_val_map.iteritems() if v is None) + if deleted: + db.conn.executemany('DELETE FROM %s WHERE book=?'%field.metadata['table'], + deleted) + for book_id in book_id_val_map: + field.table.book_col_map.pop(book_id, None) + updated = {k:v for k, v in book_id_val_map.iteritems() if v is not None} + if updated: + db.conn.executemany('INSERT OR REPLACE INTO %s(book,%s) VALUES (?,?)'%( + field.metadata['table'], field.metadata['column']), + tuple((k, sqlite_datetime(v)) for k, v in updated.iteritems())) + field.table.book_col_map.update(updated) + return set(book_id_val_map) + +def dummy(book_id_val_map, *args): + return set() + +class Writer(object): + + def __init__(self, field): + self.adapter = get_adapter(field.name, field.metadata) + self.name = field.name + self.field = field + dt = field.metadata['datatype'] + self.filter_vals = lambda x: x + if dt == 'composite' or field.name in {'cover', 'size', 'path'}: + self.set_books_func = dummy + elif field.is_many: + # TODO: Implement this + pass + else: + self.set_books_func = (one_one_in_books if field.metadata['table'] + == 'books' else one_one_in_other) + if self.name in {'timestamp', 'uuid'}: + self.filter_vals = bool + + def set_books(self, book_id_val_map, db): + book_id_val_map = {k:self.adapter(v) for k, v in + book_id_val_map.iteritems() if self.filter_vals(v)} + if not book_id_val_map: + return set() + dirtied = self.set_books_func(book_id_val_map, db, self.field) + return dirtied + diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 892d5dfbde..e64d1429ae 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -34,7 +34,7 @@ from calibre import isbytestring from calibre.utils.filenames import (ascii_filename, samefile, WindowsAtomicFolderMove, hardlink_file) from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp, - parse_only_date, UNDEFINED_DATE) + parse_only_date, UNDEFINED_DATE, parse_date) from calibre.utils.config import prefs, tweaks, from_json, to_json from calibre.utils.icu import sort_key, strcmp, lower from calibre.utils.search_query_parser import saved_searches, set_saved_searches @@ -2567,6 +2567,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def set_timestamp(self, id, dt, notify=True, commit=True): if dt: + if isinstance(dt, (unicode, bytes)): + dt = parse_date(dt, as_utc=True, assume_utc=False) self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id)) self.data.set(id, self.FIELD_MAP['timestamp'], dt, row_is_id=True) self.dirtied([id], commit=False)