Merge branch 'kovidgoyal/master'

This commit is contained in:
Charles Haley 2013-07-14 10:27:15 +02:00
commit d27197820f
7 changed files with 268 additions and 94 deletions

View File

@ -117,4 +117,5 @@ Various things that require other things before they can be migrated:
5. In the new API refresh() does not re-read from disk. That might break a
few things, for example content server reloading on db change as well as
dump/restore of db?
6. grep the sources for TODO
'''

View File

@ -7,7 +7,6 @@ __license__ = 'GPL v3'
__copyright__ = '2013, Kovid Goyal <kovid at kovidgoyal.net>'
import os, traceback, types
from functools import partial
from future_builtins import zip
from calibre import force_unicode
@ -19,6 +18,7 @@ from calibre.db.backend import DB
from calibre.db.cache import Cache
from calibre.db.categories import CATEGORY_SORTS
from calibre.db.view import View
from calibre.db.write import clean_identifier
from calibre.utils.date import utcnow
class LibraryDatabase(object):
@ -52,76 +52,8 @@ class LibraryDatabase(object):
self.get_property = self.data.get_property
for prop in (
'author_sort', 'authors', 'comment', 'comments',
'publisher', 'rating', 'series', 'series_index', 'tags',
'title', 'timestamp', 'uuid', 'pubdate', 'ondevice',
'metadata_last_modified', 'languages',
):
fm = {'comment':'comments', 'metadata_last_modified':
'last_modified', 'title_sort':'sort'}.get(prop, prop)
setattr(self, prop, partial(self.get_property,
loc=self.FIELD_MAP[fm]))
MT = lambda func: types.MethodType(func, self, LibraryDatabase)
for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'):
setattr(self, meth, getattr(self.new_api, meth))
# Legacy API to get information about many-(one, many) fields
for field in ('authors', 'tags', 'publisher', 'series'):
def getter(field):
def func(self):
return self.new_api.all_field_names(field)
return func
name = field[:-1] if field in {'authors', 'tags'} else field
setattr(self, 'all_%s_names' % name, MT(getter(field)))
self.all_formats = MT(lambda self:self.new_api.all_field_names('formats'))
for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems():
setattr(self, func, partial(self.field_id_map, field))
self.all_tags = MT(lambda self: list(self.all_tag_names()))
self.get_authors_with_ids = MT(
lambda self: [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()])
for field in ('tags', 'series', 'publishers', 'ratings', 'languages'):
def getter(field):
fname = field[:-1] if field in {'publishers', 'ratings'} else field
def func(self):
return [[tid, tag] for tid, tag in self.new_api.get_id_map(fname).iteritems()]
return func
setattr(self, 'get_%s_with_ids' % field,
MT(getter(field)))
for field in ('author', 'tag', 'series'):
def getter(field):
field = field if field == 'series' else (field+'s')
def func(self, item_id):
return self.new_api.get_item_name(field, item_id)
return func
setattr(self, '%s_name' % field, MT(getter(field)))
for field in ('publisher', 'series', 'tag'):
def getter(field):
fname = 'tags' if field == 'tag' else field
def func(self, item_id):
self.new_api.remove_items(fname, (item_id,))
return func
setattr(self, 'delete_%s_using_id' % field, MT(getter(field)))
# Legacy field API
for func in (
'standard_field_keys', 'custom_field_keys', 'all_field_keys',
'searchable_fields', 'sortable_field_keys',
'search_term_to_field_key', 'custom_field_metadata',
'all_metadata'):
setattr(self, func, getattr(self.field_metadata, func))
self.metadata_for_field = self.field_metadata.get
self.last_update_check = self.last_modified()
self.book_on_device_func = None
# Cleaning is not required anymore
self.clean = self.clean_custom = MT(lambda self:None)
self.clean_standard_field = MT(lambda self, field, commit=False:None)
# apsw operates in autocommit mode
self.commit = MT(lambda self:None)
def close(self):
self.backend.close()
@ -409,6 +341,36 @@ class LibraryDatabase(object):
finally:
self.notify('metadata', [book_id])
def set_identifier(self, book_id, typ, val, notify=True, commit=True):
with self.new_api.write_lock:
identifiers = self.new_api._field_for('identifiers', book_id)
typ, val = clean_identifier(typ, val)
if typ:
identifiers[typ] = val
self.new_api._set_field('identifiers', {book_id:identifiers})
self.notify('metadata', [book_id])
def set_isbn(self, book_id, isbn, notify=True, commit=True):
self.set_identifier(book_id, 'isbn', isbn, notify=notify, commit=commit)
def set_tags(self, book_id, tags, append=False, notify=True, commit=True, allow_case_change=False):
tags = tags or []
with self.new_api.write_lock:
if append:
otags = self.new_api._field_for('tags', book_id)
existing = {icu_lower(x) for x in otags}
tags = list(otags) + [x for x in tags if icu_lower(x) not in existing]
ret = self.new_api._set_field('tags', {book_id:tags}, allow_case_change=allow_case_change)
if notify:
self.notify('metadata', [book_id])
return ret
def set_metadata(self, book_id, mi, ignore_errors=False, set_title=True,
set_authors=True, commit=True, force_changes=False, notify=True):
self.new_api.set_metadata(book_id, mi, ignore_errors=ignore_errors, set_title=set_title, set_authors=set_authors, force_changes=force_changes)
if notify:
self.notify('metadata', [book_id])
# Private interface {{{
def __iter__(self):
for row in self.data.iterall():
@ -422,3 +384,143 @@ class LibraryDatabase(object):
# }}}
MT = lambda func: types.MethodType(func, None, LibraryDatabase)
# Legacy getter API {{{
for prop in ('author_sort', 'authors', 'comment', 'comments', 'publisher',
'rating', 'series', 'series_index', 'tags', 'title', 'title_sort',
'timestamp', 'uuid', 'pubdate', 'ondevice', 'metadata_last_modified', 'languages',):
def getter(prop):
fm = {'comment':'comments', 'metadata_last_modified':
'last_modified', 'title_sort':'sort'}.get(prop, prop)
def func(self, index, index_is_id=False):
return self.get_property(index, index_is_id=index_is_id, loc=self.FIELD_MAP[fm])
return func
setattr(LibraryDatabase, prop, MT(getter(prop)))
LibraryDatabase.has_cover = MT(lambda self, book_id:self.new_api.field_for('cover', book_id))
LibraryDatabase.get_identifiers = MT(
lambda self, index, index_is_id=False: self.new_api.field_for('identifiers', index if index_is_id else self.data.index_to_id(index)))
# }}}
# Legacy setter API {{{
for field in (
'!authors', 'author_sort', 'comment', 'has_cover', 'identifiers', 'languages',
'pubdate', '!publisher', 'rating', '!series', 'series_index', 'timestamp', 'uuid',
'title', 'title_sort',
):
def setter(field):
has_case_change = field.startswith('!')
field = {'comment':'comments', 'title_sort':'sort'}.get(field, field)
if has_case_change:
field = field[1:]
acc = field == 'series'
def func(self, book_id, val, notify=True, commit=True, allow_case_change=acc):
ret = self.new_api.set_field(field, {book_id:val}, allow_case_change=allow_case_change)
if notify:
self.notify([book_id])
return ret
elif field == 'has_cover':
def func(self, book_id, val):
self.new_api.set_field('cover', {book_id:bool(val)})
else:
null_field = field in {'title', 'sort', 'uuid'}
retval = (True if field == 'sort' else None)
def func(self, book_id, val, notify=True, commit=True):
if not val and null_field:
return (False if field == 'sort' else None)
ret = self.new_api.set_field(field, {book_id:val})
if notify:
self.notify([book_id])
return ret if field == 'languages' else retval
return func
setattr(LibraryDatabase, 'set_%s' % field.replace('!', ''), MT(setter(field)))
# }}}
# Legacy API to get information about many-(one, many) fields {{{
for field in ('authors', 'tags', 'publisher', 'series'):
def getter(field):
def func(self):
return self.new_api.all_field_names(field)
return func
name = field[:-1] if field in {'authors', 'tags'} else field
setattr(LibraryDatabase, 'all_%s_names' % name, MT(getter(field)))
LibraryDatabase.all_formats = MT(lambda self:self.new_api.all_field_names('formats'))
for func, field in {'all_authors':'authors', 'all_titles':'title', 'all_tags2':'tags', 'all_series':'series', 'all_publishers':'publisher'}.iteritems():
def getter(field):
def func(self):
return self.field_id_map(field)
return func
setattr(LibraryDatabase, func, MT(getter(field)))
LibraryDatabase.all_tags = MT(lambda self: list(self.all_tag_names()))
LibraryDatabase.get_all_identifier_types = MT(lambda self: list(self.new_api.fields['identifiers'].table.all_identifier_types()))
LibraryDatabase.get_authors_with_ids = MT(
lambda self: [[aid, adata['name'], adata['sort'], adata['link']] for aid, adata in self.new_api.author_data().iteritems()])
for field in ('tags', 'series', 'publishers', 'ratings', 'languages'):
def getter(field):
fname = field[:-1] if field in {'publishers', 'ratings'} else field
def func(self):
return [[tid, tag] for tid, tag in self.new_api.get_id_map(fname).iteritems()]
return func
setattr(LibraryDatabase, 'get_%s_with_ids' % field, MT(getter(field)))
for field in ('author', 'tag', 'series'):
def getter(field):
field = field if field == 'series' else (field+'s')
def func(self, item_id):
return self.new_api.get_item_name(field, item_id)
return func
setattr(LibraryDatabase, '%s_name' % field, MT(getter(field)))
for field in ('publisher', 'series', 'tag'):
def getter(field):
fname = 'tags' if field == 'tag' else field
def func(self, item_id):
self.new_api.remove_items(fname, (item_id,))
return func
setattr(LibraryDatabase, 'delete_%s_using_id' % field, MT(getter(field)))
# }}}
# Legacy field API {{{
for func in (
'standard_field_keys', '!custom_field_keys', 'all_field_keys',
'searchable_fields', 'sortable_field_keys',
'search_term_to_field_key', '!custom_field_metadata',
'all_metadata'):
def getter(func):
if func.startswith('!'):
func = func[1:]
def meth(self, include_composites=True):
return getattr(self.field_metadata, func)(include_composites=include_composites)
elif func == 'search_term_to_field_key':
def meth(self, term):
return self.field_metadata.search_term_to_field_key(term)
else:
def meth(self):
return getattr(self.field_metadata, func)()
return meth
setattr(LibraryDatabase, func.replace('!', ''), MT(getter(func)))
LibraryDatabase.metadata_for_field = MT(lambda self, field:self.field_metadata.get(field))
# }}}
# Miscellaneous API {{{
for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'):
def getter(meth):
def func(self, x):
return getattr(self.new_api, meth)(x)
return func
setattr(LibraryDatabase, meth, MT(getter(meth)))
# Cleaning is not required anymore
LibraryDatabase.clean = LibraryDatabase.clean_custom = MT(lambda self:None)
LibraryDatabase.clean_standard_field = MT(lambda self, field, commit=False:None)
# apsw operates in autocommit mode
LibraryDatabase.commit = MT(lambda self:None)
# }}}
del MT

View File

@ -420,3 +420,6 @@ class IdentifiersTable(ManyToManyTable):
def remove_items(self, item_ids, db):
raise NotImplementedError('Direct deletion of identifiers is not implemented')
def all_identifier_types(self):
return frozenset(k for k, v in self.col_book_map.iteritems() if v)

View File

@ -47,11 +47,15 @@ def run_funcs(self, db, ndb, funcs):
meth(*args)
else:
fmt = lambda x:x
if meth[0] in {'!', '@', '#'}:
fmt = {'!':dict, '@':frozenset, '#':lambda x:set((x or '').split(','))}[meth[0]]
if meth[0] in {'!', '@', '#', '+'}:
if meth[0] != '+':
fmt = {'!':dict, '@':lambda x:frozenset(x or ()), '#':lambda x:set((x or '').split(','))}[meth[0]]
else:
fmt = args[-1]
args = args[:-1]
meth = meth[1:]
self.assertEqual(fmt(getattr(db, meth)(*args)), fmt(getattr(ndb, meth)(*args)),
'The method: %s() returned different results for argument %s' % (meth, args))
res1, res2 = fmt(getattr(db, meth)(*args)), fmt(getattr(ndb, meth)(*args))
self.assertEqual(res1, res2, 'The method: %s() returned different results for argument %s' % (meth, args))
class LegacyTest(BaseTest):
@ -129,7 +133,7 @@ class LegacyTest(BaseTest):
def test_legacy_getters(self): # {{{
' Test various functions to get individual bits of metadata '
old = self.init_old()
getters = ('path', 'abspath', 'title', 'authors', 'series',
getters = ('path', 'abspath', 'title', 'title_sort', 'authors', 'series',
'publisher', 'author_sort', 'authors', 'comments',
'comment', 'publisher', 'rating', 'series_index', 'tags',
'timestamp', 'uuid', 'pubdate', 'ondevice',
@ -164,6 +168,7 @@ class LegacyTest(BaseTest):
'!all_authors':[()],
'!all_tags2':[()],
'@all_tags':[()],
'@get_all_identifier_types':[()],
'!all_publishers':[()],
'!all_titles':[()],
'!all_series':[()],
@ -327,9 +332,10 @@ class LegacyTest(BaseTest):
'construct_path_name', 'clear_dirtied', 'commit_dirty_cache', 'initialize_database', 'initialize_dynamic',
'run_import_plugins', 'vacuum', 'set_path', 'row', 'row_factory', 'rows', 'rmtree', 'series_index_pat',
'import_old_database', 'dirtied_lock', 'dirtied_cache', 'dirty_queue_length', 'dirty_books_referencing',
'windows_check_if_files_in_use', 'get_metadata_for_dump', 'get_a_dirtied_book',
}
SKIP_ARGSPEC = {
'__init__', 'get_next_series_num_for', 'has_book', 'author_sort_from_authors',
'__init__',
}
missing = []
@ -397,6 +403,67 @@ class LegacyTest(BaseTest):
def test_legacy_setters(self): # {{{
'Test methods that are directly equivalent in the old and new interface'
from calibre.ebooks.metadata.book.base import Metadata
ndb = self.init_legacy(self.cloned_library)
db = self.init_old(self.cloned_library)
run_funcs(self, db, ndb, (
('set_authors', 1, ('author one',),), ('set_authors', 2, ('author two',), True, True, True),
('set_author_sort', 3, 'new_aus'),
('set_comment', 1, ''), ('set_comment', 2, None), ('set_comment', 3, '<p>a comment</p>'),
('set_has_cover', 1, True), ('set_has_cover', 2, True), ('set_has_cover', 3, 1),
('set_identifiers', 2, {'test':'', 'a':'b'}), ('set_identifiers', 3, {'id':'1', 'url':'http://acme.com'}), ('set_identifiers', 1, {}),
('set_languages', 1, ('en',)),
('set_languages', 2, ()),
('set_languages', 3, ('deu', 'spa', 'fra')),
('set_pubdate', 1, None), ('set_pubdate', 2, '2011-1-7'),
('set_series', 1, 'a series one'), ('set_series', 2, 'another series [7]'), ('set_series', 3, 'a third series'),
('set_publisher', 1, 'publisher two'), ('set_publisher', 2, None), ('set_publisher', 3, 'a third puB'),
('set_rating', 1, 2.3), ('set_rating', 2, 0), ('set_rating', 3, 8),
('set_timestamp', 1, None), ('set_timestamp', 2, '2011-1-7'),
('set_uuid', 1, None), ('set_uuid', 2, 'a test uuid'),
('set_title', 1, 'title two'), ('set_title', 2, None), ('set_title', 3, 'The Test Title'),
('set_tags', 1, ['a1', 'a2'], True), ('set_tags', 2, ['b1', 'tag one'], False, False, False, True), ('set_tags', 3, ['A1']),
(db.refresh,),
('title', 0), ('title', 1), ('title', 2),
('title_sort', 0), ('title_sort', 1), ('title_sort', 2),
('authors', 0), ('authors', 1), ('authors', 2),
('author_sort', 0), ('author_sort', 1), ('author_sort', 2),
('has_cover', 3), ('has_cover', 1), ('has_cover', 2),
('get_identifiers', 0), ('get_identifiers', 1), ('get_identifiers', 2),
('pubdate', 0), ('pubdate', 1), ('pubdate', 2),
('timestamp', 0), ('timestamp', 1), ('timestamp', 2),
('publisher', 0), ('publisher', 1), ('publisher', 2),
('rating', 0), ('+rating', 1, lambda x: x or 0), ('rating', 2),
('series', 0), ('series', 1), ('series', 2),
('series_index', 0), ('series_index', 1), ('series_index', 2),
('uuid', 0), ('uuid', 1), ('uuid', 2),
('@tags', 0), ('@tags', 1), ('@tags', 2),
('@all_tags',),
('@get_all_identifier_types',),
('set_title_sort', 1, 'Title Two'), ('set_title_sort', 2, None), ('set_title_sort', 3, 'The Test Title_sort'),
('set_series_index', 1, 2.3), ('set_series_index', 2, 0), ('set_series_index', 3, 8),
('set_identifier', 1, 'moose', 'val'), ('set_identifier', 2, 'test', ''), ('set_identifier', 3, '', ''),
(db.refresh,),
('series_index', 0), ('series_index', 1), ('series_index', 2),
('title_sort', 0), ('title_sort', 1), ('title_sort', 2),
('get_identifiers', 0), ('get_identifiers', 1), ('get_identifiers', 2),
('@get_all_identifier_types',),
('set_metadata', 1, Metadata('title', ('a1',)), False, False, False, True, True),
('set_metadata', 3, Metadata('title', ('a1',))),
(db.refresh,),
('title', 0), ('title', 1), ('title', 2),
('title_sort', 0), ('title_sort', 1), ('title_sort', 2),
('authors', 0), ('authors', 1), ('authors', 2),
('author_sort', 0), ('author_sort', 1), ('author_sort', 2),
('@tags', 0), ('@tags', 1), ('@tags', 2),
('@all_tags',),
('@get_all_identifier_types',),
))
db.close()
ndb = self.init_legacy(self.cloned_library)
db = self.init_old(self.cloned_library)
@ -405,7 +472,7 @@ class LegacyTest(BaseTest):
('set', 0, 'tags', 't1,t2,tag one', True),
('set', 0, 'authors', 'author one & Author Two', True),
('set', 0, 'rating', 3.2),
('set', 0, 'publisher', 'publisher one', True),
('set', 0, 'publisher', 'publisher one', False),
(db.refresh,),
('title', 0),
('rating', 0),
@ -413,6 +480,4 @@ class LegacyTest(BaseTest):
('authors', 0), ('authors', 1), ('authors', 2),
('publisher', 0), ('publisher', 1), ('publisher', 2),
))
db.close()
# }}}

View File

@ -107,8 +107,8 @@ def adapt_languages(to_tuple, x):
return tuple(ans)
def clean_identifier(typ, val):
typ = icu_lower(typ).strip().replace(':', '').replace(',', '')
val = val.strip().replace(',', '|').replace(':', '|')
typ = icu_lower(typ or '').strip().replace(':', '').replace(',', '')
val = (val or '').strip().replace(',', '|').replace(':', '|')
return typ, val
def adapt_identifiers(to_tuple, x):

View File

@ -257,8 +257,8 @@ class EmailMixin(object): # {{{
else:
autos = [self.library_view.model().db.title(id, index_is_id=True) for id in auto]
if self.auto_convert_question(
_('Auto convert the following books before sending via '
'email?'), autos):
_('Auto convert the following books to %s before sending via '
'email?') % format.upper(), autos):
self.iactions['Convert Books'].auto_convert_mail(to, fmts, delete_from_library, auto, format, subject)
if bad:

View File

@ -2423,7 +2423,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if not authors:
authors = [_('Unknown')]
self.conn.execute('DELETE FROM books_authors_link WHERE book=?',(id,))
books_to_refresh = set([])
books_to_refresh = {id}
final_authors = []
for a in authors:
case_change = False
@ -2615,10 +2615,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def set_publisher(self, id, publisher, notify=True, commit=True,
allow_case_change=False):
self.conn.execute('DELETE FROM books_publishers_link WHERE book=?',(id,))
self.conn.execute('''DELETE FROM publishers WHERE (SELECT COUNT(id)
FROM books_publishers_link
WHERE publisher=publishers.id) < 1''')
books_to_refresh = set([])
books_to_refresh = {id}
if publisher:
case_change = False
if not isinstance(publisher, unicode):
@ -2634,6 +2631,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
case_change = True
else:
publisher = cur_name
books_to_refresh = set()
else:
aid = self.conn.execute('''INSERT INTO publishers(name)
VALUES (?)''', (publisher,)).lastrowid
@ -2643,6 +2641,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
bks = self.conn.get('''SELECT book FROM books_publishers_link
WHERE publisher=?''', (aid,))
books_to_refresh |= set([bk[0] for bk in bks])
self.conn.execute('''DELETE FROM publishers WHERE (SELECT COUNT(id)
FROM books_publishers_link
WHERE publisher=publishers.id) < 1''')
self.dirtied(set([id])|books_to_refresh, commit=False)
if commit:
self.conn.commit()
@ -3054,11 +3056,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
tags = []
if not append:
self.conn.execute('DELETE FROM books_tags_link WHERE book=?', (id,))
self.conn.execute('''DELETE FROM tags WHERE (SELECT COUNT(id)
FROM books_tags_link WHERE tag=tags.id) < 1''')
otags = self.get_tags(id)
tags = self.cleanup_tags(tags)
books_to_refresh = set([])
books_to_refresh = {id}
for tag in (set(tags)-otags):
case_changed = False
tag = tag.strip()
@ -3089,6 +3089,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
bks = self.conn.get('SELECT book FROM books_tags_link WHERE tag=?',
(tid,))
books_to_refresh |= set([bk[0] for bk in bks])
self.conn.execute('''DELETE FROM tags WHERE (SELECT COUNT(id)
FROM books_tags_link WHERE tag=tags.id) < 1''')
self.dirtied(set([id])|books_to_refresh, commit=False)
if commit:
self.conn.commit()
@ -3139,11 +3141,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
def set_series(self, id, series, notify=True, commit=True, allow_case_change=True):
self.conn.execute('DELETE FROM books_series_link WHERE book=?',(id,))
self.conn.execute('''DELETE FROM series
WHERE (SELECT COUNT(id) FROM books_series_link
WHERE series=series.id) < 1''')
(series, idx) = self._get_series_values(series)
books_to_refresh = set([])
books_to_refresh = {id}
if series:
case_change = False
if not isinstance(series, unicode):
@ -3159,6 +3158,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
case_change = True
else:
series = cur_name
books_to_refresh = set()
else:
aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid
self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid))
@ -3168,6 +3168,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
bks = self.conn.get('SELECT book FROM books_series_link WHERE series=?',
(aid,))
books_to_refresh |= set([bk[0] for bk in bks])
self.conn.execute('''DELETE FROM series
WHERE (SELECT COUNT(id) FROM books_series_link
WHERE series=series.id) < 1''')
self.dirtied([id], commit=False)
if commit:
self.conn.commit()