From c950518da25890d5a9f30b1f18b7fb1cbe21f15e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 12:18:24 +0530 Subject: [PATCH 01/21] Avoid unnecessary string formatting --- src/calibre/utils/filenames.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index 54ce568539..d6193c068b 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -259,10 +259,10 @@ def samefile(src, dst): def windows_hardlink(src, dest): import win32file, pywintypes - msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) try: win32file.CreateHardLink(dest, src) except pywintypes.error as e: + msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) raise Exception(msg % e) # We open and close dest, to ensure its directory entry is updated # see http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx @@ -273,6 +273,7 @@ def windows_hardlink(src, dest): win32file.CloseHandle(h) if sz != os.path.getsize(src): + msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) raise Exception(msg % ('hardlink size: %d not the same as source size' % sz)) class WindowsAtomicFolderMove(object): From c7033beb98340ed2577e9ff6a6515a0f77cebf6c Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 12:46:12 +0530 Subject: [PATCH 02/21] Make ads in user manual async --- manual/templates/layout.html | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/manual/templates/layout.html b/manual/templates/layout.html index 188e829469..ff2e7b0113 100644 --- a/manual/templates/layout.html +++ b/manual/templates/layout.html @@ -16,16 +16,13 @@
{% if not embedded %}
- - + +
{% endif %} From 7e4a124d527d7fd408209dfe70917c0c2a8f456d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 13:00:08 +0530 Subject: [PATCH 03/21] Ensure we dont leak a file handle --- src/calibre/utils/filenames.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/filenames.py b/src/calibre/utils/filenames.py index d6193c068b..d756978040 100644 --- a/src/calibre/utils/filenames.py +++ b/src/calibre/utils/filenames.py @@ -269,8 +269,10 @@ def windows_hardlink(src, dest): h = win32file.CreateFile( dest, 0, win32file.FILE_SHARE_READ | win32file.FILE_SHARE_WRITE | win32file.FILE_SHARE_DELETE, None, win32file.OPEN_EXISTING, 0, None) - sz = win32file.GetFileSize(h) - win32file.CloseHandle(h) + try: + sz = win32file.GetFileSize(h) + finally: + win32file.CloseHandle(h) if sz != os.path.getsize(src): msg = u'Creating hardlink from %s to %s failed: %%s' % (src, dest) From f1ce0eb75c38f0f4fc3f4dea50f77d6666766925 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 4 Jul 2013 13:16:09 +0530 Subject: [PATCH 04/21] Fix test failures due to file locking on windows --- src/calibre/db/legacy.py | 4 +++- src/calibre/db/tests/legacy.py | 32 +++++++++++++++++++------------- src/calibre/db/tests/writing.py | 3 +++ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 874707fa2e..287d2d47db 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -64,9 +64,11 @@ class LibraryDatabase(object): self.backend.close() def break_cycles(self): + delattr(self.backend, 'field_metadata') self.data.cache.backend = None self.data.cache = None - self.data = self.backend = self.new_api = self.field_metadata = self.prefs = self.listeners = self.refresh_ondevice = None + for x in ('data', 'backend', 'new_api', 'listeners',): + delattr(self, x) # Library wide properties {{{ @property diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index af6b977aef..1fe719e31e 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -117,18 +117,24 @@ class LegacyTest(BaseTest): '__init__', } - for attr in dir(db): - if attr in SKIP_ATTRS: - continue - if not hasattr(ndb, attr): - raise AssertionError('The attribute %s is missing' % attr) - obj, nobj = getattr(db, attr), getattr(ndb, attr) - if attr not in SKIP_ARGSPEC: - try: - argspec = inspect.getargspec(obj) - except TypeError: - pass - else: - self.assertEqual(argspec, inspect.getargspec(nobj), 'argspec for %s not the same' % attr) + try: + for attr in dir(db): + if attr in SKIP_ATTRS: + continue + if not hasattr(ndb, attr): + raise AssertionError('The attribute %s is missing' % attr) + obj, nobj = getattr(db, attr), getattr(ndb, attr) + if attr not in SKIP_ARGSPEC: + try: + argspec = inspect.getargspec(obj) + except TypeError: + pass + else: + self.assertEqual(argspec, inspect.getargspec(nobj), 'argspec for %s not the same' % attr) + finally: + for db in (ndb, db): + db.close() + db.break_cycles() + # }}} diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 5d04c11def..597c98a771 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -374,6 +374,9 @@ class WritingTest(BaseTest): ae(cache.field_for('cover', book_id), 1) ae(old.cover(book_id, index_is_id=True), img, 'Cover was not set correctly for book %d' % book_id) self.assertTrue(old.has_cover(book_id)) + old.close() + old.break_cycles() + del old # }}} def test_set_metadata(self): # {{{ From 4bba2cbf926bfaee59d97576fea6a70a29d9a282 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 08:01:15 +0530 Subject: [PATCH 05/21] Initial implementation of add_format() --- src/calibre/db/backend.py | 18 +++++++++++++ src/calibre/db/cache.py | 54 ++++++++++++++++++++++++++++++++++++++- src/calibre/db/tables.py | 19 +++++++++++--- 3 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index c6aa2e646f..97dc673ecc 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1059,6 +1059,24 @@ class DB(object): if wam is not None: wam.close_handles() + def add_format(self, book_id, fmt, stream, title, author, path): + fname = self.construct_file_name(book_id, title, author) + path = os.path.join(self.library_path, path) + fmt = ('.' + fmt.lower()) if fmt else '' + dest = os.path.join(path, fname + fmt) + if not os.path.exists(path): + os.makedirs(path) + size = 0 + + if (not getattr(stream, 'name', False) or not samefile(dest, stream.name)): + with lopen(dest, 'wb') as f: + shutil.copyfileobj(stream, f) + size = f.tell() + elif os.path.exists(dest): + size = os.path.getsize(dest) + + return size, fname + def update_path(self, book_id, title, author, path_field, formats_field): path = self.construct_path_name(book_id, title, author) current_path = path_field.for_book(book_id) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 88f06b43ba..4f7de11269 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -7,12 +7,13 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, random +import os, traceback, random, shutil from io import BytesIO from collections import defaultdict from functools import wraps, partial from calibre.constants import iswindows +from calibre.customize.ui import run_plugins_on_import, run_plugins_on_postimport from calibre.db import SPOOL_SIZE from calibre.db.categories import get_categories from calibre.db.locking import create_locks @@ -22,6 +23,7 @@ from calibre.db.search import Search from calibre.db.tables import VirtualTable from calibre.db.write import get_series_values from calibre.db.lazy import FormatMetadata, FormatsList +from calibre.ebooks import check_ebook_format from calibre.ebooks.metadata import string_to_authors from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf @@ -51,6 +53,18 @@ def wrap_simple(lock, func): return func(*args, **kwargs) return ans +def run_import_plugins(path_or_stream, fmt): + fmt = fmt.lower() + if hasattr(path_or_stream, 'seek'): + path_or_stream.seek(0) + pt = PersistentTemporaryFile('_import_plugin.'+fmt) + shutil.copyfileobj(path_or_stream, pt, 1024**2) + pt.close() + path = pt.name + else: + path = path_or_stream + return run_plugins_on_import(path, fmt) + class Cache(object): @@ -943,6 +957,43 @@ class Cache(object): if extra is not None or force_changes: protected_set_field(idx, extra) + @write_api + def add_format(self, book_id, fmt, stream_or_path, replace=True, run_hooks=True, dbapi=None): + if run_hooks: + # Run import plugins + npath = run_import_plugins(stream_or_path, fmt) + fmt = os.path.splitext(npath)[-1].lower().replace('.', '').upper() + stream_or_path = lopen(npath, 'rb') + fmt = check_ebook_format(stream_or_path, fmt) + + fmt = (fmt or '').upper() + self.format_metadata_cache[book_id].pop(fmt, None) + try: + name = self.fields['formats'].format_fname(book_id, fmt) + except: + name = None + + if name and not replace: + return False + + path = self._field_for('path', book_id).replace('/', os.sep) + title = self._field_for('title', book_id, default_value=_('Unknown')) + author = self._field_for('authors', book_id, default_value=(_('Unknown'),))[0] + stream = stream_or_path if hasattr(stream_or_path, 'read') else lopen(stream_or_path, 'rb') + size, fname = self.backend.add_format(book_id, fmt, stream, title, author, path) + del stream + + max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) + self.fields['size'].table.update_size(book_id, max_size) + self._update_last_modified((book_id,)) + + if run_hooks: + # Run post import plugins + run_plugins_on_postimport(dbapi or self, book_id, fmt) + stream_or_path.close() + + return True + # }}} class SortKey(object): # {{{ @@ -959,3 +1010,4 @@ class SortKey(object): # {{{ return 0 # }}} + diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 83d4b23712..4dba10abff 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -8,6 +8,7 @@ __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' from datetime import datetime +from collections import defaultdict from dateutil.tz import tzoffset @@ -98,6 +99,9 @@ class SizeTable(OneToOneTable): 'WHERE data.book=books.id) FROM books'): self.book_col_map[row[0]] = self.unserialize(row[1]) + def update_size(self, book_id, size): + self.book_col_map[book_id] = size + class UUIDTable(OneToOneTable): def read(self, db): @@ -194,8 +198,9 @@ class FormatsTable(ManyToManyTable): pass def read_maps(self, db): - self.fname_map = {} - for row in db.conn.execute('SELECT book, format, name FROM data'): + self.fname_map = defaultdict(dict) + self.size_map = defaultdict(dict) + for row in db.conn.execute('SELECT book, format, name, uncompressed_size FROM data'): if row[1] is not None: fmt = row[1].upper() if fmt not in self.col_book_map: @@ -204,9 +209,8 @@ class FormatsTable(ManyToManyTable): if row[0] not in self.book_col_map: self.book_col_map[row[0]] = [] self.book_col_map[row[0]].append(fmt) - if row[0] not in self.fname_map: - self.fname_map[row[0]] = {} self.fname_map[row[0]][fmt] = row[2] + self.size_map[row[0]][fmt] = row[3] for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(sorted(self.book_col_map[key])) @@ -216,6 +220,13 @@ class FormatsTable(ManyToManyTable): db.conn.execute('UPDATE data SET name=? WHERE book=? AND format=?', (fname, book_id, fmt)) + def update_fmt(self, book_id, fmt, fname, size, db): + self.fname_map[book_id][fmt] = fname + self.size_map[book_id][fmt] = size + db.conn.execute('INSERT OR REPLACE INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)', + (book_id, fmt, size, fname)) + return max(self.size_map[book_id].itervalues()) + class IdentifiersTable(ManyToManyTable): def read_id_maps(self, db): From 93264786e1bb9fb4d14b1f76bf536ffd47b21a67 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 08:03:47 +0530 Subject: [PATCH 06/21] Fix detection of SD Card in some PRS-T2N devices Fixes #1197970 [Sony PRS-T2 SD card not recognized](https://bugs.launchpad.net/calibre/+bug/1197970) --- src/calibre/devices/prst1/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/prst1/driver.py b/src/calibre/devices/prst1/driver.py index 0431ca7bfd..9c76eb096f 100644 --- a/src/calibre/devices/prst1/driver.py +++ b/src/calibre/devices/prst1/driver.py @@ -53,7 +53,7 @@ class PRST1(USBMS): r'(PRS-T(1|2|2N)&)' ) WINDOWS_CARD_A_MEM = re.compile( - r'(PRS-T(1|2|2N)__SD&)' + r'(PRS-T(1|2|2N)_{1,2}SD&)' ) MAIN_MEMORY_VOLUME_LABEL = 'SONY Reader Main Memory' STORAGE_CARD_VOLUME_LABEL = 'SONY Reader Storage Card' From 849465ccce3832958251d5ee2c69c350b67f6c10 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 08:27:29 +0530 Subject: [PATCH 07/21] version 0.9.38 --- Changelog.yaml | 50 ++++++++++++++++++++++++++++++++++++++++ src/calibre/constants.py | 2 +- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/Changelog.yaml b/Changelog.yaml index 7439a02986..f68617fb3a 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -20,6 +20,56 @@ # new recipes: # - title: +- version: 0.9.38 + date: 2013-07-05 + + new features: + - title: "Book polishing: Add option to embed all referenced fonts when polishing books using the 'Polish Books' tool." + tickets: [1196038] + + - title: "DOCX Input: Add support for clickable (hyperlinked) images" + tickets: [1196728] + + - title: "DOCX Input: Insert page breaks at the start of every new section" + tickets: [1196728] + + - title: "Drivers for Trekstor Pyrus Maxi and PocketBook Surfpad 2" + tickets: [1196931, 1182850] + + - title: "DOCX Input: Add support for horizontal rules created by typing three hyphens and pressing enter." + + bug fixes: + - title: "Fix detection of SD Card in some PRS-T2N devices" + tickets: [1197970] + + - title: "MOBI Input: Fix a regression that broke parsing of MOBI files with malformed markup that also used entities for apostrophes." + ticket: [1197585] + + - title: "Get Books: Update Woblink store plugin" + + - title: "Metadata download dialog: Prevent the buttons from being re-ordered when the Next button is clicked." + + - title: "PDF Output: Fix links that point to URLs with query parameters being mangled by the conversion process." + tickets: [1197006] + + - title: "DOCX Input: Fix links pointing to locations in the same document that contain multiple, redundant bookmarks not working." + + - title: "EPUB/AZW3 Output: Fix splitting on page-break-after with plain text immediately following the split point causing the text to be added before rather than after the split point." + tickets: [1196728] + + - title: "DOCX Input: handle bookmarks defined at the paragraph level" + tickets: [1196728] + + - title: "DOCX Input: Handle hyperlinks created as fields" + tickets: [1196728] + + improved recipes: + - iprofessional + + new recipes: + - title: Democracy Now + author: Antoine Beaupre + - version: 0.9.37 date: 2013-06-28 diff --git a/src/calibre/constants.py b/src/calibre/constants.py index a4edca6bd5..99146e206c 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -4,7 +4,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' __appname__ = u'calibre' -numeric_version = (0, 9, 37) +numeric_version = (0, 9, 38) __version__ = u'.'.join(map(unicode, numeric_version)) __author__ = u"Kovid Goyal " From 8c1bb3d4cb6655a7b6a3d58fbad35c66bb869050 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 10:12:23 +0530 Subject: [PATCH 08/21] Add tests for add_format() --- src/calibre/db/tables.py | 13 ++++ src/calibre/db/tests/add_remove.py | 101 +++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 src/calibre/db/tests/add_remove.py diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 4dba10abff..140c95eb88 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -221,6 +221,19 @@ class FormatsTable(ManyToManyTable): (fname, book_id, fmt)) def update_fmt(self, book_id, fmt, fname, size, db): + fmts = list(self.book_col_map.get(book_id, [])) + try: + fmts.remove(fmt) + except ValueError: + pass + fmts.append(fmt) + self.book_col_map[book_id] = tuple(fmts) + + try: + self.col_book_map[fmt].add(book_id) + except KeyError: + self.col_book_map[fmt] = {book_id} + self.fname_map[book_id][fmt] = fname self.size_map[book_id][fmt] = size db.conn.execute('INSERT OR REPLACE INTO data (book,format,uncompressed_size,name) VALUES (?,?,?,?)', diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py new file mode 100644 index 0000000000..c5b165bd02 --- /dev/null +++ b/src/calibre/db/tests/add_remove.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2013, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from io import BytesIO +from tempfile import NamedTemporaryFile + +from calibre.db.tests.base import BaseTest +from calibre.ptempfile import PersistentTemporaryFile + +def import_test(replacement_data, replacement_fmt=None): + def func(path, fmt): + if not path.endswith('.'+fmt.lower()): + raise AssertionError('path extension does not match format') + ext = (replacement_fmt or fmt).lower() + with PersistentTemporaryFile('.'+ext) as f: + f.write(replacement_data) + return f.name + return func + +class AddRemoveTest(BaseTest): + + def test_add_format(self): # {{{ + 'Test adding formats to an existing book record' + af, ae, at = self.assertFalse, self.assertEqual, self.assertTrue + + cache = self.init_cache() + table = cache.fields['formats'].table + NF = b'test_add_formatxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' + + # Test that replace=False works + previous = cache.format(1, 'FMT1') + af(cache.add_format(1, 'FMT1', BytesIO(NF), replace=False)) + ae(previous, cache.format(1, 'FMT1')) + + # Test that replace=True works + lm = cache.field_for('last_modified', 1) + at(cache.add_format(1, 'FMT1', BytesIO(NF), replace=True)) + ae(NF, cache.format(1, 'FMT1')) + ae(cache.format_metadata(1, 'FMT1')['size'], len(NF)) + at(cache.field_for('size', 1) >= len(NF)) + at(cache.field_for('last_modified', 1) > lm) + ae(('FMT2','FMT1'), cache.formats(1)) + at(1 in table.col_book_map['FMT1']) + + # Test adding a format to a record with no formats + at(cache.add_format(3, 'FMT1', BytesIO(NF), replace=True)) + ae(NF, cache.format(3, 'FMT1')) + ae(cache.format_metadata(3, 'FMT1')['size'], len(NF)) + ae(('FMT1',), cache.formats(3)) + at(3 in table.col_book_map['FMT1']) + at(cache.add_format(3, 'FMTX', BytesIO(NF), replace=True)) + at(3 in table.col_book_map['FMTX']) + ae(('FMT1','FMTX'), cache.formats(3)) + + # Test running on import plugins + import calibre.db.cache as c + orig = c.run_plugins_on_import + try: + c.run_plugins_on_import = import_test(b'replacement data') + at(cache.add_format(3, 'REPL', BytesIO(NF))) + ae(b'replacement data', cache.format(3, 'REPL')) + c.run_plugins_on_import = import_test(b'replacement data2', 'REPL2') + with NamedTemporaryFile(suffix='_test_add_format.repl') as f: + f.write(NF) + f.seek(0) + at(cache.add_format(3, 'REPL', BytesIO(NF))) + ae(b'replacement data', cache.format(3, 'REPL')) + ae(b'replacement data2', cache.format(3, 'REPL2')) + + finally: + c.run_plugins_on_import = orig + + # Test adding FMT with path + with NamedTemporaryFile(suffix='_test_add_format.fmt9') as f: + f.write(NF) + f.seek(0) + at(cache.add_format(2, 'FMT9', f)) + ae(NF, cache.format(2, 'FMT9')) + ae(cache.format_metadata(2, 'FMT9')['size'], len(NF)) + at(cache.field_for('size', 2) >= len(NF)) + at(2 in table.col_book_map['FMT9']) + + del cache + # Test that the old interface also shows correct format data + db = self.init_old() + ae(db.formats(3, index_is_id=True), ','.join(['FMT1', 'FMTX', 'REPL', 'REPL2'])) + ae(db.format(3, 'FMT1', index_is_id=True), NF) + ae(db.format(1, 'FMT1', index_is_id=True), NF) + + db.close() + del db + + # }}} + + From df2c497a13afd39b747b3937c857dd0165d44e55 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 11:42:30 +0530 Subject: [PATCH 09/21] ... --- setup/translations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/translations.py b/setup/translations.py index 786d44a6d6..3474e82acb 100644 --- a/setup/translations.py +++ b/setup/translations.py @@ -165,7 +165,7 @@ class Translations(POT): # {{{ subprocess.check_call(['msgfmt', '-o', dest, iso639]) elif locale not in ('en_GB', 'en_CA', 'en_AU', 'si', 'ur', 'sc', 'ltg', 'nds', 'te', 'yi', 'fo', 'sq', 'ast', 'ml', 'ku', - 'fr_CA', 'him', 'jv', 'ka', 'fur', 'ber'): + 'fr_CA', 'him', 'jv', 'ka', 'fur', 'ber', 'my'): self.warn('No ISO 639 translations for locale:', locale) if self.iso639_errors: From 65e31eee1231c1fe1bcc072c7571b7d80c1947ba Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 12:44:19 +0530 Subject: [PATCH 10/21] Implement remove_formats() --- src/calibre/db/backend.py | 11 +++++++- src/calibre/db/cache.py | 29 +++++++++++++++++++++- src/calibre/db/tables.py | 25 +++++++++++++++++-- src/calibre/db/tests/add_remove.py | 40 ++++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 4 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 97dc673ecc..b8963fc49d 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -26,7 +26,7 @@ from calibre.utils.date import utcfromtimestamp, parse_date from calibre.utils.filenames import (is_case_sensitive, samefile, hardlink_file, ascii_filename, WindowsAtomicFolderMove) from calibre.utils.magick.draw import save_cover_data_to -from calibre.utils.recycle_bin import delete_tree +from calibre.utils.recycle_bin import delete_tree, delete_file from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable, CompositeTable, LanguagesTable, UUIDTable) @@ -940,6 +940,15 @@ class DB(object): def has_format(self, book_id, fmt, fname, path): return self.format_abspath(book_id, fmt, fname, path) is not None + def remove_format(self, book_id, fmt, fname, path): + path = self.format_abspath(book_id, fmt, fname, path) + if path is not None: + try: + delete_file(path) + except: + import traceback + traceback.print_exc() + def copy_cover_to(self, path, dest, windows_atomic_move=None, use_hardlink=False): path = os.path.abspath(os.path.join(self.library_path, path, 'cover.jpg')) if windows_atomic_move is not None: diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 4f7de11269..4c18dde6cd 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -984,7 +984,7 @@ class Cache(object): del stream max_size = self.fields['formats'].table.update_fmt(book_id, fmt, fname, size, self.backend) - self.fields['size'].table.update_size(book_id, max_size) + self.fields['size'].table.update_sizes({book_id: max_size}) self._update_last_modified((book_id,)) if run_hooks: @@ -994,6 +994,33 @@ class Cache(object): return True + @write_api + def remove_formats(self, formats_map, db_only=False): + table = self.fields['formats'].table + formats_map = {book_id:frozenset((f or '').upper() for f in fmts) for book_id, fmts in formats_map.iteritems()} + size_map = table.remove_formats(formats_map, self.backend) + self.fields['size'].table.update_sizes(size_map) + + for book_id, fmts in formats_map.iteritems(): + for fmt in fmts: + self.format_metadata_cache[book_id].pop(fmt, None) + + if not db_only: + for book_id, fmts in formats_map.iteritems(): + try: + path = self._field_for('path', book_id).replace('/', os.sep) + except: + continue + for fmt in fmts: + try: + name = self.fields['formats'].format_fname(book_id, fmt) + except: + continue + if name and path: + self.backend.remove_format(book_id, fmt, name, path) + + self._update_last_modified(tuple(formats_map.iterkeys())) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 140c95eb88..fce8d429ba 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -99,8 +99,8 @@ class SizeTable(OneToOneTable): 'WHERE data.book=books.id) FROM books'): self.book_col_map[row[0]] = self.unserialize(row[1]) - def update_size(self, book_id, size): - self.book_col_map[book_id] = size + def update_sizes(self, size_map): + self.book_col_map.update(size_map) class UUIDTable(OneToOneTable): @@ -220,6 +220,26 @@ class FormatsTable(ManyToManyTable): db.conn.execute('UPDATE data SET name=? WHERE book=? AND format=?', (fname, book_id, fmt)) + def remove_formats(self, formats_map, db): + for book_id, fmts in formats_map.iteritems(): + self.book_col_map[book_id] = [fmt for fmt in self.book_col_map.get(book_id, []) if fmt not in fmts] + for m in (self.fname_map, self.size_map): + m[book_id] = {k:v for k, v in m[book_id].iteritems() if k not in fmts} + for fmt in fmts: + try: + self.col_book_map[fmt].discard(book_id) + except KeyError: + pass + db.conn.executemany('DELETE FROM data WHERE book=? AND format=?', + [(book_id, fmt) for book_id, fmts in formats_map.iteritems() for fmt in fmts]) + def zero_max(book_id): + try: + return max(self.size_map[book_id].itervalues()) + except ValueError: + return 0 + + return {book_id:zero_max(book_id) for book_id in formats_map} + def update_fmt(self, book_id, fmt, fname, size, db): fmts = list(self.book_col_map.get(book_id, [])) try: @@ -259,3 +279,4 @@ class LanguagesTable(ManyToManyTable): def read_id_maps(self, db): ManyToManyTable.read_id_maps(self, db) + diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index c5b165bd02..c411daa826 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -7,6 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2013, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os from io import BytesIO from tempfile import NamedTemporaryFile @@ -98,4 +99,43 @@ class AddRemoveTest(BaseTest): # }}} + def test_remove_formats(self): # {{{ + 'Test removal of formats from book records' + af, ae, at = self.assertFalse, self.assertEqual, self.assertTrue + + cache = self.init_cache() + + # Test removal of non-existing format does nothing + formats = {bid:tuple(cache.formats(bid)) for bid in (1, 2, 3)} + cache.remove_formats({1:{'NF'}, 2:{'NF'}, 3:{'NF'}}) + nformats = {bid:tuple(cache.formats(bid)) for bid in (1, 2, 3)} + ae(formats, nformats) + + # Test full removal of format + af(cache.format(1, 'FMT1') is None) + at(cache.has_format(1, 'FMT1')) + cache.remove_formats({1:{'FMT1'}}) + at(cache.format(1, 'FMT1') is None) + af(bool(cache.format_metadata(1, 'FMT1'))) + af(bool(cache.format_metadata(1, 'FMT1', allow_cache=False))) + af('FMT1' in cache.formats(1)) + af(cache.has_format(1, 'FMT1')) + + # Test db only removal + at(cache.has_format(1, 'FMT2')) + ap = cache.format_abspath(1, 'FMT2') + if ap and os.path.exists(ap): + cache.remove_formats({1:{'FMT2'}}) + af(bool(cache.format_metadata(1, 'FMT2'))) + af(cache.has_format(1, 'FMT2')) + at(os.path.exists(ap)) + + # Test that the old interface agrees + db = self.init_old() + at(db.format(1, 'FMT1', index_is_id=True) is None) + + db.close() + del db + # }}} + From e7bf1c7b7d399737563e0ce1985df3d1db35306e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 20:32:26 +0530 Subject: [PATCH 11/21] Clarify use of OPF in calibredb set_metadata --- src/calibre/library/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index b1131525d8..547cc5bc08 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -543,13 +543,14 @@ def do_set_metadata(db, id, stream): def set_metadata_option_parser(): return get_parser(_( ''' -%prog set_metadata [options] id /path/to/metadata.opf +%prog set_metadata [options] id [/path/to/metadata.opf] Set the metadata stored in the calibre database for the book identified by id from the OPF file metadata.opf. id is an id number from the list command. You can get a quick feel for the OPF format by using the --as-opf switch to the show_metadata command. You can also set the metadata of individual fields with -the --field option. +the --field option. If you use the --field option, there is no need to specify +an OPF file. ''')) def command_set_metadata(args, dbpath): From 9a8d31ee96cba490ace1d4d2d09427833ecf8247 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Jul 2013 20:48:31 +0530 Subject: [PATCH 12/21] Implement create_book_entry(), with tests --- src/calibre/db/__init__.py | 5 +- src/calibre/db/backend.py | 2 +- src/calibre/db/cache.py | 110 +++++++++++++++++++++++++++-- src/calibre/db/tables.py | 7 +- src/calibre/db/tests/add_remove.py | 51 ++++++++++++- src/calibre/db/tests/base.py | 2 + src/calibre/db/tests/reading.py | 62 +++++++++++++--- src/calibre/db/tests/writing.py | 4 +- 8 files changed, 223 insertions(+), 20 deletions(-) diff --git a/src/calibre/db/__init__.py b/src/calibre/db/__init__.py index 65beebc1fb..eded760cde 100644 --- a/src/calibre/db/__init__.py +++ b/src/calibre/db/__init__.py @@ -9,14 +9,15 @@ __docformat__ = 'restructuredtext en' SPOOL_SIZE = 30*1024*1024 -def _get_next_series_num_for_list(series_indices): +def _get_next_series_num_for_list(series_indices, unwrap=True): from calibre.utils.config_base import tweaks from math import ceil, floor if not series_indices: if isinstance(tweaks['series_index_auto_increment'], (int, float)): return float(tweaks['series_index_auto_increment']) return 1.0 - series_indices = [x[0] for x in series_indices] + if unwrap: + series_indices = [x[0] for x in series_indices] if tweaks['series_index_auto_increment'] == 'next': return floor(series_indices[-1]) + 1 if tweaks['series_index_auto_increment'] == 'first_free': diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index b8963fc49d..f998b91ccb 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -1088,7 +1088,7 @@ class DB(object): def update_path(self, book_id, title, author, path_field, formats_field): path = self.construct_path_name(book_id, title, author) - current_path = path_field.for_book(book_id) + current_path = path_field.for_book(book_id, default_value='') formats = formats_field.for_book(book_id, default_value=()) fname = self.construct_file_name(book_id, title, author) # Check if the metadata used to construct paths has changed diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 4c18dde6cd..a6647d5027 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -12,9 +12,10 @@ from io import BytesIO from collections import defaultdict from functools import wraps, partial -from calibre.constants import iswindows +from calibre import isbytestring +from calibre.constants import iswindows, preferred_encoding from calibre.customize.ui import run_plugins_on_import, run_plugins_on_postimport -from calibre.db import SPOOL_SIZE +from calibre.db import SPOOL_SIZE, _get_next_series_num_for_list from calibre.db.categories import get_categories from calibre.db.locking import create_locks from calibre.db.errors import NoSuchFormat @@ -24,12 +25,13 @@ from calibre.db.tables import VirtualTable from calibre.db.write import get_series_values from calibre.db.lazy import FormatMetadata, FormatsList from calibre.ebooks import check_ebook_format -from calibre.ebooks.metadata import string_to_authors +from calibre.ebooks.metadata import string_to_authors, author_to_author_sort from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ptempfile import (base_dir, PersistentTemporaryFile, SpooledTemporaryFile) -from calibre.utils.date import now as nowf +from calibre.utils.config import prefs +from calibre.utils.date import now as nowf, utcnow, UNDEFINED_DATE from calibre.utils.icu import sort_key def api(f): @@ -65,6 +67,16 @@ def run_import_plugins(path_or_stream, fmt): path = path_or_stream return run_plugins_on_import(path, fmt) +def _add_newbook_tag(mi): + tags = prefs['new_book_tags'] + if tags: + for tag in [t.strip() for t in tags]: + if tag: + if not mi.tags: + mi.tags = [tag] + elif tag not in mi.tags: + mi.tags.append(tag) + class Cache(object): @@ -1021,6 +1033,95 @@ class Cache(object): self._update_last_modified(tuple(formats_map.iterkeys())) + @read_api + def get_next_series_num_for(self, series): + books = () + sf = self.fields['series'] + if series: + q = icu_lower(series) + for val, book_ids in sf.iter_searchable_values(self._get_metadata, frozenset(self.all_book_ids())): + if q == icu_lower(val): + books = book_ids + break + series_indices = sorted(self._field_for('series_index', book_id) for book_id in books) + return _get_next_series_num_for_list(tuple(series_indices), unwrap=False) + + @read_api + def author_sort_from_authors(self, authors): + '''Given a list of authors, return the author_sort string for the authors, + preferring the author sort associated with the author over the computed + string. ''' + table = self.fields['authors'].table + result = [] + rmap = {icu_lower(v):k for k, v in table.id_map.iteritems()} + for aut in authors: + aid = rmap.get(icu_lower(aut), None) + result.append(author_to_author_sort(aut) if aid is None else table.asort_map[aid]) + return ' & '.join(result) + + @read_api + def has_book(self, mi): + title = mi.title + if title: + if isbytestring(title): + title = title.decode(preferred_encoding, 'replace') + q = icu_lower(title) + for title in self.fields['title'].table.book_col_map.itervalues(): + if q == icu_lower(title): + return True + return False + + @write_api + def create_book_entry(self, mi, cover=None, add_duplicates=True, force_id=None, apply_import_tags=True, preserve_uuid=False): + if mi.tags: + mi.tags = list(mi.tags) + if apply_import_tags: + _add_newbook_tag(mi) + if not add_duplicates and self._has_book(mi): + return + series_index = (self._get_next_series_num_for(mi.series) if mi.series_index is None else mi.series_index) + if not mi.authors: + mi.authors = (_('Unknown'),) + aus = mi.author_sort if mi.author_sort else self._author_sort_from_authors(mi.authors) + mi.title = mi.title or _('Unknown') + if isbytestring(aus): + aus = aus.decode(preferred_encoding, 'replace') + if isbytestring(mi.title): + mi.title = mi.title.decode(preferred_encoding, 'replace') + conn = self.backend.conn + if force_id is None: + conn.execute('INSERT INTO books(title, series_index, author_sort) VALUES (?, ?, ?)', + (mi.title, series_index, aus)) + else: + conn.execute('INSERT INTO books(id, title, series_index, author_sort) VALUES (?, ?, ?, ?)', + (force_id, mi.title, series_index, aus)) + book_id = conn.last_insert_rowid() + + mi.timestamp = utcnow() if mi.timestamp is None else mi.timestamp + mi.pubdate = UNDEFINED_DATE if mi.pubdate is None else mi.pubdate + if cover is not None: + mi.cover, mi.cover_data = None, (None, cover) + self._set_metadata(book_id, mi, ignore_errors=True) + if preserve_uuid and mi.uuid: + self._set_field('uuid', {book_id:mi.uuid}) + # Update the caches for fields from the books table + self.fields['size'].table.book_col_map[book_id] = 0 + row = next(conn.execute('SELECT sort, series_index, author_sort, uuid, has_cover FROM books WHERE id=?', (book_id,))) + for field, val in zip(('sort', 'series_index', 'author_sort', 'uuid', 'cover'), row): + if field == 'cover': + val = bool(val) + elif field == 'uuid': + self.fields[field].table.uuid_to_id_map[val] = book_id + self.fields[field].table.book_col_map[book_id] = val + + return book_id + + @write_api + def add_books(self, books, add_duplicates=True): + duplicates, ids = [], [] + for mi, format_map in books: + pass + # }}} class SortKey(object): # {{{ @@ -1038,3 +1139,4 @@ class SortKey(object): # {{{ # }}} + diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index fce8d429ba..6f3343ba12 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -110,7 +110,7 @@ class UUIDTable(OneToOneTable): def update_uuid_cache(self, book_id_val_map): for book_id, uuid in book_id_val_map.iteritems(): - self.uuid_to_id_map.pop(self.book_col_map[book_id], None) # discard old uuid + self.uuid_to_id_map.pop(self.book_col_map.get(book_id, None), None) # discard old uuid self.uuid_to_id_map[uuid] = book_id class CompositeTable(OneToOneTable): @@ -192,6 +192,11 @@ class AuthorsTable(ManyToManyTable): author_to_author_sort(row[1])) self.alink_map[row[0]] = row[3] + def set_sort_names(self, aus_map, db): + self.asort_map.update(aus_map) + db.conn.executemany('UPDATE authors SET sort=? WHERE id=?', + [(v, k) for k, v in aus_map.iteritems()]) + class FormatsTable(ManyToManyTable): def read_id_maps(self, db): diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index c411daa826..c5845f01da 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -10,9 +10,11 @@ __docformat__ = 'restructuredtext en' import os from io import BytesIO from tempfile import NamedTemporaryFile +from datetime import timedelta -from calibre.db.tests.base import BaseTest +from calibre.db.tests.base import BaseTest, IMG from calibre.ptempfile import PersistentTemporaryFile +from calibre.utils.date import now, UNDEFINED_DATE def import_test(replacement_data, replacement_fmt=None): def func(path, fmt): @@ -138,4 +140,51 @@ class AddRemoveTest(BaseTest): del db # }}} + def test_create_book_entry(self): # {{{ + 'Test the creation of new book entries' + from calibre.ebooks.metadata.book.base import Metadata + cache = self.init_cache() + mi = Metadata('Created One', authors=('Creator One', 'Creator Two')) + + book_id = cache.create_book_entry(mi) + self.assertIsNot(book_id, None) + + def do_test(cache, book_id): + for field in ('path', 'uuid', 'author_sort', 'timestamp', 'pubdate', 'title', 'authors', 'series_index', 'sort'): + self.assertTrue(cache.field_for(field, book_id)) + for field in ('size', 'cover'): + self.assertFalse(cache.field_for(field, book_id)) + self.assertEqual(book_id, cache.fields['uuid'].table.uuid_to_id_map[cache.field_for('uuid', book_id)]) + self.assertLess(now() - cache.field_for('timestamp', book_id), timedelta(seconds=30)) + self.assertEqual(('Created One', ('Creator One', 'Creator Two')), (cache.field_for('title', book_id), cache.field_for('authors', book_id))) + self.assertEqual(cache.field_for('series_index', book_id), 1.0) + self.assertEqual(cache.field_for('pubdate', book_id), UNDEFINED_DATE) + + do_test(cache, book_id) + # Test that the db contains correct data + cache = self.init_cache() + do_test(cache, book_id) + + self.assertIs(None, cache.create_book_entry(mi, add_duplicates=False), 'Duplicate added incorrectly') + book_id = cache.create_book_entry(mi, cover=IMG) + self.assertIsNot(book_id, None) + self.assertEqual(IMG, cache.cover(book_id)) + + import calibre.db.cache as c + orig = c.prefs + c.prefs = {'new_book_tags':('newbook', 'newbook2')} + try: + book_id = cache.create_book_entry(mi) + self.assertEqual(('newbook', 'newbook2'), cache.field_for('tags', book_id)) + mi.tags = ('one', 'two') + book_id = cache.create_book_entry(mi) + self.assertEqual(('one', 'two') + ('newbook', 'newbook2'), cache.field_for('tags', book_id)) + mi.tags = () + finally: + c.prefs = orig + + mi.uuid = 'a preserved uuid' + book_id = cache.create_book_entry(mi, preserve_uuid=True) + self.assertEqual(mi.uuid, cache.field_for('uuid', book_id)) + # }}} diff --git a/src/calibre/db/tests/base.py b/src/calibre/db/tests/base.py index b57b017ba3..b94faf6b28 100644 --- a/src/calibre/db/tests/base.py +++ b/src/calibre/db/tests/base.py @@ -14,6 +14,8 @@ from future_builtins import map rmtree = partial(shutil.rmtree, ignore_errors=True) +IMG = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00`\x00`\x00\x00\xff\xe1\x00\x16Exif\x00\x00II*\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xdb\x00C\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\xff\xdb\x00C\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\xff\xc0\x00\x11\x08\x00\x01\x00\x01\x03\x01"\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x15\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xff\xc4\x00\x14\x10\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x11\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbf\x80\x01\xff\xd9' # noqa {{{ }}} + class BaseTest(unittest.TestCase): longMessage = True diff --git a/src/calibre/db/tests/reading.py b/src/calibre/db/tests/reading.py index 979e2e9247..24d80d33c7 100644 --- a/src/calibre/db/tests/reading.py +++ b/src/calibre/db/tests/reading.py @@ -15,7 +15,7 @@ from calibre.db.tests.base import BaseTest class ReadingTest(BaseTest): - def test_read(self): # {{{ + def test_read(self): # {{{ 'Test the reading of data from the database' cache = self.init_cache(self.library_path) tests = { @@ -123,7 +123,7 @@ class ReadingTest(BaseTest): book_id, field, expected_val, val)) # }}} - def test_sorting(self): # {{{ + def test_sorting(self): # {{{ 'Test sorting' cache = self.init_cache(self.library_path) for field, order in { @@ -165,7 +165,7 @@ class ReadingTest(BaseTest): ('title', True)]), 'Subsort failed') # }}} - def test_get_metadata(self): # {{{ + def test_get_metadata(self): # {{{ 'Test get_metadata() returns the same data for both backends' from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) @@ -188,7 +188,7 @@ class ReadingTest(BaseTest): self.compare_metadata(mi1, mi2) # }}} - def test_get_cover(self): # {{{ + def test_get_cover(self): # {{{ 'Test cover() returns the same data for both backends' from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) @@ -212,7 +212,7 @@ class ReadingTest(BaseTest): # }}} - def test_searching(self): # {{{ + def test_searching(self): # {{{ 'Test searching returns the same data for both backends' from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) @@ -267,7 +267,7 @@ class ReadingTest(BaseTest): # }}} - def test_get_categories(self): # {{{ + def test_get_categories(self): # {{{ 'Check that get_categories() returns the same data for both backends' from calibre.library.database2 import LibraryDatabase2 old = LibraryDatabase2(self.library_path) @@ -286,9 +286,9 @@ class ReadingTest(BaseTest): oval, nval = getattr(old, attr), getattr(new, attr) if ( (category in {'rating', '#rating'} and attr in {'id_set', 'sort'}) or - (category == 'series' and attr == 'sort') or # Sorting is wrong in old + (category == 'series' and attr == 'sort') or # Sorting is wrong in old (category == 'identifiers' and attr == 'id_set') or - (category == '@Good Series') or # Sorting is wrong in old + (category == '@Good Series') or # Sorting is wrong in old (category == 'news' and attr in {'count', 'id_set'}) or (category == 'formats' and attr == 'id_set') ): @@ -306,7 +306,7 @@ class ReadingTest(BaseTest): # }}} - def test_get_formats(self): # {{{ + def test_get_formats(self): # {{{ 'Test reading ebook formats using the format() method' from calibre.library.database2 import LibraryDatabase2 from calibre.db.cache import NoSuchFormat @@ -343,3 +343,47 @@ class ReadingTest(BaseTest): # }}} + def test_author_sort_for_authors(self): # {{{ + 'Test getting the author sort for authors from the db' + cache = self.init_cache() + table = cache.fields['authors'].table + table.set_sort_names({next(table.id_map.iterkeys()): 'Fake Sort'}, cache.backend) + + authors = tuple(table.id_map.itervalues()) + nval = cache.author_sort_from_authors(authors) + self.assertIn('Fake Sort', nval) + + db = self.init_old() + self.assertEqual(db.author_sort_from_authors(authors), nval) + db.close() + del db + + # }}} + + def test_get_next_series_num(self): # {{{ + 'Test getting the next series number for a series' + cache = self.init_cache() + cache.set_field('series', {3:'test series'}) + cache.set_field('series_index', {3:13}) + table = cache.fields['series'].table + series = tuple(table.id_map.itervalues()) + nvals = {s:cache.get_next_series_num_for(s) for s in series} + db = self.init_old() + self.assertEqual({s:db.get_next_series_num_for(s) for s in series}, nvals) + db.close() + + # }}} + + def test_has_book(self): # {{{ + 'Test detecting duplicates' + from calibre.ebooks.metadata.book.base import Metadata + cache = self.init_cache() + db = self.init_old() + for title in cache.fields['title'].table.book_col_map.itervalues(): + for x in (db, cache): + self.assertTrue(x.has_book(Metadata(title))) + self.assertTrue(x.has_book(Metadata(title.upper()))) + self.assertFalse(x.has_book(Metadata(title + 'XXX'))) + self.assertFalse(x.has_book(Metadata(title[:1]))) + db.close() + # }}} diff --git a/src/calibre/db/tests/writing.py b/src/calibre/db/tests/writing.py index 597c98a771..cb525900ee 100644 --- a/src/calibre/db/tests/writing.py +++ b/src/calibre/db/tests/writing.py @@ -13,7 +13,7 @@ from io import BytesIO from calibre.ebooks.metadata import author_to_author_sort from calibre.utils.date import UNDEFINED_DATE -from calibre.db.tests.base import BaseTest +from calibre.db.tests.base import BaseTest, IMG class WritingTest(BaseTest): @@ -364,8 +364,8 @@ class WritingTest(BaseTest): ae(cache.field_for('cover', 1), 1) ae(cache.set_cover({1:None}), set([1])) ae(cache.field_for('cover', 1), 0) + img = IMG - img = b'\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x01\x00`\x00`\x00\x00\xff\xe1\x00\x16Exif\x00\x00II*\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xdb\x00C\x00\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\xff\xdb\x00C\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\x01\xff\xc0\x00\x11\x08\x00\x01\x00\x01\x03\x01"\x00\x02\x11\x01\x03\x11\x01\xff\xc4\x00\x15\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\n\xff\xc4\x00\x14\x10\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xc4\x00\x14\x11\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xda\x00\x0c\x03\x01\x00\x02\x11\x03\x11\x00?\x00\xbf\x80\x01\xff\xd9' # noqa {{{ }}} # Test setting a cover ae(cache.set_cover({bid:img for bid in (1, 2, 3)}), {1, 2, 3}) old = self.init_old() From 42202faae823d3052150af72ea9ce90d2fd71a35 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 09:30:37 +0530 Subject: [PATCH 13/21] Metadata download dialog: Have the OK button enabled in the results screen as well. See #1198288 --- src/calibre/gui2/metadata/single_download.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index ed378745a5..fc883bd88f 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -1019,7 +1019,6 @@ class FullFetch(QDialog): # {{{ self.log_button = self.bb.addButton(_('View log'), self.bb.ActionRole) self.log_button.clicked.connect(self.view_log) self.log_button.setIcon(QIcon(I('debug.png'))) - self.ok_button.setEnabled(False) self.prev_button.setVisible(False) self.identify_widget = IdentifyWidget(self.log, self) @@ -1044,7 +1043,6 @@ class FullFetch(QDialog): # {{{ def book_selected(self, book, caches): self.next_button.setVisible(False) - self.ok_button.setEnabled(True) self.prev_button.setVisible(True) self.book = book self.stack.setCurrentIndex(1) @@ -1055,7 +1053,6 @@ class FullFetch(QDialog): # {{{ def back_clicked(self): self.next_button.setVisible(True) - self.ok_button.setEnabled(False) self.prev_button.setVisible(False) self.next_button.setFocus() self.stack.setCurrentIndex(0) @@ -1063,11 +1060,14 @@ class FullFetch(QDialog): # {{{ self.covers_widget.reset_covers() def accept(self): - gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) - if self.stack.currentIndex() == 1: - return QDialog.accept(self) # Prevent the usual dialog accept mechanisms from working - pass + gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) + if DEBUG_DIALOG: + if self.stack.currentIndex() == 2: + return QDialog.accept(self) + else: + if self.stack.currentIndex() == 1: + return QDialog.accept(self) def reject(self): gprefs['metadata_single_gui_geom'] = bytearray(self.saveGeometry()) @@ -1087,6 +1087,9 @@ class FullFetch(QDialog): # {{{ def ok_clicked(self, *args): self.cover_pixmap = self.covers_widget.cover_pixmap() + if self.stack.currentIndex() == 0: + self.next_clicked() + return if DEBUG_DIALOG: if self.cover_pixmap is not None: self.w = QLabel() From 7c9ae4ebc918d10fe2defb1d294ce8af0ab26b3e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 11:13:55 +0530 Subject: [PATCH 14/21] ... --- manual/customize.rst | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/manual/customize.rst b/manual/customize.rst index ceee4ece62..59475e91f2 100644 --- a/manual/customize.rst +++ b/manual/customize.rst @@ -46,17 +46,31 @@ The default values for the tweaks are reproduced below Overriding icons, templates, et cetera ---------------------------------------- -|app| allows you to override the static resources, like icons, templates, javascript, etc. with customized versions that you like. -All static resources are stored in the resources sub-folder of the calibre install location. On Windows, this is usually -:file:`C:/Program Files/Calibre2/resources`. On OS X, :file:`/Applications/calibre.app/Contents/Resources/resources/`. On linux, if you are using the binary installer -from the calibre website it will be :file:`/opt/calibre/resources`. These paths can change depending on where you choose to install |app|. +|app| allows you to override the static resources, like icons, javascript and +templates for the metadata jacket, catalogs, etc. with customized versions that +you like. All static resources are stored in the resources sub-folder of the +calibre install location. On Windows, this is usually :file:`C:/Program Files/Calibre2/resources`. +On OS X, :file:`/Applications/calibre.app/Contents/Resources/resources/`. On linux, if +you are using the binary installer from the calibre website it will be +:file:`/opt/calibre/resources`. These paths can change depending on where you +choose to install |app|. -You should not change the files in this resources folder, as your changes will get overwritten the next time you update |app|. Instead, go to -:guilabel:`Preferences->Advanced->Miscellaneous` and click :guilabel:`Open calibre configuration directory`. In this configuration directory, create a sub-folder called resources and place the files you want to override in it. Place the files in the appropriate sub folders, for example place images in :file:`resources/images`, etc. -|app| will automatically use your custom file in preference to the built-in one the next time it is started. +You should not change the files in this resources folder, as your changes will +get overwritten the next time you update |app|. Instead, go to +:guilabel:`Preferences->Advanced->Miscellaneous` and click +:guilabel:`Open calibre configuration directory`. In this configuration directory, create a +sub-folder called resources and place the files you want to override in it. +Place the files in the appropriate sub folders, for example place images in +:file:`resources/images`, etc. |app| will automatically use your custom file +in preference to the built-in one the next time it is started. -For example, if you wanted to change the icon for the :guilabel:`Remove books` action, you would first look in the built-in resources folder and see that the relevant file is -:file:`resources/images/trash.png`. Assuming you have an alternate icon in PNG format called :file:`mytrash.png` you would save it in the configuration directory as :file:`resources/images/trash.png`. All the icons used by the calibre user interface are in :file:`resources/images` and its sub-folders. +For example, if you wanted to change the icon for the :guilabel:`Remove books` +action, you would first look in the built-in resources folder and see that the +relevant file is :file:`resources/images/trash.png`. Assuming you have an +alternate icon in PNG format called :file:`mytrash.png` you would save it in +the configuration directory as :file:`resources/images/trash.png`. All the +icons used by the calibre user interface are in :file:`resources/images` and +its sub-folders. Customizing |app| with plugins -------------------------------- From 20920b37b2ae8655aeb135f9c82ae6eb58cacf2e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 11:24:19 +0530 Subject: [PATCH 15/21] Implement add_books() --- src/calibre/db/cache.py | 11 +++++++++-- src/calibre/db/tests/add_remove.py | 16 ++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index a6647d5027..e6044ce6de 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1117,10 +1117,17 @@ class Cache(object): return book_id @write_api - def add_books(self, books, add_duplicates=True): + def add_books(self, books, add_duplicates=True, apply_import_tags=True, preserve_uuid=False, dbapi=None): duplicates, ids = [], [] for mi, format_map in books: - pass + book_id = self._create_book_entry(mi, add_duplicates=add_duplicates, apply_import_tags=apply_import_tags, preserve_uuid=preserve_uuid) + if book_id is None: + duplicates.append((mi, format_map)) + else: + ids.append(book_id) + for fmt, stream_or_path in format_map.iteritems(): + self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi) + return ids, duplicates # }}} diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index c5845f01da..5744d63635 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -188,3 +188,19 @@ class AddRemoveTest(BaseTest): self.assertEqual(mi.uuid, cache.field_for('uuid', book_id)) # }}} + def test_add_books(self): # {{{ + 'Test the adding of new books' + from calibre.ebooks.metadata.book.base import Metadata + cache = self.init_cache() + mi = Metadata('Created One', authors=('Creator One', 'Creator Two')) + FMT1, FMT2 = b'format1', b'format2' + format_map = {'FMT1':BytesIO(FMT1), 'FMT2':BytesIO(FMT2)} + ids, duplicates = cache.add_books([(mi, format_map)]) + self.assertTrue(len(ids) == 1) + self.assertFalse(duplicates) + book_id = ids[0] + self.assertEqual(set(cache.formats(book_id)), {'FMT1', 'FMT2'}) + self.assertEqual(cache.format(book_id, 'FMT1'), FMT1) + self.assertEqual(cache.format(book_id, 'FMT2'), FMT2) + # }}} + From 0711e03bd4686262ba35b28326897360fd7cb440 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 12:43:02 +0530 Subject: [PATCH 16/21] Initial implementation of remove_book(), needs testing --- src/calibre/db/backend.py | 14 +++++- src/calibre/db/cache.py | 13 +++++ src/calibre/db/tables.py | 103 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 125 insertions(+), 5 deletions(-) diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index f998b91ccb..d75106209f 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -29,7 +29,7 @@ from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_tree, delete_file from calibre.db.tables import (OneToOneTable, ManyToOneTable, ManyToManyTable, SizeTable, FormatsTable, AuthorsTable, IdentifiersTable, PathTable, - CompositeTable, LanguagesTable, UUIDTable) + CompositeTable, UUIDTable) # }}} ''' @@ -711,7 +711,6 @@ class DB(object): 'authors':AuthorsTable, 'formats':FormatsTable, 'identifiers':IdentifiersTable, - 'languages':LanguagesTable, }.get(col, ManyToManyTable) tables[col] = cls(col, self.field_metadata[col].copy()) @@ -1165,5 +1164,16 @@ class DB(object): with lopen(path, 'rb') as f: return f.read() + def remove_books(self, path_map, permanent=False): + for book_id, path in path_map.iteritems(): + if path: + path = os.path.join(self.library_path, path) + if os.path.exists(path): + self.rmtree(path, permanent=permanent) + parent = os.path.dirname(path) + if len(os.listdir(parent)) == 0: + self.rmtree(parent, permanent=permanent) + self.conn.executemany( + 'DELETE FROM books WHERE id=?', [(x,) for x in path_map]) # }}} diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index e6044ce6de..ce0582f893 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1129,6 +1129,19 @@ class Cache(object): self._add_format(book_id, fmt, stream_or_path, dbapi=dbapi) return ids, duplicates + @write_api + def remove_books(self, book_ids, permanent=False): + path_map = {} + for book_id in book_ids: + try: + path = self._field_for('path', book_id).replace('/', os.sep) + except: + path = None + path_map[book_id] = path + self.backend.remove_books(path_map, permanent=permanent) + for field in self.fields.itervalues(): + field.table.remove_books(book_ids, self.backend) + # }}} class SortKey(object): # {{{ diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 6f3343ba12..19c4ade10c 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -20,6 +20,10 @@ _c_speedup = plugins['speedup'][0] ONE_ONE, MANY_ONE, MANY_MANY = xrange(3) +class Null: + pass +null = Null() + def _c_convert_timestamp(val): if not val: return None @@ -55,6 +59,9 @@ class Table(object): self.link_table = (link_table if link_table else 'books_%s_link'%self.metadata['table']) + def remove_books(self, book_ids, db): + return set() + class VirtualTable(Table): ''' @@ -83,6 +90,14 @@ class OneToOneTable(Table): self.metadata['column'], self.metadata['table'])): self.book_col_map[row[0]] = self.unserialize(row[1]) + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + val = self.book_col_map.pop(book_id, null) + if val is not null: + clean.add(val) + return clean + class PathTable(OneToOneTable): def set_path(self, book_id, path, db): @@ -113,6 +128,15 @@ class UUIDTable(OneToOneTable): self.uuid_to_id_map.pop(self.book_col_map.get(book_id, None), None) # discard old uuid self.uuid_to_id_map[uuid] = book_id + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + val = self.book_col_map.pop(book_id, null) + if val is not null: + self.uuid_to_id_map.pop(val, None) + clean.add(val) + return clean + class CompositeTable(OneToOneTable): def read(self, db): @@ -124,6 +148,9 @@ class CompositeTable(OneToOneTable): self.composite_sort = d.get('composite_sort', False) self.use_decorations = d.get('use_decorations', False) + def remove_books(self, book_ids, db): + return set() + class ManyToOneTable(Table): ''' @@ -156,6 +183,27 @@ class ManyToOneTable(Table): self.col_book_map[row[1]].add(row[0]) self.book_col_map[row[0]] = row[1] + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + item_id = self.book_col_map.pop(book_id, None) + if item_id is not None: + try: + self.col_book_map[item_id].discard(book_id) + except KeyError: + if self.id_map.pop(item_id, null) is not null: + clean.add(item_id) + else: + if not self.col_book_map[item_id]: + del self.col_book_map[item_id] + if self.id_map.pop(item_id, null) is not null: + clean.add(item_id) + if clean: + db.conn.executemany( + 'DELETE FROM {0} WHERE id=?'.format(self.metadata['table']), + [(x,) for x in clean]) + return clean + class ManyToManyTable(ManyToOneTable): ''' @@ -166,6 +214,7 @@ class ManyToManyTable(ManyToOneTable): table_type = MANY_MANY selectq = 'SELECT book, {0} FROM {1} ORDER BY id' + do_clean_on_remove = True def read_maps(self, db): for row in db.conn.execute( @@ -180,6 +229,27 @@ class ManyToManyTable(ManyToOneTable): for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(self.book_col_map[key]) + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + item_ids = self.book_col_map.pop(book_id, ()) + for item_id in item_ids: + try: + self.col_book_map[item_id].discard(book_id) + except KeyError: + if self.id_map.pop(item_id, null) is not null: + clean.add(item_id) + else: + if not self.col_book_map[item_id]: + del self.col_book_map[item_id] + if self.id_map.pop(item_id, null) is not null: + clean.add(item_id) + if clean and self.do_clean_on_remove: + db.conn.executemany( + 'DELETE FROM {0} WHERE id=?'.format(self.metadata['table']), + [(x,) for x in clean]) + return clean + class AuthorsTable(ManyToManyTable): def read_id_maps(self, db): @@ -197,8 +267,17 @@ class AuthorsTable(ManyToManyTable): db.conn.executemany('UPDATE authors SET sort=? WHERE id=?', [(v, k) for k, v in aus_map.iteritems()]) + def remove_books(self, book_ids, db): + clean = ManyToManyTable.remove_books(self, book_ids, db) + for item_id in clean: + self.alink_map.pop(item_id, None) + self.asort_map.pop(item_id, None) + return clean + class FormatsTable(ManyToManyTable): + do_clean_on_remove = False + def read_id_maps(self, db): pass @@ -220,6 +299,13 @@ class FormatsTable(ManyToManyTable): for key in tuple(self.book_col_map.iterkeys()): self.book_col_map[key] = tuple(sorted(self.book_col_map[key])) + def remove_books(self, book_ids, db): + clean = ManyToManyTable.remove_books(self, book_ids, db) + for book_id in book_ids: + self.fname_map.pop(book_id, None) + self.size_map.pop(book_id, None) + return clean + def set_fname(self, book_id, fmt, fname, db): self.fname_map[book_id][fmt] = fname db.conn.execute('UPDATE data SET name=? WHERE book=? AND format=?', @@ -280,8 +366,19 @@ class IdentifiersTable(ManyToManyTable): self.book_col_map[row[0]] = {} self.book_col_map[row[0]][row[1]] = row[2] -class LanguagesTable(ManyToManyTable): + def remove_books(self, book_ids, db): + clean = set() + for book_id in book_ids: + item_map = self.book_col_map.pop(book_id, {}) + for item_id in item_map: + try: + self.col_book_map[item_id].discard(book_id) + except KeyError: + clean.add(item_id) + else: + if not self.col_book_map[item_id]: + del self.col_book_map[item_id] + clean.add(item_id) + return clean - def read_id_maps(self, db): - ManyToManyTable.read_id_maps(self, db) From e836bbd20390dcc669f90cdb06bb9ea8bb5547a4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 6 Jul 2013 19:02:00 +0530 Subject: [PATCH 17/21] DOCX Input: Fix no page break being inserted before the last section. Fixes #1198414 [Private bug](https://bugs.launchpad.net/calibre/+bug/1198414) --- src/calibre/ebooks/docx/to_html.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/ebooks/docx/to_html.py b/src/calibre/ebooks/docx/to_html.py index 01808657ea..be0576d2b9 100644 --- a/src/calibre/ebooks/docx/to_html.py +++ b/src/calibre/ebooks/docx/to_html.py @@ -203,6 +203,7 @@ class Convert(object): current.append(p) if current: + self.section_starts.append(current[0]) last = XPath('./w:body/w:sectPr')(doc) pr = PageProperties(last) for x in current: From 75cf58d0f119476da2da9530aa5d85a51b27a82e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 08:47:26 +0530 Subject: [PATCH 18/21] Glenn Brenwald and Ludwig von Mises Institute by anywho --- recipes/glenn_greenwald.recipe | 10 ++++++++++ recipes/ludwig_mises.recipe | 14 ++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 recipes/glenn_greenwald.recipe create mode 100644 recipes/ludwig_mises.recipe diff --git a/recipes/glenn_greenwald.recipe b/recipes/glenn_greenwald.recipe new file mode 100644 index 0000000000..63ed285e72 --- /dev/null +++ b/recipes/glenn_greenwald.recipe @@ -0,0 +1,10 @@ +from calibre.web.feeds.news import AutomaticNewsRecipe +class BasicUserRecipe1373130920(AutomaticNewsRecipe): + title = u'Glenn Greenwald | guardian.co.uk' + language = 'en_GB' + __author__ = 'anywho' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + + feeds = [(u'Latest', u'http://www.guardian.co.uk/profile/glenn-greenwald/rss')] diff --git a/recipes/ludwig_mises.recipe b/recipes/ludwig_mises.recipe new file mode 100644 index 0000000000..7e46a9a7db --- /dev/null +++ b/recipes/ludwig_mises.recipe @@ -0,0 +1,14 @@ +from calibre.web.feeds.news import AutomaticNewsRecipe + +class BasicUserRecipe1373130372(AutomaticNewsRecipe): + title = u'Ludwig von Mises Institute' + __author__ = 'anywho' + language = 'en' + oldest_article = 7 + max_articles_per_feed = 100 + auto_cleanup = True + + feeds = [(u'Daily Articles (Full text version)', + u'http://feed.mises.org/MisesFullTextArticles'), + (u'Mises Blog Posts', + u'http://mises.org/blog/index.rdf')] From 535cff6f03378a17ef26078a24bba36c5f41ad2b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 10:32:20 +0530 Subject: [PATCH 19/21] Tests for remove_books() --- src/calibre/db/cache.py | 7 +++++- src/calibre/db/tests/add_remove.py | 39 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index ce0582f893..b94258b1ef 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -1140,7 +1140,12 @@ class Cache(object): path_map[book_id] = path self.backend.remove_books(path_map, permanent=permanent) for field in self.fields.itervalues(): - field.table.remove_books(book_ids, self.backend) + try: + table = field.table + except AttributeError: + continue # Some fields like ondevice do not have tables + else: + table.remove_books(book_ids, self.backend) # }}} diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index 5744d63635..5d490a2bcb 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -204,3 +204,42 @@ class AddRemoveTest(BaseTest): self.assertEqual(cache.format(book_id, 'FMT2'), FMT2) # }}} + def test_remove_books(self): # {{{ + 'Test removal of books' + cache = self.init_cache() + af, ae, at = self.assertFalse, self.assertEqual, self.assertTrue + authors = cache.fields['authors'].table + + # Delete a single book, with no formats and check cleaning + self.assertIn(_('Unknown'), set(authors.id_map.itervalues())) + olen = len(authors.id_map) + item_id = {v:k for k, v in authors.id_map.iteritems()}[_('Unknown')] + cache.remove_books((3,)) + for c in (cache, self.init_cache()): + table = c.fields['authors'].table + self.assertNotIn(3, c.all_book_ids()) + self.assertNotIn(_('Unknown'), set(table.id_map.itervalues())) + self.assertNotIn(item_id, table.asort_map) + self.assertNotIn(item_id, table.alink_map) + ae(len(table.id_map), olen-1) + + # Check that files are removed + fmtpath = cache.format_abspath(1, 'FMT1') + bookpath = os.path.dirname(fmtpath) + authorpath = os.path.dirname(bookpath) + item_id = {v:k for k, v in cache.fields['#series'].table.id_map.iteritems()}['My Series Two'] + cache.remove_books((1,), permanent=True) + for x in (fmtpath, bookpath, authorpath): + af(os.path.exists(x)) + for c in (cache, self.init_cache()): + table = c.fields['authors'].table + self.assertNotIn(1, c.all_book_ids()) + self.assertNotIn('Author Two', set(table.id_map.itervalues())) + self.assertNotIn(6, set(c.fields['rating'].table.id_map.itervalues())) + self.assertIn('A Series One', set(c.fields['series'].table.id_map.itervalues())) + self.assertNotIn('My Series Two', set(c.fields['#series'].table.id_map.itervalues())) + self.assertNotIn(item_id, c.fields['#series'].table.col_book_map) + self.assertNotIn(1, c.fields['#series'].table.book_col_map) + # }}} + + From 3b438889a4fecc1de39c7407ef5b30f973d69501 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 11:17:57 +0530 Subject: [PATCH 20/21] Test emptying db --- src/calibre/db/tests/add_remove.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/calibre/db/tests/add_remove.py b/src/calibre/db/tests/add_remove.py index 5d490a2bcb..76349df1c5 100644 --- a/src/calibre/db/tests/add_remove.py +++ b/src/calibre/db/tests/add_remove.py @@ -240,6 +240,15 @@ class AddRemoveTest(BaseTest): self.assertNotIn('My Series Two', set(c.fields['#series'].table.id_map.itervalues())) self.assertNotIn(item_id, c.fields['#series'].table.col_book_map) self.assertNotIn(1, c.fields['#series'].table.book_col_map) + + # Test emptying the db + cache.remove_books(cache.all_book_ids(), permanent=True) + for f in ('authors', 'series', '#series', 'tags'): + table = cache.fields[f].table + self.assertFalse(table.id_map) + self.assertFalse(table.book_col_map) + self.assertFalse(table.col_book_map) + # }}} From e64ff83e070b21461e119f913837c7ea361feeff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 7 Jul 2013 11:44:30 +0530 Subject: [PATCH 21/21] Legacy implementations of a few direct methods --- src/calibre/db/legacy.py | 3 +++ src/calibre/db/tests/legacy.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/calibre/db/legacy.py b/src/calibre/db/legacy.py index 287d2d47db..2ad5da61b8 100644 --- a/src/calibre/db/legacy.py +++ b/src/calibre/db/legacy.py @@ -58,6 +58,9 @@ class LibraryDatabase(object): setattr(self, prop, partial(self.get_property, loc=self.FIELD_MAP[fm])) + for meth in ('get_next_series_num_for', 'has_book', 'author_sort_from_authors'): + setattr(self, meth, getattr(self.new_api, meth)) + self.last_update_check = self.last_modified() def close(self): diff --git a/src/calibre/db/tests/legacy.py b/src/calibre/db/tests/legacy.py index 1fe719e31e..ae99d8190f 100644 --- a/src/calibre/db/tests/legacy.py +++ b/src/calibre/db/tests/legacy.py @@ -103,6 +103,22 @@ class LegacyTest(BaseTest): # }}} + def test_legacy_direct(self): # {{{ + 'Test methods that are directly equivalent in the old and new interface' + from calibre.ebooks.metadata.book.base import Metadata + ndb = self.init_legacy() + db = self.init_old() + for meth, args in { + 'get_next_series_num_for': [('A Series One',)], + 'author_sort_from_authors': [(['Author One', 'Author Two', 'Unknown'],)], + 'has_book':[(Metadata('title one'),), (Metadata('xxxx1111'),)], + }.iteritems(): + for a in args: + self.assertEqual(getattr(db, meth)(*a), getattr(ndb, meth)(*a), + 'The method: %s() returned different results for argument %s' % (meth, a)) + db.close() + # }}} + def test_legacy_coverage(self): # {{{ ' Check that the emulation of the legacy interface is (almost) total ' cl = self.cloned_library