From 48cb1d313802dd13dd754dea60227716bbc90b34 Mon Sep 17 00:00:00 2001 From: Timothy Legge Date: Wed, 22 Sep 2010 21:45:44 -0300 Subject: [PATCH 01/11] Fix 6902 - If Title is None set to Unknown --- src/calibre/devices/kobo/books.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py index 496162d668..5724efc4ec 100644 --- a/src/calibre/devices/kobo/books.py +++ b/src/calibre/devices/kobo/books.py @@ -41,6 +41,10 @@ class Book(MetaInformation): self.authors = [''] else: self.authors = [authors] + + if not title: + self.title = 'Unknown' + self.mime = mime self.size = size # will be set later if None From d9e5e74695e82e56b8cc0f61399036ea17a485dd Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 23 Sep 2010 01:02:03 -0600 Subject: [PATCH 02/11] Console now has history --- src/calibre/utils/pyconsole/__init__.py | 2 + src/calibre/utils/pyconsole/console.py | 62 ++++++++++++++++++++-- src/calibre/utils/pyconsole/controller.py | 1 - src/calibre/utils/pyconsole/interpreter.py | 3 +- 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py index 3be9382413..6ef9f04d4b 100644 --- a/src/calibre/utils/pyconsole/__init__.py +++ b/src/calibre/utils/pyconsole/__init__.py @@ -24,6 +24,8 @@ def console_config(): c = Config('console', desc) c.add_opt('theme', default='native', help='The color theme') + c.add_opt('scrollback', default=10000, + help='Max number of lines to keep in the scrollback buffer') return c diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py index 2611965345..14670fdb59 100644 --- a/src/calibre/utils/pyconsole/console.py +++ b/src/calibre/utils/pyconsole/console.py @@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en' import sys, textwrap, traceback, StringIO from functools import partial +from codeop import CommandCompiler from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \ QApplication, QColor, QPalette, QMenu, QActionGroup, QTimer @@ -16,8 +17,9 @@ from pygments.styles import get_all_styles from calibre.utils.pyconsole.formatter import Formatter from calibre.utils.pyconsole.controller import Controller +from calibre.utils.pyconsole.history import History from calibre.utils.pyconsole import prints, prefs, __appname__, \ - __version__, error_dialog + __version__, error_dialog, dynamic class EditBlock(object): # {{{ @@ -73,6 +75,7 @@ class ThemeMenu(QMenu): # {{{ # }}} + class Console(QTextEdit): running = pyqtSignal() @@ -114,7 +117,9 @@ class Console(QTextEdit): parent=None): QTextEdit.__init__(self, parent) self.shutting_down = False + self.compiler = CommandCompiler() self.buf = self.old_buf = [] + self.history = History([''], dynamic.get('console_history', [])) self.prompt_frame = None self.allow_output = False self.prompt_frame_format = QTextFrameFormat() @@ -122,7 +127,7 @@ class Console(QTextEdit): self.prompt_frame_format.setBorderStyle(QTextFrameFormat.BorderStyle_Solid) self.prompt_len = len(prompt) - self.doc.setMaximumBlockCount(10000) + self.doc.setMaximumBlockCount(int(prefs['scrollback'])) self.lexer = PythonLexer(ensurenl=False) self.tb_lexer = PythonTracebackLexer() @@ -139,6 +144,8 @@ class Console(QTextEdit): self.key_dispatcher = { # {{{ Qt.Key_Enter : self.enter_pressed, Qt.Key_Return : self.enter_pressed, + Qt.Key_Up : self.up_pressed, + Qt.Key_Down : self.down_pressed, Qt.Key_Home : self.home_pressed, Qt.Key_End : self.end_pressed, Qt.Key_Left : self.left_pressed, @@ -153,15 +160,17 @@ class Console(QTextEdit): '''.format(sys.version.splitlines()[0], __appname__, __version__)) + sys.excepthook = self.unhandled_exception + self.controllers = [] QTimer.singleShot(0, self.launch_controller) - sys.excepthook = self.unhandled_exception with EditBlock(self.cursor): self.render_block(motd) def shutdown(self): + dynamic.set('console_history', self.history.serialize()) self.shutton_down = True for c in self.controllers: c.kill() @@ -365,7 +374,7 @@ class Console(QTextEdit): # }}} - # Keyboard handling {{{ + # Keyboard management {{{ def keyPressEvent(self, ev): text = unicode(ev.text()) @@ -394,6 +403,20 @@ class Console(QTextEdit): self.setTextCursor(c) self.ensureCursorVisible() + def up_pressed(self): + lineno, pos = self.cursor_pos + if lineno < 0: return + if lineno == 0: + b = self.history.back() + if b is not None: + self.set_prompt(b) + else: + c = self.cursor + c.movePosition(c.Up) + self.setTextCursor(c) + self.ensureCursorVisible() + + def backspace_pressed(self): lineno, pos = self.cursor_pos if lineno < 0: return @@ -414,7 +437,6 @@ class Console(QTextEdit): lineno, pos = self.cursor_pos if lineno < 0: return c = self.cursor - lineno, pos = self.cursor_pos cp = list(self.prompt(False)) if pos < len(cp[lineno]): c.movePosition(c.NextCharacter) @@ -423,6 +445,22 @@ class Console(QTextEdit): self.setTextCursor(c) self.ensureCursorVisible() + def down_pressed(self): + lineno, pos = self.cursor_pos + if lineno < 0: return + c = self.cursor + cp = list(self.prompt(False)) + if lineno >= len(cp) - 1: + b = self.history.forward() + if b is not None: + self.set_prompt(b) + else: + c = self.cursor + c.movePosition(c.Down) + self.setTextCursor(c) + self.ensureCursorVisible() + + def home_pressed(self): if self.prompt_frame is not None: mods = QApplication.keyboardModifiers() @@ -454,6 +492,19 @@ class Console(QTextEdit): return self.no_controller_error() cp = list(self.prompt()) if cp[0]: + try: + ret = self.compiler('\n'.join(cp)) + except: + pass + else: + if ret is None: + c = self.prompt_frame.lastCursorPosition() + c.insertBlock() + self.setTextCursor(c) + self.render_current_prompt() + return + else: + self.history.enter(cp) self.execute(cp) def text_typed(self, text): @@ -461,6 +512,7 @@ class Console(QTextEdit): self.move_cursor_to_prompt() self.cursor.insertText(text) self.render_current_prompt(restore_cursor=True) + self.history.current = list(self.prompt()) # }}} diff --git a/src/calibre/utils/pyconsole/controller.py b/src/calibre/utils/pyconsole/controller.py index 368e665079..d372cb4ebc 100644 --- a/src/calibre/utils/pyconsole/controller.py +++ b/src/calibre/utils/pyconsole/controller.py @@ -104,7 +104,6 @@ class Controller(QThread): def returncode(self): return self.process.returncode - @property def interrupt(self): if hasattr(signal, 'SIGINT'): os.kill(self.process.pid, signal.SIGINT) diff --git a/src/calibre/utils/pyconsole/interpreter.py b/src/calibre/utils/pyconsole/interpreter.py index 6a1aff26c9..3cd0d94711 100644 --- a/src/calibre/utils/pyconsole/interpreter.py +++ b/src/calibre/utils/pyconsole/interpreter.py @@ -11,6 +11,7 @@ from Queue import Queue, Empty from threading import Thread from binascii import unhexlify from multiprocessing.connection import Client +from repr import repr as safe_repr from calibre.utils.pyconsole import preferred_encoding, isbytestring, \ POLL_TIMEOUT @@ -35,7 +36,7 @@ def tounicode(raw): # {{{ try: raw = raw.decode(preferred_encoding, 'replace') except: - raw = repr(raw) + raw = safe_repr(raw) if isbytestring(raw): try: From cee9c6e6dbe17eff8a7420cc2ba696cabfee552d Mon Sep 17 00:00:00 2001 From: GRiker Date: Thu, 23 Sep 2010 06:35:13 -0700 Subject: [PATCH 03/11] GwR fix regression with NCX encoding #6797 --- src/calibre/ebooks/mobi/writer.py | 48 +++++++++---------------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index 23f92d1fd2..80c1bf74eb 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -1574,15 +1574,14 @@ class MobiWriter(object): id = unicode(oeb.metadata.cover[0]) item = oeb.manifest.ids[id] href = item.href - if href in self._images: - index = self._images[href] - 1 - exth.write(pack('>III', 0xc9, 0x0c, index)) - exth.write(pack('>III', 0xcb, 0x0c, 0)) - nrecs += 2 - index = self._add_thumbnail(item) - if index is not None: - exth.write(pack('>III', 0xca, 0x0c, index - 1)) - nrecs += 1 + index = self._images[href] - 1 + exth.write(pack('>III', 0xc9, 0x0c, index)) + exth.write(pack('>III', 0xcb, 0x0c, 0)) + nrecs += 2 + index = self._add_thumbnail(item) + if index is not None: + exth.write(pack('>III', 0xca, 0x0c, index - 1)) + nrecs += 1 exth = exth.getvalue() trail = len(exth) % 4 @@ -1696,11 +1695,12 @@ class MobiWriter(object): header.write(pack('>I', 1)) # 0x1c - 0x1f : Text encoding ? - # GR: Language encoding for NCX entries (latin_1) - header.write(pack('>I', 0x4e4)) + # header.write(pack('>I', 650001)) + # GR: This needs to be either 0xFDE9 or 0x4E4 + header.write(pack('>I', 0xFDE9)) - # 0x20 - 0x23 : Mimicking kindleGen - header.write(pack('>I', 0xFFFFFFFF)) + # 0x20 - 0x23 : Language code? + header.write(iana2mobi(str(self._oeb.metadata.language[0]))) # 0x24 - 0x27 : Number of TOC entries in INDX1 header.write(pack('>I', indxt_count + 1)) @@ -1800,7 +1800,7 @@ class MobiWriter(object): text = text.strip() if not isinstance(text, unicode): text = text.decode('utf-8', 'replace') - text = text.encode('cp1252','replace') + text = text.encode('ascii','replace') return text def _add_to_ctoc(self, ctoc_str, record_offset): @@ -2150,26 +2150,6 @@ class MobiWriter(object): indxt.write(decint(self._ctoc_map[index]['titleOffset'], DECINT_FORWARD)) # vwi title offset in CNCX indxt.write(decint(0, DECINT_FORWARD)) # unknown byte - def _write_subchapter_node(self, indxt, indices, index, offset, length, count): - # This style works without a parent chapter, mimicking what KindleGen does, - # using a value of 0x0B for parentIndex - # Writes an INDX1 NCXEntry of entryType 0x1F - subchapter - if self.opts.verbose > 2: - # *** GR: Turn this off while I'm developing my code - #self._oeb.log.debug('Writing TOC node to IDXT:', node.title, 'href:', node.href) - pass - - pos = 0xc0 + indxt.tell() - indices.write(pack('>H', pos)) # Save the offset for IDXTIndices - name = "%04X"%count - indxt.write(chr(len(name)) + name) # Write the name - indxt.write(INDXT['subchapter']) # entryType [0x0F | 0xDF | 0xFF | 0x3F] - indxt.write(decint(offset, DECINT_FORWARD)) # offset - indxt.write(decint(length, DECINT_FORWARD)) # length - indxt.write(decint(self._ctoc_map[index]['titleOffset'], DECINT_FORWARD)) # vwi title offset in CNCX - indxt.write(decint(0, DECINT_FORWARD)) # unknown byte - indxt.write(decint(0xb, DECINT_FORWARD)) # parentIndex - null - def _compute_offset_length(self, i, node, entries) : h = node.href if h not in self._id_offsets: From 30c96df50546e9730ad1903ac31e54a05d09f723 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 23 Sep 2010 08:13:51 -0600 Subject: [PATCH 04/11] ... --- src/calibre/utils/pyconsole/history.py | 56 ++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 src/calibre/utils/pyconsole/history.py diff --git a/src/calibre/utils/pyconsole/history.py b/src/calibre/utils/pyconsole/history.py new file mode 100644 index 0000000000..5440e57153 --- /dev/null +++ b/src/calibre/utils/pyconsole/history.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from collections import deque + +class History(object): # {{{ + + def __init__(self, current, entries): + self.entries = deque(entries, maxlen=max(2000, len(entries))) + self.index = len(self.entries) - 1 + self.current = self.default = current + self.last_was_back = False + + def back(self, amt=1): + if self.entries: + oidx = self.index + ans = self.entries[self.index] + self.index = max(0, self.index - amt) + self.last_was_back = self.index != oidx + return ans + + def forward(self, amt=1): + if self.entries: + d = self.index + if self.last_was_back: + d += 1 + if d >= len(self.entries) - 1: + self.index = len(self.entries) - 1 + self.last_was_back = False + return self.current + if self.last_was_back: + amt += 1 + self.index = min(len(self.entries)-1, self.index + amt) + self.last_was_back = False + return self.entries[self.index] + + def enter(self, x): + try: + self.entries.remove(x) + except ValueError: + pass + self.entries.append(x) + self.index = len(self.entries) - 1 + self.current = self.default + self.last_was_back = False + + def serialize(self): + return list(self.entries) + +# }}} + + From e41a54c1d70b418f25fe6592a56bcfd68ea4ea9e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 23 Sep 2010 08:31:27 -0600 Subject: [PATCH 05/11] Neal's Nuze by Tony Stegall. Fixes #6917 (Recipe for Neal's Nuze) --- resources/images/news/boortz.png | Bin 0 -> 652 bytes resources/recipes/boortz.recipe | 43 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 resources/images/news/boortz.png create mode 100644 resources/recipes/boortz.recipe diff --git a/resources/images/news/boortz.png b/resources/images/news/boortz.png new file mode 100644 index 0000000000000000000000000000000000000000..ac806e805679606e8d5afc008d13f94e74a640d2 GIT binary patch literal 652 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`Y)RhkE)4%caKYZ?lYt_f1s;*b zK-vS0-A-oPfdtD69Mgd`SU*F|v9*U87#O#Dx;TbdoL)M4qjz+m$gz4hQTFD}e8Y|z z5AHZFX_{=P5Wy{*)V623VWZO7Yl?r_xwvHK?@-x&*mQzZw}Q$|mZi=wa_0Az^|(D! zD7BD%w0rM*H}>DU;^%gL-~GJixwW&B(jWG=|GY8t=NImLd`UZ8E{0G1lgc(b)+FaN z*8Q`1t!8f7XS3^@R73t;???IEQ{0X*+_1JwN>aO5uk!uHHrvcdK_J?haV1qC%j@K= zD^qSnx*s<=*ng?o_rfNnOS#qllden=5}Y_8zHWVtV$6xnTX*ptIRCR!#=1TEIYYr4 zne|DA4BOu4RTdVpIGlJdoA4<%@H_v9haa@V&sz1(eY`3%{c>sPJ#kBx55FIHJKTJ) zKfOftcKLP&m&uyb#KkK%Z(MkEubsVeirtk{_YQ~ueVo&iqvl&y*m?f<@1FCj3_C2p z{$f#E+?iqInwioQxARbw>B(iUH%03d6~AIMu$%ruct(%frZ#n}nI_e%yi8_Z()B-c z Date: Thu, 23 Sep 2010 08:45:37 -0600 Subject: [PATCH 06/11] Database: Fix regression that caused has_cover to create empty directories unneccessarily --- src/calibre/library/database2.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 627ab6358b..9c0397e21e 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -358,10 +358,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return row[self.FIELD_MAP['path']].replace('/', os.sep) - def abspath(self, index, index_is_id=False): + def abspath(self, index, index_is_id=False, create_dirs=True): 'Return the absolute path to the directory containing this books files as a unicode string.' path = os.path.join(self.library_path, self.path(index, index_is_id=index_is_id)) - if not os.path.exists(path): + if create_dirs and not os.path.exists(path): os.makedirs(path) return path @@ -597,7 +597,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def has_cover(self, index, index_is_id=False): id = index if index_is_id else self.id(index) try: - path = os.path.join(self.abspath(id, index_is_id=True), 'cover.jpg') + path = os.path.join(self.abspath(id, index_is_id=True, + create_dirs=False), 'cover.jpg') except: # Can happen if path has not yet been set return False From b94057050456f5b44c2a71c918e5b5ddeef351da Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 23 Sep 2010 09:33:49 -0600 Subject: [PATCH 07/11] IPC: Store results file in the calibre temp dir and also dont die if for some reason removing result file fails --- src/calibre/utils/ipc/server.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index 9f21cb638f..1d14f6a128 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -18,6 +18,7 @@ from calibre.utils.ipc.launch import Worker from calibre.utils.ipc.worker import PARALLEL_FUNCS from calibre import detect_ncpus as cpu_count from calibre.constants import iswindows +from calibre.ptempfile import base_dir _counter = 0 @@ -114,8 +115,9 @@ class Server(Thread): with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count - rfile = os.path.join(tempfile.gettempdir(), - 'calibre_ipc_result_%d_%d.pickle'%(self.id, id)) + fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_'%(self.id, id), + dir=base_dir(), suffix='.pickle') + os.close(fd) if redirect_output is None: redirect_output = not gui @@ -189,8 +191,12 @@ class Server(Thread): job.failed = True job.returncode = worker.returncode elif os.path.exists(worker.rfile): - job.result = cPickle.load(open(worker.rfile, 'rb')) - os.remove(worker.rfile) + try: + job.result = cPickle.load(open(worker.rfile, 'rb')) + os.remove(worker.rfile) + except: + import traceback + traceback.print_exc() job.duration = time.time() - job.start_time self.changed_jobs_queue.put(job) From 1ad0eebd5658c73913dc0ef4b73a95ae0c8960a5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 23 Sep 2010 23:30:16 -0600 Subject: [PATCH 08/11] API for dealing with distributed metadata backup --- src/calibre/library/custom_columns.py | 5 ++- src/calibre/library/database2.py | 47 +++++++++++++++++++++++++- src/calibre/library/schema_upgrades.py | 12 +++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index d74024280e..2d8634659b 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -382,6 +382,7 @@ class CustomColumns(object): ) # get rid of the temp tables self.conn.executescript(drops) + self.dirtied(ids, commit=False) self.conn.commit() # set the in-memory copies of the tags @@ -402,19 +403,21 @@ class CustomColumns(object): same length as ids. ''' if extras is not None and len(extras) != len(ids): - raise ValueError('Lentgh of ids and extras is not the same') + raise ValueError('Length of ids and extras is not the same') ev = None for idx,id in enumerate(ids): if extras is not None: ev = extras[idx] self._set_custom(id, val, label=label, num=num, append=append, notify=notify, extra=ev) + self.dirtied(ids, commit=False) self.conn.commit() def set_custom(self, id, val, label=None, num=None, append=False, notify=True, extra=None, commit=True): self._set_custom(id, val, label=label, num=num, append=append, notify=notify, extra=extra) + self.dirtied([id], commit=False) if commit: self.conn.commit() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 1fe77077b9..7a8aef541d 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -13,6 +13,7 @@ from math import floor from PyQt4.QtGui import QImage from calibre.ebooks.metadata import title_sort, author_to_author_sort +from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.library.database import LibraryDatabase from calibre.library.field_metadata import FieldMetadata, TagsIcons from calibre.library.schema_upgrades import SchemaUpgrade @@ -126,6 +127,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def __init__(self, library_path, row_factory=False): self.field_metadata = FieldMetadata() + self.dirtied_cache = set([]) if not os.path.exists(library_path): os.makedirs(library_path) self.listeners = set([]) @@ -337,6 +339,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): setattr(self, 'title_sort', functools.partial(self.get_property, loc=self.FIELD_MAP['sort'])) + d = self.conn.get('SELECT book FROM metadata_dirtied', all=True) + self.dirtied_cache.update(set([x[0] for x in d])) + self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) self.refresh() self.last_update_check = self.last_modified() @@ -550,6 +555,33 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def metadata_for_field(self, key): return self.field_metadata[key] + def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True): + for book_id in book_ids: + mi = self.get_metadata(book_id, index_is_id=True, get_cover=True) + # Always set cover to cover.jpg. Even if cover doesn't exist, + # no harm done. This way no need to call dirtied when + # cover is set/removed + mi.cover = 'cover.jpg' + raw = metadata_to_opf(mi) + path = self.abspath(book_id, index_is_id=True) + with open(os.path.join(path, 'metadata.opf'), 'wb') as f: + f.write(raw) + if remove_from_dirtied: + self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', + (book_id,)) + if book_id in self.dirtied_cache: + self.dirtied_cache.remove(book_id) + if commit: + self.conn.commit() + + def dirtied(self, book_ids, commit=True): + self.conn.executemany( + 'INSERT OR REPLACE INTO metadata_dirtied VALUES (?)', + [(x,) for x in book_ids]) + if commit: + self.conn.commit() + self.dirtied.update(set(book_ids)) + def get_metadata(self, idx, index_is_id=False, get_cover=False): ''' Convenience method to return metadata as a :class:`Metadata` object. @@ -583,7 +615,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.uuid = self.uuid(idx, index_is_id=index_is_id) mi.title_sort = self.title_sort(idx, index_is_id=index_is_id) mi.formats = self.formats(idx, index_is_id=index_is_id, - verify_formats=False) + verify_formats=False) if hasattr(mi.formats, 'split'): mi.formats = mi.formats.split(',') else: @@ -1242,6 +1274,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ss = self.author_sort_from_book(id, index_is_id=True) self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (ss, id)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['authors'], @@ -1268,6 +1301,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True) self.set_path(id, index_is_id=True) + self.dirtied([id], commit=False) if commit: self.conn.commit() if notify: @@ -1277,6 +1311,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if dt: self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id)) self.data.set(id, self.FIELD_MAP['timestamp'], dt, row_is_id=True) + self.dirtied([id], commit=False) if commit: self.conn.commit() if notify: @@ -1286,6 +1321,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if dt: self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id)) self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True) + self.dirtied([id], commit=False) if commit: self.conn.commit() if notify: @@ -1304,6 +1340,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: aid = self.conn.execute('INSERT INTO publishers(name) VALUES (?)', (publisher,)).lastrowid self.conn.execute('INSERT INTO books_publishers_link(book, publisher) VALUES (?,?)', (id, aid)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True) @@ -1594,6 +1631,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): '''.format(tables[0], tables[1]) ) self.conn.executescript(drops) + self.dirtied(ids, commit=False) self.conn.commit() for x in ids: @@ -1639,6 +1677,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): (id, tid), all=False): self.conn.execute('INSERT INTO books_tags_link(book, tag) VALUES (?,?)', (id, tid)) + self.dirtied([id], commit=False) if commit: self.conn.commit() tags = u','.join(self.get_tags(id)) @@ -1693,6 +1732,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['series'], series, row_is_id=True) @@ -1707,6 +1747,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: idx = 1.0 self.conn.execute('UPDATE books SET series_index=? WHERE id=?', (idx, id)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['series_index'], idx, row_is_id=True) @@ -1719,6 +1760,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): rat = self.conn.get('SELECT id FROM ratings WHERE rating=?', (rating,), all=False) rat = rat if rat else self.conn.execute('INSERT INTO ratings(rating) VALUES (?)', (rating,)).lastrowid self.conn.execute('INSERT INTO books_ratings_link(book, rating) VALUES (?,?)', (id, rat)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['rating'], rating, row_is_id=True) @@ -1731,11 +1773,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['comments'], text, row_is_id=True) + self.dirtied([id], commit=False) if notify: self.notify('metadata', [id]) def set_author_sort(self, id, sort, notify=True, commit=True): self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (sort, id)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['author_sort'], sort, row_is_id=True) @@ -1744,6 +1788,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def set_isbn(self, id, isbn, notify=True, commit=True): self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True) diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index b08161abf2..167cc0a327 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -397,3 +397,15 @@ class SchemaUpgrade(object): UNIQUE(key)); ''' self.conn.executescript(script) + + def upgrade_version_13(self): + 'Dirtied table for OPF metadata backups' + script = ''' + DROP TABLE IF EXISTS metadata_dirtied; + CREATE TABLE metadata_dirtied(id INTEGER PRIMARY KEY, + book INTEGER NOT NULL, + UNIQUE(book)); + INSERT INTO metadata_dirtied (book) SELECT id FROM books; + ''' + self.conn.executescript(script) + From f46d919c751dbccb24f21c348ca130833ffc5a7a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 23 Sep 2010 23:50:22 -0600 Subject: [PATCH 09/11] Add thread to GUI for distributed metadata backup --- src/calibre/gui2/library/models.py | 5 ++++- src/calibre/gui2/ui.py | 4 ++++ src/calibre/library/caches.py | 30 ++++++++++++++++++++++++++++-- src/calibre/library/database2.py | 15 +++++++++------ 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index fe64a33c47..9d9de358c8 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -21,7 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.utils.search_query_parser import SearchQueryParser from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ - REGEXP_MATCH, CoverCache + REGEXP_MATCH, CoverCache, MetadataBackup from calibre.library.cli import parse_series_string from calibre import strftime, isbytestring, prepare_string_for_xml from calibre.constants import filesystem_encoding @@ -153,6 +153,9 @@ class BooksModel(QAbstractTableModel): # {{{ self.cover_cache.stop() self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover)) self.cover_cache.start() + self.metadata_backup = MetadataBackup(db, + FunctionDispatcher(self.db.dump_metadata)) + self.metadata_backup.start() def refresh_cover(event, ids): if event == 'cover' and self.cover_cache is not None: self.cover_cache.refresh(ids) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 9bc504a001..88a8c68572 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -551,6 +551,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{ cc = self.library_view.model().cover_cache if cc is not None: cc.stop() + mb = self.library_view.model().metadata_backup + if mb is not None: + mb.stop() + self.hide_windows() self.emailer.stop() try: diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 7849eecb2e..2d37314896 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -21,7 +21,31 @@ from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import title_sort from calibre import fit_image -class CoverCache(Thread): +class MetadataBackup(Thread): # {{{ + + def __init__(self, db, dump_func): + Thread.__init__(self) + self.daemon = True + self.db = db + self.dump_func = dump_func + self.keep_running = True + + def stop(self): + self.keep_running = False + + def run(self): + while self.keep_running: + try: + id_ = self.db.dirtied_queue.get(True, 5) + except Empty: + continue + # If there is an exception is dump_func, we + # have no way of knowing + self.dump_func([id_]) + +# }}} + +class CoverCache(Thread): # {{{ def __init__(self, db, cover_func): Thread.__init__(self) @@ -90,6 +114,7 @@ class CoverCache(Thread): for id_ in ids: self.cache.pop(id_, None) self.load_queue.put(id_) +# }}} ### Global utility function for get_match here and in gui2/library.py CONTAINS_MATCH = 0 @@ -107,7 +132,7 @@ def _match(query, value, matchkind): pass return False -class ResultCache(SearchQueryParser): +class ResultCache(SearchQueryParser): # {{{ ''' Stores sorted and filtered metadata in memory. @@ -694,4 +719,5 @@ class SortKeyGenerator(object): # }}} +# }}} diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 7a8aef541d..92f8cca0db 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -9,6 +9,7 @@ The database used to store ebook metadata import os, sys, shutil, cStringIO, glob, time, functools, traceback, re from itertools import repeat from math import floor +from Queue import Queue from PyQt4.QtGui import QImage @@ -127,7 +128,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def __init__(self, library_path, row_factory=False): self.field_metadata = FieldMetadata() - self.dirtied_cache = set([]) + self.dirtied_queue = Queue() if not os.path.exists(library_path): os.makedirs(library_path) self.listeners = set([]) @@ -340,7 +341,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): loc=self.FIELD_MAP['sort'])) d = self.conn.get('SELECT book FROM metadata_dirtied', all=True) - self.dirtied_cache.update(set([x[0] for x in d])) + for x in d: + self.dirtied_queue.put(x[0]) self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) self.refresh() @@ -557,6 +559,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True): for book_id in book_ids: + if not self.data.has_id(book_id): + continue mi = self.get_metadata(book_id, index_is_id=True, get_cover=True) # Always set cover to cover.jpg. Even if cover doesn't exist, # no harm done. This way no need to call dirtied when @@ -569,18 +573,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if remove_from_dirtied: self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', (book_id,)) - if book_id in self.dirtied_cache: - self.dirtied_cache.remove(book_id) if commit: self.conn.commit() def dirtied(self, book_ids, commit=True): self.conn.executemany( - 'INSERT OR REPLACE INTO metadata_dirtied VALUES (?)', + 'INSERT OR REPLACE INTO metadata_dirtied (book) VALUES (?)', [(x,) for x in book_ids]) if commit: self.conn.commit() - self.dirtied.update(set(book_ids)) + for x in book_ids: + self.dirtied_queue.put(x) def get_metadata(self, idx, index_is_id=False, get_cover=False): ''' From 703ea3c77827a4053d4c020fa79e7d5e8111f24a Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 23 Sep 2010 23:55:56 -0600 Subject: [PATCH 10/11] propert indentation in generated OPF files --- src/calibre/ebooks/metadata/opf2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 8a4ff6a5bd..5c2477c3dc 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1230,7 +1230,7 @@ def metadata_to_opf(mi, as_string=True): %(id)s %(uuid)s - + '''%dict(a=__appname__, id=mi.application_id, uuid=mi.uuid))) From 87d70304bd7c96b3f65c565b1d8f8177f017f7a0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 24 Sep 2010 00:05:11 -0600 Subject: [PATCH 11/11] Make metadata backup a little more robust --- src/calibre/library/caches.py | 14 ++++++++++---- src/calibre/library/database2.py | 1 + 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 2d37314896..0b5a922209 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -19,7 +19,7 @@ from calibre.utils.date import parse_date, now, UNDEFINED_DATE from calibre.utils.search_query_parser import SearchQueryParser from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import title_sort -from calibre import fit_image +from calibre import fit_image, prints class MetadataBackup(Thread): # {{{ @@ -39,9 +39,15 @@ class MetadataBackup(Thread): # {{{ id_ = self.db.dirtied_queue.get(True, 5) except Empty: continue - # If there is an exception is dump_func, we - # have no way of knowing - self.dump_func([id_]) + except: + # Happens during interpreter shutdown + break + if self.dump_func([id_]) is None: + # An exception occured in dump_func, retry once + prints('Failed to backup metadata for id:', id_, 'once') + time.sleep(2) + if not self.dump_func([id_]): + prints('Failed to backup metadata for id:', id_, 'again, giving up') # }}} diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 92f8cca0db..6a0d442927 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -575,6 +575,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): (book_id,)) if commit: self.conn.commit() + return True def dirtied(self, book_ids, commit=True): self.conn.executemany(