diff --git a/resources/images/news/boortz.png b/resources/images/news/boortz.png new file mode 100644 index 0000000000..ac806e8056 Binary files /dev/null and b/resources/images/news/boortz.png differ diff --git a/resources/recipes/boortz.recipe b/resources/recipes/boortz.recipe new file mode 100644 index 0000000000..a3b0f5fffe --- /dev/null +++ b/resources/recipes/boortz.recipe @@ -0,0 +1,43 @@ +from calibre.web.feeds.news import BasicNewsRecipe +class AdvancedUserRecipe1282101454(BasicNewsRecipe): + title = 'Nealz Nuze' + language = 'en' + __author__ = 'TonytheBookworm' + description = 'Neal Boortz Show Radio Notes' + publisher = 'Neal Boortz' + category = 'news, politics, USA, talkshow' + oldest_article = 1 + max_articles_per_feed = 100 + linearize_tables = True + no_stylesheets = True + remove_javascript = True + + masthead_url = 'http://boortz.com/images/nuze_logo.gif' + keep_only_tags = [ + dict(name='div', attrs={'id':['SiteContent']}) + #,dict(attrs={'id':['cxArticleText']}) + ] + remove_tags = [ + dict(name='a', attrs={'class':['blogPermalink']}), + dict(name='span', attrs={'class':['blogBylineSeparator']}), + dict(name='td', attrs={'id':['nealztitle']}), + ] + remove_tags_after = [dict(name='div', attrs={'class':'blogEntryBody'}),] + feeds = [ + ('NUZE', 'http://boortz.com/nealz_nuze_rss/rss.xml') + + ] + + + + + + + + + + + + + + diff --git a/src/calibre/devices/kobo/books.py b/src/calibre/devices/kobo/books.py index cca4c005d7..24ec272bb1 100644 --- a/src/calibre/devices/kobo/books.py +++ b/src/calibre/devices/kobo/books.py @@ -19,6 +19,10 @@ class Book(Book_): self.authors = [''] else: self.authors = [authors] + + if not title: + self.title = _('Unknown') + self.mime = mime self.size = size # will be set later if None diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 929dc01aec..8791d59242 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -5,7 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import copy, re, traceback +import copy, traceback from calibre import prints from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 8a4ff6a5bd..5c2477c3dc 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -1230,7 +1230,7 @@ def metadata_to_opf(mi, as_string=True): %(id)s %(uuid)s - + '''%dict(a=__appname__, id=mi.application_id, uuid=mi.uuid))) diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index 23f92d1fd2..37936d6016 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -1696,11 +1696,12 @@ class MobiWriter(object): header.write(pack('>I', 1)) # 0x1c - 0x1f : Text encoding ? - # GR: Language encoding for NCX entries (latin_1) - header.write(pack('>I', 0x4e4)) + # header.write(pack('>I', 650001)) + # GR: This needs to be either 0xFDE9 or 0x4E4 + header.write(pack('>I', 0xFDE9)) - # 0x20 - 0x23 : Mimicking kindleGen - header.write(pack('>I', 0xFFFFFFFF)) + # 0x20 - 0x23 : Language code? + header.write(iana2mobi(str(self._oeb.metadata.language[0]))) # 0x24 - 0x27 : Number of TOC entries in INDX1 header.write(pack('>I', indxt_count + 1)) @@ -1800,7 +1801,7 @@ class MobiWriter(object): text = text.strip() if not isinstance(text, unicode): text = text.decode('utf-8', 'replace') - text = text.encode('cp1252','replace') + text = text.encode('ascii','replace') return text def _add_to_ctoc(self, ctoc_str, record_offset): @@ -2150,26 +2151,6 @@ class MobiWriter(object): indxt.write(decint(self._ctoc_map[index]['titleOffset'], DECINT_FORWARD)) # vwi title offset in CNCX indxt.write(decint(0, DECINT_FORWARD)) # unknown byte - def _write_subchapter_node(self, indxt, indices, index, offset, length, count): - # This style works without a parent chapter, mimicking what KindleGen does, - # using a value of 0x0B for parentIndex - # Writes an INDX1 NCXEntry of entryType 0x1F - subchapter - if self.opts.verbose > 2: - # *** GR: Turn this off while I'm developing my code - #self._oeb.log.debug('Writing TOC node to IDXT:', node.title, 'href:', node.href) - pass - - pos = 0xc0 + indxt.tell() - indices.write(pack('>H', pos)) # Save the offset for IDXTIndices - name = "%04X"%count - indxt.write(chr(len(name)) + name) # Write the name - indxt.write(INDXT['subchapter']) # entryType [0x0F | 0xDF | 0xFF | 0x3F] - indxt.write(decint(offset, DECINT_FORWARD)) # offset - indxt.write(decint(length, DECINT_FORWARD)) # length - indxt.write(decint(self._ctoc_map[index]['titleOffset'], DECINT_FORWARD)) # vwi title offset in CNCX - indxt.write(decint(0, DECINT_FORWARD)) # unknown byte - indxt.write(decint(0xb, DECINT_FORWARD)) # parentIndex - null - def _compute_offset_length(self, i, node, entries) : h = node.href if h not in self._id_offsets: diff --git a/src/calibre/gui2/device_drivers/configwidget.py b/src/calibre/gui2/device_drivers/configwidget.py index 1d6c84ef7c..64321e1a46 100644 --- a/src/calibre/gui2/device_drivers/configwidget.py +++ b/src/calibre/gui2/device_drivers/configwidget.py @@ -81,7 +81,6 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): return self.opt_use_author_sort.isChecked() def validate(self): - print 'here in validate' tmpl = unicode(self.opt_save_template.text()) try: validation_formatter.validate(tmpl) @@ -89,6 +88,6 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): except Exception, err: error_dialog(self, _('Invalid template'), '

'+_('The template %s is invalid:')%tmpl + \ - '
'+str(err), show=True) + '
'+unicode(err), show=True) return False diff --git a/src/calibre/gui2/library/delegates.py b/src/calibre/gui2/library/delegates.py index ceb1cf14a8..1156d1ace9 100644 --- a/src/calibre/gui2/library/delegates.py +++ b/src/calibre/gui2/library/delegates.py @@ -304,6 +304,8 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{ val = 2 if val is None else 1 if not val else 0 editor.setCurrentIndex(val) +# }}} + class CcTemplateDelegate(QStyledItemDelegate): # {{{ def __init__(self, parent): ''' diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index fe64a33c47..9d9de358c8 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -21,7 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.utils.search_query_parser import SearchQueryParser from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ - REGEXP_MATCH, CoverCache + REGEXP_MATCH, CoverCache, MetadataBackup from calibre.library.cli import parse_series_string from calibre import strftime, isbytestring, prepare_string_for_xml from calibre.constants import filesystem_encoding @@ -153,6 +153,9 @@ class BooksModel(QAbstractTableModel): # {{{ self.cover_cache.stop() self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover)) self.cover_cache.start() + self.metadata_backup = MetadataBackup(db, + FunctionDispatcher(self.db.dump_metadata)) + self.metadata_backup.start() def refresh_cover(event, ids): if event == 'cover' and self.cover_cache is not None: self.cover_cache.refresh(ids) diff --git a/src/calibre/gui2/preferences/save_template.py b/src/calibre/gui2/preferences/save_template.py index 5b3f0321b2..a7f57536d5 100644 --- a/src/calibre/gui2/preferences/save_template.py +++ b/src/calibre/gui2/preferences/save_template.py @@ -6,8 +6,6 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import string - from PyQt4.Qt import QWidget, pyqtSignal from calibre.gui2 import error_dialog diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 9bc504a001..88a8c68572 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -551,6 +551,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{ cc = self.library_view.model().cover_cache if cc is not None: cc.stop() + mb = self.library_view.model().metadata_backup + if mb is not None: + mb.stop() + self.hide_windows() self.emailer.stop() try: diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 7849eecb2e..0b5a922209 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -19,9 +19,39 @@ from calibre.utils.date import parse_date, now, UNDEFINED_DATE from calibre.utils.search_query_parser import SearchQueryParser from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import title_sort -from calibre import fit_image +from calibre import fit_image, prints -class CoverCache(Thread): +class MetadataBackup(Thread): # {{{ + + def __init__(self, db, dump_func): + Thread.__init__(self) + self.daemon = True + self.db = db + self.dump_func = dump_func + self.keep_running = True + + def stop(self): + self.keep_running = False + + def run(self): + while self.keep_running: + try: + id_ = self.db.dirtied_queue.get(True, 5) + except Empty: + continue + except: + # Happens during interpreter shutdown + break + if self.dump_func([id_]) is None: + # An exception occured in dump_func, retry once + prints('Failed to backup metadata for id:', id_, 'once') + time.sleep(2) + if not self.dump_func([id_]): + prints('Failed to backup metadata for id:', id_, 'again, giving up') + +# }}} + +class CoverCache(Thread): # {{{ def __init__(self, db, cover_func): Thread.__init__(self) @@ -90,6 +120,7 @@ class CoverCache(Thread): for id_ in ids: self.cache.pop(id_, None) self.load_queue.put(id_) +# }}} ### Global utility function for get_match here and in gui2/library.py CONTAINS_MATCH = 0 @@ -107,7 +138,7 @@ def _match(query, value, matchkind): pass return False -class ResultCache(SearchQueryParser): +class ResultCache(SearchQueryParser): # {{{ ''' Stores sorted and filtered metadata in memory. @@ -694,4 +725,5 @@ class SortKeyGenerator(object): # }}} +# }}} diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index d74024280e..2d8634659b 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -382,6 +382,7 @@ class CustomColumns(object): ) # get rid of the temp tables self.conn.executescript(drops) + self.dirtied(ids, commit=False) self.conn.commit() # set the in-memory copies of the tags @@ -402,19 +403,21 @@ class CustomColumns(object): same length as ids. ''' if extras is not None and len(extras) != len(ids): - raise ValueError('Lentgh of ids and extras is not the same') + raise ValueError('Length of ids and extras is not the same') ev = None for idx,id in enumerate(ids): if extras is not None: ev = extras[idx] self._set_custom(id, val, label=label, num=num, append=append, notify=notify, extra=ev) + self.dirtied(ids, commit=False) self.conn.commit() def set_custom(self, id, val, label=None, num=None, append=False, notify=True, extra=None, commit=True): self._set_custom(id, val, label=label, num=num, append=append, notify=notify, extra=extra) + self.dirtied([id], commit=False) if commit: self.conn.commit() diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 22de8df41f..6a0d442927 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -9,10 +9,12 @@ The database used to store ebook metadata import os, sys, shutil, cStringIO, glob, time, functools, traceback, re from itertools import repeat from math import floor +from Queue import Queue from PyQt4.QtGui import QImage from calibre.ebooks.metadata import title_sort, author_to_author_sort +from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.library.database import LibraryDatabase from calibre.library.field_metadata import FieldMetadata, TagsIcons from calibre.library.schema_upgrades import SchemaUpgrade @@ -126,6 +128,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def __init__(self, library_path, row_factory=False): self.field_metadata = FieldMetadata() + self.dirtied_queue = Queue() if not os.path.exists(library_path): os.makedirs(library_path) self.listeners = set([]) @@ -337,6 +340,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): setattr(self, 'title_sort', functools.partial(self.get_property, loc=self.FIELD_MAP['sort'])) + d = self.conn.get('SELECT book FROM metadata_dirtied', all=True) + for x in d: + self.dirtied_queue.put(x[0]) + self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self) self.refresh() self.last_update_check = self.last_modified() @@ -363,10 +370,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return row[self.FIELD_MAP['path']].replace('/', os.sep) - def abspath(self, index, index_is_id=False): + def abspath(self, index, index_is_id=False, create_dirs=True): 'Return the absolute path to the directory containing this books files as a unicode string.' path = os.path.join(self.library_path, self.path(index, index_is_id=index_is_id)) - if not os.path.exists(path): + if create_dirs and not os.path.exists(path): os.makedirs(path) return path @@ -550,6 +557,35 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def metadata_for_field(self, key): return self.field_metadata[key] + def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True): + for book_id in book_ids: + if not self.data.has_id(book_id): + continue + mi = self.get_metadata(book_id, index_is_id=True, get_cover=True) + # Always set cover to cover.jpg. Even if cover doesn't exist, + # no harm done. This way no need to call dirtied when + # cover is set/removed + mi.cover = 'cover.jpg' + raw = metadata_to_opf(mi) + path = self.abspath(book_id, index_is_id=True) + with open(os.path.join(path, 'metadata.opf'), 'wb') as f: + f.write(raw) + if remove_from_dirtied: + self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?', + (book_id,)) + if commit: + self.conn.commit() + return True + + def dirtied(self, book_ids, commit=True): + self.conn.executemany( + 'INSERT OR REPLACE INTO metadata_dirtied (book) VALUES (?)', + [(x,) for x in book_ids]) + if commit: + self.conn.commit() + for x in book_ids: + self.dirtied_queue.put(x) + def get_metadata(self, idx, index_is_id=False, get_cover=False): ''' Convenience method to return metadata as a :class:`Metadata` object. @@ -583,7 +619,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.uuid = self.uuid(idx, index_is_id=index_is_id) mi.title_sort = self.title_sort(idx, index_is_id=index_is_id) mi.formats = self.formats(idx, index_is_id=index_is_id, - verify_formats=False) + verify_formats=False) if hasattr(mi.formats, 'split'): mi.formats = mi.formats.split(',') else: @@ -657,7 +693,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def has_cover(self, index, index_is_id=False): id = index if index_is_id else self.id(index) try: - path = os.path.join(self.abspath(id, index_is_id=True), 'cover.jpg') + path = os.path.join(self.abspath(id, index_is_id=True, + create_dirs=False), 'cover.jpg') except: # Can happen if path has not yet been set return False @@ -1241,6 +1278,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): ss = self.author_sort_from_book(id, index_is_id=True) self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (ss, id)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['authors'], @@ -1267,6 +1305,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True) self.set_path(id, index_is_id=True) + self.dirtied([id], commit=False) if commit: self.conn.commit() if notify: @@ -1276,6 +1315,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if dt: self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id)) self.data.set(id, self.FIELD_MAP['timestamp'], dt, row_is_id=True) + self.dirtied([id], commit=False) if commit: self.conn.commit() if notify: @@ -1285,6 +1325,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if dt: self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id)) self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True) + self.dirtied([id], commit=False) if commit: self.conn.commit() if notify: @@ -1303,6 +1344,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: aid = self.conn.execute('INSERT INTO publishers(name) VALUES (?)', (publisher,)).lastrowid self.conn.execute('INSERT INTO books_publishers_link(book, publisher) VALUES (?,?)', (id, aid)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True) @@ -1593,6 +1635,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): '''.format(tables[0], tables[1]) ) self.conn.executescript(drops) + self.dirtied(ids, commit=False) self.conn.commit() for x in ids: @@ -1638,6 +1681,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): (id, tid), all=False): self.conn.execute('INSERT INTO books_tags_link(book, tag) VALUES (?,?)', (id, tid)) + self.dirtied([id], commit=False) if commit: self.conn.commit() tags = u','.join(self.get_tags(id)) @@ -1692,6 +1736,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): else: aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['series'], series, row_is_id=True) @@ -1706,6 +1751,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: idx = 1.0 self.conn.execute('UPDATE books SET series_index=? WHERE id=?', (idx, id)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['series_index'], idx, row_is_id=True) @@ -1718,6 +1764,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): rat = self.conn.get('SELECT id FROM ratings WHERE rating=?', (rating,), all=False) rat = rat if rat else self.conn.execute('INSERT INTO ratings(rating) VALUES (?)', (rating,)).lastrowid self.conn.execute('INSERT INTO books_ratings_link(book, rating) VALUES (?,?)', (id, rat)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['rating'], rating, row_is_id=True) @@ -1730,11 +1777,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['comments'], text, row_is_id=True) + self.dirtied([id], commit=False) if notify: self.notify('metadata', [id]) def set_author_sort(self, id, sort, notify=True, commit=True): self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (sort, id)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['author_sort'], sort, row_is_id=True) @@ -1743,6 +1792,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def set_isbn(self, id, isbn, notify=True, commit=True): self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id)) + self.dirtied([id], commit=False) if commit: self.conn.commit() self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 2d0a3d1277..a58686f709 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, cStringIO, re, string +import os, traceback, cStringIO, re from calibre.utils.config import Config, StringConfig, tweaks from calibre.utils.formatter import TemplateFormatter diff --git a/src/calibre/library/schema_upgrades.py b/src/calibre/library/schema_upgrades.py index b08161abf2..167cc0a327 100644 --- a/src/calibre/library/schema_upgrades.py +++ b/src/calibre/library/schema_upgrades.py @@ -397,3 +397,15 @@ class SchemaUpgrade(object): UNIQUE(key)); ''' self.conn.executescript(script) + + def upgrade_version_13(self): + 'Dirtied table for OPF metadata backups' + script = ''' + DROP TABLE IF EXISTS metadata_dirtied; + CREATE TABLE metadata_dirtied(id INTEGER PRIMARY KEY, + book INTEGER NOT NULL, + UNIQUE(book)); + INSERT INTO metadata_dirtied (book) SELECT id FROM books; + ''' + self.conn.executescript(script) + diff --git a/src/calibre/utils/formatter.py b/src/calibre/utils/formatter.py index 95870d9c61..f1c2a2cb4d 100644 --- a/src/calibre/utils/formatter.py +++ b/src/calibre/utils/formatter.py @@ -47,10 +47,12 @@ class TemplateFormatter(string.Formatter): 'shorten' : (3, _shorten), } + format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') + compress_spaces = re.compile(r'\s+') + def get_value(self, key, args): raise Exception('get_value must be implemented in the subclass') - format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$') def _explode_format_string(self, fmt): try: @@ -64,8 +66,10 @@ class TemplateFormatter(string.Formatter): return fmt, '', '' def format_field(self, val, fmt): + # Handle conditional text fmt, prefix, suffix = self._explode_format_string(fmt) + # Handle functions p = fmt.find('(') if p >= 0 and fmt[-1] == ')' and fmt[0:p] in self.functions: field = fmt[0:p] @@ -73,7 +77,7 @@ class TemplateFormatter(string.Formatter): args = fmt[p+1:-1].split(',') if (func[0] == 0 and (len(args) != 1 or args[0])) or \ (func[0] > 0 and func[0] != len(args)): - raise Exception ('Incorrect number of arguments for function '+ fmt[0:p]) + raise ValueError('Incorrect number of arguments for function '+ fmt[0:p]) if func[0] == 0: val = func[1](self, val) else: @@ -84,8 +88,6 @@ class TemplateFormatter(string.Formatter): return '' return prefix + val + suffix - compress_spaces = re.compile(r'\s+') - def vformat(self, fmt, args, kwargs): ans = string.Formatter.vformat(self, fmt, args, kwargs) return self.compress_spaces.sub(' ', ans).strip() @@ -105,7 +107,7 @@ class ValidateFormat(TemplateFormatter): Provides a format function that substitutes '' for any missing value ''' def get_value(self, key, args, kwargs): - return 'this is some text that should be long enough' + return 'this is some text that should be long enough' def validate(self, x): return self.vformat(x, [], {}) diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index 9f21cb638f..1d14f6a128 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -18,6 +18,7 @@ from calibre.utils.ipc.launch import Worker from calibre.utils.ipc.worker import PARALLEL_FUNCS from calibre import detect_ncpus as cpu_count from calibre.constants import iswindows +from calibre.ptempfile import base_dir _counter = 0 @@ -114,8 +115,9 @@ class Server(Thread): with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count - rfile = os.path.join(tempfile.gettempdir(), - 'calibre_ipc_result_%d_%d.pickle'%(self.id, id)) + fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_'%(self.id, id), + dir=base_dir(), suffix='.pickle') + os.close(fd) if redirect_output is None: redirect_output = not gui @@ -189,8 +191,12 @@ class Server(Thread): job.failed = True job.returncode = worker.returncode elif os.path.exists(worker.rfile): - job.result = cPickle.load(open(worker.rfile, 'rb')) - os.remove(worker.rfile) + try: + job.result = cPickle.load(open(worker.rfile, 'rb')) + os.remove(worker.rfile) + except: + import traceback + traceback.print_exc() job.duration = time.time() - job.start_time self.changed_jobs_queue.put(job) diff --git a/src/calibre/utils/pyconsole/__init__.py b/src/calibre/utils/pyconsole/__init__.py index 3be9382413..6ef9f04d4b 100644 --- a/src/calibre/utils/pyconsole/__init__.py +++ b/src/calibre/utils/pyconsole/__init__.py @@ -24,6 +24,8 @@ def console_config(): c = Config('console', desc) c.add_opt('theme', default='native', help='The color theme') + c.add_opt('scrollback', default=10000, + help='Max number of lines to keep in the scrollback buffer') return c diff --git a/src/calibre/utils/pyconsole/console.py b/src/calibre/utils/pyconsole/console.py index 2611965345..14670fdb59 100644 --- a/src/calibre/utils/pyconsole/console.py +++ b/src/calibre/utils/pyconsole/console.py @@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en' import sys, textwrap, traceback, StringIO from functools import partial +from codeop import CommandCompiler from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \ QApplication, QColor, QPalette, QMenu, QActionGroup, QTimer @@ -16,8 +17,9 @@ from pygments.styles import get_all_styles from calibre.utils.pyconsole.formatter import Formatter from calibre.utils.pyconsole.controller import Controller +from calibre.utils.pyconsole.history import History from calibre.utils.pyconsole import prints, prefs, __appname__, \ - __version__, error_dialog + __version__, error_dialog, dynamic class EditBlock(object): # {{{ @@ -73,6 +75,7 @@ class ThemeMenu(QMenu): # {{{ # }}} + class Console(QTextEdit): running = pyqtSignal() @@ -114,7 +117,9 @@ class Console(QTextEdit): parent=None): QTextEdit.__init__(self, parent) self.shutting_down = False + self.compiler = CommandCompiler() self.buf = self.old_buf = [] + self.history = History([''], dynamic.get('console_history', [])) self.prompt_frame = None self.allow_output = False self.prompt_frame_format = QTextFrameFormat() @@ -122,7 +127,7 @@ class Console(QTextEdit): self.prompt_frame_format.setBorderStyle(QTextFrameFormat.BorderStyle_Solid) self.prompt_len = len(prompt) - self.doc.setMaximumBlockCount(10000) + self.doc.setMaximumBlockCount(int(prefs['scrollback'])) self.lexer = PythonLexer(ensurenl=False) self.tb_lexer = PythonTracebackLexer() @@ -139,6 +144,8 @@ class Console(QTextEdit): self.key_dispatcher = { # {{{ Qt.Key_Enter : self.enter_pressed, Qt.Key_Return : self.enter_pressed, + Qt.Key_Up : self.up_pressed, + Qt.Key_Down : self.down_pressed, Qt.Key_Home : self.home_pressed, Qt.Key_End : self.end_pressed, Qt.Key_Left : self.left_pressed, @@ -153,15 +160,17 @@ class Console(QTextEdit): '''.format(sys.version.splitlines()[0], __appname__, __version__)) + sys.excepthook = self.unhandled_exception + self.controllers = [] QTimer.singleShot(0, self.launch_controller) - sys.excepthook = self.unhandled_exception with EditBlock(self.cursor): self.render_block(motd) def shutdown(self): + dynamic.set('console_history', self.history.serialize()) self.shutton_down = True for c in self.controllers: c.kill() @@ -365,7 +374,7 @@ class Console(QTextEdit): # }}} - # Keyboard handling {{{ + # Keyboard management {{{ def keyPressEvent(self, ev): text = unicode(ev.text()) @@ -394,6 +403,20 @@ class Console(QTextEdit): self.setTextCursor(c) self.ensureCursorVisible() + def up_pressed(self): + lineno, pos = self.cursor_pos + if lineno < 0: return + if lineno == 0: + b = self.history.back() + if b is not None: + self.set_prompt(b) + else: + c = self.cursor + c.movePosition(c.Up) + self.setTextCursor(c) + self.ensureCursorVisible() + + def backspace_pressed(self): lineno, pos = self.cursor_pos if lineno < 0: return @@ -414,7 +437,6 @@ class Console(QTextEdit): lineno, pos = self.cursor_pos if lineno < 0: return c = self.cursor - lineno, pos = self.cursor_pos cp = list(self.prompt(False)) if pos < len(cp[lineno]): c.movePosition(c.NextCharacter) @@ -423,6 +445,22 @@ class Console(QTextEdit): self.setTextCursor(c) self.ensureCursorVisible() + def down_pressed(self): + lineno, pos = self.cursor_pos + if lineno < 0: return + c = self.cursor + cp = list(self.prompt(False)) + if lineno >= len(cp) - 1: + b = self.history.forward() + if b is not None: + self.set_prompt(b) + else: + c = self.cursor + c.movePosition(c.Down) + self.setTextCursor(c) + self.ensureCursorVisible() + + def home_pressed(self): if self.prompt_frame is not None: mods = QApplication.keyboardModifiers() @@ -454,6 +492,19 @@ class Console(QTextEdit): return self.no_controller_error() cp = list(self.prompt()) if cp[0]: + try: + ret = self.compiler('\n'.join(cp)) + except: + pass + else: + if ret is None: + c = self.prompt_frame.lastCursorPosition() + c.insertBlock() + self.setTextCursor(c) + self.render_current_prompt() + return + else: + self.history.enter(cp) self.execute(cp) def text_typed(self, text): @@ -461,6 +512,7 @@ class Console(QTextEdit): self.move_cursor_to_prompt() self.cursor.insertText(text) self.render_current_prompt(restore_cursor=True) + self.history.current = list(self.prompt()) # }}} diff --git a/src/calibre/utils/pyconsole/controller.py b/src/calibre/utils/pyconsole/controller.py index 368e665079..d372cb4ebc 100644 --- a/src/calibre/utils/pyconsole/controller.py +++ b/src/calibre/utils/pyconsole/controller.py @@ -104,7 +104,6 @@ class Controller(QThread): def returncode(self): return self.process.returncode - @property def interrupt(self): if hasattr(signal, 'SIGINT'): os.kill(self.process.pid, signal.SIGINT) diff --git a/src/calibre/utils/pyconsole/history.py b/src/calibre/utils/pyconsole/history.py new file mode 100644 index 0000000000..5440e57153 --- /dev/null +++ b/src/calibre/utils/pyconsole/history.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +from collections import deque + +class History(object): # {{{ + + def __init__(self, current, entries): + self.entries = deque(entries, maxlen=max(2000, len(entries))) + self.index = len(self.entries) - 1 + self.current = self.default = current + self.last_was_back = False + + def back(self, amt=1): + if self.entries: + oidx = self.index + ans = self.entries[self.index] + self.index = max(0, self.index - amt) + self.last_was_back = self.index != oidx + return ans + + def forward(self, amt=1): + if self.entries: + d = self.index + if self.last_was_back: + d += 1 + if d >= len(self.entries) - 1: + self.index = len(self.entries) - 1 + self.last_was_back = False + return self.current + if self.last_was_back: + amt += 1 + self.index = min(len(self.entries)-1, self.index + amt) + self.last_was_back = False + return self.entries[self.index] + + def enter(self, x): + try: + self.entries.remove(x) + except ValueError: + pass + self.entries.append(x) + self.index = len(self.entries) - 1 + self.current = self.default + self.last_was_back = False + + def serialize(self): + return list(self.entries) + +# }}} + + diff --git a/src/calibre/utils/pyconsole/interpreter.py b/src/calibre/utils/pyconsole/interpreter.py index 6a1aff26c9..3cd0d94711 100644 --- a/src/calibre/utils/pyconsole/interpreter.py +++ b/src/calibre/utils/pyconsole/interpreter.py @@ -11,6 +11,7 @@ from Queue import Queue, Empty from threading import Thread from binascii import unhexlify from multiprocessing.connection import Client +from repr import repr as safe_repr from calibre.utils.pyconsole import preferred_encoding, isbytestring, \ POLL_TIMEOUT @@ -35,7 +36,7 @@ def tounicode(raw): # {{{ try: raw = raw.decode(preferred_encoding, 'replace') except: - raw = repr(raw) + raw = safe_repr(raw) if isbytestring(raw): try: