Merge from trunk

resources/recipes/boortz.recipe

@@ -0,0 +1,43 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+class AdvancedUserRecipe1282101454(BasicNewsRecipe):
+    title = 'Nealz Nuze'
+    language = 'en'
+    __author__ = 'TonytheBookworm'
+    description = 'Neal Boortz Show Radio Notes'
+    publisher = 'Neal Boortz'
+    category = 'news, politics, USA, talkshow'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    linearize_tables = True
+    no_stylesheets = True
+    remove_javascript   = True
+
+    masthead_url = 'http://boortz.com/images/nuze_logo.gif'
+    keep_only_tags    = [
+                         dict(name='div', attrs={'id':['SiteContent']})
+                       #,dict(attrs={'id':['cxArticleText']})
+                        ]
+    remove_tags = [
+                   dict(name='a', attrs={'class':['blogPermalink']}),
+                   dict(name='span', attrs={'class':['blogBylineSeparator']}),
+                   dict(name='td', attrs={'id':['nealztitle']}),
+                   ]
+    remove_tags_after = [dict(name='div', attrs={'class':'blogEntryBody'}),]
+    feeds          = [
+                      ('NUZE', 'http://boortz.com/nealz_nuze_rss/rss.xml')
+
+                    ]
+
+
+
+
+
+
+
+
+
+
+
+
+
+

src/calibre/devices/kobo/books.py

@@ -19,6 +19,10 @@ class Book(Book_):
             self.authors = ['']
         else:
             self.authors = [authors]
+
+        if not title:
+            self.title = _('Unknown')
+
         self.mime = mime
 
         self.size = size # will be set later if None

src/calibre/ebooks/metadata/book/base.py

@@ -5,7 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import copy, re, traceback
+import copy, traceback
 
 from calibre import prints
 from calibre.ebooks.metadata.book import SC_COPYABLE_FIELDS

src/calibre/ebooks/mobi/writer.py

@@ -1696,11 +1696,12 @@ class MobiWriter(object):
         header.write(pack('>I', 1))
 
         # 0x1c - 0x1f : Text encoding ?
-        # GR: Language encoding for NCX entries (latin_1)
+        # header.write(pack('>I', 650001))
-        header.write(pack('>I', 0x4e4))
+        # GR: This needs to be either 0xFDE9 or 0x4E4
+        header.write(pack('>I', 0xFDE9))
 
-        # 0x20 - 0x23 : Mimicking kindleGen
+        # 0x20 - 0x23 : Language code?
-        header.write(pack('>I', 0xFFFFFFFF))
+        header.write(iana2mobi(str(self._oeb.metadata.language[0])))
 
         # 0x24 - 0x27 : Number of TOC entries in INDX1
         header.write(pack('>I', indxt_count + 1))
@@ -1800,7 +1801,7 @@ class MobiWriter(object):
         text = text.strip()
         if not isinstance(text, unicode):
             text = text.decode('utf-8', 'replace')
-        text = text.encode('cp1252','replace')
+        text = text.encode('ascii','replace')
         return text
 
     def _add_to_ctoc(self, ctoc_str, record_offset):
@@ -2150,26 +2151,6 @@ class MobiWriter(object):
         indxt.write(decint(self._ctoc_map[index]['titleOffset'], DECINT_FORWARD))	# vwi title offset in CNCX
         indxt.write(decint(0, DECINT_FORWARD))						# unknown byte
 
-    def _write_subchapter_node(self, indxt, indices, index, offset, length, count):
-        # This style works without a parent chapter, mimicking what KindleGen does,
-        # using a value of 0x0B for parentIndex
-        # Writes an INDX1 NCXEntry of entryType 0x1F - subchapter
-        if self.opts.verbose > 2:
-            # *** GR: Turn this off while I'm developing my code
-            #self._oeb.log.debug('Writing TOC node to IDXT:', node.title, 'href:', node.href)
-            pass
-
-        pos = 0xc0 + indxt.tell()
-        indices.write(pack('>H', pos))								# Save the offset for IDXTIndices
-        name = "%04X"%count
-        indxt.write(chr(len(name)) + name)							# Write the name
-        indxt.write(INDXT['subchapter'])						    # entryType [0x0F | 0xDF | 0xFF | 0x3F]
-        indxt.write(decint(offset, DECINT_FORWARD))					# offset
-        indxt.write(decint(length, DECINT_FORWARD))					# length
-        indxt.write(decint(self._ctoc_map[index]['titleOffset'], DECINT_FORWARD))	# vwi title offset in CNCX
-        indxt.write(decint(0, DECINT_FORWARD))						# unknown byte
-        indxt.write(decint(0xb, DECINT_FORWARD))				    # parentIndex - null
-
     def _compute_offset_length(self, i, node, entries) :
         h = node.href
         if h not in self._id_offsets:

src/calibre/gui2/device_drivers/configwidget.py

@@ -81,7 +81,6 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
         return self.opt_use_author_sort.isChecked()
 
     def validate(self):
-        print 'here in validate'
         tmpl = unicode(self.opt_save_template.text())
         try:
             validation_formatter.validate(tmpl)
@@ -89,6 +88,6 @@ class ConfigWidget(QWidget, Ui_ConfigWidget):
         except Exception, err:
             error_dialog(self, _('Invalid template'),
                     '<p>'+_('The template %s is invalid:')%tmpl + \
-                    '<br>'+str(err), show=True)
+                    '<br>'+unicode(err), show=True)
 
             return False

src/calibre/gui2/library/delegates.py

@@ -304,6 +304,8 @@ class CcBoolDelegate(QStyledItemDelegate): # {{{
             val = 2 if val is None else 1 if not val else 0
         editor.setCurrentIndex(val)
 
+# }}}
+
 class CcTemplateDelegate(QStyledItemDelegate): # {{{
     def __init__(self, parent):
         '''

src/calibre/gui2/library/models.py

@@ -21,7 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
-    REGEXP_MATCH, CoverCache
+    REGEXP_MATCH, CoverCache, MetadataBackup
 from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
 from calibre.constants import filesystem_encoding
@@ -153,6 +153,9 @@ class BooksModel(QAbstractTableModel): # {{{
             self.cover_cache.stop()
         self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
         self.cover_cache.start()
+        self.metadata_backup = MetadataBackup(db,
+                FunctionDispatcher(self.db.dump_metadata))
+        self.metadata_backup.start()
         def refresh_cover(event, ids):
             if event == 'cover' and self.cover_cache is not None:
                 self.cover_cache.refresh(ids)

src/calibre/gui2/preferences/save_template.py

@@ -6,8 +6,6 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import string
-
 from PyQt4.Qt import QWidget, pyqtSignal
 
 from calibre.gui2 import error_dialog

src/calibre/gui2/ui.py

@@ -551,6 +551,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
         cc = self.library_view.model().cover_cache
         if cc is not None:
             cc.stop()
+        mb = self.library_view.model().metadata_backup
+        if mb is not None:
+            mb.stop()
+
         self.hide_windows()
         self.emailer.stop()
         try:

src/calibre/library/caches.py

@@ -19,9 +19,39 @@ from calibre.utils.date import parse_date, now, UNDEFINED_DATE
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import title_sort
-from calibre import fit_image
+from calibre import fit_image, prints
 
-class CoverCache(Thread):
+class MetadataBackup(Thread): # {{{
+
+    def __init__(self, db, dump_func):
+        Thread.__init__(self)
+        self.daemon = True
+        self.db = db
+        self.dump_func = dump_func
+        self.keep_running = True
+
+    def stop(self):
+        self.keep_running = False
+
+    def run(self):
+        while self.keep_running:
+            try:
+                id_ = self.db.dirtied_queue.get(True, 5)
+            except Empty:
+                continue
+            except:
+                # Happens during interpreter shutdown
+                break
+            if self.dump_func([id_]) is None:
+                # An exception occured in dump_func, retry once
+                prints('Failed to backup metadata for id:', id_, 'once')
+                time.sleep(2)
+                if not self.dump_func([id_]):
+                    prints('Failed to backup metadata for id:', id_, 'again, giving up')
+
+# }}}
+
+class CoverCache(Thread): # {{{
 
     def __init__(self, db, cover_func):
         Thread.__init__(self)
@@ -90,6 +120,7 @@ class CoverCache(Thread):
             for id_ in ids:
                 self.cache.pop(id_, None)
                 self.load_queue.put(id_)
+# }}}
 
 ### Global utility function for get_match here and in gui2/library.py
 CONTAINS_MATCH = 0
@@ -107,7 +138,7 @@ def _match(query, value, matchkind):
             pass
     return False
 
-class ResultCache(SearchQueryParser):
+class ResultCache(SearchQueryParser): # {{{
 
     '''
     Stores sorted and filtered metadata in memory.
@@ -694,4 +725,5 @@ class SortKeyGenerator(object):
 
     # }}}
 
+# }}}
 

src/calibre/library/custom_columns.py

@@ -382,6 +382,7 @@ class CustomColumns(object):
             )
         # get rid of the temp tables
         self.conn.executescript(drops)
+        self.dirtied(ids, commit=False)
         self.conn.commit()
 
         # set the in-memory copies of the tags
@@ -402,19 +403,21 @@ class CustomColumns(object):
         same length as ids.
         '''
         if extras is not None and len(extras) != len(ids):
-            raise ValueError('Lentgh of ids and extras is not the same')
+            raise ValueError('Length of ids and extras is not the same')
         ev = None
         for idx,id in enumerate(ids):
             if extras is not None:
                 ev = extras[idx]
             self._set_custom(id, val, label=label, num=num, append=append,
                              notify=notify, extra=ev)
+        self.dirtied(ids, commit=False)
         self.conn.commit()
 
     def set_custom(self, id, val, label=None, num=None,
                    append=False, notify=True, extra=None, commit=True):
         self._set_custom(id, val, label=label, num=num, append=append,
                          notify=notify, extra=extra)
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
 

src/calibre/library/database2.py

@@ -9,10 +9,12 @@ The database used to store ebook metadata
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re
 from itertools import repeat
 from math import floor
+from Queue import Queue
 
 from PyQt4.QtGui import QImage
 
 from calibre.ebooks.metadata import title_sort, author_to_author_sort
+from calibre.ebooks.metadata.opf2 import metadata_to_opf
 from calibre.library.database import LibraryDatabase
 from calibre.library.field_metadata import FieldMetadata, TagsIcons
 from calibre.library.schema_upgrades import SchemaUpgrade
@@ -126,6 +128,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def __init__(self, library_path, row_factory=False):
         self.field_metadata = FieldMetadata()
+        self.dirtied_queue = Queue()
         if not os.path.exists(library_path):
             os.makedirs(library_path)
         self.listeners = set([])
@@ -337,6 +340,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         setattr(self, 'title_sort', functools.partial(self.get_property,
                 loc=self.FIELD_MAP['sort']))
 
+        d = self.conn.get('SELECT book FROM metadata_dirtied', all=True)
+        for x in d:
+            self.dirtied_queue.put(x[0])
+
         self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
         self.refresh()
         self.last_update_check = self.last_modified()
@@ -363,10 +370,10 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         return row[self.FIELD_MAP['path']].replace('/', os.sep)
 
 
-    def abspath(self, index, index_is_id=False):
+    def abspath(self, index, index_is_id=False, create_dirs=True):
         'Return the absolute path to the directory containing this books files as a unicode string.'
         path = os.path.join(self.library_path, self.path(index, index_is_id=index_is_id))
-        if not os.path.exists(path):
+        if create_dirs and not os.path.exists(path):
             os.makedirs(path)
         return path
 
@@ -550,6 +557,35 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def metadata_for_field(self, key):
         return self.field_metadata[key]
 
+    def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True):
+        for book_id in book_ids:
+            if not self.data.has_id(book_id):
+                continue
+            mi = self.get_metadata(book_id, index_is_id=True, get_cover=True)
+            # Always set cover to cover.jpg. Even if cover doesn't exist,
+            # no harm done. This way no need to call dirtied when
+            # cover is set/removed
+            mi.cover = 'cover.jpg'
+            raw = metadata_to_opf(mi)
+            path = self.abspath(book_id, index_is_id=True)
+            with open(os.path.join(path, 'metadata.opf'), 'wb') as f:
+                f.write(raw)
+            if remove_from_dirtied:
+                self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
+                        (book_id,))
+        if commit:
+            self.conn.commit()
+        return True
+
+    def dirtied(self, book_ids, commit=True):
+        self.conn.executemany(
+            'INSERT OR REPLACE INTO metadata_dirtied (book) VALUES (?)',
+                [(x,) for x in book_ids])
+        if commit:
+            self.conn.commit()
+        for x in book_ids:
+            self.dirtied_queue.put(x)
+
     def get_metadata(self, idx, index_is_id=False, get_cover=False):
         '''
         Convenience method to return metadata as a :class:`Metadata` object.
@@ -657,7 +693,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
     def has_cover(self, index, index_is_id=False):
         id = index if index_is_id else self.id(index)
         try:
-            path = os.path.join(self.abspath(id, index_is_id=True), 'cover.jpg')
+            path = os.path.join(self.abspath(id, index_is_id=True,
+                create_dirs=False), 'cover.jpg')
         except:
             # Can happen if path has not yet been set
             return False
@@ -1241,6 +1278,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         ss = self.author_sort_from_book(id, index_is_id=True)
         self.conn.execute('UPDATE books SET author_sort=? WHERE id=?',
                           (ss, id))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['authors'],
@@ -1267,6 +1305,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         else:
             self.data.set(id, self.FIELD_MAP['sort'], title, row_is_id=True)
         self.set_path(id, index_is_id=True)
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         if notify:
@@ -1276,6 +1315,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if dt:
             self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['timestamp'], dt, row_is_id=True)
+            self.dirtied([id], commit=False)
             if commit:
                 self.conn.commit()
             if notify:
@@ -1285,6 +1325,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if dt:
             self.conn.execute('UPDATE books SET pubdate=? WHERE id=?', (dt, id))
             self.data.set(id, self.FIELD_MAP['pubdate'], dt, row_is_id=True)
+            self.dirtied([id], commit=False)
             if commit:
                 self.conn.commit()
             if notify:
@@ -1303,6 +1344,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 aid = self.conn.execute('INSERT INTO publishers(name) VALUES (?)', (publisher,)).lastrowid
             self.conn.execute('INSERT INTO books_publishers_link(book, publisher) VALUES (?,?)', (id, aid))
+            self.dirtied([id], commit=False)
             if commit:
                 self.conn.commit()
             self.data.set(id, self.FIELD_MAP['publisher'], publisher, row_is_id=True)
@@ -1593,6 +1635,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             '''.format(tables[0], tables[1])
             )
         self.conn.executescript(drops)
+        self.dirtied(ids, commit=False)
         self.conn.commit()
 
         for x in ids:
@@ -1638,6 +1681,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                                         (id, tid), all=False):
                 self.conn.execute('INSERT INTO books_tags_link(book, tag) VALUES (?,?)',
                               (id, tid))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         tags = u','.join(self.get_tags(id))
@@ -1692,6 +1736,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             else:
                 aid = self.conn.execute('INSERT INTO series(name) VALUES (?)', (series,)).lastrowid
             self.conn.execute('INSERT INTO books_series_link(book, series) VALUES (?,?)', (id, aid))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['series'], series, row_is_id=True)
@@ -1706,6 +1751,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         except:
             idx = 1.0
         self.conn.execute('UPDATE books SET series_index=? WHERE id=?', (idx, id))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['series_index'], idx, row_is_id=True)
@@ -1718,6 +1764,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         rat = self.conn.get('SELECT id FROM ratings WHERE rating=?', (rating,), all=False)
         rat = rat if rat else self.conn.execute('INSERT INTO ratings(rating) VALUES (?)', (rating,)).lastrowid
         self.conn.execute('INSERT INTO books_ratings_link(book, rating) VALUES (?,?)', (id, rat))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['rating'], rating, row_is_id=True)
@@ -1730,11 +1777,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['comments'], text, row_is_id=True)
+        self.dirtied([id], commit=False)
         if notify:
             self.notify('metadata', [id])
 
     def set_author_sort(self, id, sort, notify=True, commit=True):
         self.conn.execute('UPDATE books SET author_sort=? WHERE id=?', (sort, id))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['author_sort'], sort, row_is_id=True)
@@ -1743,6 +1792,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def set_isbn(self, id, isbn, notify=True, commit=True):
         self.conn.execute('UPDATE books SET isbn=? WHERE id=?', (isbn, id))
+        self.dirtied([id], commit=False)
         if commit:
             self.conn.commit()
         self.data.set(id, self.FIELD_MAP['isbn'], isbn, row_is_id=True)

src/calibre/library/save_to_disk.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, traceback, cStringIO, re, string
+import os, traceback, cStringIO, re
 
 from calibre.utils.config import Config, StringConfig, tweaks
 from calibre.utils.formatter import TemplateFormatter

src/calibre/library/schema_upgrades.py

@@ -397,3 +397,15 @@ class SchemaUpgrade(object):
                                  UNIQUE(key));
         '''
         self.conn.executescript(script)
+
+    def upgrade_version_13(self):
+        'Dirtied table for OPF metadata backups'
+        script = '''
+        DROP TABLE IF EXISTS metadata_dirtied;
+        CREATE TABLE metadata_dirtied(id INTEGER PRIMARY KEY,
+                             book INTEGER NOT NULL,
+                             UNIQUE(book));
+        INSERT INTO metadata_dirtied (book) SELECT id FROM books;
+        '''
+        self.conn.executescript(script)
+

src/calibre/utils/formatter.py

@@ -47,10 +47,12 @@ class TemplateFormatter(string.Formatter):
                     'shorten'       : (3, _shorten),
         }
 
+    format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
+    compress_spaces = re.compile(r'\s+')
+
     def get_value(self, key, args):
         raise Exception('get_value must be implemented in the subclass')
 
-    format_string_re = re.compile(r'^(.*)\|(.*)\|(.*)$')
 
     def _explode_format_string(self, fmt):
         try:
@@ -64,8 +66,10 @@ class TemplateFormatter(string.Formatter):
             return fmt, '', ''
 
     def format_field(self, val, fmt):
+        # Handle conditional text
         fmt, prefix, suffix = self._explode_format_string(fmt)
 
+        # Handle functions
         p = fmt.find('(')
         if p >= 0 and fmt[-1] == ')' and fmt[0:p] in self.functions:
             field = fmt[0:p]
@@ -73,7 +77,7 @@ class TemplateFormatter(string.Formatter):
             args = fmt[p+1:-1].split(',')
             if (func[0] == 0 and (len(args) != 1 or args[0])) or \
                     (func[0] > 0 and func[0] != len(args)):
-                raise Exception ('Incorrect number of arguments for function '+ fmt[0:p])
+                raise ValueError('Incorrect number of arguments for function '+ fmt[0:p])
             if func[0] == 0:
                 val = func[1](self, val)
             else:
@@ -84,8 +88,6 @@ class TemplateFormatter(string.Formatter):
             return ''
         return prefix + val + suffix
 
-    compress_spaces = re.compile(r'\s+')
-
     def vformat(self, fmt, args, kwargs):
         ans = string.Formatter.vformat(self, fmt, args, kwargs)
         return self.compress_spaces.sub(' ', ans).strip()

src/calibre/utils/ipc/server.py

@@ -18,6 +18,7 @@ from calibre.utils.ipc.launch import Worker
 from calibre.utils.ipc.worker import PARALLEL_FUNCS
 from calibre import detect_ncpus as cpu_count
 from calibre.constants import iswindows
+from calibre.ptempfile import base_dir
 
 _counter = 0
 
@@ -114,8 +115,9 @@ class Server(Thread):
         with self._worker_launch_lock:
             self.launched_worker_count += 1
             id = self.launched_worker_count
-        rfile = os.path.join(tempfile.gettempdir(),
+        fd, rfile = tempfile.mkstemp(prefix='ipc_result_%d_%d_'%(self.id, id),
-        'calibre_ipc_result_%d_%d.pickle'%(self.id, id))
+                dir=base_dir(), suffix='.pickle')
+        os.close(fd)
         if redirect_output is None:
             redirect_output = not gui
 
@@ -189,8 +191,12 @@ class Server(Thread):
                     job.failed   = True
                     job.returncode = worker.returncode
                 elif os.path.exists(worker.rfile):
+                    try:
                         job.result = cPickle.load(open(worker.rfile, 'rb'))
                         os.remove(worker.rfile)
+                    except:
+                        import traceback
+                        traceback.print_exc()
                 job.duration = time.time() - job.start_time
                 self.changed_jobs_queue.put(job)
 

src/calibre/utils/pyconsole/__init__.py

@@ -24,6 +24,8 @@ def console_config():
     c = Config('console', desc)
 
     c.add_opt('theme', default='native', help='The color theme')
+    c.add_opt('scrollback', default=10000,
+            help='Max number of lines to keep in the scrollback buffer')
 
     return c
 

src/calibre/utils/pyconsole/console.py

@@ -7,6 +7,7 @@ __docformat__ = 'restructuredtext en'
 
 import sys, textwrap, traceback, StringIO
 from functools import partial
+from codeop import CommandCompiler
 
 from PyQt4.Qt import QTextEdit, Qt, QTextFrameFormat, pyqtSignal, \
     QApplication, QColor, QPalette, QMenu, QActionGroup, QTimer
@@ -16,8 +17,9 @@ from pygments.styles import get_all_styles
 
 from calibre.utils.pyconsole.formatter import Formatter
 from calibre.utils.pyconsole.controller import Controller
+from calibre.utils.pyconsole.history import History
 from calibre.utils.pyconsole import prints, prefs, __appname__, \
-        __version__, error_dialog
+        __version__, error_dialog, dynamic
 
 class EditBlock(object): # {{{
 
@@ -73,6 +75,7 @@ class ThemeMenu(QMenu): # {{{
 
 # }}}
 
+
 class Console(QTextEdit):
 
     running = pyqtSignal()
@@ -114,7 +117,9 @@ class Console(QTextEdit):
             parent=None):
         QTextEdit.__init__(self, parent)
         self.shutting_down = False
+        self.compiler = CommandCompiler()
         self.buf = self.old_buf = []
+        self.history = History([''], dynamic.get('console_history', []))
         self.prompt_frame = None
         self.allow_output = False
         self.prompt_frame_format = QTextFrameFormat()
@@ -122,7 +127,7 @@ class Console(QTextEdit):
         self.prompt_frame_format.setBorderStyle(QTextFrameFormat.BorderStyle_Solid)
         self.prompt_len = len(prompt)
 
-        self.doc.setMaximumBlockCount(10000)
+        self.doc.setMaximumBlockCount(int(prefs['scrollback']))
         self.lexer = PythonLexer(ensurenl=False)
         self.tb_lexer = PythonTracebackLexer()
 
@@ -139,6 +144,8 @@ class Console(QTextEdit):
         self.key_dispatcher = { # {{{
                 Qt.Key_Enter : self.enter_pressed,
                 Qt.Key_Return : self.enter_pressed,
+                Qt.Key_Up : self.up_pressed,
+                Qt.Key_Down : self.down_pressed,
                 Qt.Key_Home : self.home_pressed,
                 Qt.Key_End : self.end_pressed,
                 Qt.Key_Left : self.left_pressed,
@@ -153,15 +160,17 @@ class Console(QTextEdit):
         '''.format(sys.version.splitlines()[0], __appname__,
             __version__))
 
+        sys.excepthook = self.unhandled_exception
+
         self.controllers = []
         QTimer.singleShot(0, self.launch_controller)
 
-        sys.excepthook = self.unhandled_exception
 
         with EditBlock(self.cursor):
             self.render_block(motd)
 
     def shutdown(self):
+        dynamic.set('console_history', self.history.serialize())
         self.shutton_down = True
         for c in self.controllers:
             c.kill()
@@ -365,7 +374,7 @@ class Console(QTextEdit):
 
     # }}}
 
-    # Keyboard handling {{{
+    # Keyboard management {{{
 
     def keyPressEvent(self, ev):
         text = unicode(ev.text())
@@ -394,6 +403,20 @@ class Console(QTextEdit):
             self.setTextCursor(c)
         self.ensureCursorVisible()
 
+    def up_pressed(self):
+        lineno, pos = self.cursor_pos
+        if lineno < 0: return
+        if lineno == 0:
+            b = self.history.back()
+            if b is not None:
+                self.set_prompt(b)
+        else:
+            c = self.cursor
+            c.movePosition(c.Up)
+            self.setTextCursor(c)
+        self.ensureCursorVisible()
+
+
     def backspace_pressed(self):
         lineno, pos = self.cursor_pos
         if lineno < 0: return
@@ -414,7 +437,6 @@ class Console(QTextEdit):
         lineno, pos = self.cursor_pos
         if lineno < 0: return
         c = self.cursor
-        lineno, pos = self.cursor_pos
         cp = list(self.prompt(False))
         if pos < len(cp[lineno]):
             c.movePosition(c.NextCharacter)
@@ -423,6 +445,22 @@ class Console(QTextEdit):
         self.setTextCursor(c)
         self.ensureCursorVisible()
 
+    def down_pressed(self):
+        lineno, pos = self.cursor_pos
+        if lineno < 0: return
+        c = self.cursor
+        cp = list(self.prompt(False))
+        if lineno >= len(cp) - 1:
+            b = self.history.forward()
+            if b is not None:
+                self.set_prompt(b)
+        else:
+            c = self.cursor
+            c.movePosition(c.Down)
+            self.setTextCursor(c)
+        self.ensureCursorVisible()
+
+
     def home_pressed(self):
         if self.prompt_frame is not None:
             mods = QApplication.keyboardModifiers()
@@ -454,6 +492,19 @@ class Console(QTextEdit):
             return self.no_controller_error()
         cp = list(self.prompt())
         if cp[0]:
+            try:
+                ret = self.compiler('\n'.join(cp))
+            except:
+                pass
+            else:
+                if ret is None:
+                    c = self.prompt_frame.lastCursorPosition()
+                    c.insertBlock()
+                    self.setTextCursor(c)
+                    self.render_current_prompt()
+                    return
+                else:
+                    self.history.enter(cp)
             self.execute(cp)
 
     def text_typed(self, text):
@@ -461,6 +512,7 @@ class Console(QTextEdit):
             self.move_cursor_to_prompt()
             self.cursor.insertText(text)
             self.render_current_prompt(restore_cursor=True)
+            self.history.current = list(self.prompt())
 
     # }}}
 

src/calibre/utils/pyconsole/controller.py

@@ -104,7 +104,6 @@ class Controller(QThread):
     def returncode(self):
         return self.process.returncode
 
-    @property
     def interrupt(self):
         if hasattr(signal, 'SIGINT'):
             os.kill(self.process.pid, signal.SIGINT)

src/calibre/utils/pyconsole/history.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+__license__   = 'GPL v3'
+__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from collections import deque
+
+class History(object): # {{{
+
+    def __init__(self, current, entries):
+        self.entries = deque(entries, maxlen=max(2000, len(entries)))
+        self.index = len(self.entries) - 1
+        self.current = self.default = current
+        self.last_was_back = False
+
+    def back(self, amt=1):
+        if self.entries:
+            oidx = self.index
+            ans = self.entries[self.index]
+            self.index = max(0, self.index - amt)
+            self.last_was_back = self.index != oidx
+            return ans
+
+    def forward(self, amt=1):
+        if self.entries:
+            d = self.index
+            if self.last_was_back:
+                d += 1
+            if d >= len(self.entries) - 1:
+                self.index = len(self.entries) - 1
+                self.last_was_back = False
+                return self.current
+            if self.last_was_back:
+                amt += 1
+            self.index = min(len(self.entries)-1, self.index + amt)
+            self.last_was_back = False
+            return self.entries[self.index]
+
+    def enter(self, x):
+        try:
+            self.entries.remove(x)
+        except ValueError:
+            pass
+        self.entries.append(x)
+        self.index = len(self.entries) - 1
+        self.current = self.default
+        self.last_was_back = False
+
+    def serialize(self):
+        return list(self.entries)
+
+# }}}
+
+

src/calibre/utils/pyconsole/interpreter.py

@@ -11,6 +11,7 @@ from Queue import Queue, Empty
 from threading import Thread
 from binascii import unhexlify
 from multiprocessing.connection import Client
+from repr import repr as safe_repr
 
 from calibre.utils.pyconsole import preferred_encoding, isbytestring, \
         POLL_TIMEOUT
@@ -35,7 +36,7 @@ def tounicode(raw): # {{{
         try:
             raw = raw.decode(preferred_encoding, 'replace')
         except:
-            raw = repr(raw)
+            raw = safe_repr(raw)
 
         if isbytestring(raw):
             try: