sync with Kovid's branch

recipes/aif_ru.recipe

@@ -1,5 +1,5 @@
 __license__   = 'GPL v3'
-__copyright__ = '2010, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2010 - 2012, Darko Miletic <darko.miletic at gmail.com>'
 '''
 www.aif.ru
 '''
@@ -19,12 +19,19 @@ class AIF_ru(BasicNewsRecipe):
     encoding              = 'cp1251'
     language              = 'ru'
     publication_type      = 'magazine'
-    extra_css             = ' @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: Verdana,Arial,Helvetica,sans1,sans-serif} '
+    masthead_url          = 'http://static3.aif.ru/glossy/index/i/logo.png'
-    keep_only_tags     = [dict(name='div',attrs={'id':'inner'})]
+    extra_css             = """ 
+                                @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} 
+                                body{font-family: Verdana,Arial,Helvetica,sans1,sans-serif}
+                                img{display: block}
+                            """
+    keep_only_tags     = [
+                            dict(name='div',attrs={'class':['content-header', 'zoom']})
+                           ,dict(name='div',attrs={'id':'article-text'})
+                         ]
     remove_tags        = [
-                            dict(name=['iframe','object','link','base','input','img'])
+                            dict(name=['iframe','object','link','base','input','meta'])
-                           ,dict(name='div',attrs={'class':'photo'})
+                           ,dict(name='div',attrs={'class':'in-topic'})
-                           ,dict(name='p',attrs={'class':'resizefont'})
                          ]
 
     feeds       = [(u'News', u'http://www.aif.ru/rss/all.php')]

setup/installer/windows/freeze.py

@@ -19,8 +19,8 @@ QT_DIR = os.environ.get('QT_DIR', 'Q:\\Qt\\4.8.2')
 QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
 LIBUNRAR         = os.environ.get('UNRARDLL', 'C:\\Program Files\\UnrarDLL\\unrar.dll')
 SW               = r'C:\cygwin\home\kovid\sw'
-IMAGEMAGICK      = os.path.join(SW, 'build', 'ImageMagick-6.7.6',
+IMAGEMAGICK = os.path.join(SW, 'build',
-        'VisualMagick', 'bin')
+                            'ImageMagick-*\\VisualMagick\\bin')
 CRT = r'C:\Microsoft.VC90.CRT'
 LZMA = r'Q:\easylzma\build\easylzma-0.0.8'
 
@@ -283,8 +283,9 @@ class Win32Freeze(Command, WixMixIn):
                 shutil.copy2(msrc, self.dll_dir)
 
         # Copy ImageMagick
+        impath = glob.glob(IMAGEMAGICK)[-1]
         for pat in ('*.dll', '*.xml'):
-            for f in glob.glob(self.j(IMAGEMAGICK, pat)):
+            for f in glob.glob(self.j(impath, pat)):
                 ok = True
                 for ex in ('magick++', 'x11.dll', 'xext.dll'):
                     if ex in f.lower(): ok = False

setup/installer/windows/notes.rst

@@ -420,16 +420,32 @@ Run::
 Python Imaging Library
 ------------------------
 
+For 32-bit:
 Install as normal using installer at http://www.lfd.uci.edu/~gohlke/pythonlibs/
 
+For 64-bit:
+Download from http://pypi.python.org/pypi/Pillow/
+Edit setup.py setting the ROOT values, like this::
+
+    SW = r'C:\cygwin\home\kovid\sw'
+    JPEG_ROOT = ZLIB_ROOT = FREETYPE_ROOT = (SW+r'\lib', SW+r'\include')
+
+Build and install with::
+    python setup.py build
+    python setup.py install
+
+Note that the lcms module will not be built. PIL requires lcms-1.x but only
+lcms-2.x can be compiled as a 64 bit library.
+
 Test it on the target system with
 
-calibre-debug -c "from PIL import _imaging, _imagingmath, _imagingft, _imagingcms"
+calibre-debug -c "from PIL import Image; import _imaging, _imagingmath, _imagingft"
-
 
 kdewin32-msvc
 ----------------
 
+I dont think this is needed any more, I've left it here just in case I'm wrong.
+
 Get it from http://www.winkde.org/pub/kde/ports/win32/repository/kdesupport/
 mkdir build
 Run cmake
@@ -448,29 +464,34 @@ cp build/kdewin32-msvc-0.3.9/include/*.h include/
 poppler
 -------------
 
-In Cmake: disable GTK, Qt, OPenjpeg, cpp, lcms, gtk_tests, qt_tests. Enable qt4, jpeg, png and zlib
+mkdir build
 
-NOTE: poppler must be built as a static library, unless you build the qt4 bindings
+Run the cmake GUI which will find the various dependencies automatically.
+On 64 bit cmake might not let you choose Visual Studio 2008, in whcih case
+leave the source field blank, click configure choose Visual Studio 2008 and
+then enter the source field.
 
-cp build/utils/Release/*.exe ../../bin/
+In Cmake: disable GTK, Qt, OPenjpeg, cpp, lcms, gtk_tests, qt_tests. Enable
+jpeg, png and zlib::
 
+    cp build/utils/Release/*.exe ../../bin/
 
 podofo
 ----------
 
+Download from http://podofo.sourceforge.net/download.html
+
 Add the following three lines near the top of CMakeLists.txt
 SET(WANT_LIB64 FALSE)
 SET(PODOFO_BUILD_SHARED TRUE)
 SET(PODOFO_BUILD_STATIC FALSE)
 
-cp build/podofo-*/build/src/Release/podofo.dll bin/
+Run::
-cp build/podofo-*/build/src/Release/podofo.lib lib/
+    cp "`find . -name *.dll`" ~/sw/bin/
-cp build/podofo-*/build/src/Release/podofo.exp lib/
+    cp "`find . -name *.lib`" ~/sw/lib/
-
+    mkdir ~/sw/include/podofo
-cp build/podofo-*/build/podofo_config.h include/podofo/
+    cp build/podofo_config.h ~/sw/include/podofo
-cp -r build/podofo-*/src/* include/podofo/
+    cp -r src/* ~/sw/include/podofo/
-
-You have to use >=0.9.1
 
 
 ImageMagick
@@ -493,7 +514,7 @@ Undefine ProvideDllMain and MAGICKCORE_X11_DELEGATE
 Now open VisualMagick/VisualDynamicMT.sln set to Release
 Remove the CORE_xlib, UTIL_Imdisplay and CORE_Magick++ projects.
 
-F7 for build project, you will get one error due to the removal of xlib, ignore
+F7 for build solution, you will get one error due to the removal of xlib, ignore
 it.
 
 netifaces
@@ -503,10 +524,10 @@ Download the source tarball from http://alastairs-place.net/projects/netifaces/
 
 Rename netifaces.c to netifaces.cpp and make the same change in setup.py
 
-Run 
+Run:: 
+    python setup.py build
+    cp `find build/ -name *.pyd` /cygdrive/c/Python27/Lib/site-packages/
 
-python setup.py build
-cp build/lib.win32-2.7/netifaces.pyd /cygdrive/c/Python27/Lib/site-packages/
 
 psutil
 --------

setup/installer/windows/util.c

@@ -217,19 +217,15 @@ wchar_t* get_app_dirw() {
 
 
 void load_python_dll() {
-    char *app_dir, *fc_dir, *fc_file, *dll_dir, *qt_plugin_dir;
+    char *app_dir, *dll_dir, *qt_plugin_dir;
     size_t l;
 
     app_dir = get_app_dir();
     l = strlen(app_dir)+20;
     dll_dir = (char*) calloc(l, sizeof(char));
-    fc_dir  = (char*) calloc(l, sizeof(char));
-    fc_file = (char*) calloc(l, sizeof(char));
     qt_plugin_dir = (char*) calloc(l, sizeof(char));
-    if (!dll_dir || !qt_plugin_dir || !fc_dir) ExitProcess(_show_error(L"Out of memory", L"", 1));
+    if (!dll_dir || !qt_plugin_dir) ExitProcess(_show_error(L"Out of memory", L"", 1));
     _snprintf_s(dll_dir, l, _TRUNCATE, "%sDLLs", app_dir);
-    _snprintf_s(fc_dir,  l, _TRUNCATE, "%sfontconfig", app_dir);
-    _snprintf_s(fc_file, l, _TRUNCATE, "%s\\fonts.conf", fc_dir);
     _snprintf_s(qt_plugin_dir, l, _TRUNCATE, "%sqt_plugins", app_dir);
     free(app_dir);
 
@@ -237,8 +233,6 @@ void load_python_dll() {
     _putenv_s("MAGICK_CONFIGURE_PATH", dll_dir);
     _putenv_s("MAGICK_CODER_MODULE_PATH", dll_dir);
     _putenv_s("MAGICK_FILTER_MODULE_PATH", dll_dir);
-    _putenv_s("FC_CONFIG_DIR", fc_dir);
-    _putenv_s("FC_CONFIG_FILE", fc_file);
     _putenv_s("QT_PLUGIN_PATH", qt_plugin_dir);
 
     if (!SetDllDirectoryA(dll_dir)) ExitProcess(show_last_error(L"Failed to set DLL directory."));

src/calibre/constants.py

@@ -14,14 +14,6 @@ Various run time constants.
 
 import sys, locale, codecs, os, importlib, collections
 
-_tc = None
-def terminal_controller():
-    global _tc
-    if _tc is None:
-        from calibre.utils.terminfo import TerminalController
-        _tc = TerminalController(sys.stdout)
-    return _tc
-
 _plat = sys.platform.lower()
 iswindows = 'win32' in _plat or 'win64' in _plat
 isosx     = 'darwin' in _plat
@@ -37,6 +29,8 @@ isportable = os.environ.get('CALIBRE_PORTABLE_BUILD', None) is not None
 ispy3 = sys.version_info.major > 2
 isxp = iswindows and sys.getwindowsversion().major < 6
 isworker = os.environ.has_key('CALIBRE_WORKER') or os.environ.has_key('CALIBRE_SIMPLE_WORKER')
+if isworker:
+    os.environ.pop('CALIBRE_FORCE_ANSI', None)
 
 try:
     preferred_encoding = locale.getpreferredencoding()

src/calibre/customize/builtins.py

@@ -1716,7 +1716,7 @@ if __name__ == '__main__':
         ret = 0
 
         for x in ('lxml', 'calibre.ebooks.BeautifulSoup', 'uuid',
-            'calibre.utils.terminfo', 'calibre.utils.magick', 'PIL', 'Image',
+            'calibre.utils.terminal', 'calibre.utils.magick', 'PIL', 'Image',
             'sqlite3', 'mechanize', 'httplib', 'xml'):
             if x in sys.modules:
                 ret = 1

src/calibre/debug.py

@@ -37,9 +37,6 @@ Run an embedded python interpreter.
                       help='Run the ebook viewer',)
     parser.add_option('--paths', default=False, action='store_true',
             help='Output the paths necessary to setup the calibre environment')
-    parser.add_option('--migrate', action='store_true', default=False,
-                      help='Migrate old database. Needs two arguments. Path '
-                           'to library1.db and path to new library folder.')
     parser.add_option('--add-simple-plugin', default=None,
             help='Add a simple plugin (i.e. a plugin that consists of only a '
             '.py file), by specifying the path to the py file containing the '
@@ -118,28 +115,6 @@ def reinit_db(dbpath, callback=None, sql_dump=None):
             os.remove(dest)
     prints('Database successfully re-initialized')
 
-def migrate(old, new):
-    from calibre.utils.config import prefs
-    from calibre.library.database import LibraryDatabase
-    from calibre.library.database2 import LibraryDatabase2
-    from calibre.utils.terminfo import ProgressBar
-    from calibre.constants import terminal_controller
-    class Dummy(ProgressBar):
-        def setLabelText(self, x): pass
-        def setAutoReset(self, y): pass
-        def reset(self): pass
-        def setRange(self, min, max):
-            self.min = min
-            self.max = max
-        def setValue(self, val):
-            self.update(float(val)/getattr(self, 'max', 1))
-
-    db = LibraryDatabase(old)
-    db2 = LibraryDatabase2(new)
-    db2.migrate_old(db, Dummy(terminal_controller(), 'Migrating database...'))
-    prefs['library_path'] = os.path.abspath(new)
-    print 'Database migrated to', os.path.abspath(new)
-
 def debug_device_driver():
     from calibre.devices import debug
     debug(ioreg_to_tmp=True, buf=sys.stdout)
@@ -249,11 +224,6 @@ def main(args=sys.argv):
         exec opts.command
     elif opts.debug_device_driver:
         debug_device_driver()
-    elif opts.migrate:
-        if len(args) < 3:
-            print 'You must specify the path to library1.db and the path to the new library folder'
-            return 1
-        migrate(args[1], args[2])
     elif opts.add_simple_plugin is not None:
         add_simple_plugin(opts.add_simple_plugin)
     elif opts.paths:

src/calibre/devices/apple/driver.py

@@ -240,6 +240,7 @@ class ITUNES(DriverBase):
 
     # iTunes enumerations
     Audiobooks = [
+        'AAC audio file',
         'Audible file',
         'MPEG audio file',
         'Protected AAC audio file'

src/calibre/devices/cli.py

@@ -11,7 +11,6 @@ from optparse import OptionParser
 
 from calibre import __version__, __appname__, human_readable
 from calibre.devices.errors import PathError
-from calibre.utils.terminfo import TerminalController
 from calibre.devices.errors import ArgumentError, DeviceError, DeviceLocked
 from calibre.customize.ui import device_plugins
 from calibre.devices.scanner import DeviceScanner
@@ -20,8 +19,7 @@ from calibre.utils.config import device_prefs
 MINIMUM_COL_WIDTH = 12 #: Minimum width of columns in ls output
 
 class FileFormatter(object):
-    def __init__(self, file, term):
+    def __init__(self, file):
-        self.term = term
         self.is_dir      = file.is_dir
         self.is_readonly = file.is_readonly
         self.size        = file.size
@@ -94,7 +92,7 @@ def info(dev):
     print "Software version:", info[2]
     print "Mime type:       ", info[3]
 
-def ls(dev, path, term, recurse=False, color=False, human_readable_size=False, ll=False, cols=0):
+def ls(dev, path, recurse=False, human_readable_size=False, ll=False, cols=0):
     def col_split(l, cols): # split list l into columns
         rows = len(l) / cols
         if len(l) % cols:
@@ -126,14 +124,13 @@ def ls(dev, path, term, recurse=False, color=False, human_readable_size=False, l
             for file in files:
                 size = len(str(file.size))
                 if human_readable_size:
-                    file = FileFormatter(file, term)
+                    file = FileFormatter(file)
                     size = len(file.human_readable_size)
                 if size > maxlen: maxlen = size
         for file in files:
-            file = FileFormatter(file, term)
+            file = FileFormatter(file)
             name = file.name if ll else file.isdir_name
             lsoutput.append(name)
-            if color: name = file.name_in_color
             lscoloutput.append(name)
             if ll:
                 size = str(file.size)
@@ -173,10 +170,8 @@ def shutdown_plugins():
             pass
 
 def main():
-    term = TerminalController()
+    from calibre.utils.terminal import geometry
-    cols = term.COLS
+    cols = geometry()[0]
-    if not cols: # On windows terminal width is unknown
-        cols = 80
 
     parser = OptionParser(usage="usage: %prog [options] command args\n\ncommand "+
             "is one of: info, books, df, ls, cp, mkdir, touch, cat, rm, eject, test_file\n\n"+
@@ -260,7 +255,6 @@ def main():
             dev.mkdir(args[0])
         elif command == "ls":
             parser = OptionParser(usage="usage: %prog ls [options] path\nList files on the device\n\npath must begin with / or card:/")
-            parser.add_option("--color", help="show ls output in color", dest="color", action="store_true", default=False)
             parser.add_option("-l", help="In addition to the name of each file, print the file type, permissions, and  timestamp  (the  modification time, in the local timezone). Times are local.", dest="ll", action="store_true", default=False)
             parser.add_option("-R", help="Recursively list subdirectories encountered. /dev and /proc are omitted", dest="recurse", action="store_true", default=False)
             parser.remove_option("-h")
@@ -269,7 +263,7 @@ def main():
             if len(args) != 1:
                 parser.print_help()
                 return 1
-            print ls(dev, args[0], term, color=options.color, recurse=options.recurse, ll=options.ll, human_readable_size=options.hrs, cols=cols),
+            print ls(dev, args[0], recurse=options.recurse, ll=options.ll, human_readable_size=options.hrs, cols=cols),
         elif command == "info":
             info(dev)
         elif command == "cp":

src/calibre/ebooks/oeb/display/extract.coffee

@@ -0,0 +1,53 @@
+#!/usr/bin/env coffee
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+
+###
+ Copyright 2012, Kovid Goyal <kovid at kovidgoyal.net>
+ Released under the GPLv3 License
+###
+
+if window?.calibre_utils
+    log = window.calibre_utils.log
+
+merge = (node, cnode) ->
+    rules = node.ownerDocument.defaultView.getMatchedCSSRules(node, '')
+    if rules
+        for rule in rules
+            style = rule.style
+            for name in style
+                val = style.getPropertyValue(name)
+                if val and not cnode.style.getPropertyValue(name)
+                    cnode.style.setProperty(name, val)
+
+inline_styles = (node) ->
+    cnode = node.cloneNode(true)
+    merge(node, cnode)
+    nl = node.getElementsByTagName('*')
+    cnl = cnode.getElementsByTagName('*')
+    for node, i in nl
+        merge(node, cnl[i])
+
+    return cnode
+
+class CalibreExtract
+    # This class is a namespace to expose functions via the
+    # window.calibre_extract object.
+
+    constructor: () ->
+        if not this instanceof arguments.callee
+            throw new Error('CalibreExtract constructor called as function')
+        this.marked_node = null
+
+    mark: (node) =>
+        this.marked_node = node
+
+    extract: (node=null) =>
+        if node == null
+            node = this.marked_node
+        cnode = inline_styles(node)
+        return cnode.outerHTML
+
+if window?
+    window.calibre_extract = new CalibreExtract()
+
+

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -313,7 +313,7 @@ class CSSFlattener(object):
                 if val in ('middle', 'bottom', 'top'):
                     cssdict['vertical-align'] = val
                 elif val in ('left', 'right'):
-                    cssdict['text-align'] = val
+                    cssdict['float'] = val
             del node.attrib['align']
         if node.tag == XHTML('font'):
             tags = ['descendant::h:%s'%x for x in ('p', 'div', 'table', 'h1',

src/calibre/gui2/viewer/documentview.py

@@ -12,7 +12,7 @@ from PyQt4.Qt import (QSize, QSizePolicy, QUrl, SIGNAL, Qt, pyqtProperty,
         QPainter, QPalette, QBrush, QDialog, QColor, QPoint, QImage, QRegion,
         QIcon, pyqtSignature, QAction, QMenu, QString, pyqtSignal,
         QSwipeGesture, QApplication, pyqtSlot)
-from PyQt4.QtWebKit import QWebPage, QWebView, QWebSettings
+from PyQt4.QtWebKit import QWebPage, QWebView, QWebSettings, QWebElement
 
 from calibre.gui2.viewer.flip import SlideFlip
 from calibre.gui2.shortcuts import Shortcuts
@@ -24,6 +24,7 @@ from calibre.gui2.viewer.javascript import JavaScriptLoader
 from calibre.gui2.viewer.position import PagePosition
 from calibre.gui2.viewer.config import config, ConfigDialog, load_themes
 from calibre.gui2.viewer.image_popup import ImagePopup
+from calibre.gui2.viewer.table_popup import TablePopup
 from calibre.ebooks.oeb.display.webview import load_html
 from calibre.constants import isxp, iswindows
 # }}}
@@ -31,6 +32,7 @@ from calibre.constants import isxp, iswindows
 class Document(QWebPage): # {{{
 
     page_turn = pyqtSignal(object)
+    mark_element = pyqtSignal(QWebElement)
 
     def set_font_settings(self, opts):
         settings = self.settings()
@@ -182,6 +184,7 @@ class Document(QWebPage): # {{{
             ensure_ascii=False)))
         for pl in self.all_viewer_plugins:
             pl.load_javascript(evaljs)
+        evaljs('py_bridge.mark_element.connect(window.calibre_extract.mark)')
 
     @pyqtSignature("")
     def animated_scroll_done(self):
@@ -448,6 +451,10 @@ class Document(QWebPage): # {{{
                 self.height+amount)
         self.setPreferredContentsSize(s)
 
+    def extract_node(self):
+        return unicode(self.mainFrame().evaluateJavaScript(
+            'window.calibre_extract.extract()').toString())
+
 # }}}
 
 class DocumentView(QWebView): # {{{
@@ -496,8 +503,11 @@ class DocumentView(QWebView): # {{{
         self.dictionary_action.triggered.connect(self.lookup)
         self.addAction(self.dictionary_action)
         self.image_popup = ImagePopup(self)
+        self.table_popup = TablePopup(self)
         self.view_image_action = QAction(QIcon(I('view-image.png')), _('View &image...'), self)
         self.view_image_action.triggered.connect(self.image_popup)
+        self.view_table_action = QAction(QIcon(I('view.png')), _('View &table...'), self)
+        self.view_table_action.triggered.connect(self.popup_table)
         self.search_action = QAction(QIcon(I('dictionary.png')),
                 _('&Search for next occurrence'), self)
         self.search_action.setShortcut(Qt.CTRL+Qt.Key_S)
@@ -603,10 +613,26 @@ class DocumentView(QWebView): # {{{
         t = t.replace(u'&', u'&&')
         return _("S&earch Google for '%s'")%t
 
+    def popup_table(self):
+        html = self.document.extract_node()
+        self.table_popup(html, QUrl.fromLocalFile(self.last_loaded_path),
+                         self.document.font_magnification_step)
+
     def contextMenuEvent(self, ev):
         mf = self.document.mainFrame()
         r = mf.hitTestContent(ev.pos())
         img = r.pixmap()
+        elem = r.element()
+        if elem.isNull():
+            elem = r.enclosingBlockElement()
+        table = None
+        parent = elem
+        while not parent.isNull():
+            if (unicode(parent.tagName()) == u'table' or
+                unicode(parent.localName()) == u'table'):
+                table = parent
+                break
+            parent = parent.parent()
         self.image_popup.current_img = img
         self.image_popup.current_url = r.imageUrl()
         menu = self.document.createStandardContextMenu()
@@ -615,6 +641,9 @@ class DocumentView(QWebView): # {{{
 
         if not img.isNull():
             menu.addAction(self.view_image_action)
+        if table is not None:
+            self.document.mark_element.emit(table)
+            menu.addAction(self.view_table_action)
 
         text = self._selectedText()
         if text and img.isNull():

src/calibre/gui2/viewer/javascript.py

@@ -34,11 +34,12 @@ class JavaScriptLoader(object):
             'utils':'ebooks.oeb.display.utils',
             'fs':'ebooks.oeb.display.full_screen',
             'math': 'ebooks.oeb.display.mathjax',
+            'extract': 'ebooks.oeb.display.extract',
         }
 
     ORDER = ('jquery', 'jquery_scrollTo', 'bookmarks', 'referencing', 'images',
             'hyphenation', 'hyphenator', 'utils', 'cfi', 'indexing', 'paged',
-            'fs', 'math')
+            'fs', 'math', 'extract')
 
 
     def __init__(self, dynamic_coffeescript=False):

src/calibre/gui2/viewer/table_popup.py

@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QDialog, QDialogButtonBox, QVBoxLayout, QApplication,
+                      QSize, QIcon, Qt)
+from PyQt4.QtWebKit import QWebView
+
+from calibre.gui2 import gprefs, error_dialog
+
+class TableView(QDialog):
+
+    def __init__(self, parent, font_magnification_step):
+        QDialog.__init__(self, parent)
+        self.font_magnification_step = font_magnification_step
+        dw = QApplication.instance().desktop()
+        self.avail_geom = dw.availableGeometry(parent)
+
+        self.view = QWebView(self)
+        self.bb = bb = QDialogButtonBox(QDialogButtonBox.Close)
+        bb.accepted.connect(self.accept)
+        bb.rejected.connect(self.reject)
+        self.zi_button = zi = bb.addButton(_('Zoom &in'), bb.ActionRole)
+        self.zo_button = zo = bb.addButton(_('Zoom &out'), bb.ActionRole)
+        zi.setIcon(QIcon(I('plus.png')))
+        zo.setIcon(QIcon(I('minus.png')))
+        zi.clicked.connect(self.zoom_in)
+        zo.clicked.connect(self.zoom_out)
+
+        self.l = l = QVBoxLayout()
+        self.setLayout(l)
+        l.addWidget(self.view)
+        l.addWidget(bb)
+
+    def zoom_in(self):
+        self.view.setZoomFactor(self.view.zoomFactor() +
+                                self.font_magnification_step)
+
+    def zoom_out(self):
+        self.view.setZoomFactor(max(0.1, self.view.zoomFactor()
+                                    -self.font_magnification_step))
+
+    def __call__(self, html, baseurl):
+        self.view.setHtml(
+            '<!DOCTYPE html><html><body bgcolor="white">%s<body></html>'%html,
+            baseurl)
+        geom = self.avail_geom
+        self.resize(QSize(int(geom.width()/2.5), geom.height()-50))
+        geom = gprefs.get('viewer_table_popup_geometry', None)
+        if geom is not None:
+            self.restoreGeometry(geom)
+        self.setWindowTitle(_('View Table'))
+        self.show()
+
+    def done(self, e):
+        gprefs['viewer_table_popup_geometry'] = bytearray(self.saveGeometry())
+        return QDialog.done(self, e)
+
+class TablePopup(object):
+
+    def __init__(self, parent):
+        self.parent = parent
+        self.dialogs = []
+
+    def __call__(self, html, baseurl, font_magnification_step):
+        if not html:
+            return error_dialog(self.parent, _('No table found'),
+                _('No table was found'), show=True)
+        d = TableView(self.parent, font_magnification_step)
+        self.dialogs.append(d)
+        d.finished.connect(self.cleanup, type=Qt.QueuedConnection)
+        d(html, baseurl)
+
+    def cleanup(self):
+        for d in tuple(self.dialogs):
+            if not d.isVisible():
+                self.dialogs.remove(d)
+

src/calibre/library/cli.py

@@ -65,8 +65,7 @@ def get_db(dbpath, options):
 
 def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator,
             prefix, subtitle='Books in the calibre database'):
-    from calibre.constants import terminal_controller as tc
+    from calibre.utils.terminal import ColoredStream, geometry
-    terminal_controller = tc()
     if sort_by:
         db.sort(sort_by, ascending)
     if search_text:
@@ -101,7 +100,7 @@ def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, se
         for j, field in enumerate(fields):
             widths[j] = max(widths[j], len(unicode(i[field])))
 
-    screen_width = terminal_controller.COLS if line_width < 0 else line_width
+    screen_width = geometry()[0] if line_width < 0 else line_width
     if not screen_width:
         screen_width = 80
     field_width = screen_width//len(fields)
@@ -120,7 +119,8 @@ def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, se
     widths = list(base_widths)
     titles = map(lambda x, y: '%-*s%s'%(x-len(separator), y, separator),
             widths, title_fields)
-    print terminal_controller.GREEN + ''.join(titles)+terminal_controller.NORMAL
+    with ColoredStream(sys.stdout, fg='green'):
+        print ''.join(titles)
 
     wrappers = map(lambda x: TextWrapper(x-1), widths)
     o = cStringIO.StringIO()
@@ -1288,8 +1288,7 @@ def command_list_categories(args, dbpath):
     fields = ['category', 'tag_name', 'count', 'rating']
 
     def do_list():
-        from calibre.constants import terminal_controller as tc
+        from calibre.utils.terminal import geometry, ColoredStream
-        terminal_controller = tc()
 
         separator = ' '
         widths = list(map(lambda x : 0, fields))
@@ -1297,7 +1296,7 @@ def command_list_categories(args, dbpath):
             for j, field in enumerate(fields):
                 widths[j] = max(widths[j], max(len(field), len(unicode(i[field]))))
 
-        screen_width = terminal_controller.COLS if opts.width < 0 else opts.width
+        screen_width = geometry()[0]
         if not screen_width:
             screen_width = 80
         field_width = screen_width//len(fields)
@@ -1316,7 +1315,8 @@ def command_list_categories(args, dbpath):
         widths = list(base_widths)
         titles = map(lambda x, y: '%-*s%s'%(x-len(separator), y, separator),
                 widths, fields)
-        print terminal_controller.GREEN + ''.join(titles)+terminal_controller.NORMAL
+        with ColoredStream(sys.stdout, fg='green'):
+            print ''.join(titles)
 
         wrappers = map(lambda x: TextWrapper(x-1), widths)
         o = cStringIO.StringIO()

src/calibre/test_build.py

@@ -74,8 +74,13 @@ def test_imaging():
         print ('ImageMagick OK!')
     else:
         raise RuntimeError('ImageMagick choked!')
-    from PIL import Image, _imaging, _imagingmath, _imagingft, _imagingcms
+    from PIL import Image
-    _imaging, _imagingmath, _imagingft, _imagingcms
+    try:
+        import _imaging, _imagingmath, _imagingft
+        _imaging, _imagingmath, _imagingft
+    except ImportError:
+        from PIL import _imaging, _imagingmath, _imagingft
+    _imaging, _imagingmath, _imagingft
     i = Image.open(cStringIO.StringIO(data))
     if i.size < (20, 20):
         raise RuntimeError('PIL choked!')

src/calibre/utils/config.py

@@ -12,7 +12,7 @@ from optparse import OptionParser as _OptionParser, OptionGroup
 from optparse import IndentedHelpFormatter
 
 from calibre.constants import (config_dir, CONFIG_DIR_MODE, __appname__,
-        __version__, __author__, terminal_controller)
+        __version__, __author__)
 from calibre.utils.lock import ExclusiveFile
 from calibre.utils.config_base import (make_config_dir, Option, OptionValues,
         OptionSet, ConfigInterface, Config, prefs, StringConfig, ConfigProxy,
@@ -30,28 +30,28 @@ def check_config_write_access():
 class CustomHelpFormatter(IndentedHelpFormatter):
 
     def format_usage(self, usage):
-        tc = terminal_controller()
+        from calibre.utils.terminal import colored
-        return "%s%s%s: %s\n" % (tc.BLUE, _('Usage'), tc.NORMAL, usage)
+        return colored(_('Usage'), fg='blue', bold=True) + ': ' + usage
 
     def format_heading(self, heading):
-        tc = terminal_controller()
+        from calibre.utils.terminal import colored
-        return "%*s%s%s%s:\n" % (self.current_indent, tc.BLUE,
+        return "%*s%s:\n" % (self.current_indent, '',
-                                 "", heading, tc.NORMAL)
+                                 colored(heading, fg='blue', bold=True))
 
     def format_option(self, option):
         import textwrap
-        tc = terminal_controller()
+        from calibre.utils.terminal import colored
 
         result = []
         opts = self.option_strings[option]
         opt_width = self.help_position - self.current_indent - 2
         if len(opts) > opt_width:
             opts = "%*s%s\n" % (self.current_indent, "",
-                                    tc.GREEN+opts+tc.NORMAL)
+                                    colored(opts, fg='green'))
             indent_first = self.help_position
         else:                       # start help on same line as opts
             opts = "%*s%-*s  " % (self.current_indent, "", opt_width +
-                    len(tc.GREEN + tc.NORMAL), tc.GREEN + opts + tc.NORMAL)
+                    len(colored('', fg='green')), colored(opts, fg='green'))
             indent_first = 0
         result.append(opts)
         if option.help:
@@ -78,11 +78,11 @@ class OptionParser(_OptionParser):
                  conflict_handler='resolve',
                  **kwds):
         import textwrap
-        tc = terminal_controller()
+        from calibre.utils.terminal import colored
 
         usage = textwrap.dedent(usage)
         if epilog is None:
-            epilog = _('Created by ')+tc.RED+__author__+tc.NORMAL
+            epilog = _('Created by ')+colored(__author__, fg='cyan')
         usage += '\n\n'+_('''Whenever you pass arguments to %prog that have spaces in them, '''
                  '''enclose the arguments in quotation marks.''')
         _OptionParser.__init__(self, usage=usage, version=version, epilog=epilog,
@@ -95,6 +95,21 @@ class OptionParser(_OptionParser):
             _("show this help message and exit")
             _("show program's version number and exit")
 
+    def print_usage(self, file=None):
+        from calibre.utils.terminal import ANSIStream
+        s = ANSIStream(file)
+        _OptionParser.print_usage(self, file=s)
+
+    def print_help(self, file=None):
+        from calibre.utils.terminal import ANSIStream
+        s = ANSIStream(file)
+        _OptionParser.print_help(self, file=s)
+
+    def print_version(self, file=None):
+        from calibre.utils.terminal import ANSIStream
+        s = ANSIStream(file)
+        _OptionParser.print_version(self, file=s)
+
     def error(self, msg):
         if self.gui_mode:
             raise Exception(msg)

src/calibre/utils/logging.py

@@ -33,21 +33,18 @@ class ANSIStream(Stream):
 
     def __init__(self, stream=sys.stdout):
         Stream.__init__(self, stream)
-        from calibre.utils.terminfo import TerminalController
-        tc = TerminalController(stream)
         self.color = {
-                      DEBUG: bytes(tc.GREEN),
+                      DEBUG: u'green',
-                      INFO: bytes(''),
+                      INFO: None,
-                      WARN: bytes(tc.YELLOW),
+                      WARN: u'yellow',
-                      ERROR: bytes(tc.RED)
+                      ERROR: u'red',
                       }
-        self.normal = bytes(tc.NORMAL)
 
     def prints(self, level, *args, **kwargs):
-        self.stream.write(self.color[level])
         kwargs['file'] = self.stream
-        self._prints(*args, **kwargs)
+        from calibre.utils.terminal import ColoredStream
-        self.stream.write(self.normal)
+        with ColoredStream(self.stream, self.color[level]):
+            self._prints(*args, **kwargs)
 
     def flush(self):
         self.stream.flush()

src/calibre/utils/terminal.py

@@ -0,0 +1,285 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:fdm=marker:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, sys, re
+from itertools import izip
+
+from calibre.constants import iswindows
+
+def fmt(code):
+    return ('\033[%dm'%code).encode('ascii')
+
+RATTRIBUTES = dict(
+        izip(xrange(1, 9), (
+            'bold',
+            'dark',
+            '',
+            'underline',
+            'blink',
+            '',
+            'reverse',
+            'concealed'
+            )
+        ))
+ATTRIBUTES = {v:fmt(k) for k, v in RATTRIBUTES.iteritems()}
+del ATTRIBUTES['']
+
+RBACKGROUNDS = dict(
+        izip(xrange(41, 48), (
+            'red',
+            'green',
+            'yellow',
+            'blue',
+            'magenta',
+            'cyan',
+            'white'
+            ),
+    ))
+BACKGROUNDS = {v:fmt(k) for k, v in RBACKGROUNDS.iteritems()}
+
+RCOLORS = dict(
+        izip(xrange(31, 38), (
+            'red',
+            'green',
+            'yellow',
+            'blue',
+            'magenta',
+            'cyan',
+            'white',
+            ),
+        ))
+COLORS = {v:fmt(k) for k, v in RCOLORS.iteritems()}
+
+RESET = fmt(0)
+
+if iswindows:
+    # From wincon.h
+    WCOLORS = {c:i for i, c in enumerate((
+        'black', 'blue', 'green', 'cyan', 'red', 'magenta', 'yellow', 'white'))}
+
+    def to_flag(fg, bg, bold):
+        val = 0
+        if bold:
+            val |= 0x08
+        if fg in WCOLORS:
+            val |= WCOLORS[fg]
+        if bg in WCOLORS:
+            val |= (WCOLORS[bg] << 4)
+        return val
+
+def colored(text, fg=None, bg=None, bold=False):
+    prefix = []
+    if fg is not None:
+        prefix.append(COLORS[fg])
+    if bg is not None:
+        prefix.append(BACKGROUNDS[bg])
+    if bold:
+        prefix.append(ATTRIBUTES['bold'])
+    prefix = b''.join(prefix)
+    suffix = RESET
+    if isinstance(text, type(u'')):
+        prefix = prefix.decode('ascii')
+        suffix = suffix.decode('ascii')
+    return prefix + text + suffix
+
+class Detect(object):
+
+    def __init__(self, stream):
+        self.stream = stream or sys.stdout
+        self.isatty = getattr(self.stream, 'isatty', lambda : False)()
+        force_ansi = os.environ.has_key('CALIBRE_FORCE_ANSI')
+        if not self.isatty and force_ansi:
+            self.isatty = True
+        self.isansi = force_ansi or not iswindows
+        self.set_console = None
+        if not self.isansi:
+            try:
+                import msvcrt
+                self.msvcrt = msvcrt
+                self.file_handle = msvcrt.get_osfhandle(self.stream.fileno())
+                from ctypes import windll
+                self.set_console = windll.kernel32.SetConsoleTextAttribute
+            except:
+                pass
+
+class ColoredStream(Detect):
+
+    def __init__(self, stream=None, fg=None, bg=None, bold=False):
+        super(ColoredStream, self).__init__(stream)
+        self.fg, self.bg, self.bold = fg, bg, bold
+        if self.set_console is not None:
+            self.wval = to_flag(self.fg, self.bg, bold)
+
+    def __enter__(self):
+        if not self.isatty:
+            return
+        if self.isansi:
+            if self.bold:
+                self.stream.write(ATTRIBUTES['bold'])
+            if self.bg is not None:
+                self.stream.write(BACKGROUNDS[self.bg])
+            if self.fg is not None:
+                self.stream.write(COLORS[self.fg])
+        elif self.set_console is not None:
+            if self.wval != 0:
+                self.set_console(self.file_handle, self.wval)
+
+    def __exit__(self, *args, **kwargs):
+        if not self.isatty:
+            return
+        if not self.fg and not self.bg and not self.bold:
+            return
+        if self.isansi:
+            self.stream.write(RESET)
+        elif self.set_console is not None:
+            self.set_console(self.file_handle, WCOLORS['white'])
+
+class ANSIStream(Detect):
+
+    ANSI_RE = re.compile(br'\033\[((?:\d|;)*)([a-zA-Z])')
+
+    def __init__(self, stream=None):
+        super(ANSIStream, self).__init__(stream)
+        self.encoding = getattr(self.stream, 'encoding', 'utf-8')
+
+    def write(self, text):
+        if isinstance(text, type(u'')):
+            text = text.encode(self.encoding, 'replace')
+
+        if not self.isatty:
+            return self.strip_and_write(text)
+
+        if self.isansi:
+            return self.stream.write(text)
+
+        if not self.isansi and self.set_console is None:
+            return self.strip_and_write(text)
+
+        self.write_and_convert(text)
+
+    def strip_and_write(self, text):
+        self.stream.write(self.ANSI_RE.sub(b'', text))
+
+    def write_and_convert(self, text):
+        '''
+        Write the given text to our wrapped stream, stripping any ANSI
+        sequences from the text, and optionally converting them into win32
+        calls.
+        '''
+        self.last_state = (None, None, False)
+        cursor = 0
+        for match in self.ANSI_RE.finditer(text):
+            start, end = match.span()
+            self.write_plain_text(text, cursor, start)
+            self.convert_ansi(*match.groups())
+            cursor = end
+        self.write_plain_text(text, cursor, len(text))
+
+    def write_plain_text(self, text, start, end):
+        if start < end:
+            self.stream.write(text[start:end])
+            self.stream.flush()
+
+    def convert_ansi(self, paramstring, command):
+        params = self.extract_params(paramstring)
+        self.call_win32(command, params)
+
+    def extract_params(self, paramstring):
+        def split(paramstring):
+            for p in paramstring.split(b';'):
+                if p:
+                    yield int(p)
+        return tuple(split(paramstring))
+
+    def call_win32(self, command, params):
+        if command != b'm': return
+        fg, bg, bold = self.last_state
+
+        for param in params:
+            if param in RCOLORS:
+                fg = RCOLORS[param]
+            elif param in RBACKGROUNDS:
+                bg = RBACKGROUNDS[param]
+            elif param == 1:
+                bold = True
+            elif param == 0:
+                fg = 'white'
+                bg, bold =  None, False
+
+        self.last_state = (fg, bg, bold)
+        if fg or bg or bold:
+            self.set_console(self.file_handle, to_flag(fg, bg, bold))
+        else:
+            self.set_console(self.file_handle, WCOLORS['white'])
+
+def windows_terminfo():
+    from ctypes import Structure, byref
+    from ctypes.wintypes import SHORT, WORD
+
+    class COORD(Structure):
+        """struct in wincon.h"""
+        _fields_ = [
+            ('X', SHORT),
+            ('Y', SHORT),
+        ]
+
+    class  SMALL_RECT(Structure):
+        """struct in wincon.h."""
+        _fields_ = [
+            ("Left", SHORT),
+            ("Top", SHORT),
+            ("Right", SHORT),
+            ("Bottom", SHORT),
+        ]
+
+    class CONSOLE_SCREEN_BUFFER_INFO(Structure):
+        """struct in wincon.h."""
+        _fields_ = [
+            ("dwSize", COORD),
+            ("dwCursorPosition", COORD),
+            ("wAttributes", WORD),
+            ("srWindow", SMALL_RECT),
+            ("dwMaximumWindowSize", COORD),
+        ]
+    csbi = CONSOLE_SCREEN_BUFFER_INFO()
+    import msvcrt
+    file_handle = msvcrt.get_osfhandle(sys.stdout.fileno())
+    from ctypes import windll
+    success = windll.kernel32.GetConsoleScreenBufferInfo(file_handle,
+                                                         byref(csbi))
+    if not success:
+        raise Exception('stdout is not a console?')
+    return csbi
+
+def geometry():
+    if iswindows:
+        try:
+
+            ti = windows_terminfo()
+            return (ti.dwSize.X or 80, ti.dwSize.Y or 80)
+        except:
+            return 80, 80
+    try:
+        import curses
+        curses.setupterm()
+    except:
+        return 80, 80
+    else:
+        width = curses.tigetnum('cols') or 80
+        height = curses.tigetnum('lines') or 80
+        return width, height
+
+def test():
+    s = ANSIStream()
+
+    text = [colored(t, fg=t)+'. '+colored(t, fg=t, bold=True)+'.' for t in
+            ('red', 'yellow', 'green', 'white', 'cyan', 'magenta', 'blue',)]
+    s.write('\n'.join(text))
+    print()
+

src/calibre/utils/terminfo.py

@@ -1,215 +0,0 @@
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import sys, re, os
-
-""" Get information about the terminal we are running in """
-
-class TerminalController:
-    """
-    A class that can be used to portably generate formatted output to
-    a terminal.
-
-    `TerminalController` defines a set of instance variables whose
-    values are initialized to the control sequence necessary to
-    perform a given action.  These can be simply included in normal
-    output to the terminal:
-
-    >>> term = TerminalController()
-    >>> print 'This is '+term.GREEN+'green'+term.NORMAL
-
-    Alternatively, the `render()` method can used, which replaces
-    '${action}' with the string required to perform 'action':
-
-    >>> term = TerminalController()
-    >>> print term.render('This is ${GREEN}green${NORMAL}')
-
-    If the terminal doesn't support a given action, then the value of
-    the corresponding instance variable will be set to ''.  As a
-    result, the above code will still work on terminals that do not
-    support color, except that their output will not be colored.
-    Also, this means that you can test whether the terminal supports a
-    given action by simply testing the truth value of the
-    corresponding instance variable:
-
-    >>> term = TerminalController()
-    >>> if term.CLEAR_SCREEN:
-    ...     print 'This terminal supports clearing the screen.'
-
-    Finally, if the width and height of the terminal are known, then
-    they will be stored in the `COLS` and `LINES` attributes.
-    """
-    # Cursor movement:
-    BOL = ''             #: Move the cursor to the beginning of the line
-    UP = ''              #: Move the cursor up one line
-    DOWN = ''            #: Move the cursor down one line
-    LEFT = ''            #: Move the cursor left one char
-    RIGHT = ''           #: Move the cursor right one char
-
-    # Deletion:
-    CLEAR_SCREEN = ''    #: Clear the screen and move to home position
-    CLEAR_EOL = ''       #: Clear to the end of the line.
-    CLEAR_BOL = ''       #: Clear to the beginning of the line.
-    CLEAR_EOS = ''       #: Clear to the end of the screen
-
-    # Output modes:
-    BOLD = ''            #: Turn on bold mode
-    BLINK = ''           #: Turn on blink mode
-    DIM = ''             #: Turn on half-bright mode
-    REVERSE = ''         #: Turn on reverse-video mode
-    NORMAL = ''          #: Turn off all modes
-
-    # Cursor display:
-    HIDE_CURSOR = ''     #: Make the cursor invisible
-    SHOW_CURSOR = ''     #: Make the cursor visible
-
-    # Terminal size:
-    COLS = None          #: Width of the terminal (None for unknown)
-    LINES = None         #: Height of the terminal (None for unknown)
-
-    # Foreground colors:
-    BLACK = BLUE = GREEN = CYAN = RED = MAGENTA = YELLOW = WHITE = ''
-
-    # Background colors:
-    BG_BLACK = BG_BLUE = BG_GREEN = BG_CYAN = ''
-    BG_RED = BG_MAGENTA = BG_YELLOW = BG_WHITE = ''
-
-    _STRING_CAPABILITIES = """
-    BOL=cr UP=cuu1 DOWN=cud1 LEFT=cub1 RIGHT=cuf1
-    CLEAR_SCREEN=clear CLEAR_EOL=el CLEAR_BOL=el1 CLEAR_EOS=ed BOLD=bold
-    BLINK=blink DIM=dim REVERSE=rev UNDERLINE=smul NORMAL=sgr0
-    HIDE_CURSOR=cinvis SHOW_CURSOR=cnorm""".split()
-    _COLORS = """BLACK BLUE GREEN CYAN RED MAGENTA YELLOW WHITE""".split()
-    _ANSICOLORS = "BLACK RED GREEN YELLOW BLUE MAGENTA CYAN WHITE".split()
-
-    def __init__(self, term_stream=sys.stdout):
-        """
-        Create a `TerminalController` and initialize its attributes
-        with appropriate values for the current terminal.
-        `term_stream` is the stream that will be used for terminal
-        output; if this stream is not a tty, then the terminal is
-        assumed to be a dumb terminal (i.e., have no capabilities).
-        """
-        # Curses isn't available on all platforms
-        try: import curses
-        except: return
-
-        # If the stream isn't a tty, then assume it has no capabilities.
-        if os.environ.get('CALIBRE_WORKER', None) is not None or not hasattr(term_stream, 'isatty') or not term_stream.isatty(): return
-
-        # Check the terminal type.  If we fail, then assume that the
-        # terminal has no capabilities.
-        try: curses.setupterm()
-        except: return
-
-        # Look up numeric capabilities.
-        self.COLS = curses.tigetnum('cols')
-        self.LINES = curses.tigetnum('lines')
-
-        # Look up string capabilities.
-        for capability in self._STRING_CAPABILITIES:
-            (attrib, cap_name) = capability.split('=')
-            setattr(self, attrib, self._tigetstr(cap_name) or '')
-
-        # Colors
-        set_fg = self._tigetstr('setf')
-        if set_fg:
-            for i,color in zip(range(len(self._COLORS)), self._COLORS):
-                setattr(self, color, curses.tparm(set_fg, i) or '')
-        set_fg_ansi = self._tigetstr('setaf')
-        if set_fg_ansi:
-            for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
-                setattr(self, color, curses.tparm(set_fg_ansi, i) or '')
-        set_bg = self._tigetstr('setb')
-        if set_bg:
-            for i,color in zip(range(len(self._COLORS)), self._COLORS):
-                setattr(self, 'BG_'+color, curses.tparm(set_bg, i) or '')
-        set_bg_ansi = self._tigetstr('setab')
-        if set_bg_ansi:
-            for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
-                setattr(self, 'BG_'+color, curses.tparm(set_bg_ansi, i) or '')
-
-    def _tigetstr(self, cap_name):
-        # String capabilities can include "delays" of the form "$<2>".
-        # For any modern terminal, we should be able to just ignore
-        # these, so strip them out.
-        import curses
-        cap = curses.tigetstr(cap_name) or ''
-        return re.sub(r'\$<\d+>[/*]?', '', cap)
-
-    def render(self, template):
-        """
-        Replace each $-substitutions in the given template string with
-        the corresponding terminal control string (if it's defined) or
-        '' (if it's not).
-        """
-        return re.sub(r'\$\$|\${\w+}', self._render_sub, template)
-
-    def _render_sub(self, match):
-        s = match.group()
-        if s == '$$': return s
-        else: return getattr(self, s[2:-1])
-
-#######################################################################
-# Example use case: progress bar
-#######################################################################
-
-class ProgressBar:
-    """
-    A 3-line progress bar, which looks like::
-
-    Header
-    20% [===========----------------------------------]
-    progress message
-
-    The progress bar is colored, if the terminal supports color
-    output; and adjusts to the width of the terminal.
-
-    If the terminal doesn't have the required capabilities, it uses a
-    simple progress bar.
-    """
-    BAR = '%3d%% ${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}\n'
-    HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n'
-
-    def __init__(self, term, header, no_progress_bar = False):
-        self.term, self.no_progress_bar = term, no_progress_bar
-        self.fancy = self.term.CLEAR_EOL and self.term.UP and self.term.BOL
-        if self.fancy:
-            self.width = self.term.COLS or 75
-            self.bar = term.render(self.BAR)
-            self.header = self.term.render(self.HEADER % header.center(self.width))
-            if isinstance(self.header, unicode):
-                self.header = self.header.encode('utf-8')
-            self.cleared = 1 #: true if we haven't drawn the bar yet.
-
-    def update(self, percent, message=''):
-        if isinstance(message, unicode):
-            message = message.encode('utf-8', 'replace')
-
-        if self.no_progress_bar:
-            if message:
-                print message
-        elif self.fancy:
-            if self.cleared:
-                sys.stdout.write(self.header)
-                self.cleared = 0
-            n = int((self.width-10)*percent)
-            msg = message.center(self.width)
-            sys.stdout.write(
-            self.term.BOL + self.term.UP + self.term.CLEAR_EOL +
-            (self.bar % (100*percent, '='*n, '-'*(self.width-10-n))) +
-            self.term.CLEAR_EOL + msg)
-            sys.stdout.flush()
-        else:
-            if not message:
-                print '%d%%'%(percent*100),
-            else:
-                print '%d%%'%(percent*100), message
-            sys.stdout.flush()
-
-
-    def clear(self):
-        if self.fancy and not self.cleared:
-            sys.stdout.write(self.term.BOL + self.term.CLEAR_EOL +
-            self.term.UP + self.term.CLEAR_EOL +
-            self.term.UP + self.term.CLEAR_EOL)
-            self.cleared = 1