IGN:Sync to trunk

linux_installer.py

@@ -74,7 +74,7 @@ f.write(hook_script)
 sys.path.insert(0, CALIBRESRC)
 from calibre.linux import entry_points
 
-executables, scripts = ['calibre_postinstall', 'parallel'], \
+executables, scripts = ['calibre_postinstall', 'calibre-parallel'], \
                        [os.path.join(CALIBRESRC, 'calibre', 'linux.py'), os.path.join(CALIBRESRC, 'calibre', 'parallel.py')]
 
 for entry in entry_points['console_scripts'] + entry_points['gui_scripts']:

osx_installer.py

@@ -51,6 +51,7 @@ def _check_symlinks_prescript():
 import os
 scripts = %(sp)s
 links = %(sp)s
+fonts_conf = %(sp)s
 os.setuid(0)
 for s, l in zip(scripts, links):
     if os.path.lexists(l):
@@ -59,6 +60,11 @@ for s, l in zip(scripts, links):
     omask = os.umask(022)
     os.symlink(s, l)
     os.umask(omask)
+if not os.path.exists('/etc/fonts/fonts.conf'):
+    print 'Creating default fonts.conf'
+    if not os.path.exists('/etc/fonts'):
+        os.makedirs('/etc/fonts')
+    os.link(fonts_conf, '/etc/fonts/fonts.conf')
 """
     
     dest_path = %(dest_path)s
@@ -66,6 +72,7 @@ for s, l in zip(scripts, links):
     scripts = %(scripts)s    
     links   = [os.path.join(dest_path, i) for i in scripts]
     scripts = [os.path.join(resources_path, 'loaders', i) for i in scripts]
+    fonts_conf = os.path.join(resources_path, 'fonts.conf')
     
     bad = False
     for s, l in zip(scripts, links):
@@ -73,10 +80,12 @@ for s, l in zip(scripts, links):
             continue
         bad = True
         break
+    if not bad:
+        bad = os.path.exists('/etc/fonts/fonts.conf')
     if bad:
         auth = Authorization(destroyflags=(kAuthorizationFlagDestroyRights,))
         fd, name = tempfile.mkstemp('.py')
-        os.write(fd, AUTHTOOL %(pp)s (sys.executable, repr(scripts), repr(links)))
+        os.write(fd, AUTHTOOL %(pp)s (sys.executable, repr(scripts), repr(links), repr(fonts_conf)))
         os.close(fd)
         os.chmod(name, 0700)
         try:
@@ -276,10 +285,12 @@ sys.frameworks_dir = os.path.join(os.path.dirname(os.environ['RESOURCEPATH']), '
         f.write('src/calibre/gui2/main.py', 'calibre/gui2/main.py')
         f.close()
         print
+        print 'Adding default fonts.conf'
+        open(os.path.join(self.dist_dir, APPNAME+'.app', 'Contents', 'Resources', 'fonts.conf'), 'wb').write(open('/etc/fonts/fonts.conf').read())
+        print
         print 'Building disk image'
         BuildAPP.makedmg(os.path.join(self.dist_dir, APPNAME+'.app'), APPNAME+'-'+VERSION)
 
-
 def main():
     sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
     sys.argv[1:2] = ['py2app']
@@ -295,7 +306,7 @@ def main():
                          'iconfile' : 'icons/library.icns',
                          'frameworks': ['libusb.dylib', 'libunrar.dylib'],
                          'includes' : ['sip', 'pkg_resources', 'PyQt4.QtXml', 
-                                       'PyQt4.QtSvg', 
+                                       'PyQt4.QtSvg', 'PyQt4.QtWebKit',
                                        'mechanize', 'ClientForm', 'usbobserver', 
                                        'genshi', 'calibre.web.feeds.recipes.*',
                                        'keyword', 'codeop', 'pydoc'],

src/calibre/__init__.py

@@ -88,6 +88,9 @@ def setup_cli_handlers(logger, level):
         handler = logging.StreamHandler(sys.stderr)
         handler.setLevel(logging.DEBUG)
         handler.setFormatter(logging.Formatter('[%(levelname)s] %(filename)s:%(lineno)s: %(message)s'))
+    for hdlr in logger.handlers:
+        if hdlr.__class__ == handler.__class__:
+            logger.removeHandler(hdlr)
     logger.addHandler(handler)
 
 class CustomHelpFormatter(IndentedHelpFormatter):

src/calibre/devices/prs505/driver.py

@@ -353,9 +353,16 @@ class PRS505(Device):
     def upload_books(self, files, names, on_card=False, end_session=True):
         path = os.path.join(self._card_prefix, self.CARD_PATH_PREFIX) if on_card \
                else os.path.join(self._main_prefix, 'database', 'media', 'books')
-        infiles = [file if hasattr(file, 'read') else open(file, 'rb') for file in files]
+               
-        for f in infiles: f.seek(0, 2)
+        def get_size(obj):
-        sizes = [f.tell() for f in infiles]
+            if hasattr(obj, 'seek'):
+                obj.seek(0, 2)
+                size = obj.tell()
+                obj.seek(0)
+                return size
+            return os.path.getsize(obj)
+        
+        sizes = map(get_size, files)
         size = sum(sizes)
         space = self.free_space()
         mspace = space[0]
@@ -370,13 +377,18 @@ class PRS505(Device):
         paths, ctimes = [], []
         
         names = iter(names)
-        for infile in infiles:
+        for infile in files:
+            close = False
+            if not hasattr(infile, 'read'):
+                infile, close = open(infile, 'rb'), True
             infile.seek(0)            
             name = names.next()
             paths.append(os.path.join(path, name))
             if not os.path.exists(os.path.dirname(paths[-1])):
                 os.makedirs(os.path.dirname(paths[-1]))
             self.put_file(infile, paths[-1], replace_file=True)
+            if close:
+                infile.close()
             ctimes.append(os.path.getctime(paths[-1]))
         return zip(paths, sizes, ctimes, cycle([on_card]))
     

src/calibre/ebooks/lrf/__init__.py

@@ -121,7 +121,6 @@ def option_parser(usage, gui_mode=False):
     laf.add_option('--ignore-colors', action='store_true', default=False, dest='ignore_colors',
                       help=_('Render all content as black on white instead of the colors specified by the HTML or CSS.'))
     
-    
     page = parser.add_option_group('PAGE OPTIONS')
     profiles = profile_map.keys()
     page.add_option('-p', '--profile', default=PRS500_PROFILE, dest='profile', type='choice',
@@ -139,6 +138,11 @@ def option_parser(usage, gui_mode=False):
                     help=_('''Top margin of page. Default is %default px.'''))
     page.add_option('--bottom-margin', default=0, dest='bottom_margin', type='int',
                     help=_('''Bottom margin of page. Default is %default px.'''))
+    page.add_option('--render-tables-as-images', default=False, action='store_true',
+                   help=_('Render tables in the HTML as images (useful if the document has large or complex tables)'))
+    page.add_option('--text-size-multiplier-for-rendered-tables', type='float', default=1.0,
+                   help=_('Multiply the size of text in rendered tables by this factor. Default is %default'))
+    
     link = parser.add_option_group('LINK PROCESSING OPTIONS')
     link.add_option('--link-levels', action='store', type='int', default=sys.maxint, \
                       dest='link_levels',
@@ -154,12 +158,13 @@ def option_parser(usage, gui_mode=False):
     chapter = parser.add_option_group('CHAPTER OPTIONS')
     chapter.add_option('--disable-chapter-detection', action='store_true', 
                       default=False, dest='disable_chapter_detection', 
-                      help=_('''Prevent the automatic insertion of page breaks'''
+                      help=_('''Prevent the automatic detection chapters.'''))
-                      ''' before detected chapters.'''))
     chapter.add_option('--chapter-regex', dest='chapter_regex', 
                       default='chapter|book|appendix',
                       help=_('''The regular expression used to detect chapter titles.'''
                       ''' It is searched for in heading tags (h1-h6). Defaults to %default'''))
+    chapter.add_option('--chapter-attr', default='$,,$', 
+                       help=_('Detect a chapter beginning at an element having the specified attribute. The format for this option is tagname regexp,attribute name,attribute value regexp. For example to match all heading tags that have the attribute class="chapter" you would use "h\d,class,chapter". Default is %default'''))
     chapter.add_option('--page-break-before-tag', dest='page_break', default='h[12]',
                       help=_('''If html2lrf does not find any page breaks in the '''
                       '''html file and cannot detect chapter headings, it will '''

src/calibre/ebooks/lrf/any/convert_from.py

@@ -158,7 +158,10 @@ def main(args=sys.argv, logger=None, gui_mode=False):
         print _('No file to convert specified.')
         return 1
     
-    return process_file(args[1], options, logger)
+    src = args[1]
+    if not isinstance(src, unicode):
+        src = src.decode(sys.getfilesystemencoding())
+    return process_file(src, options, logger)
 
 if __name__ == '__main__':
     sys.exit(main())

src/calibre/ebooks/lrf/html/convert_from.py

@@ -30,7 +30,7 @@ from calibre.ebooks.lrf import option_parser as lrf_option_parser
 from calibre.ebooks import ConversionError
 from calibre.ebooks.lrf.html.table import Table 
 from calibre import filename_to_utf8,  setup_cli_handlers, __appname__, \
-                    fit_image, LoggingInterface
+                    fit_image, LoggingInterface, preferred_encoding
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.ebooks.metadata.opf import OPFReader
 from calibre.devices.interface import Device
@@ -242,6 +242,7 @@ class HTMLConverter(object, LoggingInterface):
         
         self.override_css = {}
         self.override_pcss = {}
+        self.table_render_job_server = None
          
         if self._override_css is not None:
             if os.access(self._override_css, os.R_OK):
@@ -260,7 +261,9 @@ class HTMLConverter(object, LoggingInterface):
         
         
         paths = [os.path.abspath(path) for path in paths]
+        paths = [path.decode(sys.getfilesystemencoding()) if not isinstance(path, unicode) else path for path in paths]
         
+        try:
             while len(paths) > 0 and self.link_level <= self.link_levels:
                 for path in paths:
                     if path in self.processed_files:
@@ -292,6 +295,9 @@ class HTMLConverter(object, LoggingInterface):
             if self.base_font_size > 0:
                 self.log_info('\tRationalizing font sizes...')
                 self.book.rationalize_font_sizes(self.base_font_size)
+        finally:
+            if self.table_render_job_server is not None:
+                self.table_render_job_server.killall()
         
     def is_baen(self, soup):
         return bool(soup.find('meta', attrs={'name':'Publisher', 
@@ -380,12 +386,15 @@ class HTMLConverter(object, LoggingInterface):
         self.log_info(_('\tConverting to BBeB...'))
         self.current_style = {}
         self.page_break_found = False
+        if not isinstance(path, unicode):
+            path = path.decode(sys.getfilesystemencoding())
         self.target_prefix = path
         self.previous_text = '\n'
         self.tops[path] = self.parse_file(soup)
         self.processed_files.append(path)
                 
             
+        
     def parse_css(self, style):
         """
         Parse the contents of a <style> tag or .css file.
@@ -494,7 +503,9 @@ class HTMLConverter(object, LoggingInterface):
         top = self.current_block
         self.current_block.must_append = True
         
+        self.soup = soup
         self.process_children(soup, {}, {})
+        self.soup = None
         
         if self.current_para and self.current_block:
             self.current_para.append_to(self.current_block)
@@ -625,6 +636,8 @@ class HTMLConverter(object, LoggingInterface):
             para, text, path, fragment = link['para'], link['text'], link['path'], link['fragment']
             ascii_text = text
             
+            if not isinstance(path, unicode):
+                path = path.decode(sys.getfilesystemencoding())
             if path in self.processed_files:
                 if path+fragment in self.targets.keys():
                     tb = get_target_block(path+fragment, self.targets)
@@ -1424,6 +1437,18 @@ class HTMLConverter(object, LoggingInterface):
                 return
         except KeyError:
             pass
+        if not self.disable_chapter_detection and \
+           (self.chapter_attr[0].match(tagname) and \
+           tag.has_key(self.chapter_attr[1]) and \
+           self.chapter_attr[2].match(tag[self.chapter_attr[1]])):
+                self.log_debug('Detected chapter %s', tagname)
+                self.end_page() 
+                self.page_break_found = True
+
+                if self.options.add_chapters_to_toc:
+                    self.extra_toc_entries.append((self.get_text(tag,
+                        limit=1000), self.current_block))
+
         end_page = self.process_page_breaks(tag, tagname, tag_css)
         try:
             if tagname in ["title", "script", "meta", 'del', 'frameset']:            
@@ -1680,6 +1705,36 @@ class HTMLConverter(object, LoggingInterface):
                     self.previous_text = ' '
                 self.process_children(tag, tag_css, tag_pseudo_css)
             elif tagname == 'table' and not self.ignore_tables and not self.in_table:
+                if self.render_tables_as_images:
+                    if self.table_render_job_server is None:
+                        from calibre.parallel import Server
+                        self.table_render_job_server = Server(number_of_workers=1)
+                    print 'Rendering table...'
+                    from calibre.ebooks.lrf.html.table_as_image import render_table
+                    pheight = int(self.current_page.pageStyle.attrs['textheight'])
+                    pwidth  = int(self.current_page.pageStyle.attrs['textwidth'])
+                    images = render_table(self.table_render_job_server,
+                                          self.soup, tag, tag_css, 
+                                          os.path.dirname(self.target_prefix), 
+                                          pwidth, pheight, self.profile.dpi, 
+                                          self.text_size_multiplier_for_rendered_tables)
+                    for path, width, height in images:
+                        stream = ImageStream(path, encoding='PNG')
+                        im = Image(stream, x0=0, y0=0, x1=width, y1=height,\
+                               xsize=width, ysize=height)
+                        pb = self.current_block
+                        self.end_current_para()
+                        self.process_alignment(tag_css)                    
+                        self.current_para.append(Plot(im, xsize=width*720./self.profile.dpi, 
+                                                      ysize=height*720./self.profile.dpi))
+                        self.current_block.append(self.current_para)
+                        self.current_page.append(self.current_block)                    
+                        self.current_block = self.book.create_text_block(
+                                                        textStyle=pb.textStyle,
+                                                        blockStyle=pb.blockStyle)
+                        self.current_para = Paragraph()
+                        
+                else:
                     tag_css = self.tag_css(tag)[0] # Table should not inherit CSS
                     try:
                         self.process_table(tag, tag_css)
@@ -1743,6 +1798,8 @@ def process_file(path, options, logger=None):
         level = logging.DEBUG if options.verbose else logging.INFO
         logger = logging.getLogger('html2lrf')
         setup_cli_handlers(logger, level)
+    if not isinstance(path, unicode):
+        path = path.decode(sys.getfilesystemencoding())
     path = os.path.abspath(path)
     default_title = filename_to_utf8(os.path.splitext(os.path.basename(path))[0])
     dirpath = os.path.dirname(path)
@@ -1821,9 +1878,14 @@ def process_file(path, options, logger=None):
          re.compile('$')
     fpb = re.compile(options.force_page_break, re.IGNORECASE) if options.force_page_break else \
          re.compile('$')
+    cq = options.chapter_attr.split(',')
+    options.chapter_attr = [re.compile(cq[0], re.IGNORECASE), cq[1], 
+                            re.compile(cq[2], re.IGNORECASE)]
     options.force_page_break = fpb
     options.link_exclude = le
     options.page_break = pb
+    if not isinstance(options.chapter_regex, unicode):
+        options.chapter_regex = options.chapter_regex.decode(preferred_encoding)
     options.chapter_regex = re.compile(options.chapter_regex, re.IGNORECASE)
     fpba = options.force_page_break_attr.split(',')
     if len(fpba) != 3:
@@ -1940,7 +2002,8 @@ def main(args=sys.argv):
     except Exception, err:
         print >> sys.stderr, err
         return 1
-    
+    if not isinstance(src, unicode):
+        src = src.decode(sys.getfilesystemencoding())
     process_file(src, options)
     return 0
 

src/calibre/ebooks/lrf/html/table_as_image.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Render HTML tables as images.
+'''
+import os, tempfile, atexit, shutil, time
+from PyQt4.Qt import QWebPage, QUrl, QApplication, QSize, \
+                     SIGNAL, QPainter, QImage, QObject, Qt
+
+__app = None
+
+class HTMLTableRenderer(QObject):
+
+    def __init__(self, html, base_dir, width, height, dpi, factor):
+        '''
+        `width, height`: page width and height in pixels
+        `base_dir`: The directory in which the HTML file that contains the table resides 
+        '''
+        QObject.__init__(self)
+        
+        self.app = None
+        self.width, self.height, self.dpi = width, height, dpi
+        self.base_dir = base_dir
+        self.page = QWebPage()
+        self.connect(self.page, SIGNAL('loadFinished(bool)'), self.render_html)
+        self.page.mainFrame().setTextSizeMultiplier(factor)
+        self.page.mainFrame().setHtml(html, 
+                                QUrl('file:'+os.path.abspath(self.base_dir)))
+        self.images = []
+        self.tdir = tempfile.mkdtemp(prefix='calibre_render_table')
+        
+    def render_html(self, ok):
+        try:
+            if not ok:
+                return
+            cwidth, cheight = self.page.mainFrame().contentsSize().width(), self.page.mainFrame().contentsSize().height()
+            self.page.setViewportSize(QSize(cwidth, cheight))
+            factor = float(self.width)/cwidth if cwidth > self.width else 1
+            cutoff_height = int(self.height/factor)-3
+            image = QImage(self.page.viewportSize(), QImage.Format_ARGB32)
+            image.setDotsPerMeterX(self.dpi*(100/2.54))
+            image.setDotsPerMeterX(self.dpi*(100/2.54))
+            painter = QPainter(image)
+            self.page.mainFrame().render(painter)
+            painter.end()
+            cheight = image.height()
+            cwidth = image.width()
+            pos = 0
+            while pos < cheight:
+                img = image.copy(0, pos, cwidth, min(cheight-pos, cutoff_height))
+                pos += cutoff_height-20
+                if cwidth > self.width:
+                    img = img.scaledToWidth(self.width, Qt.SmoothTransform)
+                f = os.path.join(self.tdir, '%d.png'%pos)
+                img.save(f)
+                self.images.append((f, img.width(), img.height()))
+        finally:
+            QApplication.quit()
+        
+def render_table(server, soup, table, css, base_dir, width, height, dpi, factor=1.0):
+    head = ''
+    for e in soup.findAll(['link', 'style']):
+        head += unicode(e)+'\n\n'
+    style = ''
+    for key, val in css.items():
+        style += key + ':%s;'%val
+    html = u'''\
+<html>
+    <head>
+        %s
+    </head>
+    <body style="width: %dpx; background: white">
+        <style type="text/css">
+            table {%s}
+        </style>
+        %s
+    </body>
+</html>
+    '''%(head, width-10, style, unicode(table))
+    server.run_job(1, 'render_table', 
+                            args=[html, base_dir, width, height, dpi, factor])
+    res = None
+    while res is None:
+        time.sleep(2)
+        res = server.result(1)
+    result, exception, traceback = res
+    if exception:
+        print 'Failed to render table'
+        print exception
+        print traceback
+    images, tdir = result
+    atexit.register(shutil.rmtree, tdir)
+    return images
+    
+def do_render(html, base_dir, width, height, dpi, factor):
+    app = QApplication.instance()
+    if app is None:
+        app = QApplication([])
+    tr = HTMLTableRenderer(html, base_dir, width, height, dpi, factor)
+    app.exec_()
+    return tr.images, tr.tdir

src/calibre/ebooks/metadata/__init__.py

@@ -38,7 +38,7 @@ class MetaInformation(object):
                 setattr(ans, attr, getattr(mi, attr))
         
     
-    def __init__(self, title, authors=['Unknown']):
+    def __init__(self, title, authors=[_('Unknown')]):
         '''
         @param title: title or "Unknown" or a MetaInformation object
         @param authors: List of strings or []

src/calibre/ebooks/metadata/opf.py

@@ -511,6 +511,8 @@ class OPFCreator(MetaInformation):
                 path = path[len(self.base_path)+1:]
                 manifest.append((path, mt))
             self.manifest = manifest
+        if not self.authors:
+            self.authors = [_('Unknown')]
     
     def create_manifest(self, entries):
         '''

src/calibre/ebooks/mobi/reader.py

@@ -156,7 +156,7 @@ class MobiReader(object):
         
         processed_records = self.extract_text()
         self.add_anchors()
-        self.processed_html = self.processed_html.decode(self.book_header.codec)
+        self.processed_html = self.processed_html.decode(self.book_header.codec, 'ignore')
         self.extract_images(processed_records, output_dir)
         self.replace_page_breaks()
         self.cleanup()
@@ -177,7 +177,7 @@ class MobiReader(object):
             opf.render(open(os.path.splitext(htmlfile)[0]+'.opf', 'wb'))
         
     def cleanup(self):
-        self.processed_html = re.sub(r'<div height="0(em|%)"></div>', '', self.processed_html)
+        self.processed_html = re.sub(r'<div height="0(em|%){0,1}"></div>', '', self.processed_html)
     
     def create_opf(self, htmlfile):
         mi = self.book_header.exth.mi

src/calibre/gui2/dialogs/fetch_metadata.ui

@@ -9,14 +9,15 @@
     <x>0</x>
     <y>0</y>
     <width>830</width>
-    <height>700</height>
+    <height>642</height>
    </rect>
   </property>
   <property name="windowTitle" >
    <string>Fetch metadata</string>
   </property>
   <property name="windowIcon" >
-   <iconset resource="../images.qrc" >:/images/metadata.svg</iconset>
+   <iconset resource="../images.qrc" >
+    <normaloff>:/images/metadata.svg</normaloff>:/images/metadata.svg</iconset>
   </property>
   <layout class="QVBoxLayout" >
    <item>
@@ -107,7 +108,7 @@
    <item>
     <widget class="QDialogButtonBox" name="buttonBox" >
      <property name="standardButtons" >
-      <set>QDialogButtonBox::Cancel|QDialogButtonBox::NoButton|QDialogButtonBox::Ok</set>
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
      </property>
     </widget>
    </item>

src/calibre/gui2/dialogs/jobs.py

@@ -35,7 +35,7 @@ class JobsDialog(QDialog, Ui_JobsDialog):
         self.jobs_view.setModel(model)
         self.model = model
         self.setWindowModality(Qt.NonModal)
-        self.setWindowTitle(__appname__ + ' - Active Jobs')
+        self.setWindowTitle(__appname__ + _(' - Jobs'))
         QObject.connect(self.jobs_view.model(), SIGNAL('modelReset()'), 
                         self.jobs_view.resizeColumnsToContents)
         QObject.connect(self.kill_button, SIGNAL('clicked()'),

src/calibre/gui2/dialogs/lrf_single.py

@@ -1,8 +1,8 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import os, cPickle, codecs
+import os, codecs
 
-from PyQt4.QtCore import QObject, SIGNAL, Qt, QVariant, QByteArray
+from PyQt4.QtCore import QObject, SIGNAL, Qt
 from PyQt4.QtGui import QAbstractSpinBox, QLineEdit, QCheckBox, QDialog, \
                         QPixmap, QTextEdit, QListWidgetItem, QIcon
 
@@ -48,10 +48,7 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog):
         self.gui_mono_family.setModel(self.font_family_model)
         self.load_saved_global_defaults()
     
-    def __init__(self, window, db, row):
+    def populate_list(self):
-        QDialog.__init__(self, window)
-        Ui_LRFSingleDialog.__init__(self)
-        self.setupUi(self)
         self.__w = []
         self.__w.append(QIcon(':/images/dialog_information.svg'))
         self.item1 = QListWidgetItem(self.__w[-1], _("Metadata"), self.categoryList)
@@ -61,11 +58,17 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog):
         self.item3 = QListWidgetItem(self.__w[-1], _('Page Setup'), self.categoryList)
         self.__w.append(QIcon(':/images/chapters.svg'))
         self.item4 = QListWidgetItem(self.__w[-1], _('Chapter Detection'), self.categoryList)
+    
+    def __init__(self, window, db, row):
+        QDialog.__init__(self, window)
+        Ui_LRFSingleDialog.__init__(self)
+        self.setupUi(self)
+        self.populate_list()
         self.categoryList.setCurrentRow(0)
         QObject.connect(self.categoryList, SIGNAL('itemEntered(QListWidgetItem *)'),
                         self.show_category_help)
         QObject.connect(self.cover_button, SIGNAL("clicked(bool)"), self.select_cover)
-        self.categoryList.leaveEvent = self.reset_help
+        #self.categoryList.leaveEvent = self.reset_help
         self.reset_help()
         self.selected_format = None
         self.initialize_common()
@@ -277,9 +280,9 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog):
                 obj.setWhatsThis(help)
                 self.option_map[guiname] = opt
                 obj.__class__.enterEvent = show_item_help
-                obj.leaveEvent = self.reset_help
+                #obj.leaveEvent = self.reset_help
         self.preprocess.__class__.enterEvent = show_item_help
-        self.preprocess.leaveEvent = self.reset_help
+        #self.preprocess.leaveEvent = self.reset_help
             
     
     def show_category_help(self, item):
@@ -293,6 +296,7 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog):
         self.set_help(help[text])
         
     def set_help(self, msg):
+        if msg and getattr(msg, 'strip', lambda:True)():
             self.help_view.setHtml('<html><body>%s</body></html>'%(msg,))
     
     def reset_help(self, *args):
@@ -390,6 +394,7 @@ class LRFBulkDialog(LRFSingleDialog):
         QDialog.__init__(self, window)
         Ui_LRFSingleDialog.__init__(self)
         self.setupUi(self)
+        self.populate_list()
         
         self.categoryList.takeItem(0)
         self.stack.removeWidget(self.stack.widget(0))
@@ -399,7 +404,14 @@ class LRFBulkDialog(LRFSingleDialog):
         self.setWindowTitle(_('Bulk convert ebooks to LRF'))
         
     def accept(self):
-        self.cmdline = self.cmdline = [unicode(i) for i in self.build_commandline()]
+        self.cmdline = [unicode(i) for i in self.build_commandline()]
+        for meta in ('--title', '--author', '--publisher', '--comment'):
+            try:
+                index = self.cmdline.index(meta)
+                self.cmdline[index:index+2] = []
+            except ValueError:
+                continue
+                
         self.cover_file = None
         QDialog.accept(self)
     

src/calibre/gui2/dialogs/lrf_single.ui

@@ -115,7 +115,7 @@
           <item row="0" column="0" >
            <widget class="QStackedWidget" name="stack" >
             <property name="currentIndex" >
-             <number>0</number>
+             <number>3</number>
             </property>
             <widget class="QWidget" name="metadata_page" >
              <property name="geometry" >
@@ -818,6 +818,39 @@
                 </property>
                </widget>
               </item>
+              <item row="5" column="0" >
+               <widget class="QCheckBox" name="gui_render_tables_as_images" >
+                <property name="text" >
+                 <string>&amp;Convert tables to images (good for large/complex tables)</string>
+                </property>
+               </widget>
+              </item>
+              <item row="6" column="0" >
+               <widget class="QLabel" name="label_27" >
+                <property name="text" >
+                 <string>&amp;Multiplier for text size in rendered tables:</string>
+                </property>
+                <property name="buddy" >
+                 <cstring>gui_text_size_multiplier_for_rendered_tables</cstring>
+                </property>
+               </widget>
+              </item>
+              <item row="6" column="1" >
+               <widget class="QDoubleSpinBox" name="gui_text_size_multiplier_for_rendered_tables" >
+                <property name="enabled" >
+                 <bool>false</bool>
+                </property>
+                <property name="decimals" >
+                 <number>2</number>
+                </property>
+                <property name="minimum" >
+                 <double>0.100000000000000</double>
+                </property>
+                <property name="value" >
+                 <double>1.000000000000000</double>
+                </property>
+               </widget>
+              </item>
              </layout>
             </widget>
             <widget class="QWidget" name="chapterdetection_page" >
@@ -918,6 +951,19 @@
                  <item row="2" column="1" >
                   <widget class="QLineEdit" name="gui_force_page_break_before_attr" />
                  </item>
+                 <item row="3" column="0" >
+                  <widget class="QLabel" name="label_28" >
+                   <property name="text" >
+                    <string>Detect chapter &amp;at tag:</string>
+                   </property>
+                   <property name="buddy" >
+                    <cstring>gui_chapter_attr</cstring>
+                   </property>
+                  </widget>
+                 </item>
+                 <item row="3" column="1" >
+                  <widget class="QLineEdit" name="gui_chapter_attr" />
+                 </item>
                 </layout>
                </widget>
               </item>
@@ -1048,8 +1094,8 @@ p, li { white-space: pre-wrap; }
    <slot>setCurrentIndex(int)</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>191</x>
+     <x>184</x>
-     <y>236</y>
+     <y>279</y>
     </hint>
     <hint type="destinationlabel" >
      <x>368</x>
@@ -1064,8 +1110,8 @@ p, li { white-space: pre-wrap; }
    <slot>setDisabled(bool)</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>428</x>
+     <x>650</x>
-     <y>89</y>
+     <y>122</y>
     </hint>
     <hint type="destinationlabel" >
      <x>788</x>
@@ -1073,22 +1119,6 @@ p, li { white-space: pre-wrap; }
     </hint>
    </hints>
   </connection>
-  <connection>
-   <sender>gui_header</sender>
-   <signal>toggled(bool)</signal>
-   <receiver>gui_headerformat</receiver>
-   <slot>setEnabled(bool)</slot>
-   <hints>
-    <hint type="sourcelabel" >
-     <x>348</x>
-     <y>340</y>
-    </hint>
-    <hint type="destinationlabel" >
-     <x>823</x>
-     <y>372</y>
-    </hint>
-   </hints>
-  </connection>
   <connection>
    <sender>gui_disable_chapter_detection</sender>
    <signal>toggled(bool)</signal>
@@ -1096,12 +1126,60 @@ p, li { white-space: pre-wrap; }
    <slot>setDisabled(bool)</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>321</x>
+     <x>543</x>
-     <y>78</y>
+     <y>122</y>
     </hint>
     <hint type="destinationlabel" >
-     <x>322</x>
+     <x>544</x>
-     <y>172</y>
+     <y>211</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>gui_render_tables_as_images</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>gui_text_size_multiplier_for_rendered_tables</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>298</x>
+     <y>398</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>660</x>
+     <y>435</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>gui_header</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>gui_headerformat</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>330</x>
+     <y>367</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>823</x>
+     <y>372</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>gui_disable_chapter_detection</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>gui_chapter_attr</receiver>
+   <slot>setDisabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>344</x>
+     <y>107</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>489</x>
+     <y>465</y>
     </hint>
    </hints>
   </connection>

src/calibre/gui2/dialogs/user_profiles.py

@@ -84,6 +84,7 @@ class UserProfiles(QDialog, Ui_Dialog):
             self.populate_options(recipe)
             self.stacks.setCurrentIndex(0)
             self.toggle_mode_button.setText(_('Switch to Advanced mode'))
+            self.source_code.setPlainText('')
         else:
             self.source_code.setPlainText(src)
             self.highlighter = PythonHighlighter(self.source_code.document())

src/calibre/gui2/jobs.py

@@ -86,16 +86,33 @@ class DeviceJob(Job):
         
 class ConversionJob(Job):
     ''' Jobs that involve conversion of content.'''
-    def run(self):
+    def __init__(self, *args, **kwdargs):
-        last_traceback, exception = None, None
+        Job.__init__(self, *args, **kwdargs)
-        try:
+        self.log = ''
-            self.result, exception, last_traceback, self.log = \
-                self.server.run(self.id, self.func, self.args, self.kwargs)
-        except Exception, err:
-            last_traceback = traceback.format_exc()
-            exception = (exception.__class__.__name__, unicode(str(err), 'utf8', 'replace'))
         
-        self.last_traceback, self.exception = last_traceback, exception
+    def run(self):
+        result = None
+        self.server.run_job(self.id, self.func, progress=self.progress,
+                                args=self.args, kwdargs=self.kwargs, 
+                                output=self.output)
+        res = None
+        while res is None:
+            time.sleep(2)
+            res = self.server.result(self.id)
+        if res is None:
+            exception, tb = 'UnknownError: This should not have happened', ''
+        else:
+            result, exception, tb = res
+        self.result, self.last_traceback, self.exception = result, tb, exception
+        
+    def output(self, msg):
+        if self.log is None:
+            self.log = ''
+        self.log += msg
+        self.emit(SIGNAL('output_received()'))
+    
+    def formatted_log(self):
+        return '<h2>Log:</h2><pre>%s</pre>'%self.log
     
     def notify(self):
         self.emit(SIGNAL('jobdone(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject, PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'), 
@@ -113,6 +130,9 @@ class ConversionJob(Job):
         ans += '<h2>Traceback:</h2><pre>%s</pre>'%self.last_traceback
         return ans
     
+    def progress(self, percent, msg):
+        self.emit(SIGNAL('update_progress(int, PyQt_PyObject)'), self.id, percent)
+        
 class JobManager(QAbstractTableModel):
     
     PRIORITY = {'Idle'  : QThread.IdlePriority,
@@ -149,9 +169,9 @@ class JobManager(QAbstractTableModel):
             try:
                 if isinstance(job, DeviceJob):
                     job.terminate()
-                self.process_server.kill(job.id)
             except:
                 continue
+        self.process_server.killall()
     
     def timerEvent(self, event):
         if event.timerId() == self.timer_id:
@@ -241,7 +261,10 @@ class JobManager(QAbstractTableModel):
         id = self.next_id
         job = job_class(id, description, slot, priority, *args, **kwargs)
         job.server = self.process_server
-        QObject.connect(job, SIGNAL('status_update(int, int)'), self.status_update, Qt.QueuedConnection)
+        QObject.connect(job, SIGNAL('status_update(int, int)'), self.status_update, 
+                        Qt.QueuedConnection)
+        self.connect(job, SIGNAL('update_progress(int, PyQt_PyObject)'), 
+                     self.update_progress, Qt.QueuedConnection)
         self.update_lock.lock()
         self.add_queue.append(job)
         self.update_lock.unlock()            
@@ -370,11 +393,14 @@ class DetailView(QDialog, Ui_Dialog):
         self.setupUi(self)
         self.setWindowTitle(job.description)
         self.job = job
-        txt = self.job.formatted_error() + self.job.formatted_log()
+        self.connect(self.job, SIGNAL('output_received()'), self.update)
+        self.update()
         
+            
+    def update(self):
+        txt = self.job.formatted_error() + self.job.formatted_log()
         if not txt:
             txt = 'No details available'
-            
         self.log.setHtml(txt)
-            
+        vbar = self.log.verticalScrollBar()
-        
+        vbar.setValue(vbar.maximum())

src/calibre/gui2/library.py

@@ -303,7 +303,7 @@ class BooksModel(QAbstractTableModel):
             metadata.append(mi)
         return metadata
 
-    def get_preferred_formats(self, rows, formats):
+    def get_preferred_formats(self, rows, formats, paths=False):
         ans = []
         for row in (row.row() for row in rows):
             format = None
@@ -314,7 +314,8 @@ class BooksModel(QAbstractTableModel):
             if format:
                 pt = PersistentTemporaryFile(suffix='.'+format)
                 pt.write(self.db.format(row, format))
-                pt.seek(0)
+                pt.flush()
+                pt.close() if paths else pt.seek(0)
                 ans.append(pt)
             else:
                 ans.append(None)

src/calibre/gui2/main.py

@@ -77,7 +77,6 @@ class Main(MainWindow, Ui_MainWindow):
         self.conversion_jobs = {}
         self.persistent_files = []
         self.metadata_dialogs = []
-        self.viewer_job_id = 1
         self.default_thumbnail = None
         self.device_error_dialog = ConversionErrorDialog(self, _('Error communicating with device'), ' ')
         self.device_error_dialog.setModal(Qt.NonModal)
@@ -277,14 +276,6 @@ class Main(MainWindow, Ui_MainWindow):
         elif msg.startswith('refreshdb:'):
             self.library_view.model().resort()
             self.library_view.model().research()
-        elif msg.startswith('progress:'):
-            try:
-                fields = msg.split(':')
-                job_id, percent = fields[1:3]
-                job_id, percent = int(job_id), float(percent)
-                self.job_manager.update_progress(job_id, percent)
-            except:
-                pass
         else:
             print msg
 
@@ -488,7 +479,7 @@ class Main(MainWindow, Ui_MainWindow):
         else:
             self.upload_books(paths, names, infos, on_card=on_card)
 
-    def upload_books(self, files, names, metadata, on_card=False):
+    def upload_books(self, files, names, metadata, on_card=False, memory=None):
         '''
         Upload books to device.
         @param files: List of either paths to files or file like objects
@@ -499,13 +490,13 @@ class Main(MainWindow, Ui_MainWindow):
                                         files, names, on_card=on_card,
                                         job_extra_description=titles
                                         )
-        self.upload_memory[id] = (metadata, on_card)
+        self.upload_memory[id] = (metadata, on_card, memory)
 
     def books_uploaded(self, id, description, result, exception, formatted_traceback):
         '''
         Called once books have been uploaded.
         '''
-        metadata, on_card = self.upload_memory.pop(id)
+        metadata, on_card = self.upload_memory.pop(id)[:2]
         if exception:
             if isinstance(exception, FreeSpaceError):
                 where = 'in main memory.' if 'memory' in str(exception) else 'on the storage card.'
@@ -633,8 +624,9 @@ class Main(MainWindow, Ui_MainWindow):
             if cdata:
                 mi['cover'] = self.cover_to_thumbnail(cdata)
         metadata = iter(metadata)
-        files = self.library_view.model().get_preferred_formats(rows,
+        _files = self.library_view.model().get_preferred_formats(rows,
-                                    self.device_manager.device_class.FORMATS)
+                                    self.device_manager.device_class.FORMATS, paths=True)
+        files = [f.name for f in _files]
         bad, good, gf, names = [], [], [], []
         for f in files:
             mi = metadata.next()
@@ -649,7 +641,9 @@ class Main(MainWindow, Ui_MainWindow):
                 try:
                     smi = MetaInformation(mi['title'], aus2)
                     smi.comments = mi.get('comments', None)
-                    set_metadata(f, smi, f.name.rpartition('.')[2])
+                    _f = open(f, 'r+b')
+                    set_metadata(_f, smi, f.rpartition('.')[2])
+                    _f.close()
                 except:
                     print 'Error setting metadata in book:', mi['title']
                     traceback.print_exc()
@@ -666,8 +660,8 @@ class Main(MainWindow, Ui_MainWindow):
                     prefix = prefix.encode('ascii', 'ignore')
                 else:
                     prefix = prefix.decode('ascii', 'ignore').encode('ascii', 'ignore')
-                names.append('%s_%d%s'%(prefix, id, os.path.splitext(f.name)[1]))
+                names.append('%s_%d%s'%(prefix, id, os.path.splitext(f)[1]))
-        self.upload_books(gf, names, good, on_card)
+        self.upload_books(gf, names, good, on_card, memory=_files)
         self.status_bar.showMessage(_('Sending books to device.'), 5000)
         if bad:
             bad = '\n'.join('<li>%s</li>'%(i,) for i in bad)
@@ -759,6 +753,15 @@ class Main(MainWindow, Ui_MainWindow):
 
         for i, row in enumerate([r.row() for r in rows]):
             cmdline = list(d.cmdline)
+            mi = self.library_view.model().db.get_metadata(row)
+            if mi.title:
+                cmdline.extend(['--title', mi.title])
+            if mi.authors:
+                cmdline.extend(['--author', ','.join(mi.authors)])
+            if mi.publisher:
+                cmdline.extend(['--publisher', mi.publisher])
+            if mi.comments:
+                cmdline.extend(['--comment', mi.comments])
             data = None
             for fmt in LRF_PREFERRED_SOURCE_FORMATS:
                 try:
@@ -784,7 +787,7 @@ class Main(MainWindow, Ui_MainWindow):
             cmdline.append(pt.name)
             id = self.job_manager.run_conversion_job(self.book_converted,
                                                         'any2lrf', args=[cmdline],
-                                    job_description='Convert book %d of %d'%(i, len(rows)))
+                                    job_description='Convert book %d of %d'%(i+1, len(rows)))
 
 
             self.conversion_jobs[id] = (d.cover_file, pt, of, d.output_format,
@@ -864,15 +867,16 @@ class Main(MainWindow, Ui_MainWindow):
         self._view_file(result)
 
     def _view_file(self, name):
+        self.setCursor(Qt.BusyCursor)
+        try:
             if name.upper().endswith('.LRF'):
                 args = ['lrfviewer', name]
-            self.job_manager.process_server.run('viewer%d'%self.viewer_job_id,
+                self.job_manager.process_server.run_free_job('lrfviewer', kwdargs=dict(args=args))
-                                                'lrfviewer', kwdargs=dict(args=args),
-                                                monitor=False)
-            self.viewer_job_id += 1
             else:
                 QDesktopServices.openUrl(QUrl('file:'+name))#launch(name)
-        time.sleep(2) # User feedback
+            time.sleep(5) # User feedback
+        finally:
+            self.unsetCursor()
 
     def view_specific_format(self, triggered):
         rows = self.library_view.selectionModel().selectedRows()
@@ -1076,7 +1080,7 @@ class Main(MainWindow, Ui_MainWindow):
         if getattr(exception, 'only_msg', False):
             error_dialog(self, _('Conversion Error'), unicode(exception)).exec_()
             return
-        msg =  u'<p><b>%s</b>: %s</p>'%exception
+        msg =  u'<p><b>%s</b>: </p>'%exception
         msg += u'<p>Failed to perform <b>job</b>: '+description
         msg += u'<p>Detailed <b>traceback</b>:<pre>'
         msg += formatted_traceback + '</pre>'

src/calibre/gui2/status.py

@@ -1,6 +1,6 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import textwrap, re
+import re
 
 from PyQt4.QtGui import QStatusBar, QMovie, QLabel, QFrame, QHBoxLayout, QPixmap, \
                         QVBoxLayout, QSizePolicy, QToolButton, QIcon

src/calibre/library/cli.py

@@ -11,7 +11,10 @@ import sys, os
 from textwrap import TextWrapper
 
 from calibre import OptionParser, Settings, terminal_controller, preferred_encoding
-from calibre.gui2 import SingleApplication
+try:
+    from calibre.utils.single_qt_application import send_message
+except:
+    send_message = None
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.library.database2 import LibraryDatabase2
 from calibre.library.database import text_to_tokens
@@ -184,9 +187,8 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
                     print '\t', title+':'
                     print '\t\t ', path
 
-        if SingleApplication is not None:
+        if send_message is not None:
-            sa = SingleApplication('calibre GUI')
+            send_message('refreshdb:', 'calibre GUI')
-            sa.send_message('refreshdb:')
     finally:
         sys.stdout = sys.__stdout__
 
@@ -224,9 +226,9 @@ def do_remove(db, ids):
             for y in x:
                 db.delete_book(y)
 
-    if SingleApplication is not None:
+    if send_message is not None:
-        sa = SingleApplication('calibre GUI')
+        send_message('refreshdb:', 'calibre GUI')
-        sa.send_message('refreshdb:')
+
 
 def command_remove(args, dbpath):
     parser = get_parser(_(
@@ -339,9 +341,8 @@ def do_set_metadata(db, id, stream):
     mi = OPFReader(stream)
     db.set_metadata(id, mi)
     do_show_metadata(db, id, False)
-    if SingleApplication is not None:
+    if send_message is not None:
-        sa = SingleApplication('calibre GUI')
+        send_message('refreshdb:', 'calibre GUI')
-        sa.send_message('refreshdb:')
 
 def command_set_metadata(args, dbpath):
     parser = get_parser(_(

src/calibre/library/database.py

@@ -1414,11 +1414,13 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
                 mi = OPFCreator(base, self.get_metadata(idx, index_is_id=index_is_id))
                 cover = self.cover(idx, index_is_id=index_is_id)
                 if cover is not None:
-                    cname = name + '.jpg'
+                    cname = sanitize_file_name(name) + '.jpg'
                     cpath = os.path.join(base, cname)
                     open(cpath, 'wb').write(cover)
                     mi.cover = cname
                 f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb')
+                if not mi.authors:
+                    mi.authors = [_('Unknown')]
                 mi.render(f)
                 f.close()
                 

src/calibre/linux_installer.py

@@ -8,6 +8,8 @@ Download and install the linux binary.
 '''
 import sys, os, shutil, tarfile, subprocess, tempfile, urllib2, re, stat
 
+MOBILEREAD='https://dev.mobileread.com/dist/kovid/calibre/'
+
 class TerminalController:
     """
     A class that can be used to portably generate formatted output to
@@ -239,7 +241,7 @@ def do_postinstall(destdir):
 
 def download_tarball():
     pb  = ProgressBar(TerminalController(sys.stdout), 'Downloading calibre...')
-    src = urllib2.urlopen('http://calibre.kovidgoyal.net/downloads/latest-linux-binary.tar.bz2')
+    src = urllib2.urlopen(MOBILEREAD+'calibre-%version-i686.tar.bz2')
     size = int(src.info()['content-length'])
     f = tempfile.NamedTemporaryFile()
     while f.tell() < size:

src/calibre/parallel.py

@@ -1,21 +1,26 @@
+from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
 Used to run jobs in parallel in separate processes.
 '''
-import re, sys, tempfile, os, cPickle, traceback, atexit, binascii, time, subprocess
+import sys, os, gc, cPickle, traceback, atexit, cStringIO, time, \
+       subprocess, socket, collections, binascii
+from select import select
 from functools import partial
-
+from threading import RLock, Thread, Event
 
 from calibre.ebooks.lrf.any.convert_from import main as any2lrf
 from calibre.ebooks.lrf.web.convert_from import main as web2lrf
 from calibre.ebooks.lrf.feeds.convert_from import main as feeds2lrf
 from calibre.gui2.lrf_renderer.main import main as lrfviewer
-from calibre import iswindows, __appname__, islinux
+from calibre.ptempfile import PersistentTemporaryFile
+
 try:
-    from calibre.utils.single_qt_application import SingleApplication
+    from calibre.ebooks.lrf.html.table_as_image import do_render as render_table
-except:
+except: # Dont fail is PyQt4.4 not present
-    SingleApplication = None
+    render_table = None
+from calibre import iswindows, islinux, detect_ncpus
 
 sa = None
 job_id = None
@@ -25,12 +30,14 @@ def report_progress(percent, msg=''):
         msg = 'progress:%s:%f:%s'%(job_id, percent, msg)
         sa.send_message(msg)
 
+_notify = 'fskjhwseiuyweoiu987435935-0342'
 
 PARALLEL_FUNCS = {
                   'any2lrf'   : partial(any2lrf, gui_mode=True),
                   'web2lrf'   : web2lrf,
                   'lrfviewer' : lrfviewer,
-                  'feeds2lrf' : partial(feeds2lrf, notification=report_progress),
+                  'feeds2lrf' : partial(feeds2lrf, notification=_notify),
+                  'render_table': render_table,
                   }
 
 python = sys.executable
@@ -41,138 +48,463 @@ if iswindows:
         python = os.path.join(os.path.dirname(python), 'parallel.exe')
     else:
         python = os.path.join(os.path.dirname(python), 'Scripts\\parallel.exe')
-    popen = partial(subprocess.Popen, creationflags=0x08) # CREATE_NO_WINDOW=0x08 so that no ugly console is popped up
+    open = partial(subprocess.Popen, creationflags=0x08) # CREATE_NO_WINDOW=0x08 so that no ugly console is popped up
 
 if islinux and hasattr(sys, 'frozen_path'):
-    python = os.path.join(getattr(sys, 'frozen_path'), 'parallel')
+    python = os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel')
     popen = partial(subprocess.Popen, cwd=getattr(sys, 'frozen_path')) 
 
-def cleanup(tdir):
+prefix = 'import sys; sys.in_worker = True; '
-    try:
+if hasattr(sys, 'frameworks_dir'):
-        import shutil
+    fd = getattr(sys, 'frameworks_dir')
-        shutil.rmtree(tdir, True)
+    prefix += 'sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd
-    except:
+    if fd not in os.environ['PATH']:
+        os.environ['PATH'] += ':'+fd
+if 'parallel' in python:
+    executable          = [python]
+    worker_command      = '%s:%s'
+    free_spirit_command = '%s'
+else:
+    executable          = [python, '-c']
+    worker_command      = prefix + 'from calibre.parallel import worker; worker(%s, %s)'
+    free_spirit_command = prefix + 'from calibre.parallel import free_spirit; free_spirit(%s)' 
+
+def write(socket, msg, timeout=5):
+    if isinstance(msg, unicode):
+        msg = msg.encode('utf-8')
+    length = None
+    while len(msg) > 0:
+        if length is None:
+            length = len(msg)
+            chunk = ('%-12d'%length) + msg[:4096-12]
+            msg = msg[4096-12:]
+        else:
+            chunk, msg = msg[:4096], msg[4096:]
+        w = select([], [socket], [], timeout)[1]
+        if not w:
+            raise RuntimeError('Write to socket timed out')
+        if socket.sendall(chunk) is not None:
+            raise RuntimeError('Failed to write chunk to socket')
+        
+    
+def read(socket, timeout=5):
+    buf = cStringIO.StringIO()
+    length = None
+    while select([socket],[],[],timeout)[0]:
+        msg = socket.recv(4096)
+        if not msg:
+            break
+        if length is None:
+            length, msg = int(msg[:12]), msg[12:]
+        buf.write(msg)
+        if buf.tell() >= length:
+            break
+    if not length:
+        return ''
+    msg = buf.getvalue()[:length]
+    if len(msg) < length:
+        raise RuntimeError('Corrupted packet received')
+    
+    return msg    
+
+class RepeatingTimer(Thread):
+    
+    def repeat(self):
+        while True:
+            self.event.wait(self.interval)
+            if self.event.isSet():
+                break
+            self.action()
+    
+    def __init__(self, interval, func):
+        self.event    = Event()
+        self.interval = interval
+        self.action = func  
+        Thread.__init__(self, target=self.repeat)
+        self.setDaemon(True)
+    
+class ControlError(Exception):
     pass
 
-class Server(object):
+class Overseer(object):
     
-    #: Interval in seconds at which child processes are polled for status information
-    INTERVAL = 0.1
     KILL_RESULT = 'Server: job killed by user|||#@#$%&*)*(*$#$%#$@&'
+    INTERVAL    = 0.1
     
-    def __init__(self):
+    def __init__(self, server, port, timeout=5):
-        self.tdir = tempfile.mkdtemp('', '%s_IPC_'%__appname__)
+        self.cmd = worker_command%(repr('127.0.0.1'), repr(port))
-        atexit.register(cleanup, self.tdir)
+        self.process = popen(executable + [self.cmd])
-        self.kill_jobs = []
+        self.socket = server.accept()[0]
         
-    def kill(self, job_id):
+        self.working = False
-        '''
+        self.timeout = timeout
-        Kill the job identified by job_id.
+        self.last_job_time = time.time()
-        '''
+        self.job_id = None
-        self.kill_jobs.append(str(job_id))
+        self._stop = False
+        if not select([self.socket], [], [], 120)[0]:
+            raise RuntimeError(_('Could not launch worker process.'))
+        ID = self.read().split(':') 
+        if ID[0] != 'CALIBRE_WORKER':
+            raise RuntimeError('Impostor')
+        self.worker_pid = int(ID[1])
+        self.write('OK')
+        if self.read() != 'WAITING':
+            raise RuntimeError('Worker sulking')
         
-    def _terminate(self, process):
+    def terminate(self):
         '''
         Kill process.
         '''
+        try:
+            if self.socket:
+                self.write('STOP:')
+                time.sleep(1)
+                self.socket.shutdown(socket.SHUT_RDWR)
+        except:
+            pass
         if iswindows:
             win32api = __import__('win32api')
             try:
-                win32api.TerminateProcess(int(process.pid), -1)
+                handle = win32api.OpenProcess(1, False, self.worker_pid)
+                win32api.TerminateProcess(handle, -1)
             except:
                 pass
         else:
             import signal
-            os.kill(process.pid, signal.SIGKILL)
+            try:
+                os.kill(self.worker_pid, signal.SIGKILL)
                 time.sleep(0.05)
+            except:
+                pass
+    
+        
+    def write(self, msg, timeout=None):
+        write(self.socket, msg, timeout=self.timeout if timeout is None else timeout)
+        
+    def read(self, timeout=None):
+        return read(self.socket, timeout=self.timeout if timeout is None else timeout)
+        
+    def __eq__(self, other):
+        return hasattr(other, 'process') and hasattr(other, 'worker_pid') and self.worker_pid == other.worker_pid
+    
+    def __bool__(self):
+        self.process.poll()
+        return self.process.returncode is None
+    
+    def pid(self):
+        return self.worker_pid
+    
+    def select(self, timeout=0):
+        return select([self.socket], [self.socket], [self.socket], timeout)
+    
+    def initialize_job(self, job):
+        self.job_id = job.job_id
+        self.working = True
+        self.write('JOB:'+cPickle.dumps((job.func, job.args, job.kwdargs), -1))
+        msg = self.read()
+        if msg != 'OK':
+            raise ControlError('Failed to initialize job on worker %d:%s'%(self.worker_pid, msg))
+        self.output = job.output if callable(job.output) else sys.stdout.write
+        self.progress = job.progress if callable(job.progress) else None
+        self.job =  job
+    
+    def control(self):
+        try:
+            if select([self.socket],[],[],0)[0]:
+                msg = self.read()
+                word, msg = msg.partition(':')[0], msg.partition(':')[-1]
+                if word == 'RESULT':
+                    self.write('OK')
+                    return Result(cPickle.loads(msg), None, None)
+                elif word == 'OUTPUT':
+                    self.write('OK')
+                    try:
+                        self.output(''.join(cPickle.loads(msg)))
+                    except:
+                        self.output('Bad output message: '+ repr(msg))
+                elif word == 'PROGRESS':
+                    self.write('OK')
+                    percent = None
+                    try:
+                        percent, msg = cPickle.loads(msg)[-1]
+                    except:
+                        print 'Bad progress update:', repr(msg)
+                    if self.progress and percent is not None:
+                        self.progress(percent, msg)
+                elif word == 'ERROR':
+                    self.write('OK')
+                    return Result(None, *cPickle.loads(msg))
+                else:
+                    self.terminate()
+                    return Result(None, ControlError('Worker sent invalid msg: %s', repr(msg)), '')
+            self.process.poll()
+            if self.process.returncode is not None:
+                return Result(None, ControlError('Worker process died unexpectedly with returncode: %d'%self.process.returncode), '')
+        finally:
+            self.working = False
+            self.last_job_time = time.time()
+                        
+class Job(object):
+    
+    def __init__(self, job_id, func, args, kwdargs, output, progress, done):
+        self.job_id = job_id
+        self.func = func
+        self.args = args
+        self.kwdargs = kwdargs
+        self.output = output
+        self.progress = progress
+        self.done = done
+        
+class Result(object):
+    
+    def __init__(self, result, exception, traceback):
+        self.result = result
+        self.exception = exception
+        self.traceback = traceback
+        
+    def __len__(self):
+        return 3
+    
+    def __item__(self, i):
+        return (self.result, self.exception, self.traceback)[i]
+    
+    def __iter__(self):
+        return iter((self.result, self.exception, self.traceback))
+
+class Server(Thread):
+    
+    KILL_RESULT = Overseer.KILL_RESULT
+    START_PORT = 10013
+    
+    def __init__(self, number_of_workers=detect_ncpus()):
+        Thread.__init__(self)
+        self.setDaemon(True)
+        self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.port = self.START_PORT
+        while True:
+            try:
+                self.server_socket.bind(('localhost', self.port))
+                break
+            except:
+                self.port += 1
+        self.server_socket.listen(5)
+        self.number_of_workers = number_of_workers 
+        self.pool, self.jobs, self.working, self.results = [], collections.deque(), [], {}
+        atexit.register(self.killall)
+        atexit.register(self.close)
+        self.job_lock = RLock()
+        self.overseer_lock = RLock()
+        self.working_lock = RLock()
+        self.result_lock = RLock()
+        self.pool_lock = RLock()
+        self.start()
+        
+    def close(self):
+        try:
+            self.server_socket.shutdown(socket.SHUT_RDWR)
+        except:
+            pass
+    
+    def add_job(self, job):
+        with self.job_lock:
+            self.jobs.append(job)
+            
+    def store_result(self, result, id=None):
+        if id:
+            with self.job_lock:
+                self.results[id] = result
+                
+    def result(self, id):
+        with self.result_lock:
+            return self.results.pop(id, None)
+    
+    def run(self):
+        while True:
+            job = None
+            with self.job_lock:
+                if len(self.jobs) > 0 and len(self.working) < self.number_of_workers:
+                    job = self.jobs.popleft()
+                    with self.pool_lock:
+                        o = self.pool.pop() if self.pool else Overseer(self.server_socket, self.port)
+                    try:
+                        o.initialize_job(job)
+                    except Exception, err:
+                        res = Result(None, unicode(err), traceback.format_exc())
+                        job.done(res)
+                        o.terminate()
+                        o = None
+                    if o:
+                        with self.working_lock:
+                            self.working.append(o)
+                    
+            with self.working_lock:
+                done = []
+                for o in self.working:
+                    try:
+                        res = o.control()
+                    except Exception, err:
+                        res = Result(None, unicode(err), traceback.format_exc())
+                        o.terminate()
+                    if isinstance(res, Result):
+                        o.job.done(res)
+                        done.append(o)
+                for o in done:
+                    self.working.remove(o)
+                    if o:
+                        with self.pool_lock:
+                            self.pool.append(o)
+                            
+            time.sleep(1)            
+                
+    
+    def killall(self):
+        with self.pool_lock:
+            map(lambda x: x.terminate(), self.pool)
+            self.pool = []
+        
+        
+    def kill(self, job_id):
+        with self.working_lock:
+            pop = None
+            for o in self.working:
+                if o.job_id == job_id:
+                    o.terminate()
+                    o.job.done(Result(self.KILL_RESULT, None, ''))
+                    pop = o
+                    break
+            if pop is not None:
+                self.working.remove(pop)
                 
                 
         
-    def run(self, job_id, func, args=[], kwdargs={}, monitor=True):
+    def run_job(self, job_id, func, args=[], kwdargs={}, 
+                output=None, progress=None, done=None):
         '''
-        Run a job in a separate process.
+        Run a job in a separate process. Supports job control, output redirection 
-        @param job_id: A unique (per server) identifier
+        and progress reporting.
-        @param func: One of C{PARALLEL_FUNCS.keys()}
-        @param args: A list of arguments to pass of C{func}
-        @param kwdargs: A dictionary of keyword arguments to pass to C{func}
-        @param monitor: If False launch the child process and return. Do not monitor/communicate with it.
-        @return: (result, exception, formatted_traceback, log) where log is the combined
-        stdout + stderr of the child process; or None if monitor is True. If a job is killed
-        by a call to L{kill()} then result will be L{KILL_RESULT}
         '''
-        job_id = str(job_id)
+        if done is None:
-        job_dir = os.path.join(self.tdir, job_id)
+            done = partial(self.store_result, id=job_id)
-        if os.path.exists(job_dir):
+        job = Job(job_id, func, args, kwdargs, output, progress, done)
-            raise ValueError('Cannot run job. The job_id %s has already been used.'%job_id)
+        with self.job_lock:
-        os.mkdir(job_dir)
+            self.jobs.append(job)
             
-        job_data = os.path.join(job_dir, 'job_data.pickle')
+    def run_free_job(self, func, args=[], kwdargs={}):
-        cPickle.dump((job_id, func, args, kwdargs), open(job_data, 'wb'), -1)
+        pt = PersistentTemporaryFile('.pickle', '_IPC_')
-        prefix = ''
+        pt.write(cPickle.dumps((func, args, kwdargs)))
-        if hasattr(sys, 'frameworks_dir'):
+        pt.close()
-            fd = getattr(sys, 'frameworks_dir')
+        cmd = free_spirit_command%repr(binascii.hexlify(pt.name))
-            prefix = 'import sys; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd
+        popen(executable + [cmd])
-            if fd not in os.environ['PATH']:
-                os.environ['PATH'] += ':'+fd
-        cmd = prefix + 'from calibre.parallel import run_job; run_job(\'%s\')'%binascii.hexlify(job_data)
 
-        if not monitor:
+##########################################################################################
-            popen([python, '-c', cmd], stdout=subprocess.PIPE, stdin=subprocess.PIPE,
+##################################### CLIENT CODE #####################################
-                  stderr=subprocess.PIPE)
+##########################################################################################
+
+class BufferedSender(object):
+    
+    def __init__(self, socket):
+        self.socket = socket
+        self.wbuf, self.pbuf    = [], []
+        self.wlock, self.plock   = RLock(), RLock()
+        self.timer  = RepeatingTimer(0.5, self.send)
+        self.prefix = prefix
+        self.timer.start()
+        
+    def write(self, msg):
+        if not isinstance(msg, basestring):
+            msg = unicode(msg)
+        with self.wlock:
+            self.wbuf.append(msg)
+    
+    def send(self):
+        if not select([], [self.socket], [], 30)[1]:
+            print >>sys.__stderr__, 'Cannot pipe to overseer'
             return
         
-        output = open(os.path.join(job_dir, 'output.txt'), 'wb')
+        with self.wlock:
-        p = popen([python, '-c', cmd], stdout=output, stderr=output,
+            if self.wbuf:
-                             stdin=subprocess.PIPE)
+                msg = cPickle.dumps(self.wbuf, -1)
-        p.stdin.close()
+                self.wbuf = []
-        while p.returncode is None:
+                write(self.socket, 'OUTPUT:'+msg)
-            if job_id in self.kill_jobs:
+                read(self.socket, 10)
-                self._terminate(p)
-                return self.KILL_RESULT, None, None, _('Job killed by user')
-            time.sleep(0.1)
-            p.poll()
                 
+        with self.plock:
+            if self.pbuf:
+                msg = cPickle.dumps(self.pbuf, -1)
+                self.pbuf = []
+                write(self.socket, 'PROGRESS:'+msg)
+                read(self.socket, 10)        
                 
-        output.close()
+    def notify(self, percent, msg=''):
-        job_result = os.path.join(job_dir, 'job_result.pickle')
+        with self.plock:
-        if not os.path.exists(job_result):
+            self.pbuf.append((percent, msg))
-            result, exception, traceback = None, ('ParallelRuntimeError',
-                                                  'The worker process died unexpectedly.'), ''
-        else:
-            result, exception, traceback = cPickle.load(open(job_result, 'rb'))
-        log = open(output.name, 'rb').read()
         
-        return result, exception, traceback, log
+    def flush(self):
+        pass
 
-
+def work(client_socket, func, args, kwdargs):
-def run_job(job_data):
-    global sa, job_id
-    if SingleApplication is not None:
-        sa = SingleApplication('calibre GUI')
-    job_data = binascii.unhexlify(job_data)
-    base = os.path.dirname(job_data)
-    job_result = os.path.join(base, 'job_result.pickle')
-    job_id, func, args, kwdargs = cPickle.load(open(job_data, 'rb'))
     func = PARALLEL_FUNCS[func]
-    exception, tb = None, None
+    if hasattr(func, 'keywords'):
+        for key, val in func.keywords.items():
+            if val == _notify and hasattr(sys.stdout, 'notify'):
+                func.keywords[key] = sys.stdout.notify
+    res = func(*args, **kwdargs)
+    if hasattr(sys.stdout, 'send'): 
+        sys.stdout.send()
+    return res
+    
+
+def worker(host, port):
+    client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    client_socket.connect((host, port))
+    write(client_socket, 'CALIBRE_WORKER:%d'%os.getpid())
+    msg = read(client_socket, timeout=10)
+    if msg != 'OK':
+        return 1
+    write(client_socket, 'WAITING')
+    
+    sys.stdout = BufferedSender(client_socket)
+    sys.stderr = sys.stdout
+    
+    while True:
+        msg = read(client_socket, timeout=60)
+        if msg.startswith('JOB:'):
+            func, args, kwdargs = cPickle.loads(msg[4:])
+            write(client_socket, 'OK')
             try:
-        result = func(*args, **kwdargs)
+                result = work(client_socket, func, args, kwdargs)
+                write(client_socket, 'RESULT:'+ cPickle.dumps(result))
             except (Exception, SystemExit), err:
-        result = None
                 exception = (err.__class__.__name__, unicode(str(err), 'utf-8', 'replace'))
                 tb = traceback.format_exc()
+                write(client_socket, 'ERROR:'+cPickle.dumps((exception, tb),-1))
+            if read(client_socket, 10) != 'OK':
+                break
+            gc.collect()
+        elif msg == 'STOP:':
+            return 0
+        elif not msg:
+            time.sleep(1)
+        else:
+            print >>sys.__stderr__, 'Invalid protocols message', msg
+            return 1
     
-    if os.path.exists(os.path.dirname(job_result)):
+def free_spirit(path):
-        cPickle.dump((result, exception, tb), open(job_result, 'wb'))
+    func, args, kwdargs = cPickle.load(open(binascii.unhexlify(path), 'rb'))
-    
+    try:
-def main():
+        os.unlink(path)
-    src = sys.argv[2]
+    except:
-    job_data = re.search(r'run_job\(\'([a-f0-9A-F]+)\'\)', src).group(1)
+        pass
-    run_job(job_data)
+    PARALLEL_FUNCS[func](*args, **kwdargs)
     
+def main(args=sys.argv):
+    args = args[1].split(':')
+    if len(args) == 1:
+        free_spirit(args[0].replace("'", ''))
+    else:
+        worker(args[0].replace("'", ''), int(args[1])) 
     return 0
 
 if __name__ == '__main__':
     sys.exit(main())
     
-

src/calibre/terminfo.py

@@ -94,7 +94,7 @@ class TerminalController:
         except: return
         
         # If the stream isn't a tty, then assume it has no capabilities.
-        if not hasattr(term_stream, 'isatty') or not term_stream.isatty(): return
+        if hasattr(sys, 'in_worker') or not hasattr(term_stream, 'isatty') or not term_stream.isatty(): return
         
         # Check the terminal type.  If we fail, then assume that the
         # terminal has no capabilities.

src/calibre/trac/plugins/download.py

@@ -1,6 +1,6 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import re, glob
+import re
 from pkg_resources import resource_filename
 
 from trac.core import Component, implements
@@ -12,7 +12,7 @@ from trac.util import Markup
 __appname__ = 'calibre'
 DOWNLOAD_DIR = '/var/www/calibre.kovidgoyal.net/htdocs/downloads'
 LINUX_INSTALLER = '/var/www/calibre.kovidgoyal.net/calibre/src/calibre/linux_installer.py'
-
+MOBILEREAD = 'https://dev.mobileread.com/dist/kovid/calibre/'
 
 class OS(dict):
     """Dictionary with a default value for unknown keys."""
@@ -119,7 +119,7 @@ class Download(Component):
         if req.path_info == '/download':
             return self.top_level(req)
         elif req.path_info == '/download_linux_binary_installer':
-            req.send(open(LINUX_INSTALLER).read(), 'text/x-python')
+            req.send(open(LINUX_INSTALLER).read().replace('%version', self.version_from_filename()), 'text/x-python')
         else:
             match = re.match(r'\/download_(\S+)', req.path_info)
             if match:
@@ -153,8 +153,7 @@ class Download(Component):
     
     def version_from_filename(self):
         try:
-            file = glob.glob(DOWNLOAD_DIR+'/*.exe')[0]
+            return open(DOWNLOAD_DIR+'/latest_version', 'rb').read().strip()
-            return re.search(r'\S+-(\d+\.\d+\.\d+)\.', file).group(1)
         except:
             return '0.0.0'
     
@@ -165,7 +164,7 @@ class Download(Component):
             installer_name='Windows installer', 
             title='Download %s for windows'%(__appname__),
             compatibility='%s works on Windows XP and Windows Vista.'%(__appname__,),
-            path='/downloads/'+file, app=__appname__,
+            path=MOBILEREAD+file, app=__appname__,
             note=Markup(\
 '''
 <p>If you are using the <b>SONY PRS-500</b> and %(appname)s does not detect your reader, read on:</p>
@@ -203,7 +202,7 @@ You can uninstall a driver by right clicking on it and selecting uninstall.
             installer_name='OS X universal dmg', 
             title='Download %s for OS X'%(__appname__),
             compatibility='%s works on OS X Tiger and above.'%(__appname__,),
-            path='/downloads/'+file, app=__appname__,
+            path=MOBILEREAD+file, app=__appname__,
             note=Markup(\
 '''
 <ol>

src/calibre/utils/single_qt_application.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 Enforces running of only a single application instance and allows for messaging between
 applications using a local socket.
 '''
-import atexit
+import atexit, os
 
 from PyQt4.QtCore import QByteArray, QDataStream, QIODevice, SIGNAL, QObject, Qt, QString
 from PyQt4.QtNetwork import QLocalSocket, QLocalServer
@@ -94,8 +94,23 @@ class LocalServer(QLocalServer):
         for conn in pop:
             self.connections.remove(conn)
             
+    def listen(self, name):
+        if not QLocalServer.listen(self, name):
+            try:
+                os.unlink(self.fullServerName())
+            except:
+                pass
+            return QLocalServer.listen(self, name)
+        return True
         
                 
+def send_message(msg, name, server_name='calibre_server', timeout=5000):
+    socket = QLocalSocket()
+    socket.connectToServer(server_name)
+    if socket.waitForConnected(timeout_connect):
+        if read_message(socket) == name:
+            write_message(socket, name+':'+msg, timeout)
+        
 class SingleApplication(QObject):
     
     def __init__(self, name, parent=None, server_name='calibre_server'):
@@ -123,7 +138,6 @@ class SingleApplication(QObject):
             self.connect(self.server, SIGNAL('message_received(PyQt_PyObject)'), 
                          self.mr, Qt.QueuedConnection)
             
-            if not self.server.listen(self.server_name):
             if not self.server.listen(self.server_name):
                 self.server = None
         if self.server is not None:

upload.py

@@ -1,5 +1,5 @@
 #!/usr/bin/python
-import sys, os, shutil, time, tempfile, socket, fcntl, struct
+import sys, os, shutil, time, tempfile, socket, fcntl, struct, cStringIO, pycurl, re
 sys.path.append('src')
 import subprocess
 from subprocess import check_call as _check_call
@@ -24,6 +24,7 @@ DOCS = PREFIX+"/htdocs/apidocs"
 USER_MANUAL = PREFIX+'/htdocs/user_manual'
 HTML2LRF = "src/calibre/ebooks/lrf/html/demo"
 TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
+MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
 BUILD_SCRIPT ='''\
 #!/bin/bash
 cd ~/build && \
@@ -110,19 +111,72 @@ def upload_demo():
     check_call('cd src/calibre/ebooks/lrf/txt/demo/ && zip -j /tmp/txt-demo.zip * /tmp/txt2lrf.lrf')
     check_call('''scp /tmp/txt-demo.zip divok:%s/'''%(DOWNLOADS,))
 
+def curl_list_dir(url=MOBILEREAD, listonly=1):
+    c = pycurl.Curl()
+    c.setopt(pycurl.URL, url)
+    c.setopt(c.FTP_USE_EPSV, 1)
+    c.setopt(c.NETRC, c.NETRC_REQUIRED)
+    c.setopt(c.FTPLISTONLY, listonly)
+    c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
+    b = cStringIO.StringIO()
+    c.setopt(c.WRITEFUNCTION, b.write)
+    c.perform()
+    c.close()
+    return b.getvalue().split() if listonly else b.getvalue().splitlines()
+
+def curl_delete_file(path, url=MOBILEREAD):
+    c = pycurl.Curl()
+    c.setopt(pycurl.URL, url)
+    c.setopt(c.FTP_USE_EPSV, 1)
+    c.setopt(c.NETRC, c.NETRC_REQUIRED)
+    print 'Deleting file %s on %s'%(path, url)
+    c.setopt(c.QUOTE, ['dele '+ path])
+    c.perform()
+    c.close()
+    
+
+def curl_upload_file(stream, url):
+    c = pycurl.Curl()
+    c.setopt(pycurl.URL, url)
+    c.setopt(pycurl.UPLOAD, 1)
+    c.setopt(c.NETRC, c.NETRC_REQUIRED)
+    c.setopt(pycurl.READFUNCTION, stream.read)
+    stream.seek(0, 2)
+    c.setopt(pycurl.INFILESIZE_LARGE, stream.tell())
+    stream.seek(0)
+    c.setopt(c.NOPROGRESS, 0)
+    c.setopt(c.FTP_CREATE_MISSING_DIRS, 1)
+    print 'Uploading file %s to url %s' % (getattr(stream, 'name', ''), url)
+    try:
+        c.perform()
+        c.close()
+    except:
+        pass
+    files = curl_list_dir(listonly=0)
+    for line in files:
+        line = line.split()
+        if url.endswith(line[-1]):
+            size = long(line[4])
+            stream.seek(0,2)
+            if size != stream.tell():
+                raise RuntimeError('curl failed to upload %s correctly'%getattr(stream, 'name', ''))
+                             
+        
+    
+def upload_installer(name):
+    bname = os.path.basename(name)
+    pat = re.compile(bname.replace(__version__, r'\d+\.\d+\.\d+'))
+    for f in curl_list_dir():
+        if pat.search(f):
+            curl_delete_file('/calibre/'+f)
+    curl_upload_file(open(name, 'rb'), MOBILEREAD+os.path.basename(name))
+
 def upload_installers():
-    exe, dmg, tbz2 = installer_name('exe'), installer_name('dmg'), installer_name('tar.bz2')
+    for i in ('dmg', 'exe', 'tar.bz2'):
-    if exe and os.path.exists(exe):
+        upload_installer(installer_name(i))
-        check_call('''ssh divok rm -f %s/calibre\*.exe'''%(DOWNLOADS,))
+        
-        check_call('''scp %s divok:%s/'''%(exe, DOWNLOADS))
+    check_call('''ssh divok echo %s \\> %s/latest_version'''%(__version__, DOWNLOADS))
-    if dmg and os.path.exists(dmg):
+        
-        check_call('''ssh divok rm -f %s/calibre\*.dmg'''%(DOWNLOADS,)) 
-        check_call('''scp %s divok:%s/'''%(dmg, DOWNLOADS))
-    if tbz2 and os.path.exists(tbz2):
-        check_call('''ssh divok rm -f %s/calibre-\*-i686.tar.bz2 %s/latest-linux-binary.tar.bz2'''%(DOWNLOADS,DOWNLOADS))
-        check_call('''scp %s divok:%s/'''%(tbz2, DOWNLOADS))
-        check_call('''ssh divok ln -s %s/calibre-\*-i686.tar.bz2 %s/latest-linux-binary.tar.bz2'''%(DOWNLOADS,DOWNLOADS))
-    check_call('''ssh divok chmod a+r %s/\*'''%(DOWNLOADS,))
         
 def upload_docs():
     check_call('''epydoc --config epydoc.conf''')

windows_installer.py

@@ -514,6 +514,12 @@ class BuildEXE(build_exe):
         f.write('src\\calibre\\gui2\\main.py', 'calibre\\gui2\\main.py')
         f.close()
 
+        print
+        print 'Doing DLL redirection' # See http://msdn.microsoft.com/en-us/library/ms682600(VS.85).aspx
+        for f in glob.glob(os.path.join('build', 'py2exe', '*.exe')):
+            open(f + '.local', 'wb').write('\n')
+
+
         print
         print
         print 'Building Installer'
@@ -557,12 +563,12 @@ def main():
                                              'win32file', 'pythoncom', 'rtf2xml',
                                              'lxml', 'lxml._elementpath', 'genshi',
                                              'path', 'pydoc', 'IPython.Extensions.*',
-                                             'calibre.web.feeds.recipes.*', 'pydoc',
+                                             'calibre.web.feeds.recipes.*', 'PyQt4.QtWebKit',
                                              ],
                                   'packages'  : ['PIL'],
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",
-                                                 "_imagingtk", "ImageTk", "FixTk",
+                                                 "_imagingtk", "ImageTk", "FixTk"
-                                                 'pydoc'],
+                                                ],
                                   'dll_excludes' : ['mswsock.dll'],
                                  },
                     },