From 589fa1ef13879c975ed6048d4e5e27dc8a77d3fc Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 16 Jun 2008 23:17:18 -0700 Subject: [PATCH 01/18] Fix #788 --- src/calibre/library/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index c357239fa5..dc4f4e31de 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1422,7 +1422,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; mi = OPFCreator(base, self.get_metadata(idx, index_is_id=index_is_id)) cover = self.cover(idx, index_is_id=index_is_id) if cover is not None: - cname = name + '.jpg' + cname = sanitize_file_name(name) + '.jpg' cpath = os.path.join(base, cname) open(cpath, 'wb').write(cover) mi.cover = cname From ecb18a05ea423277490139f9eaade84d908bb09b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 16 Jun 2008 23:29:22 -0700 Subject: [PATCH 02/18] Fix #792 --- src/calibre/gui2/dialogs/user_profiles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/dialogs/user_profiles.py b/src/calibre/gui2/dialogs/user_profiles.py index 5cae63c6d7..a06191e48c 100644 --- a/src/calibre/gui2/dialogs/user_profiles.py +++ b/src/calibre/gui2/dialogs/user_profiles.py @@ -84,6 +84,7 @@ class UserProfiles(QDialog, Ui_Dialog): self.populate_options(recipe) self.stacks.setCurrentIndex(0) self.toggle_mode_button.setText(_('Switch to Advanced mode')) + self.source_code.setPlainText('') else: self.source_code.setPlainText(src) self.highlighter = PythonHighlighter(self.source_code.document()) From 474f2cadbb385b134d3018a3f79ba8261660f9c1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jun 2008 09:44:49 -0700 Subject: [PATCH 03/18] Add option to render tables as images (avaialble in the Page options section). This should be helpful for HTML files with complex tables. --- osx_installer.py | 13 ++- src/calibre/ebooks/lrf/__init__.py | 6 +- src/calibre/ebooks/lrf/html/convert_from.py | 55 ++++++++--- src/calibre/ebooks/lrf/html/table_as_image.py | 99 +++++++++++++++++++ src/calibre/gui2/dialogs/fetch_metadata.ui | 7 +- src/calibre/gui2/dialogs/lrf_single.py | 1 + src/calibre/gui2/dialogs/lrf_single.ui | 99 ++++++++++++++----- src/calibre/parallel.py | 44 ++++++--- windows_installer.py | 12 ++- 9 files changed, 273 insertions(+), 63 deletions(-) create mode 100644 src/calibre/ebooks/lrf/html/table_as_image.py diff --git a/osx_installer.py b/osx_installer.py index cccb46ad93..6fb8b3a4e8 100644 --- a/osx_installer.py +++ b/osx_installer.py @@ -51,6 +51,7 @@ def _check_symlinks_prescript(): import os scripts = %(sp)s links = %(sp)s +fonts_conf = %(sp)s os.setuid(0) for s, l in zip(scripts, links): if os.path.lexists(l): @@ -59,6 +60,11 @@ for s, l in zip(scripts, links): omask = os.umask(022) os.symlink(s, l) os.umask(omask) +if not os.path.exists('/etc/fonts/fonts.conf'): + print 'Creating default fonts.conf' + if not os.path.exists('/etc/fonts'): + os.makedirs('/etc/fonts') + os.link(fonts_conf, '/etc/fonts/fonts.conf') """ dest_path = %(dest_path)s @@ -66,6 +72,7 @@ for s, l in zip(scripts, links): scripts = %(scripts)s links = [os.path.join(dest_path, i) for i in scripts] scripts = [os.path.join(resources_path, 'loaders', i) for i in scripts] + fonts_conf = os.path.join(resources_path, 'fonts.conf') bad = False for s, l in zip(scripts, links): @@ -76,7 +83,7 @@ for s, l in zip(scripts, links): if bad: auth = Authorization(destroyflags=(kAuthorizationFlagDestroyRights,)) fd, name = tempfile.mkstemp('.py') - os.write(fd, AUTHTOOL %(pp)s (sys.executable, repr(scripts), repr(links))) + os.write(fd, AUTHTOOL %(pp)s (sys.executable, repr(scripts), repr(links), repr(fonts_conf))) os.close(fd) os.chmod(name, 0700) try: @@ -276,10 +283,12 @@ sys.frameworks_dir = os.path.join(os.path.dirname(os.environ['RESOURCEPATH']), ' f.write('src/calibre/gui2/main.py', 'calibre/gui2/main.py') f.close() print + print 'Adding default fonts.conf' + open(os.path.join(self.dist_dir, APPNAME+'.app', 'Contents', 'Resources', 'fonts.conf'), 'wb').write(open('/etc/fonts/fonts.conf').read()) + print print 'Building disk image' BuildAPP.makedmg(os.path.join(self.dist_dir, APPNAME+'.app'), APPNAME+'-'+VERSION) - def main(): sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src')) sys.argv[1:2] = ['py2app'] diff --git a/src/calibre/ebooks/lrf/__init__.py b/src/calibre/ebooks/lrf/__init__.py index 02882312a8..af93bca6cb 100644 --- a/src/calibre/ebooks/lrf/__init__.py +++ b/src/calibre/ebooks/lrf/__init__.py @@ -120,7 +120,6 @@ def option_parser(usage, gui_mode=False): dest='font_delta') laf.add_option('--ignore-colors', action='store_true', default=False, dest='ignore_colors', help=_('Render all content as black on white instead of the colors specified by the HTML or CSS.')) - page = parser.add_option_group('PAGE OPTIONS') profiles = profile_map.keys() @@ -139,6 +138,11 @@ def option_parser(usage, gui_mode=False): help=_('''Top margin of page. Default is %default px.''')) page.add_option('--bottom-margin', default=0, dest='bottom_margin', type='int', help=_('''Bottom margin of page. Default is %default px.''')) + page.add_option('--render-tables-as-images', default=False, action='store_true', + help=_('Render tables in the HTML as images (useful if the document has large or complex tables)')) + page.add_option('--text-size-multiplier-for-rendered-tables', type='float', default=1.0, + help=_('Multiply the size of text in rendered tables by this factor. Default is %default')) + link = parser.add_option_group('LINK PROCESSING OPTIONS') link.add_option('--link-levels', action='store', type='int', default=sys.maxint, \ dest='link_levels', diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py index 10a7137f34..6dc268e9eb 100644 --- a/src/calibre/ebooks/lrf/html/convert_from.py +++ b/src/calibre/ebooks/lrf/html/convert_from.py @@ -383,7 +383,8 @@ class HTMLConverter(object, LoggingInterface): self.target_prefix = path self.previous_text = '\n' self.tops[path] = self.parse_file(soup) - self.processed_files.append(path) + self.processed_files.append(path) + def parse_css(self, style): @@ -494,7 +495,9 @@ class HTMLConverter(object, LoggingInterface): top = self.current_block self.current_block.must_append = True + self.soup = soup self.process_children(soup, {}, {}) + self.soup = None if self.current_para and self.current_block: self.current_para.append_to(self.current_block) @@ -1680,18 +1683,44 @@ class HTMLConverter(object, LoggingInterface): self.previous_text = ' ' self.process_children(tag, tag_css, tag_pseudo_css) elif tagname == 'table' and not self.ignore_tables and not self.in_table: - tag_css = self.tag_css(tag)[0] # Table should not inherit CSS - try: - self.process_table(tag, tag_css) - except Exception, err: - self.log_warning(_('An error occurred while processing a table: %s. Ignoring table markup.'), str(err)) - self.log_debug('', exc_info=True) - self.log_debug(_('Bad table:\n%s'), str(tag)[:300]) - self.in_table = False - self.process_children(tag, tag_css, tag_pseudo_css) - finally: - if self.minimize_memory_usage: - tag.extract() + if self.render_tables_as_images: + print 'Rendering table...' + from calibre.ebooks.lrf.html.table_as_image import render_table + pheight = int(self.current_page.pageStyle.attrs['textheight']) + pwidth = int(self.current_page.pageStyle.attrs['textwidth']) + images = render_table(self.soup, tag, tag_css, + os.path.dirname(self.target_prefix), + pwidth, pheight, self.profile.dpi, + self.text_size_multiplier_for_rendered_tables) + for path, width, height in images: + stream = ImageStream(path, encoding='PNG') + im = Image(stream, x0=0, y0=0, x1=width, y1=height,\ + xsize=width, ysize=height) + pb = self.current_block + self.end_current_para() + self.process_alignment(tag_css) + self.current_para.append(Plot(im, xsize=width*720./self.profile.dpi, + ysize=height*720./self.profile.dpi)) + self.current_block.append(self.current_para) + self.current_page.append(self.current_block) + self.current_block = self.book.create_text_block( + textStyle=pb.textStyle, + blockStyle=pb.blockStyle) + self.current_para = Paragraph() + + else: + tag_css = self.tag_css(tag)[0] # Table should not inherit CSS + try: + self.process_table(tag, tag_css) + except Exception, err: + self.log_warning(_('An error occurred while processing a table: %s. Ignoring table markup.'), str(err)) + self.log_debug('', exc_info=True) + self.log_debug(_('Bad table:\n%s'), str(tag)[:300]) + self.in_table = False + self.process_children(tag, tag_css, tag_pseudo_css) + finally: + if self.minimize_memory_usage: + tag.extract() else: self.process_children(tag, tag_css, tag_pseudo_css) finally: diff --git a/src/calibre/ebooks/lrf/html/table_as_image.py b/src/calibre/ebooks/lrf/html/table_as_image.py new file mode 100644 index 0000000000..501a049832 --- /dev/null +++ b/src/calibre/ebooks/lrf/html/table_as_image.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Render HTML tables as images. +''' +import os, tempfile, atexit, shutil +from PyQt4.Qt import QWebPage, QUrl, QApplication, QSize, \ + SIGNAL, QPainter, QImage, QObject, Qt + +__app = None + +class HTMLTableRenderer(QObject): + + def __init__(self, html, base_dir, width, height, dpi, factor): + ''' + `width, height`: page width and height in pixels + `base_dir`: The directory in which the HTML file that contains the table resides + ''' + QObject.__init__(self) + + self.app = None + self.width, self.height, self.dpi = width, height, dpi + self.base_dir = base_dir + self.page = QWebPage() + self.connect(self.page, SIGNAL('loadFinished(bool)'), self.render_html) + self.page.mainFrame().setTextSizeMultiplier(factor) + self.page.mainFrame().setHtml(html, + QUrl('file:'+os.path.abspath(self.base_dir))) + self.images = [] + self.tdir = tempfile.mkdtemp(prefix='calibre_render_table') + + def render_html(self, ok): + try: + if not ok: + return + cwidth, cheight = self.page.mainFrame().contentsSize().width(), self.page.mainFrame().contentsSize().height() + self.page.setViewportSize(QSize(cwidth, cheight)) + factor = float(self.width)/cwidth if cwidth > self.width else 1 + cutoff_height = int(self.height/factor)-3 + image = QImage(self.page.viewportSize(), QImage.Format_ARGB32) + image.setDotsPerMeterX(self.dpi*(100/2.54)) + image.setDotsPerMeterX(self.dpi*(100/2.54)) + painter = QPainter(image) + self.page.mainFrame().render(painter) + painter.end() + pos = 0 + while pos < cheight: + img = image.copy(0, pos, cwidth, cutoff_height) + pos += cutoff_height-20 + if cwidth > self.width: + img = img.scaledToWidth(self.width, Qt.SmoothTransform) + f = os.path.join(self.tdir, '%d.png'%pos) + img.save(f) + self.images.append((f, img.width(), img.height())) + finally: + QApplication.quit() + +def render_table(soup, table, css, base_dir, width, height, dpi, factor=1.0): + head = '' + for e in soup.findAll(['link', 'style']): + head += unicode(e)+'\n\n' + style = '' + for key, val in css.items(): + style += key + ':%s;'%val + html = u'''\ + + + %s + + + + %s + + + '''%(head, width-10, style, unicode(table)) + from calibre.parallel import Server + s = Server() + result, exception, traceback, log = s.run(1, 'render_table', qapp=True, report_progress=False, + args=[html, base_dir, width, height, dpi, factor]) + if exception: + print 'Failed to render table' + print traceback + print log + images, tdir = result + atexit.register(shutil.rmtree, tdir) + return images + +def do_render(html, base_dir, width, height, dpi, factor): + app = QApplication.instance() + if app is None: + app = QApplication([]) + tr = HTMLTableRenderer(html, base_dir, width, height, dpi, factor) + app.exec_() + return tr.images, tr.tdir \ No newline at end of file diff --git a/src/calibre/gui2/dialogs/fetch_metadata.ui b/src/calibre/gui2/dialogs/fetch_metadata.ui index e8b4252d8d..8e5747778f 100644 --- a/src/calibre/gui2/dialogs/fetch_metadata.ui +++ b/src/calibre/gui2/dialogs/fetch_metadata.ui @@ -9,14 +9,15 @@ 0 0 830 - 700 + 642 Fetch metadata - :/images/metadata.svg + + :/images/metadata.svg:/images/metadata.svg @@ -107,7 +108,7 @@ - QDialogButtonBox::Cancel|QDialogButtonBox::NoButton|QDialogButtonBox::Ok + QDialogButtonBox::Cancel|QDialogButtonBox::Ok diff --git a/src/calibre/gui2/dialogs/lrf_single.py b/src/calibre/gui2/dialogs/lrf_single.py index e5df43a673..bc5bdcf06b 100644 --- a/src/calibre/gui2/dialogs/lrf_single.py +++ b/src/calibre/gui2/dialogs/lrf_single.py @@ -382,6 +382,7 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog): self.cmdline = [unicode(i) for i in cmdline] else: Settings().set('LRF conversion defaults', cmdline) + print self.cmdline QDialog.accept(self) class LRFBulkDialog(LRFSingleDialog): diff --git a/src/calibre/gui2/dialogs/lrf_single.ui b/src/calibre/gui2/dialogs/lrf_single.ui index 9fd3bee155..080970b96f 100644 --- a/src/calibre/gui2/dialogs/lrf_single.ui +++ b/src/calibre/gui2/dialogs/lrf_single.ui @@ -115,7 +115,7 @@ - 0 + 2 @@ -818,6 +818,39 @@ + + + + &Convert tables to images (good for large/complex tables) + + + + + + + &Multiplier for text size in rendered tables: + + + gui_text_size_multiplier_for_rendered_tables + + + + + + + false + + + 2 + + + 0.100000000000000 + + + 1.000000000000000 + + + @@ -1048,8 +1081,8 @@ p, li { white-space: pre-wrap; } setCurrentIndex(int) - 191 - 236 + 184 + 279 368 @@ -1064,8 +1097,8 @@ p, li { white-space: pre-wrap; } setDisabled(bool) - 428 - 89 + 650 + 122 788 @@ -1073,22 +1106,6 @@ p, li { white-space: pre-wrap; } - - gui_header - toggled(bool) - gui_headerformat - setEnabled(bool) - - - 348 - 340 - - - 823 - 372 - - - gui_disable_chapter_detection toggled(bool) @@ -1096,12 +1113,44 @@ p, li { white-space: pre-wrap; } setDisabled(bool) - 321 - 78 + 543 + 122 - 322 - 172 + 544 + 211 + + + + + gui_render_tables_as_images + toggled(bool) + gui_text_size_multiplier_for_rendered_tables + setEnabled(bool) + + + 298 + 398 + + + 660 + 435 + + + + + gui_header + toggled(bool) + gui_headerformat + setEnabled(bool) + + + 330 + 367 + + + 823 + 372 diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py index 3d169f4822..a9490f4922 100644 --- a/src/calibre/parallel.py +++ b/src/calibre/parallel.py @@ -3,7 +3,7 @@ __copyright__ = '2008, Kovid Goyal ' ''' Used to run jobs in parallel in separate processes. ''' -import re, sys, tempfile, os, cPickle, traceback, atexit, binascii, time, subprocess +import sys, tempfile, os, cPickle, traceback, atexit, binascii, time, subprocess from functools import partial @@ -11,6 +11,7 @@ from calibre.ebooks.lrf.any.convert_from import main as any2lrf from calibre.ebooks.lrf.web.convert_from import main as web2lrf from calibre.ebooks.lrf.feeds.convert_from import main as feeds2lrf from calibre.gui2.lrf_renderer.main import main as lrfviewer +from calibre.ebooks.lrf.html.table_as_image import do_render as render_table from calibre import iswindows, __appname__, islinux try: from calibre.utils.single_qt_application import SingleApplication @@ -31,6 +32,7 @@ PARALLEL_FUNCS = { 'web2lrf' : web2lrf, 'lrfviewer' : lrfviewer, 'feeds2lrf' : partial(feeds2lrf, notification=report_progress), + 'render_table': render_table, } python = sys.executable @@ -88,7 +90,8 @@ class Server(object): - def run(self, job_id, func, args=[], kwdargs={}, monitor=True): + def run(self, job_id, func, args=[], kwdargs={}, monitor=True, + report_progress=True, qapp=True): ''' Run a job in a separate process. @param job_id: A unique (per server) identifier @@ -96,6 +99,8 @@ class Server(object): @param args: A list of arguments to pass of C{func} @param kwdargs: A dictionary of keyword arguments to pass to C{func} @param monitor: If False launch the child process and return. Do not monitor/communicate with it. + @param report_progess: If True progress is reported to the GUI + @param qapp: If True, A QApplication is created. If False, progress reporting will also be disabled. @return: (result, exception, formatted_traceback, log) where log is the combined stdout + stderr of the child process; or None if monitor is True. If a job is killed by a call to L{kill()} then result will be L{KILL_RESULT} @@ -107,14 +112,15 @@ class Server(object): os.mkdir(job_dir) job_data = os.path.join(job_dir, 'job_data.pickle') - cPickle.dump((job_id, func, args, kwdargs), open(job_data, 'wb'), -1) + cPickle.dump((job_id, func, args, kwdargs, report_progress, qapp), + open(job_data, 'wb'), -1) prefix = '' if hasattr(sys, 'frameworks_dir'): fd = getattr(sys, 'frameworks_dir') prefix = 'import sys; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd if fd not in os.environ['PATH']: os.environ['PATH'] += ':'+fd - cmd = prefix + 'from calibre.parallel import run_job; run_job(\'%s\')'%binascii.hexlify(job_data) + cmd = prefix + 'from calibre.parallel import main; main(\'%s\')'%binascii.hexlify(job_data) if not monitor: popen([python, '-c', cmd], stdout=subprocess.PIPE, stdin=subprocess.PIPE, @@ -145,14 +151,12 @@ class Server(object): return result, exception, traceback, log -def run_job(job_data): - global sa, job_id - if SingleApplication is not None: - sa = SingleApplication('calibre GUI') - job_data = binascii.unhexlify(job_data) - base = os.path.dirname(job_data) +def run_job(base, id, func, args, kwdargs): + global job_id + job_id = id + job_result = os.path.join(base, 'job_result.pickle') - job_id, func, args, kwdargs = cPickle.load(open(job_data, 'rb')) + func = PARALLEL_FUNCS[func] exception, tb = None, None try: @@ -165,14 +169,22 @@ def run_job(job_data): if os.path.exists(os.path.dirname(job_result)): cPickle.dump((result, exception, tb), open(job_result, 'wb')) -def main(): - src = sys.argv[2] - job_data = re.search(r'run_job\(\'([a-f0-9A-F]+)\'\)', src).group(1) - run_job(job_data) +def main(src): + from PyQt4.QtGui import QApplication + job_data = binascii.unhexlify(src) + global sa + job_id, func, args, kwdargs, rp, qapp = cPickle.load(open(job_data, 'rb')) + + if qapp and QApplication.instance() is None: + QApplication([]) + if SingleApplication is not None and rp and QApplication.instance() is not None: + sa = SingleApplication('calibre GUI') + + run_job(os.path.dirname(job_data), job_id, func, args, kwdargs) return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main(sys.argv[2])) diff --git a/windows_installer.py b/windows_installer.py index c38ee3487b..06fa64d8ce 100644 --- a/windows_installer.py +++ b/windows_installer.py @@ -514,6 +514,12 @@ class BuildEXE(build_exe): f.write('src\\calibre\\gui2\\main.py', 'calibre\\gui2\\main.py') f.close() + print + print 'Doing DLL redirection' # See http://msdn.microsoft.com/en-us/library/ms682600(VS.85).aspx + for f in glob.glob(os.path.join('build', 'py2exe', '*.exe')): + open(f + '.local', 'wb').write('\n') + + print print print 'Building Installer' @@ -558,12 +564,12 @@ def main(): 'win32file', 'pythoncom', 'rtf2xml', 'lxml', 'lxml._elementpath', 'genshi', 'path', 'pydoc', 'IPython.Extensions.*', - 'calibre.web.feeds.recipes.*', 'pydoc', + 'calibre.web.feeds.recipes.*', ], 'packages' : ['PIL'], 'excludes' : ["Tkconstants", "Tkinter", "tcl", - "_imagingtk", "ImageTk", "FixTk", - 'pydoc'], + "_imagingtk", "ImageTk", "FixTk" + ], 'dll_excludes' : ['mswsock.dll'], }, }, From 2265152c777e78de862e2b1dd16ec155b25183cb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jun 2008 10:09:33 -0700 Subject: [PATCH 04/18] Fix #795 --- src/calibre/ebooks/lrf/any/convert_from.py | 5 ++++- src/calibre/ebooks/lrf/html/convert_from.py | 7 +++++-- src/calibre/parallel.py | 5 ++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/lrf/any/convert_from.py b/src/calibre/ebooks/lrf/any/convert_from.py index 4142c71c55..ab66fc1b89 100644 --- a/src/calibre/ebooks/lrf/any/convert_from.py +++ b/src/calibre/ebooks/lrf/any/convert_from.py @@ -158,7 +158,10 @@ def main(args=sys.argv, logger=None, gui_mode=False): print _('No file to convert specified.') return 1 - return process_file(args[1], options, logger) + src = args[1] + if not isinstance(src, unicode): + src = src.decode(sys.getfilesystemencoding()) + return process_file(src, options, logger) if __name__ == '__main__': sys.exit(main()) diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py index 6dc268e9eb..aca8cf96b3 100644 --- a/src/calibre/ebooks/lrf/html/convert_from.py +++ b/src/calibre/ebooks/lrf/html/convert_from.py @@ -30,7 +30,7 @@ from calibre.ebooks.lrf import option_parser as lrf_option_parser from calibre.ebooks import ConversionError from calibre.ebooks.lrf.html.table import Table from calibre import filename_to_utf8, setup_cli_handlers, __appname__, \ - fit_image, LoggingInterface + fit_image, LoggingInterface, preferred_encoding from calibre.ptempfile import PersistentTemporaryFile from calibre.ebooks.metadata.opf import OPFReader from calibre.devices.interface import Device @@ -1853,6 +1853,8 @@ def process_file(path, options, logger=None): options.force_page_break = fpb options.link_exclude = le options.page_break = pb + if not isinstance(options.chapter_regex, unicode): + options.chapter_regex = options.chapter_regex.decode(preferred_encoding) options.chapter_regex = re.compile(options.chapter_regex, re.IGNORECASE) fpba = options.force_page_break_attr.split(',') if len(fpba) != 3: @@ -1969,7 +1971,8 @@ def main(args=sys.argv): except Exception, err: print >> sys.stderr, err return 1 - + if not isinstance(src, unicode): + src = src.decode(sys.getfilesystemencoding()) process_file(src, options) return 0 diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py index a9490f4922..307c894cfe 100644 --- a/src/calibre/parallel.py +++ b/src/calibre/parallel.py @@ -11,7 +11,10 @@ from calibre.ebooks.lrf.any.convert_from import main as any2lrf from calibre.ebooks.lrf.web.convert_from import main as web2lrf from calibre.ebooks.lrf.feeds.convert_from import main as feeds2lrf from calibre.gui2.lrf_renderer.main import main as lrfviewer -from calibre.ebooks.lrf.html.table_as_image import do_render as render_table +try: + from calibre.ebooks.lrf.html.table_as_image import do_render as render_table +except: # Dont fail is PyQt4.4 not present + render_table = None from calibre import iswindows, __appname__, islinux try: from calibre.utils.single_qt_application import SingleApplication From b5b6f10c4861330652f546beb78f30173257cdf0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jun 2008 10:42:19 -0700 Subject: [PATCH 05/18] Add option to detect chapters based on tagname and attributes --- src/calibre/ebooks/lrf/__init__.py | 7 ++++--- src/calibre/ebooks/lrf/html/convert_from.py | 15 +++++++++++++++ src/calibre/gui2/dialogs/lrf_single.py | 13 +++++++------ src/calibre/gui2/dialogs/lrf_single.ui | 15 ++++++++++++++- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/calibre/ebooks/lrf/__init__.py b/src/calibre/ebooks/lrf/__init__.py index af93bca6cb..1cdba123d8 100644 --- a/src/calibre/ebooks/lrf/__init__.py +++ b/src/calibre/ebooks/lrf/__init__.py @@ -158,12 +158,13 @@ def option_parser(usage, gui_mode=False): chapter = parser.add_option_group('CHAPTER OPTIONS') chapter.add_option('--disable-chapter-detection', action='store_true', default=False, dest='disable_chapter_detection', - help=_('''Prevent the automatic insertion of page breaks''' - ''' before detected chapters.''')) + help=_('''Prevent the automatic detection chapters.''')) chapter.add_option('--chapter-regex', dest='chapter_regex', default='chapter|book|appendix', help=_('''The regular expression used to detect chapter titles.''' - ''' It is searched for in heading tags (h1-h6). Defaults to %default''')) + ''' It is searched for in heading tags (h1-h6). Defaults to %default''')) + chapter.add_option('--chapter-attr', default='$,,$', + help=_('Detect a chapter beginning at an element having the specified attribute. The format for this option is tagname regexp,attribute name,attribute value regexp. For example to match all heading tags that have the attribute class="chapter" you would use "h\d,class,chapter". Default is %default''')) chapter.add_option('--page-break-before-tag', dest='page_break', default='h[12]', help=_('''If html2lrf does not find any page breaks in the ''' '''html file and cannot detect chapter headings, it will ''' diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py index aca8cf96b3..218ace2bf7 100644 --- a/src/calibre/ebooks/lrf/html/convert_from.py +++ b/src/calibre/ebooks/lrf/html/convert_from.py @@ -1427,6 +1427,18 @@ class HTMLConverter(object, LoggingInterface): return except KeyError: pass + if not self.disable_chapter_detection and \ + (self.chapter_attr[0].match(tagname) and \ + tag.has_key(self.chapter_attr[1]) and \ + self.chapter_attr[2].match(tag[self.chapter_attr[1]])): + self.log_debug('Detected chapter %s', tagname) + self.end_page() + self.page_break_found = True + + if self.options.add_chapters_to_toc: + self.extra_toc_entries.append((self.get_text(tag, + limit=1000), self.current_block)) + end_page = self.process_page_breaks(tag, tagname, tag_css) try: if tagname in ["title", "script", "meta", 'del', 'frameset']: @@ -1850,6 +1862,9 @@ def process_file(path, options, logger=None): re.compile('$') fpb = re.compile(options.force_page_break, re.IGNORECASE) if options.force_page_break else \ re.compile('$') + cq = options.chapter_attr.split(',') + options.chapter_attr = [re.compile(cq[0], re.IGNORECASE), cq[1], + re.compile(cq[2], re.IGNORECASE)] options.force_page_break = fpb options.link_exclude = le options.page_break = pb diff --git a/src/calibre/gui2/dialogs/lrf_single.py b/src/calibre/gui2/dialogs/lrf_single.py index bc5bdcf06b..c8895a4243 100644 --- a/src/calibre/gui2/dialogs/lrf_single.py +++ b/src/calibre/gui2/dialogs/lrf_single.py @@ -1,8 +1,8 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import os, cPickle, codecs +import os, codecs -from PyQt4.QtCore import QObject, SIGNAL, Qt, QVariant, QByteArray +from PyQt4.QtCore import QObject, SIGNAL, Qt from PyQt4.QtGui import QAbstractSpinBox, QLineEdit, QCheckBox, QDialog, \ QPixmap, QTextEdit, QListWidgetItem, QIcon @@ -65,7 +65,7 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog): QObject.connect(self.categoryList, SIGNAL('itemEntered(QListWidgetItem *)'), self.show_category_help) QObject.connect(self.cover_button, SIGNAL("clicked(bool)"), self.select_cover) - self.categoryList.leaveEvent = self.reset_help + #self.categoryList.leaveEvent = self.reset_help self.reset_help() self.selected_format = None self.initialize_common() @@ -277,9 +277,9 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog): obj.setWhatsThis(help) self.option_map[guiname] = opt obj.__class__.enterEvent = show_item_help - obj.leaveEvent = self.reset_help + #obj.leaveEvent = self.reset_help self.preprocess.__class__.enterEvent = show_item_help - self.preprocess.leaveEvent = self.reset_help + #self.preprocess.leaveEvent = self.reset_help def show_category_help(self, item): @@ -293,7 +293,8 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog): self.set_help(help[text]) def set_help(self, msg): - self.help_view.setHtml('%s'%(msg,)) + if msg and getattr(msg, 'strip', lambda:True)(): + self.help_view.setHtml('%s'%(msg,)) def reset_help(self, *args): self.set_help(_('No help available')) diff --git a/src/calibre/gui2/dialogs/lrf_single.ui b/src/calibre/gui2/dialogs/lrf_single.ui index 080970b96f..d0d304ac6e 100644 --- a/src/calibre/gui2/dialogs/lrf_single.ui +++ b/src/calibre/gui2/dialogs/lrf_single.ui @@ -115,7 +115,7 @@ - 2 + 0 @@ -951,6 +951,19 @@ + + + + Detect chapter &at tag: + + + gui_chapter_attr + + + + + + From 37cc4b13ea4a5644b0df663a6162b3ee8af4346d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jun 2008 12:52:58 -0700 Subject: [PATCH 06/18] Fix #797 --- src/calibre/gui2/dialogs/lrf_single.py | 24 +++++++++++++++++------- src/calibre/gui2/dialogs/lrf_single.ui | 18 +++++++++++++++++- src/calibre/gui2/main.py | 9 +++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/dialogs/lrf_single.py b/src/calibre/gui2/dialogs/lrf_single.py index c8895a4243..e8f3d61b7c 100644 --- a/src/calibre/gui2/dialogs/lrf_single.py +++ b/src/calibre/gui2/dialogs/lrf_single.py @@ -48,10 +48,7 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog): self.gui_mono_family.setModel(self.font_family_model) self.load_saved_global_defaults() - def __init__(self, window, db, row): - QDialog.__init__(self, window) - Ui_LRFSingleDialog.__init__(self) - self.setupUi(self) + def populate_list(self): self.__w = [] self.__w.append(QIcon(':/images/dialog_information.svg')) self.item1 = QListWidgetItem(self.__w[-1], _("Metadata"), self.categoryList) @@ -61,6 +58,12 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog): self.item3 = QListWidgetItem(self.__w[-1], _('Page Setup'), self.categoryList) self.__w.append(QIcon(':/images/chapters.svg')) self.item4 = QListWidgetItem(self.__w[-1], _('Chapter Detection'), self.categoryList) + + def __init__(self, window, db, row): + QDialog.__init__(self, window) + Ui_LRFSingleDialog.__init__(self) + self.setupUi(self) + self.populate_list() self.categoryList.setCurrentRow(0) QObject.connect(self.categoryList, SIGNAL('itemEntered(QListWidgetItem *)'), self.show_category_help) @@ -383,15 +386,15 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog): self.cmdline = [unicode(i) for i in cmdline] else: Settings().set('LRF conversion defaults', cmdline) - print self.cmdline QDialog.accept(self) class LRFBulkDialog(LRFSingleDialog): def __init__(self, window): QDialog.__init__(self, window) - Ui_LRFSingleDialog.__init__(self) + Ui_LRFSingleDialog.__init__(self) self.setupUi(self) + self.populate_list() self.categoryList.takeItem(0) self.stack.removeWidget(self.stack.widget(0)) @@ -401,7 +404,14 @@ class LRFBulkDialog(LRFSingleDialog): self.setWindowTitle(_('Bulk convert ebooks to LRF')) def accept(self): - self.cmdline = self.cmdline = [unicode(i) for i in self.build_commandline()] + self.cmdline = [unicode(i) for i in self.build_commandline()] + for meta in ('--title', '--author', '--publisher', '--comment'): + try: + index = self.cmdline.index(meta) + self.cmdline[index:index+2] = [] + except ValueError: + continue + self.cover_file = None QDialog.accept(self) diff --git a/src/calibre/gui2/dialogs/lrf_single.ui b/src/calibre/gui2/dialogs/lrf_single.ui index d0d304ac6e..c63e4915b7 100644 --- a/src/calibre/gui2/dialogs/lrf_single.ui +++ b/src/calibre/gui2/dialogs/lrf_single.ui @@ -115,7 +115,7 @@ - 0 + 3 @@ -1167,5 +1167,21 @@ p, li { white-space: pre-wrap; } + + gui_disable_chapter_detection + toggled(bool) + gui_chapter_attr + setDisabled(bool) + + + 344 + 107 + + + 489 + 465 + + + diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index 06c4eaff19..6d63e0ad67 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -746,6 +746,15 @@ class Main(MainWindow, Ui_MainWindow): for i, row in enumerate([r.row() for r in rows]): cmdline = list(d.cmdline) + mi = self.library_view.model().db.get_metadata(row) + if mi.title: + cmdline.extend(['--title', mi.title]) + if mi.authors: + cmdline.extend(['--author', ','.join(mi.authors)]) + if mi.publisher: + cmdline.extend(['--publisher', mi.publisher]) + if mi.comments: + cmdline.extend(['--comment', mi.comments]) data = None for fmt in LRF_PREFERRED_SOURCE_FORMATS: try: From 69695a599c41fc61e3b0954d7948f2f3514ba3ab Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jun 2008 13:12:09 -0700 Subject: [PATCH 07/18] IGN:... --- src/calibre/parallel.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py index 307c894cfe..405aa36b0a 100644 --- a/src/calibre/parallel.py +++ b/src/calibre/parallel.py @@ -101,7 +101,9 @@ class Server(object): @param func: One of C{PARALLEL_FUNCS.keys()} @param args: A list of arguments to pass of C{func} @param kwdargs: A dictionary of keyword arguments to pass to C{func} - @param monitor: If False launch the child process and return. Do not monitor/communicate with it. + @param monitor: If False launch the child process and return. + Do not monitor/communicate with it. Automatically sets + `report_progress` and `qapp` to False. @param report_progess: If True progress is reported to the GUI @param qapp: If True, A QApplication is created. If False, progress reporting will also be disabled. @return: (result, exception, formatted_traceback, log) where log is the combined @@ -115,6 +117,8 @@ class Server(object): os.mkdir(job_dir) job_data = os.path.join(job_dir, 'job_data.pickle') + if not monitor: + report_progress = qapp = False cPickle.dump((job_id, func, args, kwdargs, report_progress, qapp), open(job_data, 'wb'), -1) prefix = '' @@ -126,8 +130,7 @@ class Server(object): cmd = prefix + 'from calibre.parallel import main; main(\'%s\')'%binascii.hexlify(job_data) if not monitor: - popen([python, '-c', cmd], stdout=subprocess.PIPE, stdin=subprocess.PIPE, - stderr=subprocess.PIPE) + popen([python, '-c', cmd]) return output = open(os.path.join(job_dir, 'output.txt'), 'wb') @@ -157,7 +160,6 @@ class Server(object): def run_job(base, id, func, args, kwdargs): global job_id job_id = id - job_result = os.path.join(base, 'job_result.pickle') func = PARALLEL_FUNCS[func] From 36c525c0d26737c37b62ca30e0f92dd9e2c424f5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jun 2008 22:24:38 -0700 Subject: [PATCH 08/18] IGN:Another unicode fix --- src/calibre/ebooks/lrf/html/convert_from.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py index 218ace2bf7..5b5628424c 100644 --- a/src/calibre/ebooks/lrf/html/convert_from.py +++ b/src/calibre/ebooks/lrf/html/convert_from.py @@ -1784,6 +1784,8 @@ def process_file(path, options, logger=None): level = logging.DEBUG if options.verbose else logging.INFO logger = logging.getLogger('html2lrf') setup_cli_handlers(logger, level) + if not isinstance(path, unicode): + path = path.decode(sys.getfilesystemencoding()) path = os.path.abspath(path) default_title = filename_to_utf8(os.path.splitext(os.path.basename(path))[0]) dirpath = os.path.dirname(path) From 8fcf04a9be76b5e138992a834930d355e737c63e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 18 Jun 2008 22:29:53 -0700 Subject: [PATCH 09/18] IGN:... --- src/calibre/ebooks/lrf/html/convert_from.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py index 5b5628424c..457837aa30 100644 --- a/src/calibre/ebooks/lrf/html/convert_from.py +++ b/src/calibre/ebooks/lrf/html/convert_from.py @@ -260,6 +260,7 @@ class HTMLConverter(object, LoggingInterface): paths = [os.path.abspath(path) for path in paths] + paths = [path.decode(sys.getfilesystemencoding()) if not isinstance(path, unicode) else path for path in paths] while len(paths) > 0 and self.link_level <= self.link_levels: for path in paths: @@ -380,6 +381,8 @@ class HTMLConverter(object, LoggingInterface): self.log_info(_('\tConverting to BBeB...')) self.current_style = {} self.page_break_found = False + if not isinstance(path, unicode): + path = path.decode(sys.getfilesystemencoding()) self.target_prefix = path self.previous_text = '\n' self.tops[path] = self.parse_file(soup) @@ -628,6 +631,8 @@ class HTMLConverter(object, LoggingInterface): para, text, path, fragment = link['para'], link['text'], link['path'], link['fragment'] ascii_text = text + if not isinstance(path, unicode): + path = path.decode(sys.getfilesystemencoding()) if path in self.processed_files: if path+fragment in self.targets.keys(): tb = get_target_block(path+fragment, self.targets) From 9f1daa37d646dcc6addf99e742b9beb69bd2bcea Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 19 Jun 2008 23:23:34 -0700 Subject: [PATCH 10/18] IGN:Initial implementation of process pool. Tested on linux --- osx_installer.py | 2 + src/calibre/__init__.py | 3 + src/calibre/ebooks/lrf/html/convert_from.py | 71 +-- src/calibre/ebooks/lrf/html/table_as_image.py | 17 +- src/calibre/gui2/dialogs/jobs.py | 2 +- src/calibre/gui2/jobs.py | 58 +- src/calibre/gui2/main.py | 32 +- src/calibre/gui2/status.py | 2 +- src/calibre/parallel.py | 527 +++++++++++++----- src/calibre/terminfo.py | 2 +- src/calibre/utils/single_qt_application.py | 15 +- 11 files changed, 527 insertions(+), 204 deletions(-) diff --git a/osx_installer.py b/osx_installer.py index 6fb8b3a4e8..c42fe5f0a1 100644 --- a/osx_installer.py +++ b/osx_installer.py @@ -80,6 +80,8 @@ if not os.path.exists('/etc/fonts/fonts.conf'): continue bad = True break + if not bad: + bad = os.path.exists('/etc/fonts/fonts.conf') if bad: auth = Authorization(destroyflags=(kAuthorizationFlagDestroyRights,)) fd, name = tempfile.mkstemp('.py') diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 12eeb2f625..d70637d84f 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -88,6 +88,9 @@ def setup_cli_handlers(logger, level): handler = logging.StreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) handler.setFormatter(logging.Formatter('[%(levelname)s] %(filename)s:%(lineno)s: %(message)s')) + for hdlr in logger.handlers: + if hdlr.__class__ == handler.__class__: + logger.removeHandler(hdlr) logger.addHandler(handler) class CustomHelpFormatter(IndentedHelpFormatter): diff --git a/src/calibre/ebooks/lrf/html/convert_from.py b/src/calibre/ebooks/lrf/html/convert_from.py index 457837aa30..4d93b1a56b 100644 --- a/src/calibre/ebooks/lrf/html/convert_from.py +++ b/src/calibre/ebooks/lrf/html/convert_from.py @@ -242,6 +242,7 @@ class HTMLConverter(object, LoggingInterface): self.override_css = {} self.override_pcss = {} + self.table_render_job_server = None if self._override_css is not None: if os.access(self._override_css, os.R_OK): @@ -262,37 +263,41 @@ class HTMLConverter(object, LoggingInterface): paths = [os.path.abspath(path) for path in paths] paths = [path.decode(sys.getfilesystemencoding()) if not isinstance(path, unicode) else path for path in paths] - while len(paths) > 0 and self.link_level <= self.link_levels: - for path in paths: - if path in self.processed_files: - continue - try: - self.add_file(path) - except KeyboardInterrupt: - raise - except: - if self.link_level == 0: # Die on errors in the first level + try: + while len(paths) > 0 and self.link_level <= self.link_levels: + for path in paths: + if path in self.processed_files: + continue + try: + self.add_file(path) + except KeyboardInterrupt: raise - for link in self.links: - if link['path'] == path: - self.links.remove(link) - break - self.log_warn('Could not process '+path) - if self.verbose: - self.log_exception(' ') - self.links = self.process_links() - self.link_level += 1 - paths = [link['path'] for link in self.links] - - if self.current_page is not None and self.current_page.has_text(): - self.book.append(self.current_page) - - for text, tb in self.extra_toc_entries: - self.book.addTocEntry(text, tb) - - if self.base_font_size > 0: - self.log_info('\tRationalizing font sizes...') - self.book.rationalize_font_sizes(self.base_font_size) + except: + if self.link_level == 0: # Die on errors in the first level + raise + for link in self.links: + if link['path'] == path: + self.links.remove(link) + break + self.log_warn('Could not process '+path) + if self.verbose: + self.log_exception(' ') + self.links = self.process_links() + self.link_level += 1 + paths = [link['path'] for link in self.links] + + if self.current_page is not None and self.current_page.has_text(): + self.book.append(self.current_page) + + for text, tb in self.extra_toc_entries: + self.book.addTocEntry(text, tb) + + if self.base_font_size > 0: + self.log_info('\tRationalizing font sizes...') + self.book.rationalize_font_sizes(self.base_font_size) + finally: + if self.table_render_job_server is not None: + self.table_render_job_server.killall() def is_baen(self, soup): return bool(soup.find('meta', attrs={'name':'Publisher', @@ -1701,11 +1706,15 @@ class HTMLConverter(object, LoggingInterface): self.process_children(tag, tag_css, tag_pseudo_css) elif tagname == 'table' and not self.ignore_tables and not self.in_table: if self.render_tables_as_images: + if self.table_render_job_server is None: + from calibre.parallel import Server + self.table_render_job_server = Server(number_of_workers=1) print 'Rendering table...' from calibre.ebooks.lrf.html.table_as_image import render_table pheight = int(self.current_page.pageStyle.attrs['textheight']) pwidth = int(self.current_page.pageStyle.attrs['textwidth']) - images = render_table(self.soup, tag, tag_css, + images = render_table(self.table_render_job_server, + self.soup, tag, tag_css, os.path.dirname(self.target_prefix), pwidth, pheight, self.profile.dpi, self.text_size_multiplier_for_rendered_tables) diff --git a/src/calibre/ebooks/lrf/html/table_as_image.py b/src/calibre/ebooks/lrf/html/table_as_image.py index 501a049832..4c5a79eab8 100644 --- a/src/calibre/ebooks/lrf/html/table_as_image.py +++ b/src/calibre/ebooks/lrf/html/table_as_image.py @@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en' ''' Render HTML tables as images. ''' -import os, tempfile, atexit, shutil +import os, tempfile, atexit, shutil, time from PyQt4.Qt import QWebPage, QUrl, QApplication, QSize, \ SIGNAL, QPainter, QImage, QObject, Qt @@ -58,7 +58,7 @@ class HTMLTableRenderer(QObject): finally: QApplication.quit() -def render_table(soup, table, css, base_dir, width, height, dpi, factor=1.0): +def render_table(server, soup, table, css, base_dir, width, height, dpi, factor=1.0): head = '' for e in soup.findAll(['link', 'style']): head += unicode(e)+'\n\n' @@ -78,14 +78,17 @@ def render_table(soup, table, css, base_dir, width, height, dpi, factor=1.0): '''%(head, width-10, style, unicode(table)) - from calibre.parallel import Server - s = Server() - result, exception, traceback, log = s.run(1, 'render_table', qapp=True, report_progress=False, - args=[html, base_dir, width, height, dpi, factor]) + server.run_job(1, 'render_table', + args=[html, base_dir, width, height, dpi, factor]) + res = None + while res is None: + time.sleep(2) + res = server.result(1) + result, exception, traceback = res if exception: print 'Failed to render table' + print exception print traceback - print log images, tdir = result atexit.register(shutil.rmtree, tdir) return images diff --git a/src/calibre/gui2/dialogs/jobs.py b/src/calibre/gui2/dialogs/jobs.py index b601cf2c7c..e0682b6bd8 100644 --- a/src/calibre/gui2/dialogs/jobs.py +++ b/src/calibre/gui2/dialogs/jobs.py @@ -35,7 +35,7 @@ class JobsDialog(QDialog, Ui_JobsDialog): self.jobs_view.setModel(model) self.model = model self.setWindowModality(Qt.NonModal) - self.setWindowTitle(__appname__ + ' - Active Jobs') + self.setWindowTitle(__appname__ + _(' - Jobs')) QObject.connect(self.jobs_view.model(), SIGNAL('modelReset()'), self.jobs_view.resizeColumnsToContents) QObject.connect(self.kill_button, SIGNAL('clicked()'), diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index d135b7d6cd..794a9d8f17 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -86,17 +86,34 @@ class DeviceJob(Job): class ConversionJob(Job): ''' Jobs that involve conversion of content.''' - def run(self): - last_traceback, exception = None, None - try: - self.result, exception, last_traceback, self.log = \ - self.server.run(self.id, self.func, self.args, self.kwargs) - except Exception, err: - last_traceback = traceback.format_exc() - exception = (exception.__class__.__name__, unicode(str(err), 'utf8', 'replace')) - - self.last_traceback, self.exception = last_traceback, exception + def __init__(self, *args, **kwdargs): + Job.__init__(self, *args, **kwdargs) + self.log = '' + def run(self): + result = None + self.server.run_job(self.id, self.func, progress=self.progress, + args=self.args, kwdargs=self.kwargs, + output=self.output) + res = None + while res is None: + time.sleep(2) + res = self.server.result(self.id) + if res is None: + exception, tb = 'UnknownError: This should not have happened', '' + else: + result, exception, tb = res + self.result, self.last_traceback, self.exception = result, tb, exception + + def output(self, msg): + if self.log is None: + self.log = '' + self.log += msg + self.emit(SIGNAL('output_received()')) + + def formatted_log(self): + return '

Log:

%s
'%self.log + def notify(self): self.emit(SIGNAL('jobdone(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject, PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'), self.id, self.description, self.result, self.exception, self.last_traceback, self.log) @@ -112,6 +129,9 @@ class ConversionJob(Job): ans = u'

%s: %s

'%self.exception ans += '

Traceback:

%s
'%self.last_traceback return ans + + def progress(self, percent, msg): + self.emit(SIGNAL('update_progress(int, PyQt_PyObject)'), self.id, percent) class JobManager(QAbstractTableModel): @@ -149,9 +169,9 @@ class JobManager(QAbstractTableModel): try: if isinstance(job, DeviceJob): job.terminate() - self.process_server.kill(job.id) except: continue + self.process_server.killall() def timerEvent(self, event): if event.timerId() == self.timer_id: @@ -241,7 +261,10 @@ class JobManager(QAbstractTableModel): id = self.next_id job = job_class(id, description, slot, priority, *args, **kwargs) job.server = self.process_server - QObject.connect(job, SIGNAL('status_update(int, int)'), self.status_update, Qt.QueuedConnection) + QObject.connect(job, SIGNAL('status_update(int, int)'), self.status_update, + Qt.QueuedConnection) + self.connect(job, SIGNAL('update_progress(int, PyQt_PyObject)'), + self.update_progress, Qt.QueuedConnection) self.update_lock.lock() self.add_queue.append(job) self.update_lock.unlock() @@ -370,11 +393,14 @@ class DetailView(QDialog, Ui_Dialog): self.setupUi(self) self.setWindowTitle(job.description) self.job = job - txt = self.job.formatted_error() + self.job.formatted_log() + self.connect(self.job, SIGNAL('output_received()'), self.update) + self.update() + + def update(self): + txt = self.job.formatted_error() + self.job.formatted_log() if not txt: txt = 'No details available' - self.log.setHtml(txt) - - + vbar = self.log.verticalScrollBar() + vbar.setValue(vbar.maximum()) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index 6d63e0ad67..a567bc567d 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -77,7 +77,6 @@ class Main(MainWindow, Ui_MainWindow): self.conversion_jobs = {} self.persistent_files = [] self.metadata_dialogs = [] - self.viewer_job_id = 1 self.default_thumbnail = None self.device_error_dialog = ConversionErrorDialog(self, _('Error communicating with device'), ' ') self.device_error_dialog.setModal(Qt.NonModal) @@ -264,14 +263,6 @@ class Main(MainWindow, Ui_MainWindow): elif msg.startswith('refreshdb:'): self.library_view.model().resort() self.library_view.model().research() - elif msg.startswith('progress:'): - try: - fields = msg.split(':') - job_id, percent = fields[1:3] - job_id, percent = int(job_id), float(percent) - self.job_manager.update_progress(job_id, percent) - except: - pass else: print msg @@ -780,7 +771,7 @@ class Main(MainWindow, Ui_MainWindow): cmdline.append(pt.name) id = self.job_manager.run_conversion_job(self.book_converted, 'any2lrf', args=[cmdline], - job_description='Convert book %d of %d'%(i, len(rows))) + job_description='Convert book %d of %d'%(i+1, len(rows))) self.conversion_jobs[id] = (d.cover_file, pt, of, d.output_format, @@ -860,15 +851,16 @@ class Main(MainWindow, Ui_MainWindow): self._view_file(result) def _view_file(self, name): - if name.upper().endswith('.LRF'): - args = ['lrfviewer', name] - self.job_manager.process_server.run('viewer%d'%self.viewer_job_id, - 'lrfviewer', kwdargs=dict(args=args), - monitor=False) - self.viewer_job_id += 1 - else: - QDesktopServices.openUrl(QUrl('file:'+name))#launch(name) - time.sleep(2) # User feedback + self.setCursor(Qt.BusyCursor) + try: + if name.upper().endswith('.LRF'): + args = ['lrfviewer', name] + self.job_manager.process_server.run_free_job('lrfviewer', kwdargs=dict(args=args)) + else: + QDesktopServices.openUrl(QUrl('file:'+name))#launch(name) + time.sleep(5) # User feedback + finally: + self.unsetCursor() def view_specific_format(self, triggered): rows = self.library_view.selectionModel().selectedRows() @@ -1084,7 +1076,7 @@ class Main(MainWindow, Ui_MainWindow): if getattr(exception, 'only_msg', False): error_dialog(self, _('Conversion Error'), unicode(exception)).exec_() return - msg = u'

%s: %s

'%exception + msg = u'

%s:

'%exception msg += u'

Failed to perform job: '+description msg += u'

Detailed traceback:

'
         msg += formatted_traceback + '
' diff --git a/src/calibre/gui2/status.py b/src/calibre/gui2/status.py index 55e5c3f901..8b059f5711 100644 --- a/src/calibre/gui2/status.py +++ b/src/calibre/gui2/status.py @@ -1,6 +1,6 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import textwrap, re +import re from PyQt4.QtGui import QStatusBar, QMovie, QLabel, QFrame, QHBoxLayout, QPixmap, \ QVBoxLayout, QSizePolicy, QToolButton, QIcon diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py index 405aa36b0a..1c9593b94f 100644 --- a/src/calibre/parallel.py +++ b/src/calibre/parallel.py @@ -1,25 +1,25 @@ +from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' ''' Used to run jobs in parallel in separate processes. ''' -import sys, tempfile, os, cPickle, traceback, atexit, binascii, time, subprocess +import sys, os, gc, cPickle, traceback, atexit, cStringIO, time, subprocess, socket, collections +from select import select from functools import partial - +from threading import RLock, Thread, Event from calibre.ebooks.lrf.any.convert_from import main as any2lrf from calibre.ebooks.lrf.web.convert_from import main as web2lrf from calibre.ebooks.lrf.feeds.convert_from import main as feeds2lrf from calibre.gui2.lrf_renderer.main import main as lrfviewer +from calibre.ptempfile import PersistentTemporaryFile + try: from calibre.ebooks.lrf.html.table_as_image import do_render as render_table except: # Dont fail is PyQt4.4 not present render_table = None -from calibre import iswindows, __appname__, islinux -try: - from calibre.utils.single_qt_application import SingleApplication -except: - SingleApplication = None +from calibre import iswindows, islinux, detect_ncpus sa = None job_id = None @@ -29,12 +29,13 @@ def report_progress(percent, msg=''): msg = 'progress:%s:%f:%s'%(job_id, percent, msg) sa.send_message(msg) +_notify = 'fskjhwseiuyweoiu987435935-0342' PARALLEL_FUNCS = { 'any2lrf' : partial(any2lrf, gui_mode=True), 'web2lrf' : web2lrf, 'lrfviewer' : lrfviewer, - 'feeds2lrf' : partial(feeds2lrf, notification=report_progress), + 'feeds2lrf' : partial(feeds2lrf, notification=_notify), 'render_table': render_table, } @@ -52,144 +53,422 @@ if islinux and hasattr(sys, 'frozen_path'): python = os.path.join(getattr(sys, 'frozen_path'), 'parallel') popen = partial(subprocess.Popen, cwd=getattr(sys, 'frozen_path')) -def cleanup(tdir): - try: - import shutil - shutil.rmtree(tdir, True) - except: - pass +prefix = 'import sys; sys.in_worker = True; ' +if hasattr(sys, 'frameworks_dir'): + fd = getattr(sys, 'frameworks_dir') + prefix += 'sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd + if fd not in os.environ['PATH']: + os.environ['PATH'] += ':'+fd -class Server(object): + +def write(socket, msg, timeout=5): + if isinstance(msg, unicode): + msg = msg.encode('utf-8') + length = None + while len(msg) > 0: + if length is None: + length = len(msg) + chunk = ('%-12d'%length) + msg[:4096-12] + msg = msg[4096-12:] + else: + chunk, msg = msg[:4096], msg[4096:] + w = select([], [socket], [], timeout)[1] + if not w: + raise RuntimeError('Write to socket timed out') + if socket.sendall(chunk) is not None: + raise RuntimeError('Failed to write chunk to socket') + + +def read(socket, timeout=5): + buf = cStringIO.StringIO() + length = None + while select([socket],[],[],timeout)[0]: + msg = socket.recv(4096) + if not msg: + break + if length is None: + length, msg = int(msg[:12]), msg[12:] + buf.write(msg) + if buf.tell() >= length: + break + if not length: + return '' + msg = buf.getvalue()[:length] + if len(msg) < length: + raise RuntimeError('Corrupted packet received') + + return msg + +class RepeatingTimer(Thread): + + def repeat(self): + while True: + self.event.wait(self.interval) + if self.event.isSet(): + break + self.action() + + def __init__(self, interval, func): + self.event = Event() + self.interval = interval + self.action = func + Thread.__init__(self, target=self.repeat) + self.setDaemon(True) + +class ControlError(Exception): + pass + +class Overseer(object): - #: Interval in seconds at which child processes are polled for status information - INTERVAL = 0.1 KILL_RESULT = 'Server: job killed by user|||#@#$%&*)*(*$#$%#$@&' + INTERVAL = 0.1 - def __init__(self): - self.tdir = tempfile.mkdtemp('', '%s_IPC_'%__appname__) - atexit.register(cleanup, self.tdir) - self.kill_jobs = [] + def __init__(self, server, port, timeout=5): + self.cmd = prefix + 'from calibre.parallel import worker; worker(%s, %s)'%(repr('localhost'), repr(port)) + self.process = popen([python, '-c', self.cmd]) + self.socket = server.accept()[0] - def kill(self, job_id): - ''' - Kill the job identified by job_id. - ''' - self.kill_jobs.append(str(job_id)) + self.working = False + self.timeout = timeout + self.last_job_time = time.time() + self.job_id = None + self._stop = False + if not select([self.socket], [], [], 120)[0]: + raise RuntimeError(_('Could not launch worker process.')) + if int(self.read()) != self.process.pid: + raise RuntimeError('PID mismatch') + self.write('OK') + if self.read() != 'WAITING': + raise RuntimeError('Worker sulking') - def _terminate(self, process): + def terminate(self): ''' Kill process. ''' + try: + if self.socket: + self.socket.close() + except: + pass if iswindows: win32api = __import__('win32api') try: - win32api.TerminateProcess(int(process.pid), -1) + win32api.TerminateProcess(int(self.process.pid), -1) except: pass else: import signal - os.kill(process.pid, signal.SIGKILL) - time.sleep(0.05) - - + try: + os.kill(self.process.pid, signal.SIGKILL) + time.sleep(0.05) + except: + pass - def run(self, job_id, func, args=[], kwdargs={}, monitor=True, - report_progress=True, qapp=True): - ''' - Run a job in a separate process. - @param job_id: A unique (per server) identifier - @param func: One of C{PARALLEL_FUNCS.keys()} - @param args: A list of arguments to pass of C{func} - @param kwdargs: A dictionary of keyword arguments to pass to C{func} - @param monitor: If False launch the child process and return. - Do not monitor/communicate with it. Automatically sets - `report_progress` and `qapp` to False. - @param report_progess: If True progress is reported to the GUI - @param qapp: If True, A QApplication is created. If False, progress reporting will also be disabled. - @return: (result, exception, formatted_traceback, log) where log is the combined - stdout + stderr of the child process; or None if monitor is True. If a job is killed - by a call to L{kill()} then result will be L{KILL_RESULT} - ''' - job_id = str(job_id) - job_dir = os.path.join(self.tdir, job_id) - if os.path.exists(job_dir): - raise ValueError('Cannot run job. The job_id %s has already been used.'%job_id) - os.mkdir(job_dir) - job_data = os.path.join(job_dir, 'job_data.pickle') - if not monitor: - report_progress = qapp = False - cPickle.dump((job_id, func, args, kwdargs, report_progress, qapp), - open(job_data, 'wb'), -1) - prefix = '' - if hasattr(sys, 'frameworks_dir'): - fd = getattr(sys, 'frameworks_dir') - prefix = 'import sys; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd - if fd not in os.environ['PATH']: - os.environ['PATH'] += ':'+fd - cmd = prefix + 'from calibre.parallel import main; main(\'%s\')'%binascii.hexlify(job_data) + def write(self, msg, timeout=None): + write(self.socket, msg, timeout=self.timeout if timeout is None else timeout) - if not monitor: - popen([python, '-c', cmd]) + def read(self, timeout=None): + return read(self.socket, timeout=self.timeout if timeout is None else timeout) + + def __eq__(self, other): + return hasattr(other, 'process') and hasattr(other.process, 'pid') and self.process.pid == other.process.pid + + def __bool__(self): + self.process.poll() + return self.process.returncode is None + + def pid(self): + return self.process.pid + + def select(self, timeout=0): + return select([self.socket], [self.socket], [self.socket], timeout) + + def initialize_job(self, job): + self.job_id = job.job_id + self.working = True + self.write('JOB:'+cPickle.dumps((job.func, job.args, job.kwdargs), -1)) + msg = self.read() + if msg != 'OK': + raise ControlError('Failed to initialize job on worker %d:%s'%(self.process.pid, msg)) + self.output = job.output if callable(job.output) else sys.stdout.write + self.progress = job.progress if callable(job.progress) else None + self.job = job + + def control(self): + try: + if select([self.socket],[],[],0)[0]: + msg = self.read() + word, msg = msg.partition(':')[0], msg.partition(':')[-1] + if word == 'RESULT': + self.write('OK') + return Result(cPickle.loads(msg), None, None) + elif word == 'OUTPUT': + self.write('OK') + try: + self.output(''.join(cPickle.loads(msg))) + except: + self.output('Bad output message: '+ repr(msg)) + elif word == 'PROGRESS': + self.write('OK') + percent = None + try: + percent, msg = cPickle.loads(msg)[-1] + except: + print 'Bad progress update:', repr(msg) + if self.progress and percent is not None: + self.progress(percent, msg) + elif word == 'ERROR': + self.write('OK') + return Result(None, *cPickle.loads(msg)) + else: + self.terminate() + return Result(None, ControlError('Worker sent invalid msg: %s', repr(msg)), '') + self.process.poll() + if self.process.returncode is not None: + return Result(None, ControlError('Worker process died unexpectedly with returncode: %d'%self.process.returncode), '') + finally: + self.working = False + self.last_job_time = time.time() + +class Job(object): + + def __init__(self, job_id, func, args, kwdargs, output, progress, done): + self.job_id = job_id + self.func = func + self.args = args + self.kwdargs = kwdargs + self.output = output + self.progress = progress + self.done = done + +class Result(object): + + def __init__(self, result, exception, traceback): + self.result = result + self.exception = exception + self.traceback = traceback + + def __len__(self): + return 3 + + def __item__(self, i): + return (self.result, self.exception, self.traceback)[i] + + def __iter__(self): + return iter((self.result, self.exception, self.traceback)) + +class Server(Thread): + + KILL_RESULT = Overseer.KILL_RESULT + START_PORT = 10013 + + def __init__(self, number_of_workers=detect_ncpus()): + Thread.__init__(self) + self.setDaemon(True) + self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = self.START_PORT + while True: + try: + self.server_socket.bind(('localhost', self.port)) + break + except: + self.port += 1 + self.server_socket.listen(5) + self.number_of_workers = number_of_workers + self.pool, self.jobs, self.working, self.results = [], collections.deque(), [], {} + atexit.register(self.killall) + self.job_lock = RLock() + self.overseer_lock = RLock() + self.working_lock = RLock() + self.result_lock = RLock() + self.pool_lock = RLock() + self.start() + + def add_job(self, job): + with self.job_lock: + self.jobs.append(job) + + def store_result(self, result, id=None): + if id: + with self.job_lock: + self.results[id] = result + + def result(self, id): + with self.result_lock: + return self.results.pop(id, None) + + def run(self): + while True: + job = None + with self.job_lock: + if len(self.jobs) > 0 and len(self.working) < self.number_of_workers: + job = self.jobs.popleft() + with self.pool_lock: + o = self.pool.pop() if self.pool else Overseer(self.server_socket, self.port) + try: + o.initialize_job(job) + except Exception, err: + res = Result(None, unicode(err), traceback.format_exc()) + job.done(res) + o.terminate() + o = None + if o: + with self.working_lock: + self.working.append(o) + + with self.working_lock: + done = [] + for o in self.working: + try: + res = o.control() + except Exception, err: + res = Result(None, unicode(err), traceback.format_exc()) + o.terminate() + if isinstance(res, Result): + o.job.done(res) + done.append(o) + for o in done: + self.working.remove(o) + if o: + with self.pool_lock: + self.pool.append(o) + + time.sleep(1) + + + def killall(self): + with self.pool_lock: + map(lambda x: x.terminate(), self.pool) + self.pool = [] + + + def kill(self, job_id): + with self.working_lock: + pop = None + for o in self.working: + if o.job_id == job_id: + o.terminate() + o.job.done(Result(self.KILL_RESULT, None, '')) + pop = o + break + if pop is not None: + self.working.remove(pop) + + + + def run_job(self, job_id, func, args=[], kwdargs={}, + output=None, progress=None, done=None): + ''' + Run a job in a separate process. Supports job control, output redirection + and progress reporting. + ''' + if done is None: + done = partial(self.store_result, id=job_id) + job = Job(job_id, func, args, kwdargs, output, progress, done) + with self.job_lock: + self.jobs.append(job) + + def run_free_job(self, func, args=[], kwdargs={}): + pt = PersistentTemporaryFile('.pickle', '_IPC_') + pt.write(cPickle.dumps((func, args, kwdargs))) + pt.close() + cmd = prefix + 'from calibre.parallel import free_spirit; free_spirit(%s)'%repr(pt.name) + popen([python, '-c', cmd]) + +########################################################################################## +##################################### CLIENT CODE ##################################### +########################################################################################## + +class BufferedSender(object): + + def __init__(self, socket): + self.socket = socket + self.wbuf, self.pbuf = [], [] + self.wlock, self.plock = RLock(), RLock() + self.timer = RepeatingTimer(0.5, self.send) + self.prefix = prefix + self.timer.start() + + def write(self, msg): + if not isinstance(msg, basestring): + msg = unicode(msg) + with self.wlock: + self.wbuf.append(msg) + + def send(self): + if not select([], [self.socket], [], 30)[1]: + print >>sys.__stderr__, 'Cannot pipe to overseer' return - output = open(os.path.join(job_dir, 'output.txt'), 'wb') - p = popen([python, '-c', cmd], stdout=output, stderr=output, - stdin=subprocess.PIPE) - p.stdin.close() - while p.returncode is None: - if job_id in self.kill_jobs: - self._terminate(p) - return self.KILL_RESULT, None, None, _('Job killed by user') - time.sleep(0.1) - p.poll() + with self.wlock: + if self.wbuf: + msg = cPickle.dumps(self.wbuf, -1) + self.wbuf = [] + write(self.socket, 'OUTPUT:'+msg) + read(self.socket, 10) + + with self.plock: + if self.pbuf: + msg = cPickle.dumps(self.pbuf, -1) + self.pbuf = [] + write(self.socket, 'PROGRESS:'+msg) + read(self.socket, 10) + + def notify(self, percent, msg=''): + with self.plock: + self.pbuf.append((percent, msg)) - - output.close() - job_result = os.path.join(job_dir, 'job_result.pickle') - if not os.path.exists(job_result): - result, exception, traceback = None, ('ParallelRuntimeError', - 'The worker process died unexpectedly.'), '' - else: - result, exception, traceback = cPickle.load(open(job_result, 'rb')) - log = open(output.name, 'rb').read() - - return result, exception, traceback, log - + def flush(self): + pass -def run_job(base, id, func, args, kwdargs): - global job_id - job_id = id - job_result = os.path.join(base, 'job_result.pickle') - +def work(client_socket, func, args, kwdargs): func = PARALLEL_FUNCS[func] - exception, tb = None, None + if hasattr(func, 'keywords'): + for key, val in func.keywords.items(): + if val == _notify: + func.keywords[key] = sys.stdout.notify + res = func(*args, **kwdargs) + sys.stdout.send() + return res + + + + +def worker(host, port): + client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client_socket.connect((host, port)) + write(client_socket, str(os.getpid())) + msg = read(client_socket, timeout=10) + if msg != 'OK': + return 1 + write(client_socket, 'WAITING') + sys.stdout = BufferedSender(client_socket) + sys.stderr = sys.stdout + + while True: + msg = read(client_socket, timeout=60) + if msg.startswith('JOB:'): + func, args, kwdargs = cPickle.loads(msg[4:]) + write(client_socket, 'OK') + try: + result = work(client_socket, func, args, kwdargs) + write(client_socket, 'RESULT:'+ cPickle.dumps(result)) + except (Exception, SystemExit), err: + exception = (err.__class__.__name__, unicode(str(err), 'utf-8', 'replace')) + tb = traceback.format_exc() + write(client_socket, 'ERROR:'+cPickle.dumps((exception, tb),-1)) + if read(client_socket, 10) != 'OK': + break + gc.collect() + elif msg == 'STOP:': + return 0 + +def free_spirit(path): + func, args, kwdargs = cPickle.load(open(path, 'rb')) try: - result = func(*args, **kwdargs) - except (Exception, SystemExit), err: - result = None - exception = (err.__class__.__name__, unicode(str(err), 'utf-8', 'replace')) - tb = traceback.format_exc() - - if os.path.exists(os.path.dirname(job_result)): - cPickle.dump((result, exception, tb), open(job_result, 'wb')) - -def main(src): - from PyQt4.QtGui import QApplication - job_data = binascii.unhexlify(src) - global sa - job_id, func, args, kwdargs, rp, qapp = cPickle.load(open(job_data, 'rb')) - - if qapp and QApplication.instance() is None: - QApplication([]) - if SingleApplication is not None and rp and QApplication.instance() is not None: - sa = SingleApplication('calibre GUI') - - run_job(os.path.dirname(job_data), job_id, func, args, kwdargs) - - return 0 - -if __name__ == '__main__': - sys.exit(main(sys.argv[2])) - - + os.unlink(path) + except: + pass + PARALLEL_FUNCS[func](*args, **kwdargs) + \ No newline at end of file diff --git a/src/calibre/terminfo.py b/src/calibre/terminfo.py index 2141b9115b..2ed03a3077 100644 --- a/src/calibre/terminfo.py +++ b/src/calibre/terminfo.py @@ -94,7 +94,7 @@ class TerminalController: except: return # If the stream isn't a tty, then assume it has no capabilities. - if not hasattr(term_stream, 'isatty') or not term_stream.isatty(): return + if hasattr(sys, 'in_worker') or not hasattr(term_stream, 'isatty') or not term_stream.isatty(): return # Check the terminal type. If we fail, then assume that the # terminal has no capabilities. diff --git a/src/calibre/utils/single_qt_application.py b/src/calibre/utils/single_qt_application.py index 0c1eb0936a..78171a3dec 100644 --- a/src/calibre/utils/single_qt_application.py +++ b/src/calibre/utils/single_qt_application.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' Enforces running of only a single application instance and allows for messaging between applications using a local socket. ''' -import atexit +import atexit, os from PyQt4.QtCore import QByteArray, QDataStream, QIODevice, SIGNAL, QObject, Qt, QString from PyQt4.QtNetwork import QLocalSocket, QLocalServer @@ -93,6 +93,16 @@ class LocalServer(QLocalServer): for conn in pop: self.connections.remove(conn) + + def listen(self, name): + if not QLocalServer.listen(self, name): + try: + os.unlink(self.fullServerName()) + except: + pass + return QLocalServer.listen(self, name) + return True + @@ -124,8 +134,7 @@ class SingleApplication(QObject): self.mr, Qt.QueuedConnection) if not self.server.listen(self.server_name): - if not self.server.listen(self.server_name): - self.server = None + self.server = None if self.server is not None: atexit.register(self.server.close) From 103f8d91d949ead38735d7fde2076db5a92e560d Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Thu, 19 Jun 2008 23:33:22 -0700 Subject: [PATCH 11/18] IGN:Remove uneccessary dependency on Qt mainloop when messaging the GUI from a commandline app --- src/calibre/library/cli.py | 27 +++++++++++----------- src/calibre/utils/single_qt_application.py | 7 +++++- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index edda1c8502..f76fb5bba7 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -11,7 +11,10 @@ import sys, os from textwrap import TextWrapper from calibre import OptionParser, Settings, terminal_controller, preferred_encoding -from calibre.gui2 import SingleApplication +try: + from calibre.utils.single_qt_application import send_message +except: + send_message = None from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata.opf import OPFCreator, OPFReader from calibre.library.database import LibraryDatabase, text_to_tokens @@ -181,10 +184,9 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates): title = title.encode(preferred_encoding) print '\t', title+':' print '\t\t ', path - - if SingleApplication is not None: - sa = SingleApplication('calibre GUI') - sa.send_message('refreshdb:') + + if send_message is not None: + send_message('refreshdb:', 'calibre GUI') finally: sys.stdout = sys.__stdout__ @@ -222,10 +224,10 @@ def do_remove(db, ids): for y in x: db.delete_book(y) - if SingleApplication is not None: - sa = SingleApplication('calibre GUI') - sa.send_message('refreshdb:') - + if send_message is not None: + send_message('refreshdb:', 'calibre GUI') + + def command_remove(args, dbpath): parser = get_parser(_( '''\ @@ -337,10 +339,9 @@ def do_set_metadata(db, id, stream): mi = OPFReader(stream) db.set_metadata(id, mi) do_show_metadata(db, id, False) - if SingleApplication is not None: - sa = SingleApplication('calibre GUI') - sa.send_message('refreshdb:') - + if send_message is not None: + send_message('refreshdb:', 'calibre GUI') + def command_set_metadata(args, dbpath): parser = get_parser(_( ''' diff --git a/src/calibre/utils/single_qt_application.py b/src/calibre/utils/single_qt_application.py index 78171a3dec..846736c507 100644 --- a/src/calibre/utils/single_qt_application.py +++ b/src/calibre/utils/single_qt_application.py @@ -104,7 +104,12 @@ class LocalServer(QLocalServer): return True - +def send_message(msg, name, server_name='calibre_server', timeout=5000): + socket = QLocalSocket() + socket.connectToServer(server_name) + if socket.waitForConnected(timeout_connect): + if read_message(socket) == name: + write_message(socket, name+':'+msg, timeout) class SingleApplication(QObject): From d3075d163642a8dd2d467ee7408f7f3b69ddfedb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 20 Jun 2008 09:14:52 -0700 Subject: [PATCH 12/18] IGN:Move to using mobileread for hosting the binaries --- osx_installer.py | 2 +- src/calibre/linux_installer.py | 4 +- src/calibre/trac/plugins/download.py | 13 +++-- upload.py | 80 +++++++++++++++++++++++----- windows_installer.py | 2 +- 5 files changed, 78 insertions(+), 23 deletions(-) diff --git a/osx_installer.py b/osx_installer.py index c42fe5f0a1..bd8b57a000 100644 --- a/osx_installer.py +++ b/osx_installer.py @@ -306,7 +306,7 @@ def main(): 'iconfile' : 'icons/library.icns', 'frameworks': ['libusb.dylib', 'libunrar.dylib'], 'includes' : ['sip', 'pkg_resources', 'PyQt4.QtXml', - 'PyQt4.QtSvg', + 'PyQt4.QtSvg', 'PyQt4.QtWebKit', 'mechanize', 'ClientForm', 'usbobserver', 'genshi', 'calibre.web.feeds.recipes.*', 'keyword', 'codeop', 'pydoc'], diff --git a/src/calibre/linux_installer.py b/src/calibre/linux_installer.py index 7a55d134d3..a4fc1cff77 100644 --- a/src/calibre/linux_installer.py +++ b/src/calibre/linux_installer.py @@ -8,6 +8,8 @@ Download and install the linux binary. ''' import sys, os, shutil, tarfile, subprocess, tempfile, urllib2, re, stat +MOBILEREAD='https://dev.mobileread.com/dist/kovid/calibre/' + class TerminalController: """ A class that can be used to portably generate formatted output to @@ -239,7 +241,7 @@ def do_postinstall(destdir): def download_tarball(): pb = ProgressBar(TerminalController(sys.stdout), 'Downloading calibre...') - src = urllib2.urlopen('http://calibre.kovidgoyal.net/downloads/latest-linux-binary.tar.bz2') + src = urllib2.urlopen(MOBILEREAD+'calibre-%version-i686.tar.bz2') size = int(src.info()['content-length']) f = tempfile.NamedTemporaryFile() while f.tell() < size: diff --git a/src/calibre/trac/plugins/download.py b/src/calibre/trac/plugins/download.py index e586a11f50..fe66dad363 100644 --- a/src/calibre/trac/plugins/download.py +++ b/src/calibre/trac/plugins/download.py @@ -1,6 +1,6 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' -import re, glob +import re from pkg_resources import resource_filename from trac.core import Component, implements @@ -12,7 +12,7 @@ from trac.util import Markup __appname__ = 'calibre' DOWNLOAD_DIR = '/var/www/calibre.kovidgoyal.net/htdocs/downloads' LINUX_INSTALLER = '/var/www/calibre.kovidgoyal.net/calibre/src/calibre/linux_installer.py' - +MOBILEREAD = 'https://dev.mobileread.com/dist/kovid/calibre/' class OS(dict): """Dictionary with a default value for unknown keys.""" @@ -119,7 +119,7 @@ class Download(Component): if req.path_info == '/download': return self.top_level(req) elif req.path_info == '/download_linux_binary_installer': - req.send(open(LINUX_INSTALLER).read(), 'text/x-python') + req.send(open(LINUX_INSTALLER).read().replace('%version', self.version_from_filename()), 'text/x-python') else: match = re.match(r'\/download_(\S+)', req.path_info) if match: @@ -153,8 +153,7 @@ class Download(Component): def version_from_filename(self): try: - file = glob.glob(DOWNLOAD_DIR+'/*.exe')[0] - return re.search(r'\S+-(\d+\.\d+\.\d+)\.', file).group(1) + return open(DOWNLOAD_DIR+'/latest_version', 'rb').read().strip() except: return '0.0.0' @@ -165,7 +164,7 @@ class Download(Component): installer_name='Windows installer', title='Download %s for windows'%(__appname__), compatibility='%s works on Windows XP and Windows Vista.'%(__appname__,), - path='/downloads/'+file, app=__appname__, + path=MOBILEREAD+file, app=__appname__, note=Markup(\ '''

If you are using the SONY PRS-500 and %(appname)s does not detect your reader, read on:

@@ -203,7 +202,7 @@ You can uninstall a driver by right clicking on it and selecting uninstall. installer_name='OS X universal dmg', title='Download %s for OS X'%(__appname__), compatibility='%s works on OS X Tiger and above.'%(__appname__,), - path='/downloads/'+file, app=__appname__, + path=MOBILEREAD+file, app=__appname__, note=Markup(\ '''
    diff --git a/upload.py b/upload.py index 3ad07aa28a..8ec158c1db 100644 --- a/upload.py +++ b/upload.py @@ -1,5 +1,5 @@ #!/usr/bin/python -import sys, os, shutil, time, tempfile, socket, fcntl, struct +import sys, os, shutil, time, tempfile, socket, fcntl, struct, cStringIO, pycurl, re sys.path.append('src') import subprocess from subprocess import check_call as _check_call @@ -24,6 +24,7 @@ DOCS = PREFIX+"/htdocs/apidocs" USER_MANUAL = PREFIX+'/htdocs/user_manual' HTML2LRF = "src/calibre/ebooks/lrf/html/demo" TXT2LRF = "src/calibre/ebooks/lrf/txt/demo" +MOBILEREAD = 'ftp://dev.mobileread.com/calibre/' BUILD_SCRIPT ='''\ #!/bin/bash cd ~/build && \ @@ -110,19 +111,72 @@ def upload_demo(): check_call('cd src/calibre/ebooks/lrf/txt/demo/ && zip -j /tmp/txt-demo.zip * /tmp/txt2lrf.lrf') check_call('''scp /tmp/txt-demo.zip divok:%s/'''%(DOWNLOADS,)) +def curl_list_dir(url=MOBILEREAD, listonly=1): + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(c.FTP_USE_EPSV, 1) + c.setopt(c.NETRC, c.NETRC_REQUIRED) + c.setopt(c.FTPLISTONLY, listonly) + c.setopt(c.FTP_CREATE_MISSING_DIRS, 1) + b = cStringIO.StringIO() + c.setopt(c.WRITEFUNCTION, b.write) + c.perform() + c.close() + return b.getvalue().split() if listonly else b.getvalue().splitlines() + +def curl_delete_file(path, url=MOBILEREAD): + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(c.FTP_USE_EPSV, 1) + c.setopt(c.NETRC, c.NETRC_REQUIRED) + print 'Deleting file %s on %s'%(path, url) + c.setopt(c.QUOTE, ['dele '+ path]) + c.perform() + c.close() + + +def curl_upload_file(stream, url): + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(pycurl.UPLOAD, 1) + c.setopt(c.NETRC, c.NETRC_REQUIRED) + c.setopt(pycurl.READFUNCTION, stream.read) + stream.seek(0, 2) + c.setopt(pycurl.INFILESIZE_LARGE, stream.tell()) + stream.seek(0) + c.setopt(c.NOPROGRESS, 0) + c.setopt(c.FTP_CREATE_MISSING_DIRS, 1) + print 'Uploading file %s to url %s' % (getattr(stream, 'name', ''), url) + try: + c.perform() + c.close() + except: + pass + files = curl_list_dir(listonly=0) + for line in files: + line = line.split() + if url.endswith(line[-1]): + size = long(line[4]) + stream.seek(0,2) + if size != stream.tell(): + raise RuntimeError('curl failed to upload %s correctly'%getattr(stream, 'name', '')) + + + +def upload_installer(name): + bname = os.path.basename(name) + pat = re.compile(bname.replace(__version__, r'\d+\.\d+\.\d+')) + for f in curl_list_dir(): + if pat.search(f): + curl_delete_file('/calibre/'+f) + curl_upload_file(open(name, 'rb'), MOBILEREAD+os.path.basename(name)) + def upload_installers(): - exe, dmg, tbz2 = installer_name('exe'), installer_name('dmg'), installer_name('tar.bz2') - if exe and os.path.exists(exe): - check_call('''ssh divok rm -f %s/calibre\*.exe'''%(DOWNLOADS,)) - check_call('''scp %s divok:%s/'''%(exe, DOWNLOADS)) - if dmg and os.path.exists(dmg): - check_call('''ssh divok rm -f %s/calibre\*.dmg'''%(DOWNLOADS,)) - check_call('''scp %s divok:%s/'''%(dmg, DOWNLOADS)) - if tbz2 and os.path.exists(tbz2): - check_call('''ssh divok rm -f %s/calibre-\*-i686.tar.bz2 %s/latest-linux-binary.tar.bz2'''%(DOWNLOADS,DOWNLOADS)) - check_call('''scp %s divok:%s/'''%(tbz2, DOWNLOADS)) - check_call('''ssh divok ln -s %s/calibre-\*-i686.tar.bz2 %s/latest-linux-binary.tar.bz2'''%(DOWNLOADS,DOWNLOADS)) - check_call('''ssh divok chmod a+r %s/\*'''%(DOWNLOADS,)) + for i in ('dmg', 'exe', 'tar.bz2'): + upload_installer(installer_name(i)) + + check_call('''ssh divok echo %s \\> %s/latest_version'''%(__version__, DOWNLOADS)) + def upload_docs(): check_call('''epydoc --config epydoc.conf''') diff --git a/windows_installer.py b/windows_installer.py index 06fa64d8ce..afabe531f8 100644 --- a/windows_installer.py +++ b/windows_installer.py @@ -564,7 +564,7 @@ def main(): 'win32file', 'pythoncom', 'rtf2xml', 'lxml', 'lxml._elementpath', 'genshi', 'path', 'pydoc', 'IPython.Extensions.*', - 'calibre.web.feeds.recipes.*', + 'calibre.web.feeds.recipes.*', 'PyQt4.QtWebKit', ], 'packages' : ['PIL'], 'excludes' : ["Tkconstants", "Tkinter", "tcl", From d0106bd4a0764b59980ee90b7dc92c035c68666b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 20 Jun 2008 11:41:44 -0700 Subject: [PATCH 13/18] IGN:Fix rendering of tables to images to not always create full page images --- src/calibre/ebooks/lrf/html/table_as_image.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/lrf/html/table_as_image.py b/src/calibre/ebooks/lrf/html/table_as_image.py index 4c5a79eab8..f4bdfa973d 100644 --- a/src/calibre/ebooks/lrf/html/table_as_image.py +++ b/src/calibre/ebooks/lrf/html/table_as_image.py @@ -46,9 +46,11 @@ class HTMLTableRenderer(QObject): painter = QPainter(image) self.page.mainFrame().render(painter) painter.end() + cheight = image.height() + cwidth = image.width() pos = 0 while pos < cheight: - img = image.copy(0, pos, cwidth, cutoff_height) + img = image.copy(0, pos, cwidth, min(cheight-pos, cutoff_height)) pos += cutoff_height-20 if cwidth > self.width: img = img.scaledToWidth(self.width, Qt.SmoothTransform) @@ -70,7 +72,7 @@ def render_table(server, soup, table, css, base_dir, width, height, dpi, factor= %s - + From 15305e3bea074f765a209d893d204986833d1e25 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 20 Jun 2008 11:42:33 -0700 Subject: [PATCH 14/18] IGN:Fix new IPC framework on windows --- linux_installer.py | 2 +- src/calibre/parallel.py | 74 +++++++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/linux_installer.py b/linux_installer.py index 315b2d8486..7556f438fb 100644 --- a/linux_installer.py +++ b/linux_installer.py @@ -74,7 +74,7 @@ f.write(hook_script) sys.path.insert(0, CALIBRESRC) from calibre.linux import entry_points -executables, scripts = ['calibre_postinstall', 'parallel'], \ +executables, scripts = ['calibre_postinstall', 'calibre-parallel'], \ [os.path.join(CALIBRESRC, 'calibre', 'linux.py'), os.path.join(CALIBRESRC, 'calibre', 'parallel.py')] for entry in entry_points['console_scripts'] + entry_points['gui_scripts']: diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py index 1c9593b94f..5ba20d28aa 100644 --- a/src/calibre/parallel.py +++ b/src/calibre/parallel.py @@ -47,10 +47,10 @@ if iswindows: python = os.path.join(os.path.dirname(python), 'parallel.exe') else: python = os.path.join(os.path.dirname(python), 'Scripts\\parallel.exe') - popen = partial(subprocess.Popen, creationflags=0x08) # CREATE_NO_WINDOW=0x08 so that no ugly console is popped up + open = partial(subprocess.Popen, creationflags=0x08) # CREATE_NO_WINDOW=0x08 so that no ugly console is popped up if islinux and hasattr(sys, 'frozen_path'): - python = os.path.join(getattr(sys, 'frozen_path'), 'parallel') + python = os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel') popen = partial(subprocess.Popen, cwd=getattr(sys, 'frozen_path')) prefix = 'import sys; sys.in_worker = True; ' @@ -59,7 +59,14 @@ if hasattr(sys, 'frameworks_dir'): prefix += 'sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd if fd not in os.environ['PATH']: os.environ['PATH'] += ':'+fd - +if 'parallel' in python: + executable = [python] + worker_command = '%s:%s' + free_spirit_command = '%s' +else: + executable = [python, '-c'] + worker_command = prefix + 'from calibre.parallel import worker; worker(%s, %s)' + free_spirit_command = prefix + 'from calibre.parallel import free_spirit; free_spirit(%s)' def write(socket, msg, timeout=5): if isinstance(msg, unicode): @@ -124,8 +131,8 @@ class Overseer(object): INTERVAL = 0.1 def __init__(self, server, port, timeout=5): - self.cmd = prefix + 'from calibre.parallel import worker; worker(%s, %s)'%(repr('localhost'), repr(port)) - self.process = popen([python, '-c', self.cmd]) + self.cmd = worker_command%(repr('127.0.0.1'), repr(port)) + self.process = popen(executable + [self.cmd]) self.socket = server.accept()[0] self.working = False @@ -135,8 +142,10 @@ class Overseer(object): self._stop = False if not select([self.socket], [], [], 120)[0]: raise RuntimeError(_('Could not launch worker process.')) - if int(self.read()) != self.process.pid: - raise RuntimeError('PID mismatch') + ID = self.read().split(':') + if ID[0] != 'CALIBRE_WORKER': + raise RuntimeError('Impostor') + self.worker_pid = int(ID[1]) self.write('OK') if self.read() != 'WAITING': raise RuntimeError('Worker sulking') @@ -147,19 +156,22 @@ class Overseer(object): ''' try: if self.socket: - self.socket.close() + self.write('STOP:') + time.sleep(1) + self.socket.shutdown(socket.SHUT_RDWR) except: pass if iswindows: win32api = __import__('win32api') try: - win32api.TerminateProcess(int(self.process.pid), -1) + handle = win32api.OpenProcess(1, False, self.worker_pid) + win32api.TerminateProcess(handle, -1) except: pass else: import signal try: - os.kill(self.process.pid, signal.SIGKILL) + os.kill(self.worker_pid, signal.SIGKILL) time.sleep(0.05) except: pass @@ -172,14 +184,14 @@ class Overseer(object): return read(self.socket, timeout=self.timeout if timeout is None else timeout) def __eq__(self, other): - return hasattr(other, 'process') and hasattr(other.process, 'pid') and self.process.pid == other.process.pid + return hasattr(other, 'process') and hasattr(other, 'worker_pid') and self.worker_pid == other.worker_pid def __bool__(self): self.process.poll() return self.process.returncode is None def pid(self): - return self.process.pid + return self.worker_pid def select(self, timeout=0): return select([self.socket], [self.socket], [self.socket], timeout) @@ -190,7 +202,7 @@ class Overseer(object): self.write('JOB:'+cPickle.dumps((job.func, job.args, job.kwdargs), -1)) msg = self.read() if msg != 'OK': - raise ControlError('Failed to initialize job on worker %d:%s'%(self.process.pid, msg)) + raise ControlError('Failed to initialize job on worker %d:%s'%(self.worker_pid, msg)) self.output = job.output if callable(job.output) else sys.stdout.write self.progress = job.progress if callable(job.progress) else None self.job = job @@ -278,6 +290,7 @@ class Server(Thread): self.number_of_workers = number_of_workers self.pool, self.jobs, self.working, self.results = [], collections.deque(), [], {} atexit.register(self.killall) + atexit.register(self.close) self.job_lock = RLock() self.overseer_lock = RLock() self.working_lock = RLock() @@ -285,6 +298,12 @@ class Server(Thread): self.pool_lock = RLock() self.start() + def close(self): + try: + self.server_socket.shutdown(socket.SHUT_RDWR) + except: + pass + def add_job(self, job): with self.job_lock: self.jobs.append(job) @@ -373,8 +392,8 @@ class Server(Thread): pt = PersistentTemporaryFile('.pickle', '_IPC_') pt.write(cPickle.dumps((func, args, kwdargs))) pt.close() - cmd = prefix + 'from calibre.parallel import free_spirit; free_spirit(%s)'%repr(pt.name) - popen([python, '-c', cmd]) + cmd = free_spirit_command%repr(pt.name) + popen(executable + [cmd]) ########################################################################################## ##################################### CLIENT CODE ##################################### @@ -426,26 +445,26 @@ def work(client_socket, func, args, kwdargs): func = PARALLEL_FUNCS[func] if hasattr(func, 'keywords'): for key, val in func.keywords.items(): - if val == _notify: + if val == _notify and hasattr(sys.stdout, 'notify'): func.keywords[key] = sys.stdout.notify res = func(*args, **kwdargs) - sys.stdout.send() + if hasattr(sys.stdout, 'send'): + sys.stdout.send() return res - - def worker(host, port): client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) client_socket.connect((host, port)) - write(client_socket, str(os.getpid())) + write(client_socket, 'CALIBRE_WORKER:%d'%os.getpid()) msg = read(client_socket, timeout=10) if msg != 'OK': return 1 write(client_socket, 'WAITING') + sys.stdout = BufferedSender(client_socket) sys.stderr = sys.stdout - + while True: msg = read(client_socket, timeout=60) if msg.startswith('JOB:'): @@ -463,6 +482,8 @@ def worker(host, port): gc.collect() elif msg == 'STOP:': return 0 + elif not msg: + time.sleep(1) def free_spirit(path): func, args, kwdargs = cPickle.load(open(path, 'rb')) @@ -471,4 +492,15 @@ def free_spirit(path): except: pass PARALLEL_FUNCS[func](*args, **kwdargs) + +def main(args=sys.argv): + args = args[1].split(':') + if len(args) == 1: + free_spirit(args[0]) + else: + worker(args[0].replace("'", ''), int(args[1])) + return 0 + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file From af4be77ae28337bf4d1ea742613c16252a3c06c1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 20 Jun 2008 13:02:54 -0700 Subject: [PATCH 15/18] Fix #799 --- src/calibre/devices/prs505/driver.py | 20 ++++++++++++++++---- src/calibre/gui2/library.py | 5 +++-- src/calibre/gui2/main.py | 19 +++++++++++-------- src/calibre/parallel.py | 12 ++++++++---- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/calibre/devices/prs505/driver.py b/src/calibre/devices/prs505/driver.py index 0050e091d3..4f2e98b4f7 100644 --- a/src/calibre/devices/prs505/driver.py +++ b/src/calibre/devices/prs505/driver.py @@ -353,9 +353,16 @@ class PRS505(Device): def upload_books(self, files, names, on_card=False, end_session=True): path = os.path.join(self._card_prefix, self.CARD_PATH_PREFIX) if on_card \ else os.path.join(self._main_prefix, 'database', 'media', 'books') - infiles = [file if hasattr(file, 'read') else open(file, 'rb') for file in files] - for f in infiles: f.seek(0, 2) - sizes = [f.tell() for f in infiles] + + def get_size(obj): + if hasattr(obj, 'seek'): + obj.seek(0, 2) + size = obj.tell() + obj.seek(0) + return size + return os.path.getsize(obj) + + sizes = map(get_size, files) size = sum(sizes) space = self.free_space() mspace = space[0] @@ -370,13 +377,18 @@ class PRS505(Device): paths, ctimes = [], [] names = iter(names) - for infile in infiles: + for infile in files: + close = False + if not hasattr(infile, 'read'): + infile, close = open(infile, 'rb'), True infile.seek(0) name = names.next() paths.append(os.path.join(path, name)) if not os.path.exists(os.path.dirname(paths[-1])): os.makedirs(os.path.dirname(paths[-1])) self.put_file(infile, paths[-1], replace_file=True) + if close: + infile.close() ctimes.append(os.path.getctime(paths[-1])) return zip(paths, sizes, ctimes, cycle([on_card])) diff --git a/src/calibre/gui2/library.py b/src/calibre/gui2/library.py index 5ede2b49a0..ce9db76eed 100644 --- a/src/calibre/gui2/library.py +++ b/src/calibre/gui2/library.py @@ -312,7 +312,7 @@ class BooksModel(QAbstractTableModel): metadata.append(mi) return metadata - def get_preferred_formats(self, rows, formats): + def get_preferred_formats(self, rows, formats, paths=False): ans = [] for row in (row.row() for row in rows): format = None @@ -323,7 +323,8 @@ class BooksModel(QAbstractTableModel): if format: pt = PersistentTemporaryFile(suffix='.'+format) pt.write(self.db.format(row, format)) - pt.seek(0) + pt.flush() + pt.close() if paths else pt.seek(0) ans.append(pt) else: ans.append(None) diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index a567bc567d..e856d748f0 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -466,7 +466,7 @@ class Main(MainWindow, Ui_MainWindow): else: self.upload_books(paths, names, infos, on_card=on_card) - def upload_books(self, files, names, metadata, on_card=False): + def upload_books(self, files, names, metadata, on_card=False, memory=None): ''' Upload books to device. @param files: List of either paths to files or file like objects @@ -477,13 +477,13 @@ class Main(MainWindow, Ui_MainWindow): files, names, on_card=on_card, job_extra_description=titles ) - self.upload_memory[id] = (metadata, on_card) + self.upload_memory[id] = (metadata, on_card, memory) def books_uploaded(self, id, description, result, exception, formatted_traceback): ''' Called once books have been uploaded. ''' - metadata, on_card = self.upload_memory.pop(id) + metadata, on_card = self.upload_memory.pop(id)[:2] if exception: if isinstance(exception, FreeSpaceError): where = 'in main memory.' if 'memory' in str(exception) else 'on the storage card.' @@ -611,8 +611,9 @@ class Main(MainWindow, Ui_MainWindow): if cdata: mi['cover'] = self.cover_to_thumbnail(cdata) metadata = iter(metadata) - files = self.library_view.model().get_preferred_formats(rows, - self.device_manager.device_class.FORMATS) + _files = self.library_view.model().get_preferred_formats(rows, + self.device_manager.device_class.FORMATS, paths=True) + files = [f.name for f in _files] bad, good, gf, names = [], [], [], [] for f in files: mi = metadata.next() @@ -627,7 +628,9 @@ class Main(MainWindow, Ui_MainWindow): try: smi = MetaInformation(mi['title'], aus2) smi.comments = mi.get('comments', None) - set_metadata(f, smi, f.name.rpartition('.')[2]) + _f = open(f, 'r+b') + set_metadata(_f, smi, f.rpartition('.')[2]) + _f.close() except: print 'Error setting metadata in book:', mi['title'] traceback.print_exc() @@ -644,8 +647,8 @@ class Main(MainWindow, Ui_MainWindow): prefix = prefix.encode('ascii', 'ignore') else: prefix = prefix.decode('ascii', 'ignore').encode('ascii', 'ignore') - names.append('%s_%d%s'%(prefix, id, os.path.splitext(f.name)[1])) - self.upload_books(gf, names, good, on_card) + names.append('%s_%d%s'%(prefix, id, os.path.splitext(f)[1])) + self.upload_books(gf, names, good, on_card, memory=_files) self.status_bar.showMessage(_('Sending books to device.'), 5000) if bad: bad = '\n'.join('
  1. %s
    2. '%(i,) for i in bad) diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py index 5ba20d28aa..7c0c997def 100644 --- a/src/calibre/parallel.py +++ b/src/calibre/parallel.py @@ -4,7 +4,8 @@ __copyright__ = '2008, Kovid Goyal ' ''' Used to run jobs in parallel in separate processes. ''' -import sys, os, gc, cPickle, traceback, atexit, cStringIO, time, subprocess, socket, collections +import sys, os, gc, cPickle, traceback, atexit, cStringIO, time, \ + subprocess, socket, collections, binascii from select import select from functools import partial from threading import RLock, Thread, Event @@ -392,7 +393,7 @@ class Server(Thread): pt = PersistentTemporaryFile('.pickle', '_IPC_') pt.write(cPickle.dumps((func, args, kwdargs))) pt.close() - cmd = free_spirit_command%repr(pt.name) + cmd = free_spirit_command%repr(binascii.hexlify(pt.name)) popen(executable + [cmd]) ########################################################################################## @@ -484,9 +485,12 @@ def worker(host, port): return 0 elif not msg: time.sleep(1) + else: + print >>sys.__stderr__, 'Invalid protocols message', msg + return 1 def free_spirit(path): - func, args, kwdargs = cPickle.load(open(path, 'rb')) + func, args, kwdargs = cPickle.load(open(binascii.unhexlify(path), 'rb')) try: os.unlink(path) except: @@ -496,7 +500,7 @@ def free_spirit(path): def main(args=sys.argv): args = args[1].split(':') if len(args) == 1: - free_spirit(args[0]) + free_spirit(args[0].replace("'", '')) else: worker(args[0].replace("'", ''), int(args[1])) return 0 From 41c447a0d24978b55ce5b90222d09b35857143d5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 20 Jun 2008 13:19:40 -0700 Subject: [PATCH 16/18] Fix #777 --- src/calibre/ebooks/mobi/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 2c96846aae..38fa1a6175 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -156,7 +156,7 @@ class MobiReader(object): processed_records = self.extract_text() self.add_anchors() - self.processed_html = self.processed_html.decode(self.book_header.codec) + self.processed_html = self.processed_html.decode(self.book_header.codec, 'ignore') self.extract_images(processed_records, output_dir) self.replace_page_breaks() self.cleanup() From 7501dbbe6504f0475cff543eb14fe5dc58492e22 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 20 Jun 2008 13:36:54 -0700 Subject: [PATCH 17/18] Fix #777 --- src/calibre/ebooks/mobi/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 38fa1a6175..99184d9244 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -177,7 +177,7 @@ class MobiReader(object): opf.render(open(os.path.splitext(htmlfile)[0]+'.opf', 'wb')) def cleanup(self): - self.processed_html = re.sub(r'
    ', '', self.processed_html) + self.processed_html = re.sub(r'
    ', '', self.processed_html) def create_opf(self, htmlfile): mi = self.book_header.exth.mi From e49db6236b58d9b0979c4f01fa0ab09127d1dd19 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 20 Jun 2008 13:43:27 -0700 Subject: [PATCH 18/18] Fix #802 --- src/calibre/ebooks/metadata/__init__.py | 2 +- src/calibre/ebooks/metadata/opf.py | 2 ++ src/calibre/library/database.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 2f14441faa..2ca67e526d 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -38,7 +38,7 @@ class MetaInformation(object): setattr(ans, attr, getattr(mi, attr)) - def __init__(self, title, authors=['Unknown']): + def __init__(self, title, authors=[_('Unknown')]): ''' @param title: title or "Unknown" or a MetaInformation object @param authors: List of strings or [] diff --git a/src/calibre/ebooks/metadata/opf.py b/src/calibre/ebooks/metadata/opf.py index 021b63ac08..bff9ed5f8a 100644 --- a/src/calibre/ebooks/metadata/opf.py +++ b/src/calibre/ebooks/metadata/opf.py @@ -511,6 +511,8 @@ class OPFCreator(MetaInformation): path = path[len(self.base_path)+1:] manifest.append((path, mt)) self.manifest = manifest + if not self.authors: + self.authors = [_('Unknown')] def create_manifest(self, entries): ''' diff --git a/src/calibre/library/database.py b/src/calibre/library/database.py index dc4f4e31de..9d54afc88c 100644 --- a/src/calibre/library/database.py +++ b/src/calibre/library/database.py @@ -1427,6 +1427,8 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE; open(cpath, 'wb').write(cover) mi.cover = cname f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb') + if not mi.authors: + mi.authors = [_('Unknown')] mi.render(f) f.close()