From c913e7902a85dc1250ad2e3954866d12b50f52fb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Sep 2009 09:48:57 -0600 Subject: [PATCH 1/5] Updated recipe for Business Week --- installer/osx/py2app/main.py | 94 ++++++++++++++++--- src/calibre/devices/prs500/cli/main.py | 2 +- src/calibre/ebooks/mobi/output.py | 5 +- .../web/feeds/recipes/recipe_business_week.py | 84 +++++++++++------ 4 files changed, 139 insertions(+), 46 deletions(-) diff --git a/installer/osx/py2app/main.py b/installer/osx/py2app/main.py index 0c19ed13d6..142d5cb763 100644 --- a/installer/osx/py2app/main.py +++ b/installer/osx/py2app/main.py @@ -66,6 +66,16 @@ def strip_files(files, argv_max=(256 * 1024)): for args in flips: flipwritable(*args) +def flush(func): + def ff(*args, **kwargs): + sys.stdout.flush() + sys.stderr.flush() + ret = func(*args, **kwargs) + sys.stdout.flush() + sys.stderr.flush() + return ret + return ff + class Py2App(object): FID = '@executable_path/../Frameworks' @@ -76,6 +86,14 @@ class Py2App(object): self.resources_dir = join(self.contents_dir, 'Resources') self.frameworks_dir = join(self.contents_dir, 'Frameworks') self.to_strip = [] + self.warnings = [] + + def warn(self, *args): + self.warnings.append(args) + prefix = '' if args and args[0].startswith('WARNING:') else 'WARNING: ' + sys.stdout.write(prefix+' '.join(args)+'\n') + sys.stdout.flush() + def run(self): self.create_skeleton() @@ -103,26 +121,40 @@ class Py2App(object): self.strip_files() self.create_launchers() - return self.makedmg(self.build_dir, APPNAME+'-'+VERSION+'-x86_64') + ret = self.makedmg(self.build_dir, APPNAME+'-'+VERSION+'-x86_64') + sys.stdout.flush() + sys.stderr.flush() + print '\nThere were', len(self.warnings), 'warnings' + for w in list(self.warnings): + print + self.warn(*w) + return ret + + @flush def create_launchers(self): - launcher = join(os.path.dirname(__file__), 'launcher.py') + print '\nCreating launchers' + all_names = basenames['console'] + basenames['gui'] + all_modules = main_modules['console'] + main_modules['gui'] + launcher = join(os.path.dirname(__file__), 'loader.py') launcher = open(launcher, 'rb').read() launcher = launcher.replace('{}##ENV##', repr(ENV)) os.mkdir(join(self.resources_dir, 'loaders')) - for module, basename in zip(main_modules, basenames): + for basename, module in zip(all_names, all_modules): raw = launcher.replace("''##MODULE##", repr(module)) path = join(self.resources_dir, 'loaders', basename) open(path, 'wb').write(raw) os.chmod(path, stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH|stat.S_IREAD\ |stat.S_IWUSR|stat.S_IROTH|stat.S_IRGRP) - + @flush def strip_files(self): print '\nStripping files...' strip_files(self.to_strip) + @flush def create_exe(self): + print '\nCreating executable' gcc = os.environ.get('CC', 'gcc') base = os.path.dirname(__file__) out = join(self.contents_dir, 'MacOS', 'calibre') @@ -130,12 +162,14 @@ class Py2App(object): 'main.c'), '-o', out]) self.to_strip.append(out) + @flush def set_id(self, path_to_lib, new_id): old_mode = flipwritable(path_to_lib) subprocess.check_call(['install_name_tool', '-id', new_id, path_to_lib]) if old_mode is not None: flipwritable(path_to_lib, old_mode) + @flush def get_dependencies(self, path_to_lib): raw = subprocess.Popen(['otool', '-L', path_to_lib], stdout=subprocess.PIPE).stdout.read() @@ -149,6 +183,7 @@ class Py2App(object): continue yield path + @flush def get_local_dependencies(self, path_to_lib): for x in self.get_dependencies(path_to_lib): for y in (SW+'/lib/', '/usr/local/lib/', SW+'/qt/lib/', @@ -157,11 +192,13 @@ class Py2App(object): yield x, x[len(y):] break + @flush def change_dep(self, old_dep, new_dep, path_to_lib): print '\tResolving dependency %s to'%old_dep, new_dep subprocess.check_call(['install_name_tool', '-change', old_dep, new_dep, path_to_lib]) + @flush def fix_dependencies_in_lib(self, path_to_lib): print '\nFixing dependencies in', path_to_lib self.to_strip.append(path_to_lib) @@ -174,6 +211,7 @@ class Py2App(object): if old_mode is not None: flipwritable(path_to_lib, old_mode) + @flush def add_python_framework(self): src = join(SW, 'python', 'Python.framework') x = join(self.frameworks_dir, 'Python.framework') @@ -186,6 +224,7 @@ class Py2App(object): self.set_id(join(currd, 'Python'), self.FID+'/Python.framework/Versions/%s/Python'%basename(curr)) + @flush def add_qt_frameworks(self): for f in ('QtCore', 'QtGui', 'QtXml', 'QtNetwork', 'QtSvg', 'QtWebkit', 'phonon'): @@ -197,6 +236,7 @@ class Py2App(object): x = os.path.relpath(l, join(self.contents_dir, 'MacOS')) self.set_id(l, '@executable_path/'+x) + @flush def add_qt_framework(self, f): libname = f f = f+'.framework' @@ -210,6 +250,7 @@ class Py2App(object): self.set_id(lib, self.FID+'/'+rpath) self.fix_dependencies_in_lib(lib) + @flush def create_skeleton(self): c = join(self.build_dir, 'Contents') for x in ('Frameworks', 'MacOS', 'Resources'): @@ -217,6 +258,7 @@ class Py2App(object): x = 'library.icns' shutil.copyfile(join('icons', x), join(self.resources_dir, x)) + @flush def add_calibre_plugins(self): dest = join(self.frameworks_dir, 'plugins') os.mkdir(dest) @@ -225,6 +267,7 @@ class Py2App(object): self.fix_dependencies_in_lib(join(dest, basename(f))) + @flush def create_plist(self): pl = dict( CFBundleDevelopmentRegion='English', @@ -248,6 +291,7 @@ class Py2App(object): ) plistlib.writePlist(pl, join(self.contents_dir, 'Info.plist')) + @flush def install_dylib(self, path, set_id=True): shutil.copy2(path, self.frameworks_dir) if set_id: @@ -255,25 +299,30 @@ class Py2App(object): self.FID+'/'+basename(path)) self.fix_dependencies_in_lib(join(self.frameworks_dir, basename(path))) + @flush def add_podofo(self): print '\nAdding PoDoFo' pdf = join(SW, 'lib', 'libpodofo.0.6.99.dylib') self.install_dylib(pdf) + @flush def add_poppler(self): print '\nAdding poppler' for x in ('libpoppler.4.dylib', 'libpoppler-qt4.3.dylib'): self.install_dylib(os.path.join(SW, 'lib', x)) self.install_dylib(os.path.join(SW, 'bin', 'pdftohtml'), False) + @flush def add_libjpeg(self): print '\nAdding libjpeg' self.install_dylib(os.path.join(SW, 'lib', 'libjpeg.7.dylib')) + @flush def add_libpng(self): print '\nAdding libpng' self.install_dylib(os.path.join(SW, 'lib', 'libpng12.0.dylib')) + @flush def add_fontconfig(self): print '\nAdding fontconfig' for x in ('fontconfig.1', 'freetype.6', 'expat.1'): @@ -297,6 +346,7 @@ class Py2App(object): ''') open(fc, 'wb').write(raw) + @flush def add_imagemagick(self): print '\nAdding ImageMagick' for x in ('Wand', 'Core'): @@ -315,11 +365,13 @@ class Py2App(object): f = join(x[0], f) self.fix_dependencies_in_lib(f) + @flush def add_misc_libraries(self): for x in ('usb', 'unrar'): print '\nAdding', x shutil.copy2(join(SW, 'lib', 'lib%s.dylib'%x), self.frameworks_dir) + @flush def add_site_packages(self): print '\nAdding site-packages' self.site_packages = join(self.resources_dir, 'Python', 'site-packages') @@ -327,18 +379,19 @@ class Py2App(object): paths = reversed(map(abspath, [x for x in sys.path if x.startswith('/')])) upaths = [] for x in paths: - if x not in upaths: - upaths.append(x) - for x in upaths: if x.endswith('/PIL') or 'site-packages' not in x: continue + if x not in upaths: + upaths.append(x) + upaths.append(os.path.expanduser('~/build/calibre/src')) + for x in upaths: tdir = None try: if not os.path.isdir(x): try: zf = zipfile.ZipFile(x) except: - print "WARNING:", x, 'is neither a directory nor a zipfile' + self.warn(x, 'is neither a directory nor a zipfile') continue tdir = tempfile.mkdtemp() zf.extractall(tdir) @@ -350,6 +403,7 @@ class Py2App(object): shutil.rmtree(tdir) self.remove_bytecode(join(self.resources_dir, 'Python', 'site-packages')) + @flush def add_modules_from_dir(self, src): for x in glob.glob(join(src, '*.py'))+glob.glob(join(src, '*.so')): dest = join(self.site_packages, basename(x)) @@ -357,6 +411,7 @@ class Py2App(object): if x.endswith('.so'): self.fix_dependencies_in_lib(x) + @flush def add_packages_from_dir(self, src): for x in os.listdir(src): x = join(src, x) @@ -365,13 +420,16 @@ class Py2App(object): continue self.add_package_dir(x) + @flush def add_package_dir(self, x, dest=None): def ignore(root, files): ans = [] for y in files: - if os.path.splitext(y) in ('.py', '.so'): - continue - ans.append(y) + ext = os.path.splitext(y)[1] + if ext not in ('', '.py', '.so') or \ + (not ext and not os.path.isdir(join(root, y))): + ans.append(y) + return ans if dest is None: dest = self.site_packages @@ -379,13 +437,16 @@ class Py2App(object): shutil.copytree(x, dest, symlinks=True, ignore=ignore) self.postprocess_package(x, dest) + @flush def filter_package(self, name): return name in ('Cython', 'modulegraph', 'macholib', 'py2app', 'bdist_mpkg', 'altgraph') + @flush def postprocess_package(self, src_path, dest_path): pass + @flush def add_stdlib(self): print '\nAdding python stdlib' src = join(SW, 'python/Python.framework/Versions/Current/lib/python') @@ -405,6 +466,7 @@ class Py2App(object): self.fix_dependencies_in_lib(dest) self.remove_bytecode(join(self.resources_dir, 'Python', 'lib')) + @flush def remove_bytecode(self, dest): for x in os.walk(dest): root = x[0] @@ -412,6 +474,7 @@ class Py2App(object): if os.path.splitext(f) in ('.pyc', '.pyo'): os.remove(join(root, f)) + @flush def compile_py_modules(self): print '\nCompiling Python modules' base = join(self.resources_dir, 'Python') @@ -425,8 +488,9 @@ class Py2App(object): py_compile.compile(y, dfile=rel, doraise=True) os.remove(y) except: - print 'WARNING: Failed to byte-compile', y + self.warn('WARNING: Failed to byte-compile', y) + @flush def create_console_app(self): print '\nCreating console.app' cc_dir = os.path.join(self.contents_dir, 'console.app', 'Contents') @@ -442,16 +506,20 @@ class Py2App(object): os.symlink(join('../..', x), join(cc_dir, x)) + @flush def copy_launcher_and_site(self): base = os.path.dirname(__file__) for x in ('launcher', 'site'): shutil.copy2(join(base, x+'.py'), self.resources_dir) + @flush def makedmg(self, d, volname, destdir='dist', internet_enable=True, format='UDBZ'): ''' Copy a directory d into a dmg named volname ''' + print '\nCreating dmg' + sys.stdout.flush() if not os.path.exists(destdir): os.makedirs(destdir) dmg = os.path.join(destdir, volname+'.dmg') @@ -461,6 +529,8 @@ class Py2App(object): '-volname', volname, '-format', format, dmg]) if internet_enable: subprocess.check_call(['/usr/bin/hdiutil', 'internet-enable', '-yes', dmg]) + size = os.stat(dmg).st_size/(1024*1024.) + print '\nInstaller size: %.2fMB\n'%size return dmg diff --git a/src/calibre/devices/prs500/cli/main.py b/src/calibre/devices/prs500/cli/main.py index 9865d383be..459f2d05e3 100755 --- a/src/calibre/devices/prs500/cli/main.py +++ b/src/calibre/devices/prs500/cli/main.py @@ -339,7 +339,7 @@ def main(): dev.touch(args[0]) else: parser.print_help() - if dev.handle: dev.close() + if getattr(dev, 'handle', False): dev.close() return 1 except DeviceLocked: print >> sys.stderr, "The device is locked. Use the --unlock option" diff --git a/src/calibre/ebooks/mobi/output.py b/src/calibre/ebooks/mobi/output.py index 2a70ace6e3..db8152b8fe 100644 --- a/src/calibre/ebooks/mobi/output.py +++ b/src/calibre/ebooks/mobi/output.py @@ -141,8 +141,9 @@ class MOBIOutput(OutputFormatPlugin): toc.nodes[0].href = toc.nodes[0].nodes[0].href # GR diagnostics - #self.dump_toc(toc) - #self.dump_manifest() + if self.opts.verbose > 3: + self.dump_toc(toc) + self.dump_manifest() def convert(self, oeb, output_path, input_plugin, opts, log): diff --git a/src/calibre/web/feeds/recipes/recipe_business_week.py b/src/calibre/web/feeds/recipes/recipe_business_week.py index d24e1c0f6a..da64eed711 100644 --- a/src/calibre/web/feeds/recipes/recipe_business_week.py +++ b/src/calibre/web/feeds/recipes/recipe_business_week.py @@ -12,46 +12,68 @@ from calibre.web.feeds.news import BasicNewsRecipe class BusinessWeek(BasicNewsRecipe): title = 'Business Week' description = 'Business News, Stock Market and Financial Advice' - __author__ = 'ChuckEggDotCom' + __author__ = 'ChuckEggDotCom and Sujata Raman' language = _('English') oldest_article = 7 max_articles_per_feed = 10 + no_stylesheets = True + + + extra_css = ''' + h1{font-family :Arial,Helvetica,sans-serif; font-size:large;} + h2{font-family :Arial,Helvetica,sans-serif; font-size:small;color:#666666;} + p{font-family :Arial,Helvetica,sans-serif; } + #lede600{font-size:x-small;} + #storybody{font-size:x-small;} + .strap{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#064599;} + .byline{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + .postedBy{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + .trackback{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + .date{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + .wrapper{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + .photoCredit{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + .tagline{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + ''' + + remove_tags = [ dict(name='div', attrs={'id':["bw2-header","column2","wrapper-bw2-footer","wrapper-mgh-footer","inset","commentForm","commentDisplay","bwExtras","bw2-umbrella","readerComments","pageNav","leg"]}), + ] - remove_tags_before = dict(name='h1') - remove_tags_after = dict(id='footer') - remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool']}), - dict(id=['footer', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']), - dict(name='h2', attrs={'class':'listspace'}), - ] - feeds = [ - (u'Top Stories', u'http://www.businessweek.com/topStories/rss/topStories.rss'), - (u'Top News', u'http://www.businessweek.com/rss/bwdaily.rss'), - (u'Asia', u'http://www.businessweek.com/rss/asia.rss'), - (u'Autos', u'http://www.businessweek.com/rss/autos/index.rss'), - (u'Classic Cars', u'http://www.businessweek.com/rss/autos/classic_cars/index.rss'), - (u'Hybrids', u'http://www.businessweek.com/rss/hybrids/index.rss'), - (u'Europe', u'http://www.businessweek.com/rss/europe.rss'), - (u'Auto Reviews', u'http://www.businessweek.com/rss/autos/reviews/index.rss'), - (u'Innovation & Design', u'http://www.businessweek.com/rss/innovate.rss'), - (u'Architecture', u'http://www.businessweek.com/rss/architecture.rss'), - (u'Brand Equity', u'http://www.businessweek.com/rss/brandequity.rss'), - (u'Auto Design', u'http://www.businessweek.com/rss/carbuff.rss'), - (u'Game Room', u'http://www.businessweek.com/rss/gameroom.rss'), - (u'Technology', u'http://www.businessweek.com/rss/technology.rss'), - (u'Investing', u'http://www.businessweek.m/rss/investor.rss'), - (u'Small Business', u'http://www.businessweek.com/rss/smallbiz.rss'), - (u'Careers', u'http://www.businessweek.com/rss/careers.rss'), - (u'B-Schools', u'http://www.businessweek.com/rss/bschools.rss'), - (u'Magazine Selections', u'http://www.businessweek.com/rss/magazine.rss'), + (u'Top Stories', u'http://www.businessweek.com/topStories/rss/topStories.rss'), + (u'Top News', u'http://www.businessweek.com/rss/bwdaily.rss'), + (u'Asia', u'http://www.businessweek.com/rss/asia.rss'), + (u'Autos', u'http://www.businessweek.com/rss/autos/index.rss'), + (u'Classic Cars', u'http://rss.businessweek.com/bw_rss/classiccars'), + (u'Hybrids', u'http://rss.businessweek.com/bw_rss/hybrids'), + (u'Europe', u'http://www.businessweek.com/rss/europe.rss'), + (u'Auto Reviews', u'http://rss.businessweek.com/bw_rss/autoreviews'), + (u'Innovation & Design', u'http://www.businessweek.com/rss/innovate.rss'), + (u'Architecture', u'http://www.businessweek.com/rss/architecture.rss'), + (u'Brand Equity', u'http://www.businessweek.com/rss/brandequity.rss'), + (u'Auto Design', u'http://www.businessweek.com/rss/carbuff.rss'), + (u'Game Room', u'http://rss.businessweek.com/bw_rss/gameroom'), + (u'Technology', u'http://www.businessweek.com/rss/technology.rss'), + (u'Investing', u'http://rss.businessweek.com/bw_rss/investor'), + (u'Small Business', u'http://www.businessweek.com/rss/smallbiz.rss'), + (u'Careers', u'http://rss.businessweek.com/bw_rss/careers'), + (u'B-Schools', u'http://www.businessweek.com/rss/bschools.rss'), + (u'Magazine Selections', u'http://www.businessweek.com/rss/magazine.rss'), (u'CEO Guide to Tech', u'http://www.businessweek.com/rss/ceo_guide_tech.rss'), ] def get_article_url(self, article): + url = article.get('guid', None) - if 'podcasts' in url: + + if 'podcasts' in url or 'surveys' in url: url = None + return url - - def print_version(self, url): - return url.replace('http://www.businessweek.com/', 'http://www.businessweek.com/print/') + + def postrocess_html(self, soup, first): + + for tag in soup.findAll(name=['ul','li']): + tag.name = 'div' + + return soup + From 0acbb330e974f71fc4926a40fb8c6443cf7f15f3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Sep 2009 15:26:38 -0600 Subject: [PATCH 2/5] Fix problem where some news downloads were showing incomplete Table of Contents on Kindle --- src/calibre/web/feeds/news.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py index db0a06886a..212578c5af 100644 --- a/src/calibre/web/feeds/news.py +++ b/src/calibre/web/feeds/news.py @@ -922,6 +922,8 @@ class BasicNewsRecipe(Recipe): desc = a.text_summary if not desc: desc = None + else: + desc = self.description_limiter(desc) entries.append('%sindex.html'%adir) po = self.play_order_map.get(entries[-1], None) if po is None: From 6ad672ae40cc42f8c7e775c8577f4dd9a044f664 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Sep 2009 16:16:40 -0600 Subject: [PATCH 3/5] IGN:More work on leopard build --- installer/osx/py2app/launcher.py | 9 ++++-- installer/osx/py2app/loader.py | 9 ++++-- installer/osx/py2app/main.c | 5 +-- installer/osx/py2app/main.py | 52 +++++++++++++++++++++----------- src/calibre/devices/libusb.py | 35 +++++++++++---------- upload.py | 3 +- 6 files changed, 72 insertions(+), 41 deletions(-) diff --git a/installer/osx/py2app/launcher.py b/installer/osx/py2app/launcher.py index c12b6bdc3a..468feca940 100644 --- a/installer/osx/py2app/launcher.py +++ b/installer/osx/py2app/launcher.py @@ -1,6 +1,5 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' @@ -39,13 +38,19 @@ def _run(): import os, sys, site sys.frozen = 'macosx_app' base = os.environ['RESOURCEPATH'] - sys.frameworks_dir = os.path.join(os.path.dirname(base, 'Frameworks')) + sys.frameworks_dir = os.path.join(os.path.dirname(base), 'Frameworks') sys.new_app_bundle = True site.addsitedir(base) site.addsitedir(os.path.join(base, 'Python', 'site-packages')) exe = os.environ.get('CALIBRE_LAUNCH_MODULE', 'calibre.gui2.main') exe = os.path.join(base, 'Python', 'site-packages', *exe.split('.')) + exe += '.py' sys.argv[0] = __file__ = exe + argv = os.environ.get('CALIBRE_LAUNCH_ARGV', None) + if argv is not None: + import cPickle + argv = cPickle.loads(argv) + sys.argv[1:] = argv execfile(exe, globals(), globals()) _run() diff --git a/installer/osx/py2app/loader.py b/installer/osx/py2app/loader.py index 533a88bbe3..ea6565c5f5 100644 --- a/installer/osx/py2app/loader.py +++ b/installer/osx/py2app/loader.py @@ -5,7 +5,7 @@ __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, sys +import os, sys, cPickle ENV = {}##ENV## MODULE = ''##MODULE## @@ -18,13 +18,16 @@ resources_dir = os.path.join(base_dir, 'Resources') frameworks_dir = os.path.join(base_dir, 'Frameworks') exe_dir = os.path.join(base_dir, 'MacOS') base_name = os.path.splitext(name)[0] -python = os.path.join(base_dir, 'MacOS', 'calibre') +python = os.path.join(base_dir, 'MacOS', 'Python') for key, val in ENV.items(): if val.startswith('@exec'): ENV[key] = os.path.normpath(val.replace('@executable_path', exe_dir)) ENV['CALIBRE_LAUNCH_MODULE'] = MODULE +ENV['CALIBRE_LAUNCH_ARGV'] = cPickle.dumps(sys.argv[1:], -1) +ENV['RESOURCEPATH'] = resources_dir os.environ.update(ENV) -args = [path] + sys.argv[1:] +launcher = os.path.join(resources_dir, 'launcher.py') +args = ['-OO', launcher] os.execv(python, args) diff --git a/installer/osx/py2app/main.c b/installer/osx/py2app/main.c index 01abe6d3a7..bc4ec1a434 100644 --- a/installer/osx/py2app/main.c +++ b/installer/osx/py2app/main.c @@ -864,9 +864,10 @@ static int py2app_main(int argc, char * const *argv, char * const *envp) { if (!getApplicationName()) return report_error(ERR_NONAME); pyLocations = (CFArrayRef)getKey("PyRuntimeLocations"); if (!pyLocations) return report_error(ERR_PYRUNTIMELOCATIONS); + printf("1111111\n;"); pyLocation = findPyLocation(pyLocations); if (!pyLocation) return report_error(ERR_NOPYTHONRUNTIME); - + printf("2222222\n"); setExecutablePath(); setResourcePath(); /* check for ':' in path, not compatible with Python due to Py_GetPath */ @@ -895,7 +896,7 @@ static int py2app_main(int argc, char * const *argv, char * const *envp) { xCFStringGetCString(pyLocation, buf, sizeof(buf), kCFStringEncodingUTF8); py_dylib = dlopen(buf, PYMACAPP_DYLD_FLAGS); if (py_dylib == NULL) return report_linkEdit_error(); - + printf("3333333\n"); #define LOOKUP_SYMBOL(NAME) \ tmpSymbol = dlsym(py_dylib, # NAME) #define LOOKUP_DEFINEADDRESS(NAME, ADDRESS) \ diff --git a/installer/osx/py2app/main.py b/installer/osx/py2app/main.py index 142d5cb763..67980053a8 100644 --- a/installer/osx/py2app/main.py +++ b/installer/osx/py2app/main.py @@ -26,6 +26,7 @@ ENV = dict( PYTHONIOENCODING='utf-8:replace', PYTHONPATH='@executable_path/../Resources/Python/site-packages', PYTHONHOME='@executable_path/../Resources/Python', + PYTHONOPTIMIZE='2', QT_PLUGIN_PATH='@executable_path' ) @@ -85,6 +86,7 @@ class Py2App(object): self.contents_dir = join(self.build_dir, 'Contents') self.resources_dir = join(self.contents_dir, 'Resources') self.frameworks_dir = join(self.contents_dir, 'Frameworks') + self.version_info = '.'.join(map(str, sys.version_info[:2])) self.to_strip = [] self.warnings = [] @@ -141,6 +143,9 @@ class Py2App(object): launcher = launcher.replace('{}##ENV##', repr(ENV)) os.mkdir(join(self.resources_dir, 'loaders')) for basename, module in zip(all_names, all_modules): + py_file = join('src', *module.split('.'))+'.py' + shutil.copy2(py_file, join(self.resources_dir, 'Python', + 'site-packages', *module.split('.'))+'.py') raw = launcher.replace("''##MODULE##", repr(module)) path = join(self.resources_dir, 'loaders', basename) open(path, 'wb').write(raw) @@ -174,13 +179,10 @@ class Py2App(object): raw = subprocess.Popen(['otool', '-L', path_to_lib], stdout=subprocess.PIPE).stdout.read() for line in raw.splitlines(): - if 'compatibility' not in line: + if 'compatibility' not in line or line.strip().endswith(':'): continue idx = line.find('(') path = line[:idx].strip() - bname = os.path.basename(path).partition('.')[0] - if bname in path_to_lib: - continue yield path @flush @@ -213,6 +215,7 @@ class Py2App(object): @flush def add_python_framework(self): + print '\nAdding Python framework' src = join(SW, 'python', 'Python.framework') x = join(self.frameworks_dir, 'Python.framework') curr = os.path.realpath(join(src, 'Versions', 'Current')) @@ -223,6 +226,10 @@ class Py2App(object): shutil.copy2(join(curr, 'Python'), currd) self.set_id(join(currd, 'Python'), self.FID+'/Python.framework/Versions/%s/Python'%basename(curr)) + python = '%s/python/Python.framework/Versions/%s/Resources/Python.app/Contents/MacOS/Python'\ + % (SW, self.version_info) + shutil.copy2(python, join(self.contents_dir, 'MacOS')) + self.fix_dependencies_in_lib(join(self.contents_dir, 'MacOS', 'Python')) @flush def add_qt_frameworks(self): @@ -264,7 +271,7 @@ class Py2App(object): os.mkdir(dest) for f in glob.glob('src/calibre/plugins/*.so'): shutil.copy2(f, dest) - self.fix_dependencies_in_lib(join(dest, basename(f))) + self.fix_dependencies_in_lib(join(dest, basename(f))) @flush @@ -279,7 +286,7 @@ class Py2App(object): CFBundleSignature='????', CFBundleExecutable='calibre', LSMinimumSystemVersion='10.5.2', - PyRuntimeLocations=[self.FID+'/Python.framework/Versions/Current/Python'], + PyRuntimeLocations=[self.FID+'/Python.framework/Versions/%s/Python'%self.version_info], LSRequiresNativeExecution=True, NSAppleScriptEnabled=False, NSHumanReadableCopyright='Copyright 2008, Kovid Goyal', @@ -369,7 +376,9 @@ class Py2App(object): def add_misc_libraries(self): for x in ('usb', 'unrar'): print '\nAdding', x - shutil.copy2(join(SW, 'lib', 'lib%s.dylib'%x), self.frameworks_dir) + x = 'lib%s.dylib'%x + shutil.copy2(join(SW, 'lib', x), self.frameworks_dir) + self.set_id(join(self.frameworks_dir, x), self.FID+'/'+x) @flush def add_site_packages(self): @@ -436,6 +445,11 @@ class Py2App(object): dest = join(dest, basename(x)) shutil.copytree(x, dest, symlinks=True, ignore=ignore) self.postprocess_package(x, dest) + for x in os.walk(dest): + for f in x[-1]: + if f.endswith('.so'): + f = join(x[0], f) + self.fix_dependencies_in_lib(f) @flush def filter_package(self, name): @@ -450,20 +464,22 @@ class Py2App(object): def add_stdlib(self): print '\nAdding python stdlib' src = join(SW, 'python/Python.framework/Versions/Current/lib/python') - src += '.'.join(map(str, sys.version_info[:2])) + src += self.version_info dest = join(self.resources_dir, 'Python', 'lib', 'python') - dest += '.'.join(map(str, sys.version_info[:2])) + dest += self.version_info + os.makedirs(dest) for x in os.listdir(src): if x in ('site-packages', 'config', 'test', 'lib2to3', 'lib-tk', 'lib-old', 'idlelib', 'plat-mac', 'plat-darwin', 'site.py'): continue + x = join(src, x) if os.path.isdir(x): - self.add_package_dir(join(src, x), dest) - elif os.path.splitext(x) in ('.so', '.py'): - shutil.copy2(join(src, x), dest) - dest = join(dest, basename(x)) - if dest.endswith('.so'): - self.fix_dependencies_in_lib(dest) + self.add_package_dir(x, dest) + elif os.path.splitext(x)[1] in ('.so', '.py'): + shutil.copy2(x, dest) + dest2 = join(dest, basename(x)) + if dest2.endswith('.so'): + self.fix_dependencies_in_lib(dest2) self.remove_bytecode(join(self.resources_dir, 'Python', 'lib')) @flush @@ -509,8 +525,10 @@ class Py2App(object): @flush def copy_launcher_and_site(self): base = os.path.dirname(__file__) - for x in ('launcher', 'site'): - shutil.copy2(join(base, x+'.py'), self.resources_dir) + shutil.copy2(join(base, 'launcher.py'), self.resources_dir) + shutil.copy2(join(base, 'site.py'), join(self.resources_dir, 'Python', + 'lib', 'python'+self.version_info)) + @flush def makedmg(self, d, volname, diff --git a/src/calibre/devices/libusb.py b/src/calibre/devices/libusb.py index a465f95029..edfec679fd 100644 --- a/src/calibre/devices/libusb.py +++ b/src/calibre/devices/libusb.py @@ -305,22 +305,25 @@ Device._fields_ = [ \ ] if _libusb is not None: - _libusb.usb_get_busses.restype = POINTER(Bus) - _libusb.usb_open.restype = POINTER(DeviceHandle) - _libusb.usb_open.argtypes = [POINTER(Device)] - _libusb.usb_close.argtypes = [POINTER(DeviceHandle)] - _libusb.usb_claim_interface.argtypes = [POINTER(DeviceHandle), c_int] - _libusb.usb_claim_interface.restype = c_int - _libusb.usb_release_interface.argtypes = [POINTER(DeviceHandle), c_int] - _libusb.usb_release_interface.restype = c_int - _libusb.usb_reset.argtypes = [POINTER(DeviceHandle)] - _libusb.usb_reset.restype = c_int - _libusb.usb_control_msg.restype = c_int - _libusb.usb_bulk_read.restype = c_int - _libusb.usb_bulk_write.restype = c_int - _libusb.usb_set_configuration.argtypes = [POINTER(DeviceHandle), c_int] - _libusb.usb_set_configuration.restype = c_int - _libusb.usb_init() + try: + _libusb.usb_get_busses.restype = POINTER(Bus) + _libusb.usb_open.restype = POINTER(DeviceHandle) + _libusb.usb_open.argtypes = [POINTER(Device)] + _libusb.usb_close.argtypes = [POINTER(DeviceHandle)] + _libusb.usb_claim_interface.argtypes = [POINTER(DeviceHandle), c_int] + _libusb.usb_claim_interface.restype = c_int + _libusb.usb_release_interface.argtypes = [POINTER(DeviceHandle), c_int] + _libusb.usb_release_interface.restype = c_int + _libusb.usb_reset.argtypes = [POINTER(DeviceHandle)] + _libusb.usb_reset.restype = c_int + _libusb.usb_control_msg.restype = c_int + _libusb.usb_bulk_read.restype = c_int + _libusb.usb_bulk_write.restype = c_int + _libusb.usb_set_configuration.argtypes = [POINTER(DeviceHandle), c_int] + _libusb.usb_set_configuration.restype = c_int + _libusb.usb_init() + except: + _libusb = None diff --git a/upload.py b/upload.py index 6e3c5fa099..9574c633b3 100644 --- a/upload.py +++ b/upload.py @@ -523,8 +523,9 @@ class VMInstaller(OptionlessCommand): --exclude "*.pyc" --exclude "*.pyo" --exclude "*.swp" --exclude "*.swo" \ rsync://%(host)s/work/%(project)s . && \ cd %(project)s && \ + rm -rf src/calibre/plugins/* && \ %%s && \ - rm -rf build/* dist/* src/calibre/plugins/* && \ + rm -rf build/* dist/* && \ %%s %%s '''%dict(host=HOST, project=__appname__)) From acf94d47866d20624d421389759d7e5dd077d476 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Sep 2009 16:21:15 -0600 Subject: [PATCH 4/5] IGN:Optimize TXT output --- src/calibre/ebooks/txt/newlines.py | 2 +- src/calibre/ebooks/txt/txtml.py | 53 +++++++++++++++++++----------- 2 files changed, 35 insertions(+), 20 deletions(-) diff --git a/src/calibre/ebooks/txt/newlines.py b/src/calibre/ebooks/txt/newlines.py index 983d356206..ae766a216f 100644 --- a/src/calibre/ebooks/txt/newlines.py +++ b/src/calibre/ebooks/txt/newlines.py @@ -19,7 +19,7 @@ class TxtNewlines(object): self.newline = self.NEWLINE_TYPES.get(newline_type.lower(), os.linesep) def specified_newlines(newline, text): - if newline == os.linesep: + if newline == '\n': return text return text.replace(os.linesep, newline) diff --git a/src/calibre/ebooks/txt/txtml.py b/src/calibre/ebooks/txt/txtml.py index 206dff50ed..28ba9eea4a 100644 --- a/src/calibre/ebooks/txt/txtml.py +++ b/src/calibre/ebooks/txt/txtml.py @@ -8,7 +8,8 @@ __docformat__ = 'restructuredtext en' Transform OEB content into plain text ''' -import os, re +import os +import re from lxml import etree @@ -43,15 +44,15 @@ class TXTMLizer(object): return self.mlize_spine() def mlize_spine(self): - output = u'' - output += self.get_toc() + output = [u''] + output.append(self.get_toc()) for item in self.oeb_book.spine: self.log.debug('Converting %s to TXT...' % item.href) stylizer = Stylizer(item.data, item.href, self.oeb_book, self.opts.output_profile) content = unicode(etree.tostring(item.data.find(XHTML('body')), encoding=unicode)) content = self.remove_newlines(content) - output += self.dump_text(etree.fromstring(content), stylizer) - output = self.cleanup_text(output) + output.append(self.dump_text(etree.fromstring(content), stylizer)) + output = self.cleanup_text(u''.join(output)) return output @@ -64,13 +65,13 @@ class TXTMLizer(object): return text def get_toc(self): - toc = u'' + toc = [u''] if getattr(self.opts, 'inline_toc', None): self.log.debug('Generating table of contents...') - toc += u'%s\n\n' % _(u'Table of Contents:') + toc.append(u'%s\n\n' % _(u'Table of Contents:')) for item in self.oeb_book.toc: - toc += u'* %s\n\n' % item.title - return toc + toc.append(u'* %s\n\n' % item.title) + return ''.join(toc) def cleanup_text(self, text): self.log.debug('\tClean up text...') @@ -99,6 +100,17 @@ class TXTMLizer(object): return text + def get_text(self, elem, stylizer): + ''' + @elem: The element in the etree that we are working on. + @stylizer: The style information attached to the element. + @end: The last two characters of the text from the previous element. + This is used to determine if a blank line is needed when starting + a new block element. + ''' + + return u''.join(self.dump_text(elem, stylizer)) + def dump_text(self, elem, stylizer, end=''): ''' @elem: The element in the etree that we are working on. @@ -110,14 +122,14 @@ class TXTMLizer(object): if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: - return u'' + return [''] - text = u'' + text = [''] style = stylizer.style(elem) if style['display'] in ('none', 'oeb-page-head', 'oeb-page-foot') \ or style['visibility'] == 'hidden': - return u'' + return [''] tag = barename(elem.tag) in_block = False @@ -125,20 +137,23 @@ class TXTMLizer(object): # Are we in a paragraph block? if tag in BLOCK_TAGS or style['display'] in BLOCK_STYLES: in_block = True - if not end.endswith(os.linesep + os.linesep) and hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '': - text += os.linesep + os.linesep + if not end.endswith('\n\n') and hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '': + text.append('\n\n') - # Proccess tags that contain text. + # Process tags that contain text. if hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '': - text += elem.text + text.append(elem.text) for item in elem: - text += self.dump_text(item, stylizer, text[-2:]) + en = u'' + if len(text) >= 2: + en = text[-1][-2:] + text += self.dump_text(item, stylizer, en) if in_block: - text += os.linesep + os.linesep + text.append('\n\n') if hasattr(elem, 'tail') and elem.tail != None and elem.tail.strip() != '': - text += elem.tail + text.append(elem.tail) return text From f6acabca5035d60238d8ed854e206b621f86ca0e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Sep 2009 16:26:16 -0600 Subject: [PATCH 5/5] TXT Output: Options to not add a blank line between paragraphs and to add a TAB at the start of each paragraph --- src/calibre/ebooks/txt/output.py | 6 ++++++ src/calibre/ebooks/txt/txtml.py | 15 +++++++++++---- src/calibre/gui2/convert/txt_output.py | 3 ++- src/calibre/gui2/convert/txt_output.ui | 18 ++++++++++++++++-- 4 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/txt/output.py index 6f0a768b8f..b3bda7fa9d 100644 --- a/src/calibre/ebooks/txt/output.py +++ b/src/calibre/ebooks/txt/output.py @@ -33,6 +33,12 @@ class TXTOutput(OutputFormatPlugin): OptionRecommendation(name='inline_toc', recommended_value=False, level=OptionRecommendation.LOW, help=_('Add Table of Contents to beginning of the book.')), + OptionRecommendation(name='flush_paras', + recommended_value=False, level=OptionRecommendation.LOW, + help=_('Do not add a blank line between paragraphs.')), + OptionRecommendation(name='indent_paras', + recommended_value=False, level=OptionRecommendation.LOW, + help=_('Add a tab at the beginning of each paragraph.')), ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): diff --git a/src/calibre/ebooks/txt/txtml.py b/src/calibre/ebooks/txt/txtml.py index 28ba9eea4a..cba383ef9a 100644 --- a/src/calibre/ebooks/txt/txtml.py +++ b/src/calibre/ebooks/txt/txtml.py @@ -41,6 +41,7 @@ class TXTMLizer(object): self.log.info('Converting XHTML to TXT...') self.oeb_book = oeb_book self.opts = opts + return self.mlize_spine() def mlize_spine(self): @@ -92,12 +93,18 @@ class TXTMLizer(object): # Remove excessive newlines. text = re.sub('\n[ ]+\n', '\n\n', text) - text = re.sub('\n{3,}', '\n\n', text) + if self.opts.flush_paras: + text = re.sub('\n{2,}', '\n', text) + else: + text = re.sub('\n{3,}', '\n\n', text) # Replace spaces at the beginning and end of lines text = re.sub('(?imu)^[ ]+', '', text) text = re.sub('(?imu)[ ]+$', '', text) + if self.opts.indent_paras: + text = re.sub('(?imu)^(?=.)', '\t', text) + return text def get_text(self, elem, stylizer): @@ -137,8 +144,8 @@ class TXTMLizer(object): # Are we in a paragraph block? if tag in BLOCK_TAGS or style['display'] in BLOCK_STYLES: in_block = True - if not end.endswith('\n\n') and hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '': - text.append('\n\n') + if not end.endswith(u'\n\n') and hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '': + text.append(u'\n\n') # Process tags that contain text. if hasattr(elem, 'text') and elem.text != None and elem.text.strip() != '': @@ -151,7 +158,7 @@ class TXTMLizer(object): text += self.dump_text(item, stylizer, en) if in_block: - text.append('\n\n') + text.append(u'\n\n') if hasattr(elem, 'tail') and elem.tail != None and elem.tail.strip() != '': text.append(elem.tail) diff --git a/src/calibre/gui2/convert/txt_output.py b/src/calibre/gui2/convert/txt_output.py index 407c308089..c2474ac4b8 100644 --- a/src/calibre/gui2/convert/txt_output.py +++ b/src/calibre/gui2/convert/txt_output.py @@ -17,7 +17,8 @@ class PluginWidget(Widget, Ui_Form): HELP = _('Options specific to')+' TXT '+_('output') def __init__(self, parent, get_option, get_help, db=None, book_id=None): - Widget.__init__(self, parent, 'txt_output', ['newline', 'inline_toc']) + Widget.__init__(self, parent, 'txt_output', ['newline', 'inline_toc', + 'flush_paras', 'indent_paras']) self.db, self.book_id = db, book_id self.initialize_options(get_option, get_help, db, book_id) diff --git a/src/calibre/gui2/convert/txt_output.ui b/src/calibre/gui2/convert/txt_output.ui index 6e62040533..900198aca9 100644 --- a/src/calibre/gui2/convert/txt_output.ui +++ b/src/calibre/gui2/convert/txt_output.ui @@ -27,7 +27,7 @@ - + Qt::Vertical @@ -40,13 +40,27 @@ - + &Inline TOC + + + + Do not add a blank line between paragraphs. + + + + + + + Add a tab at the beginning of each paragraph + + +