From c913e7902a85dc1250ad2e3954866d12b50f52fb Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 2 Sep 2009 09:48:57 -0600 Subject: [PATCH] Updated recipe for Business Week --- installer/osx/py2app/main.py | 94 ++++++++++++++++--- src/calibre/devices/prs500/cli/main.py | 2 +- src/calibre/ebooks/mobi/output.py | 5 +- .../web/feeds/recipes/recipe_business_week.py | 84 +++++++++++------ 4 files changed, 139 insertions(+), 46 deletions(-) diff --git a/installer/osx/py2app/main.py b/installer/osx/py2app/main.py index 0c19ed13d6..142d5cb763 100644 --- a/installer/osx/py2app/main.py +++ b/installer/osx/py2app/main.py @@ -66,6 +66,16 @@ def strip_files(files, argv_max=(256 * 1024)): for args in flips: flipwritable(*args) +def flush(func): + def ff(*args, **kwargs): + sys.stdout.flush() + sys.stderr.flush() + ret = func(*args, **kwargs) + sys.stdout.flush() + sys.stderr.flush() + return ret + return ff + class Py2App(object): FID = '@executable_path/../Frameworks' @@ -76,6 +86,14 @@ class Py2App(object): self.resources_dir = join(self.contents_dir, 'Resources') self.frameworks_dir = join(self.contents_dir, 'Frameworks') self.to_strip = [] + self.warnings = [] + + def warn(self, *args): + self.warnings.append(args) + prefix = '' if args and args[0].startswith('WARNING:') else 'WARNING: ' + sys.stdout.write(prefix+' '.join(args)+'\n') + sys.stdout.flush() + def run(self): self.create_skeleton() @@ -103,26 +121,40 @@ class Py2App(object): self.strip_files() self.create_launchers() - return self.makedmg(self.build_dir, APPNAME+'-'+VERSION+'-x86_64') + ret = self.makedmg(self.build_dir, APPNAME+'-'+VERSION+'-x86_64') + sys.stdout.flush() + sys.stderr.flush() + print '\nThere were', len(self.warnings), 'warnings' + for w in list(self.warnings): + print + self.warn(*w) + return ret + + @flush def create_launchers(self): - launcher = join(os.path.dirname(__file__), 'launcher.py') + print '\nCreating launchers' + all_names = basenames['console'] + basenames['gui'] + all_modules = main_modules['console'] + main_modules['gui'] + launcher = join(os.path.dirname(__file__), 'loader.py') launcher = open(launcher, 'rb').read() launcher = launcher.replace('{}##ENV##', repr(ENV)) os.mkdir(join(self.resources_dir, 'loaders')) - for module, basename in zip(main_modules, basenames): + for basename, module in zip(all_names, all_modules): raw = launcher.replace("''##MODULE##", repr(module)) path = join(self.resources_dir, 'loaders', basename) open(path, 'wb').write(raw) os.chmod(path, stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH|stat.S_IREAD\ |stat.S_IWUSR|stat.S_IROTH|stat.S_IRGRP) - + @flush def strip_files(self): print '\nStripping files...' strip_files(self.to_strip) + @flush def create_exe(self): + print '\nCreating executable' gcc = os.environ.get('CC', 'gcc') base = os.path.dirname(__file__) out = join(self.contents_dir, 'MacOS', 'calibre') @@ -130,12 +162,14 @@ class Py2App(object): 'main.c'), '-o', out]) self.to_strip.append(out) + @flush def set_id(self, path_to_lib, new_id): old_mode = flipwritable(path_to_lib) subprocess.check_call(['install_name_tool', '-id', new_id, path_to_lib]) if old_mode is not None: flipwritable(path_to_lib, old_mode) + @flush def get_dependencies(self, path_to_lib): raw = subprocess.Popen(['otool', '-L', path_to_lib], stdout=subprocess.PIPE).stdout.read() @@ -149,6 +183,7 @@ class Py2App(object): continue yield path + @flush def get_local_dependencies(self, path_to_lib): for x in self.get_dependencies(path_to_lib): for y in (SW+'/lib/', '/usr/local/lib/', SW+'/qt/lib/', @@ -157,11 +192,13 @@ class Py2App(object): yield x, x[len(y):] break + @flush def change_dep(self, old_dep, new_dep, path_to_lib): print '\tResolving dependency %s to'%old_dep, new_dep subprocess.check_call(['install_name_tool', '-change', old_dep, new_dep, path_to_lib]) + @flush def fix_dependencies_in_lib(self, path_to_lib): print '\nFixing dependencies in', path_to_lib self.to_strip.append(path_to_lib) @@ -174,6 +211,7 @@ class Py2App(object): if old_mode is not None: flipwritable(path_to_lib, old_mode) + @flush def add_python_framework(self): src = join(SW, 'python', 'Python.framework') x = join(self.frameworks_dir, 'Python.framework') @@ -186,6 +224,7 @@ class Py2App(object): self.set_id(join(currd, 'Python'), self.FID+'/Python.framework/Versions/%s/Python'%basename(curr)) + @flush def add_qt_frameworks(self): for f in ('QtCore', 'QtGui', 'QtXml', 'QtNetwork', 'QtSvg', 'QtWebkit', 'phonon'): @@ -197,6 +236,7 @@ class Py2App(object): x = os.path.relpath(l, join(self.contents_dir, 'MacOS')) self.set_id(l, '@executable_path/'+x) + @flush def add_qt_framework(self, f): libname = f f = f+'.framework' @@ -210,6 +250,7 @@ class Py2App(object): self.set_id(lib, self.FID+'/'+rpath) self.fix_dependencies_in_lib(lib) + @flush def create_skeleton(self): c = join(self.build_dir, 'Contents') for x in ('Frameworks', 'MacOS', 'Resources'): @@ -217,6 +258,7 @@ class Py2App(object): x = 'library.icns' shutil.copyfile(join('icons', x), join(self.resources_dir, x)) + @flush def add_calibre_plugins(self): dest = join(self.frameworks_dir, 'plugins') os.mkdir(dest) @@ -225,6 +267,7 @@ class Py2App(object): self.fix_dependencies_in_lib(join(dest, basename(f))) + @flush def create_plist(self): pl = dict( CFBundleDevelopmentRegion='English', @@ -248,6 +291,7 @@ class Py2App(object): ) plistlib.writePlist(pl, join(self.contents_dir, 'Info.plist')) + @flush def install_dylib(self, path, set_id=True): shutil.copy2(path, self.frameworks_dir) if set_id: @@ -255,25 +299,30 @@ class Py2App(object): self.FID+'/'+basename(path)) self.fix_dependencies_in_lib(join(self.frameworks_dir, basename(path))) + @flush def add_podofo(self): print '\nAdding PoDoFo' pdf = join(SW, 'lib', 'libpodofo.0.6.99.dylib') self.install_dylib(pdf) + @flush def add_poppler(self): print '\nAdding poppler' for x in ('libpoppler.4.dylib', 'libpoppler-qt4.3.dylib'): self.install_dylib(os.path.join(SW, 'lib', x)) self.install_dylib(os.path.join(SW, 'bin', 'pdftohtml'), False) + @flush def add_libjpeg(self): print '\nAdding libjpeg' self.install_dylib(os.path.join(SW, 'lib', 'libjpeg.7.dylib')) + @flush def add_libpng(self): print '\nAdding libpng' self.install_dylib(os.path.join(SW, 'lib', 'libpng12.0.dylib')) + @flush def add_fontconfig(self): print '\nAdding fontconfig' for x in ('fontconfig.1', 'freetype.6', 'expat.1'): @@ -297,6 +346,7 @@ class Py2App(object): ''') open(fc, 'wb').write(raw) + @flush def add_imagemagick(self): print '\nAdding ImageMagick' for x in ('Wand', 'Core'): @@ -315,11 +365,13 @@ class Py2App(object): f = join(x[0], f) self.fix_dependencies_in_lib(f) + @flush def add_misc_libraries(self): for x in ('usb', 'unrar'): print '\nAdding', x shutil.copy2(join(SW, 'lib', 'lib%s.dylib'%x), self.frameworks_dir) + @flush def add_site_packages(self): print '\nAdding site-packages' self.site_packages = join(self.resources_dir, 'Python', 'site-packages') @@ -327,18 +379,19 @@ class Py2App(object): paths = reversed(map(abspath, [x for x in sys.path if x.startswith('/')])) upaths = [] for x in paths: - if x not in upaths: - upaths.append(x) - for x in upaths: if x.endswith('/PIL') or 'site-packages' not in x: continue + if x not in upaths: + upaths.append(x) + upaths.append(os.path.expanduser('~/build/calibre/src')) + for x in upaths: tdir = None try: if not os.path.isdir(x): try: zf = zipfile.ZipFile(x) except: - print "WARNING:", x, 'is neither a directory nor a zipfile' + self.warn(x, 'is neither a directory nor a zipfile') continue tdir = tempfile.mkdtemp() zf.extractall(tdir) @@ -350,6 +403,7 @@ class Py2App(object): shutil.rmtree(tdir) self.remove_bytecode(join(self.resources_dir, 'Python', 'site-packages')) + @flush def add_modules_from_dir(self, src): for x in glob.glob(join(src, '*.py'))+glob.glob(join(src, '*.so')): dest = join(self.site_packages, basename(x)) @@ -357,6 +411,7 @@ class Py2App(object): if x.endswith('.so'): self.fix_dependencies_in_lib(x) + @flush def add_packages_from_dir(self, src): for x in os.listdir(src): x = join(src, x) @@ -365,13 +420,16 @@ class Py2App(object): continue self.add_package_dir(x) + @flush def add_package_dir(self, x, dest=None): def ignore(root, files): ans = [] for y in files: - if os.path.splitext(y) in ('.py', '.so'): - continue - ans.append(y) + ext = os.path.splitext(y)[1] + if ext not in ('', '.py', '.so') or \ + (not ext and not os.path.isdir(join(root, y))): + ans.append(y) + return ans if dest is None: dest = self.site_packages @@ -379,13 +437,16 @@ class Py2App(object): shutil.copytree(x, dest, symlinks=True, ignore=ignore) self.postprocess_package(x, dest) + @flush def filter_package(self, name): return name in ('Cython', 'modulegraph', 'macholib', 'py2app', 'bdist_mpkg', 'altgraph') + @flush def postprocess_package(self, src_path, dest_path): pass + @flush def add_stdlib(self): print '\nAdding python stdlib' src = join(SW, 'python/Python.framework/Versions/Current/lib/python') @@ -405,6 +466,7 @@ class Py2App(object): self.fix_dependencies_in_lib(dest) self.remove_bytecode(join(self.resources_dir, 'Python', 'lib')) + @flush def remove_bytecode(self, dest): for x in os.walk(dest): root = x[0] @@ -412,6 +474,7 @@ class Py2App(object): if os.path.splitext(f) in ('.pyc', '.pyo'): os.remove(join(root, f)) + @flush def compile_py_modules(self): print '\nCompiling Python modules' base = join(self.resources_dir, 'Python') @@ -425,8 +488,9 @@ class Py2App(object): py_compile.compile(y, dfile=rel, doraise=True) os.remove(y) except: - print 'WARNING: Failed to byte-compile', y + self.warn('WARNING: Failed to byte-compile', y) + @flush def create_console_app(self): print '\nCreating console.app' cc_dir = os.path.join(self.contents_dir, 'console.app', 'Contents') @@ -442,16 +506,20 @@ class Py2App(object): os.symlink(join('../..', x), join(cc_dir, x)) + @flush def copy_launcher_and_site(self): base = os.path.dirname(__file__) for x in ('launcher', 'site'): shutil.copy2(join(base, x+'.py'), self.resources_dir) + @flush def makedmg(self, d, volname, destdir='dist', internet_enable=True, format='UDBZ'): ''' Copy a directory d into a dmg named volname ''' + print '\nCreating dmg' + sys.stdout.flush() if not os.path.exists(destdir): os.makedirs(destdir) dmg = os.path.join(destdir, volname+'.dmg') @@ -461,6 +529,8 @@ class Py2App(object): '-volname', volname, '-format', format, dmg]) if internet_enable: subprocess.check_call(['/usr/bin/hdiutil', 'internet-enable', '-yes', dmg]) + size = os.stat(dmg).st_size/(1024*1024.) + print '\nInstaller size: %.2fMB\n'%size return dmg diff --git a/src/calibre/devices/prs500/cli/main.py b/src/calibre/devices/prs500/cli/main.py index 9865d383be..459f2d05e3 100755 --- a/src/calibre/devices/prs500/cli/main.py +++ b/src/calibre/devices/prs500/cli/main.py @@ -339,7 +339,7 @@ def main(): dev.touch(args[0]) else: parser.print_help() - if dev.handle: dev.close() + if getattr(dev, 'handle', False): dev.close() return 1 except DeviceLocked: print >> sys.stderr, "The device is locked. Use the --unlock option" diff --git a/src/calibre/ebooks/mobi/output.py b/src/calibre/ebooks/mobi/output.py index 2a70ace6e3..db8152b8fe 100644 --- a/src/calibre/ebooks/mobi/output.py +++ b/src/calibre/ebooks/mobi/output.py @@ -141,8 +141,9 @@ class MOBIOutput(OutputFormatPlugin): toc.nodes[0].href = toc.nodes[0].nodes[0].href # GR diagnostics - #self.dump_toc(toc) - #self.dump_manifest() + if self.opts.verbose > 3: + self.dump_toc(toc) + self.dump_manifest() def convert(self, oeb, output_path, input_plugin, opts, log): diff --git a/src/calibre/web/feeds/recipes/recipe_business_week.py b/src/calibre/web/feeds/recipes/recipe_business_week.py index d24e1c0f6a..da64eed711 100644 --- a/src/calibre/web/feeds/recipes/recipe_business_week.py +++ b/src/calibre/web/feeds/recipes/recipe_business_week.py @@ -12,46 +12,68 @@ from calibre.web.feeds.news import BasicNewsRecipe class BusinessWeek(BasicNewsRecipe): title = 'Business Week' description = 'Business News, Stock Market and Financial Advice' - __author__ = 'ChuckEggDotCom' + __author__ = 'ChuckEggDotCom and Sujata Raman' language = _('English') oldest_article = 7 max_articles_per_feed = 10 + no_stylesheets = True + + + extra_css = ''' + h1{font-family :Arial,Helvetica,sans-serif; font-size:large;} + h2{font-family :Arial,Helvetica,sans-serif; font-size:small;color:#666666;} + p{font-family :Arial,Helvetica,sans-serif; } + #lede600{font-size:x-small;} + #storybody{font-size:x-small;} + .strap{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#064599;} + .byline{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + .postedBy{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + .trackback{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + .date{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + .wrapper{font-family :Arial,Helvetica,sans-serif; font-size:x-small;} + .photoCredit{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + .tagline{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;} + ''' + + remove_tags = [ dict(name='div', attrs={'id':["bw2-header","column2","wrapper-bw2-footer","wrapper-mgh-footer","inset","commentForm","commentDisplay","bwExtras","bw2-umbrella","readerComments","pageNav","leg"]}), + ] - remove_tags_before = dict(name='h1') - remove_tags_after = dict(id='footer') - remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool']}), - dict(id=['footer', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']), - dict(name='h2', attrs={'class':'listspace'}), - ] - feeds = [ - (u'Top Stories', u'http://www.businessweek.com/topStories/rss/topStories.rss'), - (u'Top News', u'http://www.businessweek.com/rss/bwdaily.rss'), - (u'Asia', u'http://www.businessweek.com/rss/asia.rss'), - (u'Autos', u'http://www.businessweek.com/rss/autos/index.rss'), - (u'Classic Cars', u'http://www.businessweek.com/rss/autos/classic_cars/index.rss'), - (u'Hybrids', u'http://www.businessweek.com/rss/hybrids/index.rss'), - (u'Europe', u'http://www.businessweek.com/rss/europe.rss'), - (u'Auto Reviews', u'http://www.businessweek.com/rss/autos/reviews/index.rss'), - (u'Innovation & Design', u'http://www.businessweek.com/rss/innovate.rss'), - (u'Architecture', u'http://www.businessweek.com/rss/architecture.rss'), - (u'Brand Equity', u'http://www.businessweek.com/rss/brandequity.rss'), - (u'Auto Design', u'http://www.businessweek.com/rss/carbuff.rss'), - (u'Game Room', u'http://www.businessweek.com/rss/gameroom.rss'), - (u'Technology', u'http://www.businessweek.com/rss/technology.rss'), - (u'Investing', u'http://www.businessweek.m/rss/investor.rss'), - (u'Small Business', u'http://www.businessweek.com/rss/smallbiz.rss'), - (u'Careers', u'http://www.businessweek.com/rss/careers.rss'), - (u'B-Schools', u'http://www.businessweek.com/rss/bschools.rss'), - (u'Magazine Selections', u'http://www.businessweek.com/rss/magazine.rss'), + (u'Top Stories', u'http://www.businessweek.com/topStories/rss/topStories.rss'), + (u'Top News', u'http://www.businessweek.com/rss/bwdaily.rss'), + (u'Asia', u'http://www.businessweek.com/rss/asia.rss'), + (u'Autos', u'http://www.businessweek.com/rss/autos/index.rss'), + (u'Classic Cars', u'http://rss.businessweek.com/bw_rss/classiccars'), + (u'Hybrids', u'http://rss.businessweek.com/bw_rss/hybrids'), + (u'Europe', u'http://www.businessweek.com/rss/europe.rss'), + (u'Auto Reviews', u'http://rss.businessweek.com/bw_rss/autoreviews'), + (u'Innovation & Design', u'http://www.businessweek.com/rss/innovate.rss'), + (u'Architecture', u'http://www.businessweek.com/rss/architecture.rss'), + (u'Brand Equity', u'http://www.businessweek.com/rss/brandequity.rss'), + (u'Auto Design', u'http://www.businessweek.com/rss/carbuff.rss'), + (u'Game Room', u'http://rss.businessweek.com/bw_rss/gameroom'), + (u'Technology', u'http://www.businessweek.com/rss/technology.rss'), + (u'Investing', u'http://rss.businessweek.com/bw_rss/investor'), + (u'Small Business', u'http://www.businessweek.com/rss/smallbiz.rss'), + (u'Careers', u'http://rss.businessweek.com/bw_rss/careers'), + (u'B-Schools', u'http://www.businessweek.com/rss/bschools.rss'), + (u'Magazine Selections', u'http://www.businessweek.com/rss/magazine.rss'), (u'CEO Guide to Tech', u'http://www.businessweek.com/rss/ceo_guide_tech.rss'), ] def get_article_url(self, article): + url = article.get('guid', None) - if 'podcasts' in url: + + if 'podcasts' in url or 'surveys' in url: url = None + return url - - def print_version(self, url): - return url.replace('http://www.businessweek.com/', 'http://www.businessweek.com/print/') + + def postrocess_html(self, soup, first): + + for tag in soup.findAll(name=['ul','li']): + tag.name = 'div' + + return soup +