Updated recipe for Business Week

installer/osx/py2app/main.py

@@ -66,6 +66,16 @@ def strip_files(files, argv_max=(256 * 1024)):
         for args in flips:
             flipwritable(*args)
 
+def flush(func):
+    def ff(*args, **kwargs):
+        sys.stdout.flush()
+        sys.stderr.flush()
+        ret = func(*args, **kwargs)
+        sys.stdout.flush()
+        sys.stderr.flush()
+        return ret
+    return ff
+
 class Py2App(object):
 
     FID = '@executable_path/../Frameworks'
@@ -76,6 +86,14 @@ class Py2App(object):
         self.resources_dir = join(self.contents_dir, 'Resources')
         self.frameworks_dir = join(self.contents_dir, 'Frameworks')
         self.to_strip = []
+        self.warnings = []
+
+    def warn(self, *args):
+        self.warnings.append(args)
+        prefix = '' if args and args[0].startswith('WARNING:') else 'WARNING: '
+        sys.stdout.write(prefix+' '.join(args)+'\n')
+        sys.stdout.flush()
+
 
     def run(self):
         self.create_skeleton()
@@ -103,26 +121,40 @@ class Py2App(object):
         self.strip_files()
         self.create_launchers()
 
-        return self.makedmg(self.build_dir, APPNAME+'-'+VERSION+'-x86_64')
+        ret = self.makedmg(self.build_dir, APPNAME+'-'+VERSION+'-x86_64')
+        sys.stdout.flush()
+        sys.stderr.flush()
 
+        print '\nThere were', len(self.warnings), 'warnings'
+        for w in list(self.warnings):
+            print
+            self.warn(*w)
+        return ret
+
+    @flush
     def create_launchers(self):
-        launcher = join(os.path.dirname(__file__), 'launcher.py')
+        print '\nCreating launchers'
+        all_names   = basenames['console'] + basenames['gui']
+        all_modules   = main_modules['console'] + main_modules['gui']
+        launcher = join(os.path.dirname(__file__), 'loader.py')
         launcher = open(launcher, 'rb').read()
         launcher = launcher.replace('{}##ENV##', repr(ENV))
         os.mkdir(join(self.resources_dir, 'loaders'))
-        for module, basename in zip(main_modules, basenames):
+        for basename, module in zip(all_names, all_modules):
             raw = launcher.replace("''##MODULE##", repr(module))
             path = join(self.resources_dir, 'loaders', basename)
             open(path, 'wb').write(raw)
             os.chmod(path, stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH|stat.S_IREAD\
                      |stat.S_IWUSR|stat.S_IROTH|stat.S_IRGRP)
 
-
+    @flush
     def strip_files(self):
         print '\nStripping files...'
         strip_files(self.to_strip)
 
+    @flush
     def create_exe(self):
+        print '\nCreating executable'
         gcc = os.environ.get('CC', 'gcc')
         base = os.path.dirname(__file__)
         out = join(self.contents_dir, 'MacOS', 'calibre')
@@ -130,12 +162,14 @@ class Py2App(object):
             'main.c'), '-o', out])
         self.to_strip.append(out)
 
+    @flush
     def set_id(self, path_to_lib, new_id):
         old_mode = flipwritable(path_to_lib)
         subprocess.check_call(['install_name_tool', '-id', new_id, path_to_lib])
         if old_mode is not None:
             flipwritable(path_to_lib, old_mode)
 
+    @flush
     def get_dependencies(self, path_to_lib):
         raw = subprocess.Popen(['otool', '-L', path_to_lib],
                 stdout=subprocess.PIPE).stdout.read()
@@ -149,6 +183,7 @@ class Py2App(object):
                 continue
             yield path
 
+    @flush
     def get_local_dependencies(self, path_to_lib):
         for x in self.get_dependencies(path_to_lib):
             for y in (SW+'/lib/', '/usr/local/lib/', SW+'/qt/lib/',
@@ -157,11 +192,13 @@ class Py2App(object):
                     yield x, x[len(y):]
                     break
 
+    @flush
     def change_dep(self, old_dep, new_dep, path_to_lib):
         print '\tResolving dependency %s to'%old_dep, new_dep
         subprocess.check_call(['install_name_tool', '-change', old_dep, new_dep,
             path_to_lib])
 
+    @flush
     def fix_dependencies_in_lib(self, path_to_lib):
         print '\nFixing dependencies in', path_to_lib
         self.to_strip.append(path_to_lib)
@@ -174,6 +211,7 @@ class Py2App(object):
         if old_mode is not None:
             flipwritable(path_to_lib, old_mode)
 
+    @flush
     def add_python_framework(self):
         src = join(SW, 'python', 'Python.framework')
         x = join(self.frameworks_dir, 'Python.framework')
@@ -186,6 +224,7 @@ class Py2App(object):
         self.set_id(join(currd, 'Python'),
             self.FID+'/Python.framework/Versions/%s/Python'%basename(curr))
 
+    @flush
     def add_qt_frameworks(self):
         for f in ('QtCore', 'QtGui', 'QtXml', 'QtNetwork', 'QtSvg', 'QtWebkit',
                 'phonon'):
@@ -197,6 +236,7 @@ class Py2App(object):
             x = os.path.relpath(l, join(self.contents_dir, 'MacOS'))
             self.set_id(l, '@executable_path/'+x)
 
+    @flush
     def add_qt_framework(self, f):
         libname = f
         f = f+'.framework'
@@ -210,6 +250,7 @@ class Py2App(object):
         self.set_id(lib, self.FID+'/'+rpath)
         self.fix_dependencies_in_lib(lib)
 
+    @flush
     def create_skeleton(self):
         c = join(self.build_dir, 'Contents')
         for x in ('Frameworks', 'MacOS', 'Resources'):
@@ -217,6 +258,7 @@ class Py2App(object):
         x = 'library.icns'
         shutil.copyfile(join('icons', x), join(self.resources_dir, x))
 
+    @flush
     def add_calibre_plugins(self):
         dest = join(self.frameworks_dir, 'plugins')
         os.mkdir(dest)
@@ -225,6 +267,7 @@ class Py2App(object):
         self.fix_dependencies_in_lib(join(dest, basename(f)))
 
 
+    @flush
     def create_plist(self):
         pl = dict(
                 CFBundleDevelopmentRegion='English',
@@ -248,6 +291,7 @@ class Py2App(object):
         )
         plistlib.writePlist(pl, join(self.contents_dir, 'Info.plist'))
 
+    @flush
     def install_dylib(self, path, set_id=True):
         shutil.copy2(path, self.frameworks_dir)
         if set_id:
@@ -255,25 +299,30 @@ class Py2App(object):
                     self.FID+'/'+basename(path))
         self.fix_dependencies_in_lib(join(self.frameworks_dir, basename(path)))
 
+    @flush
     def add_podofo(self):
         print '\nAdding PoDoFo'
         pdf = join(SW, 'lib', 'libpodofo.0.6.99.dylib')
         self.install_dylib(pdf)
 
+    @flush
     def add_poppler(self):
         print '\nAdding poppler'
         for x in ('libpoppler.4.dylib', 'libpoppler-qt4.3.dylib'):
             self.install_dylib(os.path.join(SW, 'lib', x))
         self.install_dylib(os.path.join(SW, 'bin', 'pdftohtml'), False)
 
+    @flush
     def add_libjpeg(self):
         print '\nAdding libjpeg'
         self.install_dylib(os.path.join(SW, 'lib', 'libjpeg.7.dylib'))
 
+    @flush
     def add_libpng(self):
         print '\nAdding libpng'
         self.install_dylib(os.path.join(SW, 'lib', 'libpng12.0.dylib'))
 
+    @flush
     def add_fontconfig(self):
         print '\nAdding fontconfig'
         for x in ('fontconfig.1', 'freetype.6', 'expat.1'):
@@ -297,6 +346,7 @@ class Py2App(object):
         ''')
         open(fc, 'wb').write(raw)
 
+    @flush
     def add_imagemagick(self):
         print '\nAdding ImageMagick'
         for x in ('Wand', 'Core'):
@@ -315,11 +365,13 @@ class Py2App(object):
                     f = join(x[0], f)
                     self.fix_dependencies_in_lib(f)
 
+    @flush
     def add_misc_libraries(self):
         for x in ('usb', 'unrar'):
             print '\nAdding', x
             shutil.copy2(join(SW, 'lib', 'lib%s.dylib'%x), self.frameworks_dir)
 
+    @flush
     def add_site_packages(self):
         print '\nAdding site-packages'
         self.site_packages = join(self.resources_dir, 'Python', 'site-packages')
@@ -327,18 +379,19 @@ class Py2App(object):
         paths = reversed(map(abspath, [x for x in sys.path if x.startswith('/')]))
         upaths = []
         for x in paths:
-            if x not in upaths:
-                upaths.append(x)
-        for x in upaths:
             if x.endswith('/PIL') or 'site-packages' not in x:
                 continue
+            if x not in upaths:
+                upaths.append(x)
+        upaths.append(os.path.expanduser('~/build/calibre/src'))
+        for x in upaths:
             tdir = None
             try:
                 if not os.path.isdir(x):
                     try:
                         zf = zipfile.ZipFile(x)
                     except:
-                        print "WARNING:", x, 'is neither a directory nor a zipfile'
+                        self.warn(x, 'is neither a directory nor a zipfile')
                         continue
                     tdir = tempfile.mkdtemp()
                     zf.extractall(tdir)
@@ -350,6 +403,7 @@ class Py2App(object):
                     shutil.rmtree(tdir)
         self.remove_bytecode(join(self.resources_dir, 'Python', 'site-packages'))
 
+    @flush
     def add_modules_from_dir(self, src):
         for x in glob.glob(join(src, '*.py'))+glob.glob(join(src, '*.so')):
             dest = join(self.site_packages, basename(x))
@@ -357,6 +411,7 @@ class Py2App(object):
             if x.endswith('.so'):
                 self.fix_dependencies_in_lib(x)
 
+    @flush
     def add_packages_from_dir(self, src):
         for x in os.listdir(src):
             x = join(src, x)
@@ -365,13 +420,16 @@ class Py2App(object):
                     continue
                 self.add_package_dir(x)
 
+    @flush
     def add_package_dir(self, x, dest=None):
         def ignore(root, files):
             ans  = []
             for y in files:
-                if os.path.splitext(y) in ('.py', '.so'):
+                ext = os.path.splitext(y)[1]
-                    continue
+                if ext not in ('', '.py', '.so') or \
+                    (not ext and not os.path.isdir(join(root, y))):
                         ans.append(y)
+
             return ans
         if dest is None:
             dest = self.site_packages
@@ -379,13 +437,16 @@ class Py2App(object):
         shutil.copytree(x, dest, symlinks=True, ignore=ignore)
         self.postprocess_package(x, dest)
 
+    @flush
     def filter_package(self, name):
         return name in ('Cython', 'modulegraph', 'macholib', 'py2app',
         'bdist_mpkg', 'altgraph')
 
+    @flush
     def postprocess_package(self, src_path, dest_path):
         pass
 
+    @flush
     def add_stdlib(self):
         print '\nAdding python stdlib'
         src = join(SW, 'python/Python.framework/Versions/Current/lib/python')
@@ -405,6 +466,7 @@ class Py2App(object):
                     self.fix_dependencies_in_lib(dest)
         self.remove_bytecode(join(self.resources_dir, 'Python', 'lib'))
 
+    @flush
     def remove_bytecode(self, dest):
         for x in os.walk(dest):
             root = x[0]
@@ -412,6 +474,7 @@ class Py2App(object):
                 if os.path.splitext(f) in ('.pyc', '.pyo'):
                     os.remove(join(root, f))
 
+    @flush
     def compile_py_modules(self):
         print '\nCompiling Python modules'
         base = join(self.resources_dir, 'Python')
@@ -425,8 +488,9 @@ class Py2App(object):
                         py_compile.compile(y, dfile=rel, doraise=True)
                         os.remove(y)
                     except:
-                        print 'WARNING: Failed to byte-compile', y
+                        self.warn('WARNING: Failed to byte-compile', y)
 
+    @flush
     def create_console_app(self):
         print '\nCreating console.app'
         cc_dir = os.path.join(self.contents_dir, 'console.app', 'Contents')
@@ -442,16 +506,20 @@ class Py2App(object):
                 os.symlink(join('../..', x),
                            join(cc_dir, x))
 
+    @flush
     def copy_launcher_and_site(self):
         base = os.path.dirname(__file__)
         for x in ('launcher', 'site'):
             shutil.copy2(join(base, x+'.py'), self.resources_dir)
 
+    @flush
     def makedmg(self, d, volname,
                 destdir='dist',
                 internet_enable=True,
                 format='UDBZ'):
         ''' Copy a directory d into a dmg named volname '''
+        print '\nCreating dmg'
+        sys.stdout.flush()
         if not os.path.exists(destdir):
             os.makedirs(destdir)
         dmg = os.path.join(destdir, volname+'.dmg')
@@ -461,6 +529,8 @@ class Py2App(object):
                                '-volname', volname, '-format', format, dmg])
         if internet_enable:
            subprocess.check_call(['/usr/bin/hdiutil', 'internet-enable', '-yes', dmg])
+        size = os.stat(dmg).st_size/(1024*1024.)
+        print '\nInstaller size: %.2fMB\n'%size
         return dmg
 
 

src/calibre/devices/prs500/cli/main.py

@@ -339,7 +339,7 @@ def main():
             dev.touch(args[0])
         else:
             parser.print_help()
-            if dev.handle: dev.close()
+            if getattr(dev, 'handle', False): dev.close()
             return 1
     except DeviceLocked:
         print >> sys.stderr, "The device is locked. Use the --unlock option"

src/calibre/ebooks/mobi/output.py

@@ -141,8 +141,9 @@ class MOBIOutput(OutputFormatPlugin):
             toc.nodes[0].href = toc.nodes[0].nodes[0].href
 
             # GR diagnostics
-            #self.dump_toc(toc)
+            if self.opts.verbose > 3:
-            #self.dump_manifest()
+                self.dump_toc(toc)
+                self.dump_manifest()
 
 
     def convert(self, oeb, output_path, input_plugin, opts, log):

src/calibre/web/feeds/recipes/recipe_business_week.py

@@ -12,16 +12,30 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class BusinessWeek(BasicNewsRecipe):
     title          = 'Business Week'
     description    = 'Business News, Stock Market and Financial Advice'
-    __author__     = 'ChuckEggDotCom'
+    __author__     = 'ChuckEggDotCom and Sujata Raman'
     language = _('English')
     oldest_article = 7
     max_articles_per_feed = 10
+    no_stylesheets = True
 
-    remove_tags_before = dict(name='h1')
+
-    remove_tags_after  = dict(id='footer')
+    extra_css = '''
-    remove_tags = [dict(attrs={'class':['articleTools', 'post-tools', 'side_tool']}), 
+                h1{font-family :Arial,Helvetica,sans-serif; font-size:large;}
-                   dict(id=['footer', 'navigation', 'archive', 'side_search', 'blog_sidebar', 'side_tool', 'side_index']), 
+                h2{font-family :Arial,Helvetica,sans-serif; font-size:small;color:#666666;}
-                   dict(name='h2', attrs={'class':'listspace'}),
+                p{font-family :Arial,Helvetica,sans-serif; }
+                #lede600{font-size:x-small;}
+                #storybody{font-size:x-small;}
+                .strap{font-family :Arial,Helvetica,sans-serif; font-size:x-small; color:#064599;}
+                .byline{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
+                .postedBy{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;}
+                .trackback{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;}
+                .date{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;}
+                .wrapper{font-family :Arial,Helvetica,sans-serif; font-size:x-small;}
+                .photoCredit{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;}
+                .tagline{font-family :Arial,Helvetica,sans-serif; font-size:x-small;color:#666666;}
+                '''
+
+    remove_tags = [  dict(name='div', attrs={'id':["bw2-header","column2","wrapper-bw2-footer","wrapper-mgh-footer","inset","commentForm","commentDisplay","bwExtras","bw2-umbrella","readerComments","pageNav","leg"]}),
                     ]
 
     feeds          = [
@@ -29,29 +43,37 @@ class BusinessWeek(BasicNewsRecipe):
                       (u'Top News', u'http://www.businessweek.com/rss/bwdaily.rss'),
                       (u'Asia', u'http://www.businessweek.com/rss/asia.rss'),
                       (u'Autos', u'http://www.businessweek.com/rss/autos/index.rss'),
-                      (u'Classic Cars', u'http://www.businessweek.com/rss/autos/classic_cars/index.rss'),
+                      (u'Classic Cars', u'http://rss.businessweek.com/bw_rss/classiccars'),
-                      (u'Hybrids', u'http://www.businessweek.com/rss/hybrids/index.rss'),  
+                      (u'Hybrids', u'http://rss.businessweek.com/bw_rss/hybrids'),
                       (u'Europe', u'http://www.businessweek.com/rss/europe.rss'),
-                      (u'Auto Reviews', u'http://www.businessweek.com/rss/autos/reviews/index.rss'), 
+                      (u'Auto Reviews', u'http://rss.businessweek.com/bw_rss/autoreviews'),
                       (u'Innovation & Design', u'http://www.businessweek.com/rss/innovate.rss'),
                       (u'Architecture', u'http://www.businessweek.com/rss/architecture.rss'),
                       (u'Brand Equity', u'http://www.businessweek.com/rss/brandequity.rss'),
                       (u'Auto Design', u'http://www.businessweek.com/rss/carbuff.rss'),
-                      (u'Game Room', u'http://www.businessweek.com/rss/gameroom.rss'), 
+                      (u'Game Room', u'http://rss.businessweek.com/bw_rss/gameroom'),
                       (u'Technology', u'http://www.businessweek.com/rss/technology.rss'),
-                      (u'Investing', u'http://www.businessweek.m/rss/investor.rss'), 
+                      (u'Investing', u'http://rss.businessweek.com/bw_rss/investor'),
                       (u'Small Business', u'http://www.businessweek.com/rss/smallbiz.rss'),
-                      (u'Careers', u'http://www.businessweek.com/rss/careers.rss'), 
+                      (u'Careers', u'http://rss.businessweek.com/bw_rss/careers'),
                       (u'B-Schools', u'http://www.businessweek.com/rss/bschools.rss'),
                       (u'Magazine Selections', u'http://www.businessweek.com/rss/magazine.rss'),
                       (u'CEO Guide to Tech', u'http://www.businessweek.com/rss/ceo_guide_tech.rss'),
                       ]
 
     def get_article_url(self, article):
+
         url = article.get('guid', None)
-        if 'podcasts' in url:
+
+        if 'podcasts' in url or 'surveys' in url:
             url = None
+
         return url
 
-    def print_version(self, url):
+    def postrocess_html(self, soup, first):
-        return url.replace('http://www.businessweek.com/', 'http://www.businessweek.com/print/')
+
+            for tag in soup.findAll(name=['ul','li']):
+                tag.name = 'div'
+
+            return soup
+