More fixes to the linux binary installer

linux_installer.py

@@ -23,6 +23,12 @@ CALIBREPLUGINS = os.path.join(CALIBRESRC, 'calibre', 'plugins')
 
 sys.path.insert(0, CALIBRESRC)
 from calibre import __version__
+from calibre.parallel import PARALLEL_FUNCS
+from calibre.web.feeds.recipes import recipes
+hiddenimports = map(lambda x: x[0], PARALLEL_FUNCS.values())
+hiddenimports += ['lxml._elementpath', 'keyword', 'codeop', 'commands', 'shlex', 'pydoc']
+hiddenimports += map(lambda x: x.__module__, recipes)
+open(os.path.join(PYINSTALLER, 'hooks', 'hook-calibre.parallel.py'), 'wb').write('hiddenimports = %s'%repr(hiddenimports))
 
 def run_pyinstaller(args=sys.argv):
     subprocess.check_call(('/usr/bin/sudo', 'chown', '-R', 'kovid:users', glob.glob('/usr/lib/python*/site-packages/')[-1]))
@@ -60,18 +66,7 @@ excludes = ['gtk._gtk', 'gtk.glade', 'qt', 'matplotlib.nxutils', 'matplotlib._cn
             'matplotlib._transforms', 'matplotlib._agg', 'matplotlib.backends._backend_agg',
             'matplotlib.axes', 'matplotlib', 'matplotlib.pyparsing',
             'TKinter', 'atk', 'gobject._gobject', 'pango', 'PIL', 'Image', 'IPython']
-temp = ['keyword', 'codeop']
 
-recipes = ['calibre', 'web', 'feeds', 'recipes']
-prefix  = '.'.join(recipes)+'.'
-recipes_toc = []
-extra_toc = [
-        ('keyword', '/usr/lib/python2.5/keyword.pyo', 'PYSOURCE'),
-        ('codeop', '/usr/lib/python2.5/codeop.pyo', 'PYSOURCE')
-        ]
-for f in glob.glob(os.path.join(CALIBRESRC, *(recipes+['*.py']))):
-    py_compile.compile(f, doraise=True)
-    recipes_toc.append((prefix + os.path.basename(f).partition('.')[0], f+'o', 'PYSOURCE'))
 
 sys.path.insert(0, CALIBRESRC)
 from calibre.linux import entry_points
@@ -90,9 +85,6 @@ analyses = [Analysis([os.path.join(HOMEPATH,'support/_mountzlib.py'), os.path.jo
 pyz = TOC()
 binaries = TOC()
 
-pyz += extra_toc
-pyz += recipes_toc
-
 for a in analyses:
     pyz = a.pure + pyz
     binaries = a.binaries + binaries

src/calibre/parallel.py

@@ -121,6 +121,7 @@ class WorkerMother(object):
     def __init__(self):
         ext = 'windows' if iswindows else 'osx' if isosx else 'linux'
         self.os = os # Needed incase cleanup called when interpreter is shutting down
+        self.env = {}
         if iswindows:
             self.executable = os.path.join(os.path.dirname(sys.executable),
                    'calibre-parallel.exe' if isfrozen else 'Scripts\\calibre-parallel.exe')
@@ -135,13 +136,14 @@ class WorkerMother(object):
                 
                 self.prefix += 'import sys; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd
                 self.prefix += 'sys.path.insert(0, %s); '%repr(sp)
-                self.env = {}
                 if fd not in os.environ['PATH']:
                     self.env['PATH'] = os.environ['PATH']+':'+fd
                 self.env['PYTHONHOME'] = resources
         else:
             self.executable = os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel') \
                                 if isfrozen else 'calibre-parallel'
+            if isfrozen:
+                self.env['LD_LIBRARY_PATH'] = getattr(sys, 'frozen_path') + ':' + os.environ.get('LD_LIBRARY_PATH', '')
     
         self.spawn_worker_windows = lambda arg : self.spawn_free_spirit_windows(arg, type='worker')
         self.spawn_worker_linux   = lambda arg : self.spawn_free_spirit_linux(arg, type='worker')
@@ -176,6 +178,7 @@ class WorkerMother(object):
     def get_env(self):
         env = dict(os.environ)
         env['CALIBRE_WORKER'] = '1'
+        env['ORIGWD'] = os.path.abspath(os.getcwd())
         if hasattr(self, 'env'):
             env.update(self.env)
         return env
@@ -189,7 +192,8 @@ class WorkerMother(object):
     
     def spawn_free_spirit_linux(self, arg, type='free_spirit'):
         cmdline = [self.executable, arg]
-        child = WorkerStatus(subprocess.Popen(cmdline, env=self.get_env()))
+        child = WorkerStatus(subprocess.Popen(cmdline, 
+                        env=self.get_env(), cwd=getattr(sys, 'frozen_path', None)))
         atexit.register(self.cleanup_child_linux, child)
         return child 
     
@@ -607,7 +611,7 @@ class BufferedSender(object):
             self.wbuf.append(msg)
     
     def send(self):
-        if select([self.socket], [], [], 0)[0]:
+        if callable(select) and select([self.socket], [], [], 0)[0]:
             msg = read(self.socket)
             if msg == 'PING:':
                 write(self.socket, 'OK')

windows_installer.py

@@ -539,7 +539,7 @@ def main():
     sys.argv[1:2] = ['py2exe']
     
     console = [dict(dest_base=basenames['console'][i], script=scripts['console'][i])
-               for i in range(len(scripts['console']))]# if not 'parallel.py' in scripts['console'][i] ]
+               for i in range(len(scripts['console']))]
     sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
     setup(
           cmdclass = {'py2exe': BuildEXE},
@@ -568,7 +568,8 @@ def main():
                                              'calibre.ebooks.lrf.feeds.*',
                                              'lxml', 'lxml._elementpath', 'genshi',
                                              'path', 'pydoc', 'IPython.Extensions.*',
-                                             'calibre.web.feeds.recipes.*', 'PyQt4.QtWebKit',
+                                             'calibre.web.feeds.recipes.*', 
+                                             'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
                                              ],
                                   'packages'  : ['PIL'],
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",