From c979cb10b8a91da5d061c1bdec099510d84c0cff Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 19 Mar 2011 18:02:29 -0600 Subject: [PATCH] Windows build: Put all python files into a zip file to reduce upgrade time --- setup/installer/windows/freeze.py | 116 +++++++++++++++++++++++++++++- setup/installer/windows/site.py | 2 +- setup/installer/windows/util.c | 2 +- 3 files changed, 117 insertions(+), 3 deletions(-) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index e9e47816fd..cf4dcd5f9d 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, shutil, glob, py_compile, subprocess, re +import sys, os, shutil, glob, py_compile, subprocess, re, zipfile, time from setup import Command, modules, functions, basenames, __version__, \ __appname__ @@ -40,6 +40,13 @@ DESCRIPTIONS = { 'calibre-smtp' : 'Command line interface for sending books via email', } +def walk(dir): + ''' A nice interface to os.walk ''' + for record in os.walk(dir): + for f in record[-1]: + yield os.path.join(record[0], f) + + class Win32Freeze(Command, WixMixIn): description = 'Free windows calibre installation' @@ -63,12 +70,15 @@ class Win32Freeze(Command, WixMixIn): self.rc_template = self.j(self.d(self.a(__file__)), 'template.rc') self.py_ver = ''.join(map(str, sys.version_info[:2])) self.lib_dir = self.j(self.base, 'Lib') + self.pydlib = self.j(self.base, 'pydlib') + self.pylib = self.j(self.base, 'pylib.zip') self.initbase() self.build_launchers() self.freeze() self.embed_manifests() self.install_site_py() + self.archive_lib_dir() self.create_installer() def initbase(self): @@ -356,4 +366,108 @@ class Win32Freeze(Command, WixMixIn): dest, lib] self.run_builder(cmd) + def archive_lib_dir(self): + self.info('Putting all python code into a zip file for performance') + if os.path.exists(self.pydlib): + shutil.rmtree(self.pydlib) + os.makedirs(self.pydlib) + self.zf_timestamp = time.localtime(time.time())[:6] + self.zf_names = set() + with zipfile.ZipFile(self.pylib, 'w', zipfile.ZIP_STORED) as zf: + for x in os.listdir(self.lib_dir): + if x == 'site-packages': + continue + self.add_to_zipfile(zf, x, self.lib_dir) + + sp = self.j(self.lib_dir, 'site-packages') + handled = set(['site.pyo']) + for pth in ('PIL.pth', 'pywin32.pth'): + handled.add(pth) + shutil.copyfile(self.j(sp, pth), self.j(self.pydlib, pth)) + for d in self.get_pth_dirs(self.j(sp, pth)): + shutil.copytree(d, self.j(self.pydlib, self.b(d)), True) + handled.add(self.b(d)) + + handled.add('easy-install.pth') + for d in self.get_pth_dirs(self.j(sp, 'easy-install.pth')): + handled.add(self.b(d)) + zip_safe = self.is_zip_safe(d) + for x in os.listdir(d): + if x == 'EGG-INFO': + continue + if zip_safe: + self.add_to_zipfile(zf, x, d) + else: + absp = self.j(d, x) + dest = self.j(self.pydlib, x) + if os.path.isdir(absp): + shutil.copytree(absp, dest, True) + else: + shutil.copy2(absp, dest) + + for x in os.listdir(sp): + if x in handled or x.endswith('.egg-info'): + continue + absp = self.j(sp, x) + if os.path.isdir(absp): + if not os.listdir(absp): + continue + if self.is_zip_safe(absp): + self.add_to_zipfile(zf, x, sp) + else: + shutil.copytree(absp, self.j(self.pydlib, x), True) + else: + if x.endswith('.pyd'): + shutil.copy2(absp, self.j(self.pydlib, x)) + else: + self.add_to_zipfile(zf, x, sp) + + shutil.rmtree(self.lib_dir) + + def is_zip_safe(self, path): + for f in walk(path): + ext = os.path.splitext(f)[1].lower() + if ext in ('.pyd', '.dll', '.exe'): + return False + return True + + def get_pth_dirs(self, pth): + base = os.path.dirname(pth) + for line in open(pth).readlines(): + line = line.strip() + if not line or line.startswith('#') or line.startswith('import'): + continue + if line == 'win32\\lib': + continue + candidate = self.j(base, line) + if os.path.exists(candidate): + yield candidate + + def add_to_zipfile(self, zf, name, base, exclude=frozenset()): + abspath = self.j(base, name) + name = name.replace(os.sep, '/') + if name in self.zf_names: + raise ValueError('Already added %r to zipfile [%r]'%(name, abspath)) + zinfo = zipfile.ZipInfo(filename=name, date_time=self.zf_timestamp) + + if os.path.isdir(abspath): + if not os.listdir(abspath): + return + zinfo.external_attr = 0700 << 16 + zf.writestr(zinfo, '') + for x in os.listdir(abspath): + if x not in exclude: + self.add_to_zipfile(zf, name + os.sep + x, base) + else: + ext = os.path.splitext(name)[1].lower() + if ext in ('.pyd', '.dll', '.exe'): + raise ValueError('Cannot add %r to zipfile'%abspath) + zinfo.external_attr = 0600 << 16 + if ext in ('.py', '.pyc', '.pyo'): + with open(abspath, 'rb') as f: + zf.writestr(zinfo, f.read()) + + self.zf_names.add(name) + + diff --git a/setup/installer/windows/site.py b/setup/installer/windows/site.py index 0e770f3253..5610ff197e 100644 --- a/setup/installer/windows/site.py +++ b/setup/installer/windows/site.py @@ -96,7 +96,7 @@ def main(): abs__file__() - addsitedir(os.path.join(sys.app_dir, 'Lib', 'site-packages')) + addsitedir(os.path.join(sys.app_dir, 'pydlib')) add_calibre_vars() diff --git a/setup/installer/windows/util.c b/setup/installer/windows/util.c index fdec6d786f..329e3bf8c3 100644 --- a/setup/installer/windows/util.c +++ b/setup/installer/windows/util.c @@ -198,7 +198,7 @@ void initialize_interpreter(wchar_t *outr, wchar_t *errr, buf[strlen(buf)-1] = '\0'; _snprintf_s(python_home, MAX_PATH, _TRUNCATE, "%s", buf); - _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\DLLs;%s\\Lib;%s\\Lib\\site-packages", + _snprintf_s(path, 3*MAX_PATH, _TRUNCATE, "%s\\pylib.zip;%s\\pydlib;%s\\DLLs", buf, buf, buf); free(buf);