From 8e08834403e413adfdeda3ec5e49971dfdc078da Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 12 Nov 2012 15:32:28 +0530 Subject: [PATCH] Much nicer integration with the windows build VMs --- setup/build_environment.py | 3 +- setup/extensions.py | 5 +- setup/installer/windows/__init__.py | 10 +-- setup/installer/windows/freeze.py | 8 +-- setup/installer/windows/notes.rst | 104 ++++++++++++++++++++-------- setup/vcvars.py | 63 ++++++++++++++--- src/calibre/test_build.py | 2 + 7 files changed, 141 insertions(+), 54 deletions(-) diff --git a/setup/build_environment.py b/setup/build_environment.py index 492eca0697..e20ad778e3 100644 --- a/setup/build_environment.py +++ b/setup/build_environment.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, socket, struct, subprocess +import os, socket, struct, subprocess, sys from distutils.spawn import find_executable from PyQt4 import pyqtconfig @@ -16,6 +16,7 @@ from setup import isosx, iswindows, islinux OSX_SDK = '/Developer/SDKs/MacOSX10.5.sdk' os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.5' +is64bit = sys.maxsize > 2**32 NMAKE = RC = msvc = MT = win_inc = win_lib = win_ddk = win_ddk_lib_dirs = None if iswindows: diff --git a/setup/extensions.py b/setup/extensions.py index 12c1e435c3..381ac36435 100644 --- a/setup/extensions.py +++ b/setup/extensions.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import textwrap, os, shlex, subprocess, glob, shutil, sys +import textwrap, os, shlex, subprocess, glob, shutil from distutils import sysconfig from multiprocessing import cpu_count @@ -18,10 +18,9 @@ from setup.build_environment import (chmlib_inc_dirs, msvc, MT, win_inc, win_lib, win_ddk, magick_inc_dirs, magick_lib_dirs, magick_libs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs, icu_lib_dirs, win_ddk_lib_dirs, ft_libs, ft_lib_dirs, ft_inc_dirs, - zlib_libs, zlib_lib_dirs, zlib_inc_dirs) + zlib_libs, zlib_lib_dirs, zlib_inc_dirs, is64bit) MT isunix = islinux or isosx or isbsd -is64bit = sys.maxsize > 2**32 make = 'make' if isunix else NMAKE diff --git a/setup/installer/windows/__init__.py b/setup/installer/windows/__init__.py index f1c3e8ab70..61149a9f26 100644 --- a/setup/installer/windows/__init__.py +++ b/setup/installer/windows/__init__.py @@ -6,13 +6,11 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, shutil, subprocess, re +import os, shutil, subprocess from setup import Command, __appname__, __version__ from setup.installer import VMInstaller -SIGNTOOL = r'C:\cygwin\home\kovid\sign.bat' - class Win(Command): description = 'Build windows binary installers' @@ -38,11 +36,7 @@ class Win32(VMInstaller): def sign_msi(self): print ('Signing installers ...') - raw = open(self.VM).read() - vmx = re.search(r'''launch_vmware\(['"](.+?)['"]''', raw).group(1) - subprocess.check_call(['vmrun', '-T', 'ws', '-gu', 'kovid', '-gp', - "et tu brutus", 'runProgramInGuest', vmx, 'cmd.exe', '/C', - r'C:\cygwin\home\kovid\sign.bat']) + subprocess.check_call(['ssh', self.VM_NAME, '~/sign.sh'], shell=False) def download_installer(self): installer = self.installer() diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index 9ec8a76ada..8dff5bd167 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -13,11 +13,11 @@ from setup import (Command, modules, functions, basenames, __version__, from setup.build_environment import msvc, MT, RC from setup.installer.windows.wix import WixMixIn -ICU_DIR = r'Q:\icu' +ICU_DIR = os.environ.get('ICU_DIR', r'Q:\icu') OPENSSL_DIR = os.environ.get('OPENSSL_DIR', r'Q:\openssl') QT_DIR = os.environ.get('QT_DIR', 'Q:\\Qt\\4.8.2') QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] -LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' +LIBUNRAR = os.environ.get('UNRARDLL', 'C:\\Program Files\\UnrarDLL\\unrar.dll') SW = r'C:\cygwin\home\kovid\sw' IMAGEMAGICK = os.path.join(SW, 'build', 'ImageMagick-6.7.6', 'VisualMagick', 'bin') @@ -261,8 +261,8 @@ class Win32Freeze(Command, WixMixIn): print print 'Adding third party dependencies' print '\tAdding unrar' - shutil.copyfile(LIBUNRAR, - os.path.join(self.dll_dir, os.path.basename(LIBUNRAR))) + shutil.copyfile(LIBUNRAR, os.path.join(self.dll_dir, + os.path.basename(LIBUNRAR).replace('64', ''))) print '\tAdding misc binary deps' bindir = os.path.join(SW, 'bin') diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index b4c4c9c09e..2340d5015e 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -40,11 +40,43 @@ Cygwin This is needed for automation of the build process, you dont need it otherwise. -Install, vim, rsync, openssh, unzip, wget at a minimum. -In a cygwin terminal do: +Install, vim, rsync, openssh, unzip, wget, make at a minimum. -ssh-host-config -y -net start sshd +In order to build debug builds (.pdb files and sign files), you have to be able +to login as the normal user account with ssh. To do this, follow these steps: + + * Setup a password for your user account + * Follow the steps here: + http://pcsupport.about.com/od/windows7/ht/auto-logon-windows-7.htm or + http://pcsupport.about.com/od/windowsxp/ht/auto-logon-xp.htm to allow the + machine to bootup without having to enter the password + * First clean out any existing cygwin ssh setup with:: + net stop sshd + cygrunsrv -R sshd + net user sshd /DELETE + net user cyg_server /DELETE (delete any other cygwin users account you + can list them with net user) + rm -R /etc/ssh* + mkpasswd -cl > /etc/passwd + mkgroup --local > /etc/group + * Assign the necessary rights to the normal user account:: + editrights.exe -a SeAssignPrimaryTokenPrivilege -u kovid + editrights.exe -a SeCreateTokenPrivilege -u kovid + editrights.exe -a SeTcbPrivilege -u kovid + editrights.exe -a SeServiceLogonRight -u kovid + * Run:: + ssh-host-config + And answer (yes) to all questions. If it asks do you want to use a + different user name, specify the name of your user account and enter + username and password (it asks on Win 7 not on Win XP) + * On Windows XP, I also had to run:: + passwd -R + to allow sshd to use my normal user account even with public key + authentication. See http://cygwin.com/cygwin-ug-net/ntsec.html for + details. On Windows 7 this wasn't necessary for some reason. + * Start sshd with:: + net start sshd + * See http://www.kgx.net.nz/2010/03/cygwin-sshd-and-windows-7/ for details Pass port 22 through Windows firewall. Create ~/.ssh/authorized_keys @@ -172,9 +204,7 @@ SIP Available from: http://www.riverbankcomputing.co.uk/software/sip/download :: - python configure.py -p win32-msvc2008 - nmake - nmake install + python configure.py -p win32-msvc2008 && nmake && nmake install PyQt4 ---------- @@ -185,15 +215,6 @@ Compiling instructions:: nmake nmake install -Python Imaging Library ------------------------- - -Install as normal using installer at http://www.lfd.uci.edu/~gohlke/pythonlibs/ - -Test it on the target system with - -calibre-debug -c "import _imaging, _imagingmath, _imagingft, _imagingcms" - ICU ------- @@ -217,12 +238,30 @@ Optionally run make check Libunrar ---------- -http://www.rarlab.com/rar/UnRARDLL.exe install and add C:\Program Files\UnrarDLL to PATH +Get the source from http://www.rarlab.com/rar_add.htm -lxml +Open UnrarDll.vcproj, change build type to release. +If building 64 bit change Win32 to x64. + +Build the Solution, find the dll in the build subdir. As best as I can tell, +the vcproj already defines the SILENT preprocessor directive, but you should +test this. + +.. http://www.rarlab.com/rar/UnRARDLL.exe install and add C:\Program Files\UnrarDLL to PATH + +TODO: 64-bit check that SILENT is defined and that the ctypes bindings actuall +work + +zlib ------ -http://pypi.python.org/pypi/lxml +Build with:: + nmake -f win32/Makefile.msc + nmake -f win32/Makefile.msc test + + cp zlib1.dll* ../../bin + cp zlib.lib zdll.* ../../lib + cp zconf.h zlib.h ../../include jpeg-7 ------- @@ -252,16 +291,6 @@ cp build/jpeg-7/Release/jpeg.dll bin/ cp build/jpeg-7/Release/jpeg.lib build/jpeg-7/Release/jpeg.exp cp build/jpeg-7/jerror.h build/jpeg-7/jpeglib.h build/jpeg-7/jconfig.h build/jpeg-7/jmorecfg.h include/ -zlib ------- - -nmake -f win32/Makefile.msc -nmake -f win32/Makefile.msc test - -cp zlib1.dll* ../../bin -cp zlib.lib zdll.* ../../lib -cp zconf.h zlib.h ../../include - libpng --------- @@ -302,6 +331,21 @@ Now change configuration back to static for .lib cp build/freetype-2.3.9/objs/win32/vc2008/freetype239MT.lib lib/ cp -rf build/freetype-2.3.9/include/* include/ +lxml +------ + +http://pypi.python.org/pypi/lxml + + +Python Imaging Library +------------------------ + +Install as normal using installer at http://www.lfd.uci.edu/~gohlke/pythonlibs/ + +Test it on the target system with + +calibre-debug -c "import _imaging, _imagingmath, _imagingft, _imagingcms" + expat -------- @@ -418,6 +462,8 @@ cp -r build/lib.win32-*/* /cygdrive/c/Python27/Lib/site-packages/ easylzma ---------- +This is only needed to build the portable installer. + Get it from http://lloyd.github.com/easylzma/ (use the trunk version) Run cmake and build the Visual Studio solution (generates CLI tools and dll and diff --git a/setup/vcvars.py b/setup/vcvars.py index c45f889ebc..6314345f1a 100644 --- a/setup/vcvars.py +++ b/setup/vcvars.py @@ -7,15 +7,55 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os -from distutils.msvc9compiler import MSVCCompiler +import os, sys, subprocess +from distutils.msvc9compiler import find_vcvarsall, get_build_version -c = MSVCCompiler() -orig_path = os.environ['path'] -c.initialize() -paths = c._MSVCCompiler__paths -lib = os.environ['lib'].split(';') -include = os.environ['include'].split(';') +plat = 'amd64' if sys.maxsize > 2**32 else 'x86' + +def remove_dups(variable): + old_list = variable.split(os.pathsep) + new_list = [] + for i in old_list: + if i not in new_list: + new_list.append(i) + return os.pathsep.join(new_list) + +def query_process(cmd): + result = {} + popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise RuntimeError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.splitlines(): + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key == 'path': + if value.endswith(os.pathsep): + value = value[:-1] + value = remove_dups(value) + result[key] = value + + finally: + popen.stdout.close() + popen.stderr.close() + return result + +def query_vcvarsall(): + vcvarsall = find_vcvarsall(get_build_version()) + return query_process('"%s" %s & set' % (vcvarsall, plat)) + +env = query_vcvarsall() +paths = env['path'].split(';') +lib = env['lib'] +include = env['include'] +libpath = env['libpath'] def unix(paths): up = [] @@ -28,9 +68,14 @@ raw = '''\ #!/bin/sh export PATH="%s:$PATH" + export LIB="%s" + export INCLUDE="%s" -'''%(unix(paths), ';'.join(lib), ';'.join(include)) + +export LIBPATH="%s" + +'''%(unix(paths), lib, include, libpath) with open(os.path.expanduser('~/.vcvars'), 'wb') as f: f.write(raw.encode('utf-8')) diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index f173c46ca0..33017efc47 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -79,6 +79,8 @@ def test_imaging(): print ('ImageMagick OK!') else: raise RuntimeError('ImageMagick choked!') + import _imaging, _imagingmath, _imagingft, _imagingcms + _imaging, _imagingmath, _imagingft, _imagingcms from PIL import Image i = Image.open(cStringIO.StringIO(data)) if i.size < (20, 20):