Much nicer integration with the windows build VMs

setup/build_environment.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, socket, struct, subprocess
+import os, socket, struct, subprocess, sys
 from distutils.spawn import find_executable
 
 from PyQt4 import pyqtconfig
@@ -16,6 +16,7 @@ from setup import isosx, iswindows, islinux
 OSX_SDK = '/Developer/SDKs/MacOSX10.5.sdk'
 
 os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.5'
+is64bit = sys.maxsize > 2**32
 
 NMAKE = RC = msvc = MT = win_inc = win_lib = win_ddk = win_ddk_lib_dirs = None
 if iswindows:

setup/extensions.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import textwrap, os, shlex, subprocess, glob, shutil, sys
+import textwrap, os, shlex, subprocess, glob, shutil
 from distutils import sysconfig
 from multiprocessing import cpu_count
 
@@ -18,10 +18,9 @@ from setup.build_environment import (chmlib_inc_dirs,
         msvc, MT, win_inc, win_lib, win_ddk, magick_inc_dirs, magick_lib_dirs,
         magick_libs, chmlib_lib_dirs, sqlite_inc_dirs, icu_inc_dirs,
         icu_lib_dirs, win_ddk_lib_dirs, ft_libs, ft_lib_dirs, ft_inc_dirs,
-        zlib_libs, zlib_lib_dirs, zlib_inc_dirs)
+        zlib_libs, zlib_lib_dirs, zlib_inc_dirs, is64bit)
 MT
 isunix = islinux or isosx or isbsd
-is64bit = sys.maxsize > 2**32
 
 make = 'make' if isunix else NMAKE
 

setup/installer/windows/__init__.py

@@ -6,13 +6,11 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil, subprocess, re
+import os, shutil, subprocess
 
 from setup import Command, __appname__, __version__
 from setup.installer import VMInstaller
 
-SIGNTOOL = r'C:\cygwin\home\kovid\sign.bat'
-
 class Win(Command):
 
     description = 'Build windows binary installers'
@@ -38,11 +36,7 @@ class Win32(VMInstaller):
 
     def sign_msi(self):
         print ('Signing installers ...')
-        raw = open(self.VM).read()
-        vmx = re.search(r'''launch_vmware\(['"](.+?)['"]''', raw).group(1)
-        subprocess.check_call(['vmrun', '-T', 'ws', '-gu', 'kovid', '-gp',
-            "et tu brutus", 'runProgramInGuest', vmx, 'cmd.exe', '/C',
-            r'C:\cygwin\home\kovid\sign.bat'])
+        subprocess.check_call(['ssh', self.VM_NAME, '~/sign.sh'], shell=False)
 
     def download_installer(self):
         installer = self.installer()

setup/installer/windows/freeze.py

@@ -13,11 +13,11 @@ from setup import (Command, modules, functions, basenames, __version__,
 from setup.build_environment import msvc, MT, RC
 from setup.installer.windows.wix import WixMixIn
 
-ICU_DIR = r'Q:\icu'
+ICU_DIR = os.environ.get('ICU_DIR', r'Q:\icu')
 OPENSSL_DIR = os.environ.get('OPENSSL_DIR', r'Q:\openssl')
 QT_DIR = os.environ.get('QT_DIR', 'Q:\\Qt\\4.8.2')
 QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns']
-LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
+LIBUNRAR         = os.environ.get('UNRARDLL', 'C:\\Program Files\\UnrarDLL\\unrar.dll')
 SW               = r'C:\cygwin\home\kovid\sw'
 IMAGEMAGICK      = os.path.join(SW, 'build', 'ImageMagick-6.7.6',
         'VisualMagick', 'bin')
@@ -261,8 +261,8 @@ class Win32Freeze(Command, WixMixIn):
         print
         print 'Adding third party dependencies'
         print '\tAdding unrar'
-        shutil.copyfile(LIBUNRAR,
-                os.path.join(self.dll_dir, os.path.basename(LIBUNRAR)))
+        shutil.copyfile(LIBUNRAR, os.path.join(self.dll_dir,
+                        os.path.basename(LIBUNRAR).replace('64', '')))
 
         print '\tAdding misc binary deps'
         bindir = os.path.join(SW, 'bin')

setup/installer/windows/notes.rst

@@ -40,11 +40,43 @@ Cygwin
 
 This is needed for automation of the build process, you dont need it otherwise.
 
-Install, vim, rsync, openssh, unzip, wget at a minimum.
-In a cygwin terminal do:
+Install, vim, rsync, openssh, unzip, wget, make at a minimum.
 
-ssh-host-config -y
-net start sshd
+In order to build debug builds (.pdb files and sign files), you have to be able
+to login as the normal user account with ssh. To do this, follow these steps:
+
+    * Setup a password for your user account
+    * Follow the steps here:
+      http://pcsupport.about.com/od/windows7/ht/auto-logon-windows-7.htm or
+      http://pcsupport.about.com/od/windowsxp/ht/auto-logon-xp.htm to allow the
+      machine to bootup without having to enter the password
+    * First clean out any existing cygwin ssh setup with::
+        net stop sshd
+        cygrunsrv -R sshd
+        net user sshd /DELETE
+        net user cyg_server /DELETE (delete any other cygwin users account you
+        can list them with net user)
+        rm -R /etc/ssh*
+        mkpasswd -cl > /etc/passwd
+        mkgroup --local > /etc/group
+    * Assign the necessary rights to the normal user account::
+        editrights.exe -a SeAssignPrimaryTokenPrivilege -u kovid
+        editrights.exe -a SeCreateTokenPrivilege -u kovid
+        editrights.exe -a SeTcbPrivilege -u kovid
+        editrights.exe -a SeServiceLogonRight -u kovid
+    * Run::
+        ssh-host-config
+      And answer (yes) to all questions. If it asks do you want to use a
+      different user name, specify the name of your user account and enter
+      username and password (it asks on Win 7 not on Win XP)
+    * On Windows XP, I also had to run::
+        passwd -R
+      to allow sshd to use my normal user account even with public key
+      authentication. See http://cygwin.com/cygwin-ug-net/ntsec.html for
+      details. On Windows 7 this wasn't necessary for some reason.
+    * Start sshd with::
+        net start sshd
+    * See http://www.kgx.net.nz/2010/03/cygwin-sshd-and-windows-7/ for details
 
 Pass port 22 through Windows firewall. Create ~/.ssh/authorized_keys
 
@@ -172,9 +204,7 @@ SIP
 
 Available from: http://www.riverbankcomputing.co.uk/software/sip/download ::
 
-    python configure.py -p win32-msvc2008
-    nmake
-    nmake install
+    python configure.py -p win32-msvc2008 && nmake && nmake install
 
 PyQt4
 ----------
@@ -185,15 +215,6 @@ Compiling instructions::
     nmake
     nmake install
 
-Python Imaging Library
-------------------------
-
-Install as normal using installer at http://www.lfd.uci.edu/~gohlke/pythonlibs/
-
-Test it on the target system with
-
-calibre-debug -c "import _imaging, _imagingmath, _imagingft, _imagingcms"
-
 ICU
 -------
 
@@ -217,12 +238,30 @@ Optionally run make check
 Libunrar
 ----------
 
-http://www.rarlab.com/rar/UnRARDLL.exe install and add C:\Program Files\UnrarDLL to PATH
+Get the source from http://www.rarlab.com/rar_add.htm
 
-lxml
+Open UnrarDll.vcproj, change build type to release.
+If building 64 bit change Win32 to x64.
+
+Build the Solution, find the dll in the build subdir. As best as I can tell,
+the vcproj already defines the SILENT preprocessor directive, but you should
+test this.
+
+.. http://www.rarlab.com/rar/UnRARDLL.exe install and add C:\Program Files\UnrarDLL to PATH
+
+TODO: 64-bit check that SILENT is defined and that the ctypes bindings actuall
+work
+
+zlib
 ------
 
-http://pypi.python.org/pypi/lxml
+Build with::
+    nmake -f win32/Makefile.msc
+    nmake -f win32/Makefile.msc test
+
+    cp zlib1.dll* ../../bin
+    cp zlib.lib zdll.* ../../lib
+    cp zconf.h zlib.h ../../include
 
 jpeg-7
 -------
@@ -252,16 +291,6 @@ cp build/jpeg-7/Release/jpeg.dll bin/
 cp build/jpeg-7/Release/jpeg.lib build/jpeg-7/Release/jpeg.exp
 cp build/jpeg-7/jerror.h build/jpeg-7/jpeglib.h build/jpeg-7/jconfig.h build/jpeg-7/jmorecfg.h include/
 
-zlib
-------
-
-nmake -f win32/Makefile.msc
-nmake -f win32/Makefile.msc test
-
-cp zlib1.dll* ../../bin
-cp zlib.lib zdll.* ../../lib
-cp zconf.h zlib.h ../../include
-
 
 libpng
 ---------
@@ -302,6 +331,21 @@ Now change configuration back to static for .lib
 cp build/freetype-2.3.9/objs/win32/vc2008/freetype239MT.lib lib/
 cp -rf build/freetype-2.3.9/include/* include/
 
+lxml
+------
+
+http://pypi.python.org/pypi/lxml
+
+
+Python Imaging Library
+------------------------
+
+Install as normal using installer at http://www.lfd.uci.edu/~gohlke/pythonlibs/
+
+Test it on the target system with
+
+calibre-debug -c "import _imaging, _imagingmath, _imagingft, _imagingcms"
+
 expat
 --------
 
@@ -418,6 +462,8 @@ cp -r build/lib.win32-*/* /cygdrive/c/Python27/Lib/site-packages/
 easylzma
 ----------
 
+This is only needed to build the portable installer.
+
 Get it from http://lloyd.github.com/easylzma/ (use the trunk version)
 
 Run cmake and build the Visual Studio solution (generates CLI tools and dll and

setup/vcvars.py

@@ -7,15 +7,55 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid at kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
-from distutils.msvc9compiler import MSVCCompiler
+import os, sys, subprocess
+from distutils.msvc9compiler import find_vcvarsall, get_build_version
 
-c = MSVCCompiler()
-orig_path = os.environ['path']
-c.initialize()
-paths = c._MSVCCompiler__paths
-lib = os.environ['lib'].split(';')
-include = os.environ['include'].split(';')
+plat = 'amd64' if sys.maxsize > 2**32 else 'x86'
+
+def remove_dups(variable):
+    old_list = variable.split(os.pathsep)
+    new_list = []
+    for i in old_list:
+        if i not in new_list:
+            new_list.append(i)
+    return os.pathsep.join(new_list)
+
+def query_process(cmd):
+    result = {}
+    popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE)
+    try:
+        stdout, stderr = popen.communicate()
+        if popen.wait() != 0:
+            raise RuntimeError(stderr.decode("mbcs"))
+
+        stdout = stdout.decode("mbcs")
+        for line in stdout.splitlines():
+            if '=' not in line:
+                continue
+            line = line.strip()
+            key, value = line.split('=', 1)
+            key = key.lower()
+            if key == 'path':
+                if value.endswith(os.pathsep):
+                    value = value[:-1]
+                value = remove_dups(value)
+            result[key] = value
+
+    finally:
+        popen.stdout.close()
+        popen.stderr.close()
+    return result
+
+def query_vcvarsall():
+    vcvarsall = find_vcvarsall(get_build_version())
+    return query_process('"%s" %s & set' % (vcvarsall, plat))
+
+env = query_vcvarsall()
+paths = env['path'].split(';')
+lib = env['lib']
+include = env['include']
+libpath = env['libpath']
 
 def unix(paths):
     up = []
@@ -28,9 +68,14 @@ raw = '''\
 #!/bin/sh
 
 export PATH="%s:$PATH"
+
 export LIB="%s"
+
 export INCLUDE="%s"
-'''%(unix(paths), ';'.join(lib), ';'.join(include))
+
+export LIBPATH="%s"
+
+'''%(unix(paths), lib, include, libpath)
 
 with open(os.path.expanduser('~/.vcvars'), 'wb') as f:
     f.write(raw.encode('utf-8'))

src/calibre/test_build.py

@@ -79,6 +79,8 @@ def test_imaging():
         print ('ImageMagick OK!')
     else:
         raise RuntimeError('ImageMagick choked!')
+    import _imaging, _imagingmath, _imagingft, _imagingcms
+    _imaging, _imagingmath, _imagingft, _imagingcms
     from PIL import Image
     i = Image.open(cStringIO.StringIO(data))
     if i.size < (20, 20):