IGN:Switch to using InstallJammer to build windows installer. Also add uninstaller to linux binary installer

.bzrignore

@@ -19,3 +19,4 @@ src/calibre/gui2/pictureflow/debug/
 src/calibre/gui2/pictureflow/pictureflow_resource.rc
 src/calibre/gui2/pictureflow/release/
 src/calibre/translations/compiled.py
+installer/windows/calibre/build.log

installer/windows/build_installer.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+'''
+import sys, time, subprocess, os
+from calibre import __appname__, __version__
+
+cmdline = [
+    '/usr/local/installjammer/installjammer',
+    '--build-dir', '/tmp/calibre-installjammer',
+    '-DAppName', __appname__,
+    '-DShortAppName', __appname__,
+    '-DApplicationURL', 'http://%s.kovidgoyal.net'%__appname__,
+    '-DCopyright', time.strftime('%Y Kovid Goyal'),
+    '-DPackageDescription', '%s is an e-book library manager. It can view, convert and catalog e-books in most of the major e-book formats. It can also talk to a few e-book reader devices. It can go out to the internet and fetch metadata for your books. It can download newspapers and convert them into e-books for convenient reading.'%__appname__,
+    '-DPackageSummary', '%s: E-book library management'%__appname__,
+    '-DVersion', __version__,
+    '-DInstallVersion', __version__ + '.0',
+    '-DLicense', open(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'LICENSE')).read().replace('\n', '\r\n'),
+    '--output-dir', os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'dist'),
+    '--platform', 'Windows',    
+]
+
+def run_install_jammer(installer_name='<%AppName%>-<%Version%><%Ext%>', build_for_release=True):
+    global cmdline
+    mpi = os.path.abspath(os.path.join(os.path.dirname(__file__), 'calibre', 'calibre.mpi'))
+    cmdline.extend(['-DWindows,Executable', installer_name])
+    compression = 'zlib'
+    if build_for_release:
+        cmdline += ['--build-for-release']
+        compression = 'lzma (solid)'
+    cmdline += ['-DCompressionMethod', compression]
+    cmdline += ['--build', mpi]
+    #print 'Running installjammer with cmdline:'
+    #print cmdline
+    subprocess.check_call(cmdline)
+
+def main(args=sys.argv):
+    run_install_jammer(build_for_release=False)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

installer/windows/freeze.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Freeze app into executable using py2exe.
+'''
+QT_DIR           = 'C:\\Qt\\4.4.0'
+DEVCON           = 'C:\\devcon\\i386\\devcon.exe'
+LIBUSB_DIR       = 'C:\\libusb'
+LIBUNRAR         = 'C:\\Program Files\\UnrarDLL\\unrar.dll'
+CLIT             = 'C:\\clit\\clit.exe'
+PDFTOHTML        = 'C:\\pdftohtml\\pdftohtml.exe'
+IMAGEMAGICK_DIR  = 'C:\\ImageMagick'
+FONTCONFIG_DIR   = 'C:\\fontconfig'
+
+
+import sys, os, py2exe, shutil, zipfile, glob, subprocess
+from distutils.core import setup
+from distutils.filelist import FileList
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+sys.path.insert(0, BASE_DIR)
+from setup import VERSION, APPNAME, entry_points, scripts, basenames
+sys.path.remove(BASE_DIR)
+
+PY2EXE_DIR = os.path.join(BASE_DIR, 'build','py2exe')
+
+class BuildEXE(py2exe.build_exe.py2exe):
+    manifest_resource_id = 0
+     
+    MANIFEST_TEMPLATE = '''
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0"> 
+  <assemblyIdentity version="%(version)s"
+     processorArchitecture="x86"
+     name="net.kovidgoyal.%(prog)s"
+     type="win32"
+     /> 
+  <description>Ebook management application</description> 
+  <!-- Identify the application security requirements. -->
+  <trustInfo xmlns="urn:schemas-microsoft-com:asm.v2">
+    <security>
+      <requestedPrivileges>
+        <requestedExecutionLevel
+          level="asInvoker"
+          uiAccess="false"/>
+        </requestedPrivileges>
+       </security>
+  </trustInfo>
+</assembly>
+'''
+    def build_plugins(self):
+        cwd = os.getcwd()
+        dd = os.path.join(cwd, self.dist_dir)
+        try:
+            os.chdir(os.path.join('src', 'calibre', 'gui2', 'pictureflow'))
+            if os.path.exists('.build'):
+                shutil.rmtree('.build')
+            os.mkdir('.build')
+            os.chdir('.build')
+            subprocess.check_call(['qmake', '../pictureflow.pro'])
+            subprocess.check_call(['mingw32-make', '-f', 'Makefile.Release'])
+            shutil.copyfile('release\\pictureflow0.dll', os.path.join(dd, 'pictureflow0.dll'))
+            os.chdir('..\\PyQt')
+            if not os.path.exists('.build'):
+                os.mkdir('.build')
+            os.chdir('.build')
+            subprocess.check_call(['python', '..\\configure.py'])
+            subprocess.check_call(['mingw32-make', '-f', 'Makefile'])
+            shutil.copyfile('pictureflow.pyd', os.path.join(dd, 'pictureflow.pyd'))
+            os.chdir('..')
+            shutil.rmtree('.build', True)
+            os.chdir('..')
+            shutil.rmtree('.build', True)
+        finally:
+            os.chdir(cwd)
+    
+    def run(self):
+        if not os.path.exists(self.dist_dir):
+            os.makedirs(self.dist_dir)
+        print 'Building custom plugins...'
+        self.build_plugins()
+        py2exe.build_exe.py2exe.run(self)
+        qtsvgdll = None
+        for other in self.other_depends:
+            if 'qtsvg4.dll' in other.lower():
+                qtsvgdll = other
+                break
+        shutil.copyfile('LICENSE', os.path.join(self.dist_dir, 'LICENSE'))
+        print
+        if qtsvgdll:
+            print 'Adding', qtsvgdll
+            shutil.copyfile(qtsvgdll, os.path.join(self.dist_dir, os.path.basename(qtsvgdll)))
+            qtxmldll = os.path.join(os.path.dirname(qtsvgdll), 'QtXml4.dll')
+            print 'Adding', qtxmldll
+            shutil.copyfile(qtxmldll, 
+                            os.path.join(self.dist_dir, os.path.basename(qtxmldll)))
+        print 'Adding plugins...',
+        qt_prefix = QT_DIR
+        if qtsvgdll:
+            qt_prefix = os.path.dirname(os.path.dirname(qtsvgdll))
+        plugdir = os.path.join(qt_prefix, 'plugins')
+        for d in ('imageformats', 'codecs', 'iconengines'):
+            print d,
+            imfd = os.path.join(plugdir, d)
+            tg = os.path.join(self.dist_dir, d)        
+            if os.path.exists(tg):
+                shutil.rmtree(tg)
+            shutil.copytree(imfd, tg)
+            
+        print 
+        print 'Adding main scripts'
+        f = zipfile.ZipFile(os.path.join(PY2EXE_DIR, 'library.zip'), 'a', zipfile.ZIP_DEFLATED)
+        for i in scripts['console'] + scripts['gui']:
+            f.write(i, i.partition('\\')[-1])
+        f.close()
+        
+        print
+        print 'Adding third party dependencies'
+        print '\tAdding devcon'
+        tdir = os.path.join(PY2EXE_DIR, 'driver')
+        os.makedirs(tdir)
+        for pat in ('*.dll', '*.sys', '*.cat', '*.inf'):
+            for f in glob.glob(os.path.join(LIBUSB_DIR, pat)):
+                shutil.copyfile(f, os.path.join(tdir, os.path.basename(f)))
+        shutil.copyfile(DEVCON, os.path.join(tdir, os.path.basename(DEVCON)))
+        print '\tAdding unrar'
+        shutil.copyfile(LIBUNRAR, os.path.join(PY2EXE_DIR, os.path.basename(LIBUNRAR)))
+        print '\tAdding ConvertLIT'
+        shutil.copyfile(CLIT, os.path.join(PY2EXE_DIR, os.path.basename(CLIT)))
+        print '\tAdding pdftohtml'
+        shutil.copyfile(PDFTOHTML, os.path.join(PY2EXE_DIR, os.path.basename(PDFTOHTML)))
+        print '\tAdding ImageMagick'
+        shutil.copytree(IMAGEMAGICK_DIR, os.path.join(PY2EXE_DIR, 'ImageMagick'))
+        print '\tCopying fontconfig'
+        for f in glob.glob(os.path.join(FONTCONFIG_DIR, '*')):
+            tgt = os.path.join(PY2EXE_DIR, os.path.basename(f))
+            if os.path.isdir(f):
+                shutil.copytree(f, tgt)
+            else:
+                shutil.copyfile(f, tgt)
+        
+        print 
+        print 'Doing DLL redirection' # See http://msdn.microsoft.com/en-us/library/ms682600(VS.85).aspx
+        for f in glob.glob(os.path.join(PY2EXE_DIR, '*.exe')):
+            open(f + '.local', 'w').write('\n')
+        
+        
+    @classmethod
+    def manifest(cls, prog):
+        cls.manifest_resource_id += 1
+        return (24, cls.manifest_resource_id, 
+                cls.MANIFEST_TEMPLATE % dict(prog=prog, version=VERSION+'.0'))
+
+
+def main(args=sys.argv):
+    sys.argv[1:2] = ['py2exe']
+    if os.path.exists(PY2EXE_DIR):
+        shutil.rmtree(PY2EXE_DIR)
+
+    console = [dict(dest_base=basenames['console'][i], script=scripts['console'][i])
+               for i in range(len(scripts['console']))]
+    
+    setup(
+          cmdclass = {'py2exe': BuildEXE},
+          windows = [
+                     {'script'          : scripts['gui'][0],
+                      'dest_base'       : APPNAME,
+                      'icon_resources'  : [(1, os.path.join(BASE_DIR, 'icons', 'library.ico'))],
+                      'other_resources' : [BuildEXE.manifest(APPNAME)],
+                      },
+                      {'script'         : scripts['gui'][1],
+                      'dest_base'       : 'lrfviewer',
+                      'icon_resources'  : [(1, os.path.join(BASE_DIR, 'icons', 'viewer.ico'))],
+                      'other_resources' : [BuildEXE.manifest('lrfviewer')],
+                      },
+                      ],
+          console = console,
+          options = { 'py2exe' : {'compressed': 1,
+                                  'optimize'  : 2,
+                                  'dist_dir'  : PY2EXE_DIR,
+                                  'includes'  : [
+                                             'sip', 'pkg_resources', 'PyQt4.QtSvg',
+                                             'mechanize', 'ClientForm', 'wmi',
+                                             'win32file', 'pythoncom', 'rtf2xml',
+                                             'win32process', 'win32api', 'msvcrt',
+                                             'win32event', 'calibre.ebooks.lrf.any.*',
+                                             'calibre.ebooks.lrf.feeds.*',
+                                             'lxml', 'lxml._elementpath', 'genshi',
+                                             'path', 'pydoc', 'IPython.Extensions.*',
+                                             'calibre.web.feeds.recipes.*', 
+                                             'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
+                                             ],
+                                  'packages'  : ['PIL'],
+                                  'excludes'  : ["Tkconstants", "Tkinter", "tcl",
+                                                 "_imagingtk", "ImageTk", "FixTk"
+                                                ],
+                                  'dll_excludes' : ['mswsock.dll'],
+                                 },
+                    },
+          
+          )
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/libunrar.py

@@ -5,7 +5,7 @@ This module provides a thin ctypes based wrapper around libunrar.
 
 See  ftp://ftp.rarlabs.com/rar/unrarsrc-3.7.5.tar.gz
 """
-import os, ctypes
+import os, ctypes, sys
 from ctypes import Structure, c_char_p, c_uint, c_void_p, POINTER, \
                     byref, c_wchar_p, c_int, c_char, c_wchar
 from StringIO import StringIO
@@ -18,6 +18,8 @@ if iswindows:
     Structure._pack_ = 1
     _librar_name = 'unrar'
     cdll = ctypes.windll
+if hasattr(sys, 'frozen') and iswindows:
+    _libunrar = cdll.LoadLibrary(os.path.join(os.path.dirname(sys.executable), 'unrar.dll'))
 _libunrar = load_library(_librar_name, cdll)
 
 RAR_OM_LIST    = 0

src/calibre/linux.py

@@ -167,7 +167,9 @@ def setup_completion(fatal_errors):
         f = open_file('/etc/bash_completion.d/libprs500')
         f.close()
         os.remove(f.name)
+        manifest = []
         f = open_file('/etc/bash_completion.d/calibre')
+        manifest.append(f.name)
         
         f.write('# calibre Bash Shell Completion\n')
         f.write(opts_and_exts('html2lrf', htmlop, 
@@ -275,18 +277,22 @@ complete -o nospace  -F _prs500 prs500
         print 'failed'
         import traceback
         traceback.print_exc()
+    return manifest
         
 def setup_udev_rules(group_file, reload, fatal_errors):
     print 'Trying to setup udev rules...'
+    manifest = []
     sys.stdout.flush()
     groups = open(group_file, 'rb').read()
     group = 'plugdev' if 'plugdev' in groups else 'usb'
     udev = open_file('/etc/udev/rules.d/95-calibre.rules')
+    manifest.append(udev.name)
     udev.write('''# Sony Reader PRS-500\n'''
                '''BUS=="usb", SYSFS{idProduct}=="029b", SYSFS{idVendor}=="054c", MODE="660", GROUP="%s"\n'''%(group,)
              )
     udev.close()
     fdi = open_file('/usr/share/hal/fdi/policy/20thirdparty/10-calibre.fdi')
+    manifest.append(fdi.name)
     fdi.write('<?xml version="1.0" encoding="UTF-8"?>\n\n<deviceinfo version="0.2">\n')
     for cls in DEVICES:
         fdi.write(\
@@ -326,6 +332,7 @@ def setup_udev_rules(group_file, reload, fatal_errors):
                     if fatal_errors:
                         raise Exception("Couldn't reload udev, you may have to reboot")
                     print >>sys.stderr, "Couldn't reload udev, you may have to reboot"
+    return manifest
 
 def option_parser():
     from optparse import OptionParser
@@ -340,6 +347,8 @@ def option_parser():
                       help='If set, do not check if we are root.')
     parser.add_option('--make-errors-fatal', action='store_true', default=False, 
                       dest='fatal_errors', help='If set die on errors.')
+    parser.add_option('--save-manifest-to', default=None, 
+                      help='Save a manifest of all installed files to the specified location')
     return parser
 
 def install_man_pages(fatal_errors):
@@ -350,6 +359,7 @@ def install_man_pages(fatal_errors):
     f = open_file('/tmp/man_extra', 'wb')
     f.write('[see also]\nhttp://%s.kovidgoyal.net\n'%__appname__)
     f.close()
+    manifest = []
     for src in entry_points['console_scripts']:
         prog = src[:src.index('=')].strip()
         if prog in ('prs500', 'pdf-meta', 'epub-meta', 'lit-meta', 
@@ -360,6 +370,7 @@ def install_man_pages(fatal_errors):
                     '--section', '1', '--no-info', '--include',
                     f.name, '--manual', __appname__)
         manfile = os.path.join(manpath, prog+'.1'+__appname__+'.bz2')
+        print '\tInstalling MAN page for', prog
         try:
             p = subprocess.Popen(help2man, stdout=subprocess.PIPE)
         except OSError, err:
@@ -372,10 +383,10 @@ def install_man_pages(fatal_errors):
         if not raw.strip():
             print 'Unable to create MAN page for', prog
             continue
-        open_file(manfile).write(compress(raw))
+        f2 = open_file(manfile)
-        
+        manifest.append(f2.name) 
-     
+        f2.write(compress(raw))
-    
+    return manifest
 
 def post_install():
     parser = option_parser()
@@ -387,19 +398,21 @@ def post_install():
         
     global use_destdir
     use_destdir = opts.destdir
-    
+    manifest = []
-    setup_udev_rules(opts.group_file, not opts.dont_reload, opts.fatal_errors)
+    manifest += setup_udev_rules(opts.group_file, not opts.dont_reload, opts.fatal_errors)
-    setup_completion(opts.fatal_errors)
+    manifest += setup_completion(opts.fatal_errors)
     setup_desktop_integration(opts.fatal_errors)
-    install_man_pages(opts.fatal_errors)
+    manifest += install_man_pages(opts.fatal_errors)
         
     try:
         from PyQt4 import Qt
-        if Qt.PYQT_VERSION < int('0x40301', 16):
+        if Qt.PYQT_VERSION < int('0x40402', 16):
-            print 'WARNING: You need PyQt >= 4.3.1 for the GUI. You have', Qt.PYQT_VERSION_STR, '\nYou may experience crashes or other strange behavior.'
+            print 'WARNING: You need PyQt >= 4.4.2 for the GUI. You have', Qt.PYQT_VERSION_STR, '\nYou may experience crashes or other strange behavior.'
     except ImportError:
         print 'WARNING: You do not have PyQt4 installed. The GUI will not work.'
     
+    if opts.save_manifest_to:
+        open(opts.save_manifest_to, 'wb').write('\n'.join(manifest)+'\n')
 
     
 VIEWER = '''\

src/calibre/linux_installer.py

@@ -221,6 +221,7 @@ def extract_tarball(tar, destdir):
         tarfile.open(tar, 'r').extractall(destdir)
 
 def create_launchers(destdir, bindir='/usr/bin'):
+    manifest = []
     for launcher in open(os.path.join(destdir, 'manifest')).readlines():
         if 'postinstall' in launcher:
             continue
@@ -229,15 +230,20 @@ def create_launchers(destdir, bindir='/usr/bin'):
         print 'Creating', lp
         open(lp, 'wb').write(LAUNCHER%(destdir, launcher))
         os.chmod(lp, stat.S_IXUSR|stat.S_IXOTH|stat.S_IXGRP|stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH)
+        manifest.append(lp)
+    return manifest
         
 def do_postinstall(destdir):
     cwd = os.getcwd()
+    t = tempfile.NamedTemporaryFile()
     try:
         os.chdir(destdir)
         os.environ['LD_LIBRARY_PATH'] = destdir+':'+os.environ.get('LD_LIBRARY_PATH', '')
-        subprocess.call((os.path.join(destdir, 'calibre_postinstall'),))
+        subprocess.call((os.path.join(destdir, 'calibre_postinstall'), '--save-manifest-to', t.name))
     finally:
         os.chdir(cwd)
+    t.seek(0)
+    return list(t.readlines())
 
 def download_tarball():
     try:
@@ -271,8 +277,37 @@ def main(args=sys.argv):
     
     print 'Extracting...'
     extract_tarball(f, destdir)
-    create_launchers(destdir)
+    manifest =  create_launchers(destdir)
-    do_postinstall(destdir)
+    manifest += do_postinstall(destdir)
+    
+    manifest += ['/usr/bin/calibre-uninstall']
+    
+    UNINSTALLER = '''\
+#!/usr/bin/env python
+import os, sys
+if os.geteuid() != 0:
+    print 'You must run this uninstaller as root'
+    sys.exit(0)
+manifest = %s
+failures = []
+for path in manifest:
+    print 'Deleting', path
+    try:
+        os.unlink(path)
+    except:
+        failures.append(path)
+        
+print 'Uninstalling complete.'
+if failures:
+    print 'Failed to remove the following files:'
+    for f in failures: print f
+'''%repr(manifest)
+    
+    open('/usr/bin/calibre-uninstall', 'wb').write(UNINSTALLER)
+    os.chmod('/usr/bin/calibre-uninstall', 
+             stat.S_IXUSR|stat.S_IXOTH|stat.S_IXGRP|stat.S_IREAD|stat.S_IWRITE|stat.S_IRGRP|stat.S_IROTH)
+    
+    print 'You can uninstall calibre by running sudo calibre-uninstall'
     
     return 0
 

src/calibre/utils/zipfile.py

@@ -352,6 +352,7 @@ class ZipFile:
         elif key == 'a':
             try:                        # See if file is a zip file
                 self._RealGetContents()
+                self._calculate_file_offsets()
                 # seek to start of directory and overwrite
                 self.fp.seek(self.start_dir, 0)
             except BadZipfile:          # file is not a zip file, just append
@@ -430,7 +431,7 @@ class ZipFile:
             self.NameToInfo[x.filename] = x
             if self.debug > 2:
                 print "total", total
-        self._calculate_file_offsets()
+            
 
     def _calculate_file_offsets(self):
         for zip_info in self.filelist:

upload.py

@@ -4,7 +4,7 @@ sys.path.append('src')
 import subprocess
 from subprocess import check_call as _check_call
 from functools import partial
-#from pyvix.vix import Host, VIX_SERVICEPROVIDER_VMWARE_WORKSTATION
+
 def get_ip_address(ifname):
     s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     return socket.inet_ntoa(fcntl.ioctl(
@@ -66,12 +66,19 @@ def start_vm(vm, ssh_host, build_script, sleep=75):
 def build_windows(shutdown=True):
     installer = installer_name('exe')
     vm = '/vmware/Windows XP/Windows XP Professional.vmx'
-    start_vm(vm, 'windows', BUILD_SCRIPT%('python setup.py develop', 'python','windows_installer.py'))
+    start_vm(vm, 'windows', BUILD_SCRIPT%('python setup.py develop', 'python','installer\\\\windows\\\\freeze.py'))
-    subprocess.check_call(('scp', 'windows:build/%s/dist/*.exe'%PROJECT, 'dist'))
+    subprocess.check_call(('scp', '-rp', 'windows:build/%s/build/py2exe'%PROJECT, 'build'))
-    if not os.path.exists(installer):
+    if not os.path.exists('build/py2exe'):
-        raise Exception('Failed to build installer '+installer)
+        raise Exception('Failed to run py2exe')
     if shutdown:
         subprocess.Popen(('ssh', 'windows', 'shutdown', '-s', '-t', '0'))
+    ibp = os.path.abspath('installer/windows')
+    sys.path.insert(0, ibp)
+    import build_installer
+    sys.path.remove(ibp)
+    build_installer.run_install_jammer(installer_name=os.path.basename(installer))
+    if not os.path.exists(installer):
+        raise Exception('Failed to run installjammer')
     return os.path.basename(installer)
 
 def build_osx(shutdown=True):