Linux binaries build

setup/build_environment.py

@@ -81,6 +81,7 @@ def readvar(name):
 pyqt = {x:readvar(y) for x, y in (
     ('inc', 'QT_INSTALL_HEADERS'), ('lib', 'QT_INSTALL_LIBS')
 )}
+qt = {x:readvar(y) for x, y in {'libs':'QT_INSTALL_LIBS', 'plugins':'QT_INSTALL_PLUGINS'}.iteritems()}
 c = sipconfig.Configuration()
 pyqt['sip_bin'] = c.sip_bin + ('.exe' if iswindows and not c.sip_bin.endswith('.exe') else '')
 
@@ -194,6 +195,9 @@ else:
             '/usr/include/freetype2')
     ft_lib_dirs = pkgconfig_lib_dirs('freetype2', 'FT_LIB_DIR', '/usr/lib')
     ft_libs = pkgconfig_libs('freetype2', '', '')
+    sw = os.environ.get('SW', os.path.expanduser('~/sw'))
+    podofo_inc = os.path.join(sw, 'include', 'podofo')
+    podofo_lib = os.path.join(sw, 'lib')
 
 
 magick_error = None

setup/installer/linux/__init__.py

@@ -17,7 +17,7 @@ class Linux32(VMInstaller):
     INSTALLER_EXT = 'tar.bz2'
     VM_NAME = 'linux32-build'
     FREEZE_COMMAND = 'linux_freeze'
-    FREEZE_TEMPLATE = 'sudo python -OO setup.py {freeze_command}'
+    FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command}'
 
 
 class Linux64(Linux32):

setup/installer/linux/freeze2.py

@@ -1,77 +1,140 @@
 #!/usr/bin/env python
 # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
-from __future__ import with_statement
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
 
 __license__   = 'GPL v3'
-__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__copyright__ = '2014, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import sys, os, shutil, platform, subprocess, stat, py_compile, glob, \
-        textwrap, tarfile, re
+'''
+Setup instructions for linux build-system
 
-from setup import Command, modules, basenames, functions, __version__, \
-    __appname__
+Edit /etc/network/interfaces and add
 
-SITE_PACKAGES = ['PIL', 'dateutil', 'dns', 'PyQt5', 'mechanize',
-        'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml',
-        'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so',
-        '_dbus_glib_bindings.so', 'netifaces.so', '_psutil_posix.so',
-        '_psutil_linux.so', 'psutil', 'cssselect', 'apsw.so']
+iface eth1 inet static
+        address 192.168.xxx.xxx
 
-QTDIR          = '/usr/lib/qt4'
-QTDLLS         = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus', 'QtXmlPatterns')
-MAGICK_PREFIX = '/usr'
+Also add eth1 to the auto line (use sudo ifup eth1 to start eth1 without rebooting)
+
+sudo visudo (all no password actions for user)
+sudo apt-get install build-essential module-assistant vim zsh vim-scripts rsync \
+    htop nasm unzip libdbus-1-dev cmake libltdl-dev libudev-dev apt-file \
+    libdbus-glib-1-dev libcups2-dev "^libxcb.*" libx11-xcb-dev libglu1-mesa-dev \
+    libxrender-dev flex bison gperf libasound2-dev libgstreamer0.10-dev \
+    libgstreamer-plugins-base0.10-dev libpulse-dev
+apt-file update
+
+# For recent enough version of debian (>= sid) also install libxkbcommon-dev
+
+mkdir -p ~/bin ~/sw/sources ~/sw/build
+
+chsh -s /bin/zsh
+
+Edit /etc/default/grub and change the GRUB_TIMEOUT to 1, then run
+sudo update-grub
+
+Copy over authorized_keys, .vimrc and .zshrc
+
+Create ~/.zshenv as
+
+export SW=$HOME/sw
+export MAKEOPTS="-j2"
+export CFLAGS=-I$SW/include
+export LDFLAGS=-L$SW/lib
+export LD_LIBRARY_PATH=$SW/lib
+export PKG_CONFIG_PATH=$SW/lib/pkgconfig:$PKG_CONFIG_PATH
+
+typeset -U path
+path=($SW/bin "$path[@]")
+path=($SW/qt/bin "$path[@]")
+path=(~/bin "$path[@]")
+
+'''
+
+
+import sys, os, shutil, platform, subprocess, stat, py_compile, glob, textwrap, tarfile
+
+from setup import Command, modules, basenames, functions, __version__,  __appname__
+from setup.build_environment import QT_DLLS, QT_PLUGINS, qt, PYQT_MODULES, sw as SW
+
+j = os.path.join
 is64bit = platform.architecture()[0] == '64bit'
-binary_includes = [
-                '/usr/bin/pdftohtml',
-                '/usr/bin/pdfinfo',
-                '/usr/lib/libusb-1.0.so.0' if is64bit else '/lib/libusb-1.0.so.0',
-                '/usr/lib/libmtp.so.9',
-                '/usr/lib/libglib-2.0.so.0',
-                '/usr/bin/pdftoppm',
-                '/usr/lib/libwmflite-0.2.so.7',
-                '/usr/lib/liblcms.so.1',
-                '/usr/lib/liblzma.so.0',
-                '/usr/lib/libexpat.so.1',
-                '/usr/lib/libsqlite3.so.0',
-                '/usr/lib/libmng.so.1',
-                '/usr/lib/libpodofo.so.0.9.1',
-                '/lib/libz.so.1',
-                '/usr/lib/libtiff.so.5',
-                '/lib/libbz2.so.1',
-                '/usr/lib/libpoppler.so.37',
-                '/usr/lib/libxml2.so.2',
-                '/usr/lib/libopenjpeg.so.2',
-                '/usr/lib/libxslt.so.1',
-                '/usr/lib/libjpeg.so.8',
-                '/usr/lib/libxslt.so.1',
-                '/usr/lib/libgthread-2.0.so.0',
-                '/usr/lib/libpng14.so.14',
-                '/usr/lib/libexslt.so.0',
-                # Ensure that libimobiledevice is compiled against openssl, not gnutls
-                '/usr/lib/libimobiledevice.so.4',
-                '/usr/lib/libusbmuxd.so.2',
-                '/usr/lib/libplist.so.1',
-                MAGICK_PREFIX+'/lib/libMagickWand.so.5',
-                MAGICK_PREFIX+'/lib/libMagickCore.so.5',
-                '/usr/lib/libgcrypt.so.11',
-                '/usr/lib/libgpg-error.so.0',
-                '/usr/lib/libphonon.so.4',
-                '/usr/lib/libssl.so.1.0.0',
-                '/usr/lib/libcrypto.so.1.0.0',
-                '/lib/libreadline.so.6',
-                '/usr/lib/libchm.so.0',
-                '/usr/lib/liblcms2.so.2',
-                '/usr/lib/libicudata.so.49',
-                '/usr/lib/libicui18n.so.49',
-                '/usr/lib/libicuuc.so.49',
-                '/usr/lib/libicuio.so.49',
-                '/usr/lib/libdbus-1.so.3',
-                ]
-binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS]
+py_ver = '.'.join(map(str, sys.version_info[:2]))
+
+def binary_includes():
+    return [
+    j(SW, 'bin', x) for x in ('pdftohtml', 'pdfinfo', 'pdftoppm')] + [
+
+    j(SW, 'lib', 'lib' + x) for x in (
+        'usb-1.0.so.0', 'mtp.so.9', 'expat.so.1', 'sqlite3.so.0',
+        'podofo.so.0.9.1', 'z.so.1', 'bz2.so.1.0', 'poppler.so.46',
+        'iconv.so.2', 'xml2.so.2', 'xslt.so.1', 'jpeg.so.8', 'png16.so.16',
+        'exslt.so.0', 'imobiledevice.so.4', 'usbmuxd.so.2', 'plist.so.2',
+        'MagickCore-6.Q16.so.2', 'MagickWand-6.Q16.so.2', 'ssl.so.1.0.0',
+        'crypto.so.1.0.0', 'readline.so.6', 'chm.so.0', 'icudata.so.53',
+        'icui18n.so.53', 'icuuc.so.53', 'icuio.so.53', 'python%s.so' % py_ver
+    )] + [
+
+    glob.glob('/lib/*/lib' + x)[-1] for x in (
+        'glib-2.0.so.0', 'gcrypt.so.11', 'gpg-error.so.0', 'dbus-1.so.3', 'pcre.so.3',
+    )] + [
+
+    glob.glob('/usr/lib/*/lib' + x)[-1] for x in (
+        'gthread-2.0.so.0', 'ffi.so.5', 'stdc++.so.6',
+    )] + [
+        j(qt['libs'], 'lib%s.so.5' % x) for x in QT_DLLS]
 
 arch = 'x86_64' if is64bit else 'i686'
 
+def ignore_in_lib(base, items):
+    ans = []
+    for y in items:
+        x = os.path.join(base, y)
+        if (os.path.isfile(x) and os.path.splitext(x)[1] in ('.so',
+                '.py')) or \
+            (os.path.isdir(x) and x not in ('.svn', '.bzr', '.git', 'test', 'tests',
+                'testing')):
+            continue
+        ans.append(y)
+    return ans
+
+def import_site_packages(srcdir, dest):
+    if not os.path.exists(dest):
+        os.mkdir(dest)
+    for x in os.listdir(srcdir):
+        ext = x.rpartition('.')
+        f = j(srcdir, x)
+        if ext in ('py', 'so'):
+            shutil.copy2(f, dest)
+        elif ext == 'pth' and x != 'setuptools.pth':
+            for line in open(f, 'rb').readlines():
+                src = j(srcdir, line)
+                if os.path.exists(src) and os.path.isdir(src):
+                    import_site_packages(src, dest)
+        elif os.path.exists(j(f, '__init__.py')):
+            shutil.copytree(f, j(dest, x), ignore=ignore_in_lib)
+
+def is_elf(path):
+    with open(path, 'rb') as f:
+        return f.read(4) == b'\x7fELF'
+
+STRIPCMD = ['strip']
+def strip_files(files, argv_max=(256 * 1024)):
+    """ Strip a list of files """
+    while files:
+        cmd = list(STRIPCMD)
+        pathlen = sum(len(s) + 1 for s in cmd)
+        while pathlen < argv_max and files:
+            f = files.pop()
+            cmd.append(f)
+            pathlen += len(f) + 1
+        if len(cmd) > len(STRIPCMD):
+            all_files = cmd[len(STRIPCMD):]
+            unwritable_files = tuple(filter(None, (None if os.access(x, os.W_OK) else (x, os.stat(x).st_mode) for x in all_files)))
+            [os.chmod(x, stat.S_IWRITE | old_mode) for x, old_mode in unwritable_files]
+            subprocess.check_call(cmd)
+            [os.chmod(x, old_mode) for x, old_mode in unwritable_files]
 
 class LinuxFreeze(Command):
 
@@ -80,7 +143,6 @@ class LinuxFreeze(Command):
         self.opts = opts
         self.src_root = self.d(self.SRC)
         self.base = self.j(self.src_root, 'build', 'linfrozen')
-        self.py_ver = '.'.join(map(str, sys.version_info[:2]))
         self.lib_dir = self.j(self.base, 'lib')
         self.bin_dir = self.j(self.base, 'bin')
 
@@ -88,6 +150,7 @@ class LinuxFreeze(Command):
         self.copy_libs()
         self.copy_python()
         self.build_launchers()
+        self.strip_files()
         self.create_tarfile()
 
     def initbase(self):
@@ -100,66 +163,25 @@ class LinuxFreeze(Command):
         os.mkdir(self.lib_dir)
         os.mkdir(self.bin_dir)
 
-        gcc = subprocess.Popen(["gcc-config", "-c"], stdout=subprocess.PIPE).communicate()[0]
-        chost, _, gcc = gcc.rpartition('-')
-        gcc_lib = '/usr/lib/gcc/%s/%s/'%(chost.strip(), gcc.strip())
-        stdcpp = gcc_lib+'libstdc++.so.?'
-        stdcpp = glob.glob(stdcpp)[-1]
-        ffi_libs = [glob.glob('/usr/lib/libffi.so.?')[-1]]
-        ffi = gcc_lib+'libffi.so.?'
-        ffi = glob.glob(ffi)
-        if ffi and ffi[-1] not in ffi_libs:
-            ffi_libs.append(ffi[-1])
-
-        for x in binary_includes + [stdcpp] + ffi_libs:
+        for x in binary_includes():
             dest = self.bin_dir if '/bin/' in x else self.lib_dir
             shutil.copy2(x, dest)
-        shutil.copy2('/usr/lib/libpython%s.so.1.0'%self.py_ver, dest)
 
-        base = self.j(QTDIR, 'plugins')
+        base = qt['plugins']
         dest = self.j(self.lib_dir, 'qt_plugins')
         os.mkdir(dest)
-        for x in os.listdir(base):
-            y = self.j(base, x)
-            if x not in ('designer', 'sqldrivers', 'codecs'):
-                shutil.copytree(y, self.j(dest, x))
+        for x in QT_PLUGINS:
+            shutil.copytree(self.j(base, x), self.j(dest, x))
 
-        im = glob.glob(MAGICK_PREFIX + '/lib/ImageMagick-*')[-1]
+        im = glob.glob(SW + '/lib/ImageMagick-*')[-1]
         self.magick_base = os.path.basename(im)
         dest = self.j(self.lib_dir, self.magick_base)
-        shutil.copytree(im, dest, ignore=shutil.ignore_patterns('*.a'))
-        from calibre import walk
-        for x in walk(dest):
-            if x.endswith('.la'):
-                raw = open(x).read()
-                raw = re.sub('libdir=.*', '', raw)
-                open(x, 'wb').write(raw)
-
-        dest = self.j(dest, 'config')
-        src = self.j(MAGICK_PREFIX, 'share', self.magick_base, 'config')
-        for x in glob.glob(src+'/*'):
-            d = self.j(dest, os.path.basename(x))
-            if os.path.isdir(x):
-                shutil.copytree(x, d)
-            else:
-                shutil.copyfile(x, d)
+        shutil.copytree(im, dest, ignore=shutil.ignore_patterns('*.a', '*.la'))
 
     def copy_python(self):
         self.info('Copying python...')
 
-        def ignore_in_lib(base, items):
-            ans = []
-            for y in items:
-                x = os.path.join(base, y)
-                if (os.path.isfile(x) and os.path.splitext(x)[1] in ('.so',
-                        '.py')) or \
-                   (os.path.isdir(x) and x not in ('.svn', '.bzr', '.git', 'test', 'tests',
-                       'testing')):
-                    continue
-                ans.append(y)
-            return ans
-
-        srcdir = self.j('/usr/lib/python'+self.py_ver)
+        srcdir = self.j(SW, 'lib/python'+py_ver)
         self.py_dir = self.j(self.lib_dir, self.b(srcdir))
         if not os.path.exists(self.py_dir):
             os.mkdir(self.py_dir)
@@ -171,23 +193,18 @@ class LinuxFreeze(Command):
                     'site-packages', 'idlelib', 'lib2to3', 'dist-packages'):
                 shutil.copytree(y, self.j(self.py_dir, x),
                         ignore=ignore_in_lib)
-            if os.path.isfile(y) and ext in ('.py', '.so') and \
-                    self.b(y) not in ('pdflib_py.so',):
+            if os.path.isfile(y) and ext in ('.py', '.so'):
                 shutil.copy2(y, self.py_dir)
 
         srcdir = self.j(srcdir, 'site-packages')
         dest = self.j(self.py_dir, 'site-packages')
-        os.mkdir(dest)
-        for x in SITE_PACKAGES:
-            x = self.j(srcdir, x)
-            ext = os.path.splitext(x)[1]
-            if os.path.isdir(x):
-                shutil.copytree(x, self.j(dest, self.b(x)),
-                        ignore=ignore_in_lib)
-            elif os.path.isfile(x) and ext in ('.py', '.so'):
-                shutil.copy2(x, dest)
-            else:
-                raise ValueError('%s does not exist in site-packages' % x)
+        import_site_packages(srcdir, dest)
+
+        filter_pyqt = {x+'.so' for x in PYQT_MODULES}
+        pyqt = self.j(dest, 'PyQt5')
+        for x in os.listdir(pyqt):
+            if x.endswith('.so') and x not in filter_pyqt:
+                os.remove(self.j(pyqt, x))
 
         for x in os.listdir(self.SRC):
             if os.path.isdir(self.j(self.SRC, x)):
@@ -195,13 +212,6 @@ class LinuxFreeze(Command):
                         ignore=ignore_in_lib)
             else:
                 shutil.copy2(self.j(self.SRC, x), self.j(dest, x))
-        for x in ('trac',):
-            x = self.j(dest, 'calibre', x)
-            if os.path.exists(x):
-                shutil.rmtree(x)
-
-        for x in glob.glob(self.j(dest, 'calibre', 'translations', '*.po')):
-            os.remove(x)
 
         shutil.copytree(self.j(self.src_root, 'resources'), self.j(self.base,
                 'resources'))
@@ -220,7 +230,8 @@ class LinuxFreeze(Command):
                         if os.path.exists(z):
                             os.remove(z)
                     except:
-                        self.warn('Failed to byte-compile', y)
+                        if '/uic/port_v3/' not in y:
+                            self.warn('Failed to byte-compile', y)
 
     def run_builder(self, cmd, verbose=True):
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
@@ -237,10 +248,11 @@ class LinuxFreeze(Command):
 
     def create_tarfile(self):
         self.info('Creating archive...')
-        dist = os.path.join(self.d(self.SRC), 'dist',
-            '%s-%s-%s.tar.bz2'%(__appname__, __version__, arch))
-        with tarfile.open(dist, mode='w:bz2',
-                    format=tarfile.PAX_FORMAT) as tf:
+        base = self.j(self.d(self.SRC), 'dist')
+        if not os.path.exists(base):
+            os.mkdir(base)
+        dist = os.path.join(base, '%s-%s-%s.tar'%(__appname__, __version__, arch))
+        with tarfile.open(dist, mode='w', format=tarfile.PAX_FORMAT) as tf:
             cwd = os.getcwd()
             os.chdir(self.base)
             try:
@@ -248,6 +260,8 @@ class LinuxFreeze(Command):
                     tf.add(x)
             finally:
                 os.chdir(cwd)
+        subprocess.check_call(['bzip2', '-f', '-9', dist])
+        dist += '.bz2'
         self.info('Archive %s created: %.2f MB'%(dist,
             os.stat(dist).st_size/(1024.**2)))
 
@@ -259,8 +273,8 @@ class LinuxFreeze(Command):
         sources = [self.j(base, x) for x in ['util.c']]
         headers = [self.j(base, x) for x in ['util.h']]
         objects = [self.j(self.obj_dir, self.b(x)+'.o') for x in sources]
-        cflags  = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%self.py_ver
-        cflags  = cflags.split() + ['-I/usr/include/python'+self.py_ver]
+        cflags  = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%py_ver
+        cflags  = cflags.split() + ['-I%s/include/python%s' % (SW, py_ver)]
         for src, obj in zip(sources, objects):
             if not self.newer(obj, headers+[src, __file__]):
                 continue
@@ -270,7 +284,7 @@ class LinuxFreeze(Command):
         dll = self.j(self.lib_dir, 'libcalibre-launcher.so')
         if self.newer(dll, objects):
             cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \
-                    ['-lpython'+self.py_ver]
+                    ['-L%s/lib'%SW, '-lpython'+py_ver]
             self.info('Linking libcalibre-launcher.so')
             self.run_builder(cmd)
 
@@ -296,7 +310,6 @@ class LinuxFreeze(Command):
 
                 dest = self.j(self.obj_dir, bname+'.o')
                 if self.newer(dest, [src, __file__]+headers):
-                    self.info('Compiling', bname)
                     cmd = ['gcc'] + xflags + [src, '-o', dest]
                     self.run_builder(cmd, verbose=False)
                 exe = self.j(self.bin_dir, bname)
@@ -306,7 +319,6 @@ class LinuxFreeze(Command):
                     stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH)
 
                 if self.newer(exe, [dest, __file__]):
-                    self.info('Linking', bname)
                     cmd = ['gcc', '-O2',
                             '-o', exe,
                             dest,
@@ -316,6 +328,21 @@ class LinuxFreeze(Command):
 
                     self.run_builder(cmd, verbose=False)
 
+    def strip_files(self):
+        from calibre import walk
+        files = {self.j(self.bin_dir, x) for x in os.listdir(self.bin_dir)} | {
+            x for x in {
+            self.j(self.d(self.bin_dir), x) for x in os.listdir(self.bin_dir)} if os.path.exists(x)}
+        for x in walk(self.lib_dir):
+            x = os.path.realpath(x)
+            if x not in files and is_elf(x):
+                files.add(x)
+        self.info('Stripping %d files...' % len(files))
+        before = sum(os.path.getsize(x) for x in files)
+        strip_files(files)
+        after = sum(os.path.getsize(x) for x in files)
+        self.info('Stripped %.1f MB' % ((before - after)/(1024*1024.)))
+
     def create_site_py(self):  # {{{
         with open(self.j(self.py_dir, 'site.py'), 'wb') as f:
             f.write(textwrap.dedent('''\

src/calibre/test_build.py

@@ -202,6 +202,12 @@ def test_podofo():
     from calibre.utils.podofo import test_podofo as dotest
     dotest()
 
+def test_terminal():
+    import readline, curses
+    curses.setupterm()
+    del readline
+    print ('readline and curses OK!')
+
 def test():
     if iswindows:
         test_dlls()
@@ -225,6 +231,8 @@ def test():
     if iswindows:
         test_wpd()
         test_winutil()
+    else:
+        test_terminal()
     if islinux:
         test_dbus()