diff --git a/setup/build_environment.py b/setup/build_environment.py index f9cc75232c..36bbd537fa 100644 --- a/setup/build_environment.py +++ b/setup/build_environment.py @@ -81,6 +81,7 @@ def readvar(name): pyqt = {x:readvar(y) for x, y in ( ('inc', 'QT_INSTALL_HEADERS'), ('lib', 'QT_INSTALL_LIBS') )} +qt = {x:readvar(y) for x, y in {'libs':'QT_INSTALL_LIBS', 'plugins':'QT_INSTALL_PLUGINS'}.iteritems()} c = sipconfig.Configuration() pyqt['sip_bin'] = c.sip_bin + ('.exe' if iswindows and not c.sip_bin.endswith('.exe') else '') @@ -194,6 +195,9 @@ else: '/usr/include/freetype2') ft_lib_dirs = pkgconfig_lib_dirs('freetype2', 'FT_LIB_DIR', '/usr/lib') ft_libs = pkgconfig_libs('freetype2', '', '') + sw = os.environ.get('SW', os.path.expanduser('~/sw')) + podofo_inc = os.path.join(sw, 'include', 'podofo') + podofo_lib = os.path.join(sw, 'lib') magick_error = None diff --git a/setup/installer/linux/__init__.py b/setup/installer/linux/__init__.py index e65e0a7063..20a8e7a31a 100644 --- a/setup/installer/linux/__init__.py +++ b/setup/installer/linux/__init__.py @@ -17,7 +17,7 @@ class Linux32(VMInstaller): INSTALLER_EXT = 'tar.bz2' VM_NAME = 'linux32-build' FREEZE_COMMAND = 'linux_freeze' - FREEZE_TEMPLATE = 'sudo python -OO setup.py {freeze_command}' + FREEZE_TEMPLATE = 'python -OO setup.py {freeze_command}' class Linux64(Linux32): diff --git a/setup/installer/linux/freeze2.py b/setup/installer/linux/freeze2.py index 040e9d68f7..b25b34e400 100644 --- a/setup/installer/linux/freeze2.py +++ b/setup/installer/linux/freeze2.py @@ -1,77 +1,140 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement +from __future__ import (unicode_literals, division, absolute_import, + print_function) __license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal ' +__copyright__ = '2014, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import sys, os, shutil, platform, subprocess, stat, py_compile, glob, \ - textwrap, tarfile, re +''' +Setup instructions for linux build-system -from setup import Command, modules, basenames, functions, __version__, \ - __appname__ +Edit /etc/network/interfaces and add -SITE_PACKAGES = ['PIL', 'dateutil', 'dns', 'PyQt5', 'mechanize', - 'sip.so', 'BeautifulSoup.py', 'cssutils', 'encutils', 'lxml', - 'sipconfig.py', 'xdg', 'dbus', '_dbus_bindings.so', - '_dbus_glib_bindings.so', 'netifaces.so', '_psutil_posix.so', - '_psutil_linux.so', 'psutil', 'cssselect', 'apsw.so'] +iface eth1 inet static + address 192.168.xxx.xxx -QTDIR = '/usr/lib/qt4' -QTDLLS = ('QtCore', 'QtGui', 'QtNetwork', 'QtSvg', 'QtXml', 'QtWebKit', 'QtDBus', 'QtXmlPatterns') -MAGICK_PREFIX = '/usr' +Also add eth1 to the auto line (use sudo ifup eth1 to start eth1 without rebooting) + +sudo visudo (all no password actions for user) +sudo apt-get install build-essential module-assistant vim zsh vim-scripts rsync \ + htop nasm unzip libdbus-1-dev cmake libltdl-dev libudev-dev apt-file \ + libdbus-glib-1-dev libcups2-dev "^libxcb.*" libx11-xcb-dev libglu1-mesa-dev \ + libxrender-dev flex bison gperf libasound2-dev libgstreamer0.10-dev \ + libgstreamer-plugins-base0.10-dev libpulse-dev +apt-file update + +# For recent enough version of debian (>= sid) also install libxkbcommon-dev + +mkdir -p ~/bin ~/sw/sources ~/sw/build + +chsh -s /bin/zsh + +Edit /etc/default/grub and change the GRUB_TIMEOUT to 1, then run +sudo update-grub + +Copy over authorized_keys, .vimrc and .zshrc + +Create ~/.zshenv as + +export SW=$HOME/sw +export MAKEOPTS="-j2" +export CFLAGS=-I$SW/include +export LDFLAGS=-L$SW/lib +export LD_LIBRARY_PATH=$SW/lib +export PKG_CONFIG_PATH=$SW/lib/pkgconfig:$PKG_CONFIG_PATH + +typeset -U path +path=($SW/bin "$path[@]") +path=($SW/qt/bin "$path[@]") +path=(~/bin "$path[@]") + +''' + + +import sys, os, shutil, platform, subprocess, stat, py_compile, glob, textwrap, tarfile + +from setup import Command, modules, basenames, functions, __version__, __appname__ +from setup.build_environment import QT_DLLS, QT_PLUGINS, qt, PYQT_MODULES, sw as SW + +j = os.path.join is64bit = platform.architecture()[0] == '64bit' -binary_includes = [ - '/usr/bin/pdftohtml', - '/usr/bin/pdfinfo', - '/usr/lib/libusb-1.0.so.0' if is64bit else '/lib/libusb-1.0.so.0', - '/usr/lib/libmtp.so.9', - '/usr/lib/libglib-2.0.so.0', - '/usr/bin/pdftoppm', - '/usr/lib/libwmflite-0.2.so.7', - '/usr/lib/liblcms.so.1', - '/usr/lib/liblzma.so.0', - '/usr/lib/libexpat.so.1', - '/usr/lib/libsqlite3.so.0', - '/usr/lib/libmng.so.1', - '/usr/lib/libpodofo.so.0.9.1', - '/lib/libz.so.1', - '/usr/lib/libtiff.so.5', - '/lib/libbz2.so.1', - '/usr/lib/libpoppler.so.37', - '/usr/lib/libxml2.so.2', - '/usr/lib/libopenjpeg.so.2', - '/usr/lib/libxslt.so.1', - '/usr/lib/libjpeg.so.8', - '/usr/lib/libxslt.so.1', - '/usr/lib/libgthread-2.0.so.0', - '/usr/lib/libpng14.so.14', - '/usr/lib/libexslt.so.0', - # Ensure that libimobiledevice is compiled against openssl, not gnutls - '/usr/lib/libimobiledevice.so.4', - '/usr/lib/libusbmuxd.so.2', - '/usr/lib/libplist.so.1', - MAGICK_PREFIX+'/lib/libMagickWand.so.5', - MAGICK_PREFIX+'/lib/libMagickCore.so.5', - '/usr/lib/libgcrypt.so.11', - '/usr/lib/libgpg-error.so.0', - '/usr/lib/libphonon.so.4', - '/usr/lib/libssl.so.1.0.0', - '/usr/lib/libcrypto.so.1.0.0', - '/lib/libreadline.so.6', - '/usr/lib/libchm.so.0', - '/usr/lib/liblcms2.so.2', - '/usr/lib/libicudata.so.49', - '/usr/lib/libicui18n.so.49', - '/usr/lib/libicuuc.so.49', - '/usr/lib/libicuio.so.49', - '/usr/lib/libdbus-1.so.3', - ] -binary_includes += [os.path.join(QTDIR, 'lib%s.so.4'%x) for x in QTDLLS] +py_ver = '.'.join(map(str, sys.version_info[:2])) + +def binary_includes(): + return [ + j(SW, 'bin', x) for x in ('pdftohtml', 'pdfinfo', 'pdftoppm')] + [ + + j(SW, 'lib', 'lib' + x) for x in ( + 'usb-1.0.so.0', 'mtp.so.9', 'expat.so.1', 'sqlite3.so.0', + 'podofo.so.0.9.1', 'z.so.1', 'bz2.so.1.0', 'poppler.so.46', + 'iconv.so.2', 'xml2.so.2', 'xslt.so.1', 'jpeg.so.8', 'png16.so.16', + 'exslt.so.0', 'imobiledevice.so.4', 'usbmuxd.so.2', 'plist.so.2', + 'MagickCore-6.Q16.so.2', 'MagickWand-6.Q16.so.2', 'ssl.so.1.0.0', + 'crypto.so.1.0.0', 'readline.so.6', 'chm.so.0', 'icudata.so.53', + 'icui18n.so.53', 'icuuc.so.53', 'icuio.so.53', 'python%s.so' % py_ver + )] + [ + + glob.glob('/lib/*/lib' + x)[-1] for x in ( + 'glib-2.0.so.0', 'gcrypt.so.11', 'gpg-error.so.0', 'dbus-1.so.3', 'pcre.so.3', + )] + [ + + glob.glob('/usr/lib/*/lib' + x)[-1] for x in ( + 'gthread-2.0.so.0', 'ffi.so.5', 'stdc++.so.6', + )] + [ + j(qt['libs'], 'lib%s.so.5' % x) for x in QT_DLLS] arch = 'x86_64' if is64bit else 'i686' +def ignore_in_lib(base, items): + ans = [] + for y in items: + x = os.path.join(base, y) + if (os.path.isfile(x) and os.path.splitext(x)[1] in ('.so', + '.py')) or \ + (os.path.isdir(x) and x not in ('.svn', '.bzr', '.git', 'test', 'tests', + 'testing')): + continue + ans.append(y) + return ans + +def import_site_packages(srcdir, dest): + if not os.path.exists(dest): + os.mkdir(dest) + for x in os.listdir(srcdir): + ext = x.rpartition('.') + f = j(srcdir, x) + if ext in ('py', 'so'): + shutil.copy2(f, dest) + elif ext == 'pth' and x != 'setuptools.pth': + for line in open(f, 'rb').readlines(): + src = j(srcdir, line) + if os.path.exists(src) and os.path.isdir(src): + import_site_packages(src, dest) + elif os.path.exists(j(f, '__init__.py')): + shutil.copytree(f, j(dest, x), ignore=ignore_in_lib) + +def is_elf(path): + with open(path, 'rb') as f: + return f.read(4) == b'\x7fELF' + +STRIPCMD = ['strip'] +def strip_files(files, argv_max=(256 * 1024)): + """ Strip a list of files """ + while files: + cmd = list(STRIPCMD) + pathlen = sum(len(s) + 1 for s in cmd) + while pathlen < argv_max and files: + f = files.pop() + cmd.append(f) + pathlen += len(f) + 1 + if len(cmd) > len(STRIPCMD): + all_files = cmd[len(STRIPCMD):] + unwritable_files = tuple(filter(None, (None if os.access(x, os.W_OK) else (x, os.stat(x).st_mode) for x in all_files))) + [os.chmod(x, stat.S_IWRITE | old_mode) for x, old_mode in unwritable_files] + subprocess.check_call(cmd) + [os.chmod(x, old_mode) for x, old_mode in unwritable_files] class LinuxFreeze(Command): @@ -80,7 +143,6 @@ class LinuxFreeze(Command): self.opts = opts self.src_root = self.d(self.SRC) self.base = self.j(self.src_root, 'build', 'linfrozen') - self.py_ver = '.'.join(map(str, sys.version_info[:2])) self.lib_dir = self.j(self.base, 'lib') self.bin_dir = self.j(self.base, 'bin') @@ -88,6 +150,7 @@ class LinuxFreeze(Command): self.copy_libs() self.copy_python() self.build_launchers() + self.strip_files() self.create_tarfile() def initbase(self): @@ -100,66 +163,25 @@ class LinuxFreeze(Command): os.mkdir(self.lib_dir) os.mkdir(self.bin_dir) - gcc = subprocess.Popen(["gcc-config", "-c"], stdout=subprocess.PIPE).communicate()[0] - chost, _, gcc = gcc.rpartition('-') - gcc_lib = '/usr/lib/gcc/%s/%s/'%(chost.strip(), gcc.strip()) - stdcpp = gcc_lib+'libstdc++.so.?' - stdcpp = glob.glob(stdcpp)[-1] - ffi_libs = [glob.glob('/usr/lib/libffi.so.?')[-1]] - ffi = gcc_lib+'libffi.so.?' - ffi = glob.glob(ffi) - if ffi and ffi[-1] not in ffi_libs: - ffi_libs.append(ffi[-1]) - - for x in binary_includes + [stdcpp] + ffi_libs: + for x in binary_includes(): dest = self.bin_dir if '/bin/' in x else self.lib_dir shutil.copy2(x, dest) - shutil.copy2('/usr/lib/libpython%s.so.1.0'%self.py_ver, dest) - base = self.j(QTDIR, 'plugins') + base = qt['plugins'] dest = self.j(self.lib_dir, 'qt_plugins') os.mkdir(dest) - for x in os.listdir(base): - y = self.j(base, x) - if x not in ('designer', 'sqldrivers', 'codecs'): - shutil.copytree(y, self.j(dest, x)) + for x in QT_PLUGINS: + shutil.copytree(self.j(base, x), self.j(dest, x)) - im = glob.glob(MAGICK_PREFIX + '/lib/ImageMagick-*')[-1] + im = glob.glob(SW + '/lib/ImageMagick-*')[-1] self.magick_base = os.path.basename(im) dest = self.j(self.lib_dir, self.magick_base) - shutil.copytree(im, dest, ignore=shutil.ignore_patterns('*.a')) - from calibre import walk - for x in walk(dest): - if x.endswith('.la'): - raw = open(x).read() - raw = re.sub('libdir=.*', '', raw) - open(x, 'wb').write(raw) - - dest = self.j(dest, 'config') - src = self.j(MAGICK_PREFIX, 'share', self.magick_base, 'config') - for x in glob.glob(src+'/*'): - d = self.j(dest, os.path.basename(x)) - if os.path.isdir(x): - shutil.copytree(x, d) - else: - shutil.copyfile(x, d) + shutil.copytree(im, dest, ignore=shutil.ignore_patterns('*.a', '*.la')) def copy_python(self): self.info('Copying python...') - def ignore_in_lib(base, items): - ans = [] - for y in items: - x = os.path.join(base, y) - if (os.path.isfile(x) and os.path.splitext(x)[1] in ('.so', - '.py')) or \ - (os.path.isdir(x) and x not in ('.svn', '.bzr', '.git', 'test', 'tests', - 'testing')): - continue - ans.append(y) - return ans - - srcdir = self.j('/usr/lib/python'+self.py_ver) + srcdir = self.j(SW, 'lib/python'+py_ver) self.py_dir = self.j(self.lib_dir, self.b(srcdir)) if not os.path.exists(self.py_dir): os.mkdir(self.py_dir) @@ -171,23 +193,18 @@ class LinuxFreeze(Command): 'site-packages', 'idlelib', 'lib2to3', 'dist-packages'): shutil.copytree(y, self.j(self.py_dir, x), ignore=ignore_in_lib) - if os.path.isfile(y) and ext in ('.py', '.so') and \ - self.b(y) not in ('pdflib_py.so',): + if os.path.isfile(y) and ext in ('.py', '.so'): shutil.copy2(y, self.py_dir) srcdir = self.j(srcdir, 'site-packages') dest = self.j(self.py_dir, 'site-packages') - os.mkdir(dest) - for x in SITE_PACKAGES: - x = self.j(srcdir, x) - ext = os.path.splitext(x)[1] - if os.path.isdir(x): - shutil.copytree(x, self.j(dest, self.b(x)), - ignore=ignore_in_lib) - elif os.path.isfile(x) and ext in ('.py', '.so'): - shutil.copy2(x, dest) - else: - raise ValueError('%s does not exist in site-packages' % x) + import_site_packages(srcdir, dest) + + filter_pyqt = {x+'.so' for x in PYQT_MODULES} + pyqt = self.j(dest, 'PyQt5') + for x in os.listdir(pyqt): + if x.endswith('.so') and x not in filter_pyqt: + os.remove(self.j(pyqt, x)) for x in os.listdir(self.SRC): if os.path.isdir(self.j(self.SRC, x)): @@ -195,13 +212,6 @@ class LinuxFreeze(Command): ignore=ignore_in_lib) else: shutil.copy2(self.j(self.SRC, x), self.j(dest, x)) - for x in ('trac',): - x = self.j(dest, 'calibre', x) - if os.path.exists(x): - shutil.rmtree(x) - - for x in glob.glob(self.j(dest, 'calibre', 'translations', '*.po')): - os.remove(x) shutil.copytree(self.j(self.src_root, 'resources'), self.j(self.base, 'resources')) @@ -220,7 +230,8 @@ class LinuxFreeze(Command): if os.path.exists(z): os.remove(z) except: - self.warn('Failed to byte-compile', y) + if '/uic/port_v3/' not in y: + self.warn('Failed to byte-compile', y) def run_builder(self, cmd, verbose=True): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, @@ -237,10 +248,11 @@ class LinuxFreeze(Command): def create_tarfile(self): self.info('Creating archive...') - dist = os.path.join(self.d(self.SRC), 'dist', - '%s-%s-%s.tar.bz2'%(__appname__, __version__, arch)) - with tarfile.open(dist, mode='w:bz2', - format=tarfile.PAX_FORMAT) as tf: + base = self.j(self.d(self.SRC), 'dist') + if not os.path.exists(base): + os.mkdir(base) + dist = os.path.join(base, '%s-%s-%s.tar'%(__appname__, __version__, arch)) + with tarfile.open(dist, mode='w', format=tarfile.PAX_FORMAT) as tf: cwd = os.getcwd() os.chdir(self.base) try: @@ -248,6 +260,8 @@ class LinuxFreeze(Command): tf.add(x) finally: os.chdir(cwd) + subprocess.check_call(['bzip2', '-f', '-9', dist]) + dist += '.bz2' self.info('Archive %s created: %.2f MB'%(dist, os.stat(dist).st_size/(1024.**2))) @@ -259,8 +273,8 @@ class LinuxFreeze(Command): sources = [self.j(base, x) for x in ['util.c']] headers = [self.j(base, x) for x in ['util.h']] objects = [self.j(self.obj_dir, self.b(x)+'.o') for x in sources] - cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%self.py_ver - cflags = cflags.split() + ['-I/usr/include/python'+self.py_ver] + cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%py_ver + cflags = cflags.split() + ['-I%s/include/python%s' % (SW, py_ver)] for src, obj in zip(sources, objects): if not self.newer(obj, headers+[src, __file__]): continue @@ -270,7 +284,7 @@ class LinuxFreeze(Command): dll = self.j(self.lib_dir, 'libcalibre-launcher.so') if self.newer(dll, objects): cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \ - ['-lpython'+self.py_ver] + ['-L%s/lib'%SW, '-lpython'+py_ver] self.info('Linking libcalibre-launcher.so') self.run_builder(cmd) @@ -296,7 +310,6 @@ class LinuxFreeze(Command): dest = self.j(self.obj_dir, bname+'.o') if self.newer(dest, [src, __file__]+headers): - self.info('Compiling', bname) cmd = ['gcc'] + xflags + [src, '-o', dest] self.run_builder(cmd, verbose=False) exe = self.j(self.bin_dir, bname) @@ -306,7 +319,6 @@ class LinuxFreeze(Command): stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH) if self.newer(exe, [dest, __file__]): - self.info('Linking', bname) cmd = ['gcc', '-O2', '-o', exe, dest, @@ -316,6 +328,21 @@ class LinuxFreeze(Command): self.run_builder(cmd, verbose=False) + def strip_files(self): + from calibre import walk + files = {self.j(self.bin_dir, x) for x in os.listdir(self.bin_dir)} | { + x for x in { + self.j(self.d(self.bin_dir), x) for x in os.listdir(self.bin_dir)} if os.path.exists(x)} + for x in walk(self.lib_dir): + x = os.path.realpath(x) + if x not in files and is_elf(x): + files.add(x) + self.info('Stripping %d files...' % len(files)) + before = sum(os.path.getsize(x) for x in files) + strip_files(files) + after = sum(os.path.getsize(x) for x in files) + self.info('Stripped %.1f MB' % ((before - after)/(1024*1024.))) + def create_site_py(self): # {{{ with open(self.j(self.py_dir, 'site.py'), 'wb') as f: f.write(textwrap.dedent('''\ diff --git a/src/calibre/test_build.py b/src/calibre/test_build.py index 3a6d21415c..faec9cdd6a 100644 --- a/src/calibre/test_build.py +++ b/src/calibre/test_build.py @@ -202,6 +202,12 @@ def test_podofo(): from calibre.utils.podofo import test_podofo as dotest dotest() +def test_terminal(): + import readline, curses + curses.setupterm() + del readline + print ('readline and curses OK!') + def test(): if iswindows: test_dlls() @@ -225,6 +231,8 @@ def test(): if iswindows: test_wpd() test_winutil() + else: + test_terminal() if islinux: test_dbus()