From df9284b30d3faae533a3dcb15bfc6be8c539064e Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sun, 6 Sep 2009 19:45:40 -0600 Subject: [PATCH] build and develop commands done --- .bzrignore | 1 + INSTALL | 34 ++ README | 17 + pyqtdistutils.py | 279 -------------- session.vim | 2 +- setup.py | 340 +++-------------- setup/__init__.py | 193 ++++++++++ setup/build_environment.py | 101 +++++ setup/commands.py | 71 ++++ setup/extensions.py | 348 ++++++++++++++++++ setup/install.py | 92 +++++ .../translations => setup}/pygettext.py | 5 +- setup/translations.py | 218 +++++++++++ src/calibre/constants.py | 19 +- src/calibre/startup.py | 6 + src/calibre/utils/ipc/launch.py | 4 +- src/calibre/utils/localization.py | 18 + src/calibre/utils/resources.py | 17 + .../web/feeds/recipes/recipe_ars_technica.py | 165 +++++---- upload.py | 211 +++-------- 20 files changed, 1326 insertions(+), 815 deletions(-) create mode 100644 INSTALL create mode 100644 README delete mode 100644 pyqtdistutils.py create mode 100644 setup/__init__.py create mode 100644 setup/build_environment.py create mode 100644 setup/commands.py create mode 100644 setup/extensions.py create mode 100644 setup/install.py rename {src/calibre/translations => setup}/pygettext.py (99%) create mode 100644 setup/translations.py create mode 100644 src/calibre/utils/localization.py create mode 100644 src/calibre/utils/resources.py diff --git a/.bzrignore b/.bzrignore index 0a44159b1e..081ed9f5ca 100644 --- a/.bzrignore +++ b/.bzrignore @@ -13,6 +13,7 @@ src/calibre/manual/cli/ build dist docs +resources nbproject/ src/calibre/gui2/pictureflow/Makefile.Debug src/calibre/gui2/pictureflow/Makefile.Release diff --git a/INSTALL b/INSTALL new file mode 100644 index 0000000000..623918f09e --- /dev/null +++ b/INSTALL @@ -0,0 +1,34 @@ +calibre supports installation from source only on Linux. On Windows and OS X use the provided installers and usethe facilities provided by calibre-debug to hack on the calibre source. + +On Linux, there are two kinds of installation from source possible. Note that both kinds require lots of dependencies as well as a full development environment (compilers, headers files, etc.) + +All installation related functions are accessed by the command:: + + python setup.py + +Install +========== + +The first type of install will actually "install" calibre to your computer by putting its files into the system in the following locations: + + - Binaries (actually python wrapper scripts) in /bin + - Python and C modules in /lib/calibre + - Resources like icons, etc. in /share/calibre + +This type of install can be run by the command:: + + sudo python setup.py install + + is normally the installation prefix of python, usually /usr. It can be controlled by the --prefix +option. + +Develop +============= + +This type of install is designed to let you run calibre from your home directory, making it easy to hack on it. It will only install binaries into /usr/bin, but all the actual code and resource files will be read from the calibre source tree in your home drectory (or wherever you choose to put it). + +This type of install can be run with the command:: + + sudo python setup.py develop + + diff --git a/README b/README new file mode 100644 index 0000000000..6697b3f7f9 --- /dev/null +++ b/README @@ -0,0 +1,17 @@ +calibre is an e-book library manager. It can view, convert and catalog e-books \ +in most of the major e-book formats. It can also talk to e-book reader \ +devices. It can go out to the internet and fetch metadata for your books. \ +It can download newspapers and convert them into e-books for convenient \ +reading. It is cross platform, running on Linux, Windows and OS X. + +For screenshots: https://calibre.kovidgoyal.net/wiki/Screenshots + +For installation/usage instructions please see +http://calibre.kovidgoyal.net + +For source code access: +bzr branch lp:calibre + +To update your copy of the source code: +bzr merge + diff --git a/pyqtdistutils.py b/pyqtdistutils.py deleted file mode 100644 index 550bf46da7..0000000000 --- a/pyqtdistutils.py +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Build PyQt extensions. Integrates with distutils (but uses the PyQt build system). -''' -from distutils.core import Extension as _Extension -from distutils.command.build_ext import build_ext as _build_ext -from distutils.dep_util import newer_group -from distutils import log -from distutils.spawn import find_executable - -import sipconfig, os, sys, string, glob, shutil -from PyQt4 import pyqtconfig -iswindows = 'win32' in sys.platform -isosx = 'darwin' in sys.platform -QMAKE = '/Volumes/sw/qt/bin/qmake' if isosx else 'qmake' -if find_executable('qmake-qt4'): - QMAKE = find_executable('qmake-qt4') -elif find_executable('qmake'): - QMAKE = find_executable('qmake') -QMAKE = os.environ.get('QMAKE', QMAKE) -WINDOWS_PYTHON = ['C:/Python26/libs'] -OSX_SDK = '/Developer/SDKs/MacOSX10.5.sdk' -if not os.path.exists(OSX_SDK): - OSX_SDK = '/Developer/SDKs/MacOSX10.4u.sdk' - -leopard_build = '10.5' in OSX_SDK - -def replace_suffix(path, new_suffix): - return os.path.splitext(path)[0] + new_suffix - -class Extension(_Extension): - - def __init__(self, *args, **kwargs): - if leopard_build: - prev = kwargs.get('extra_compile_args', []) - prev.extend(['-arch', 'ppc64', '-arch', 'x86_64']) - kwargs['extra_compile_args'] = prev - _Extension.__init__(self, *args, **kwargs) - - -if iswindows: - from distutils import msvc9compiler - msvc = msvc9compiler.MSVCCompiler() - msvc.initialize() - nmake = msvc.find_exe('nmake.exe') - rc = msvc.find_exe('rc.exe') - -class PyQtExtension(Extension): - - def __init__(self, name, sources, sip_sources, **kw): - ''' - :param sources: Qt .cpp and .h files needed for this extension - :param sip_sources: List of .sip files this extension depends on. The - first .sip file will be used toactually build the extension. - ''' - self.module_makefile = pyqtconfig.QtGuiModuleMakefile - self.sip_sources = map(lambda x: x.replace('/', os.sep), sip_sources) - Extension.__init__(self, name, sources, **kw) - - -class build_ext(_build_ext): - - def make(self, makefile): - make = nmake if iswindows else 'make' - self.spawn([make, '-f', makefile]) - - def build_qt_objects(self, ext, bdir): - if not iswindows: - bdir = os.path.join(bdir, 'qt') - if not os.path.exists(bdir): - os.makedirs(bdir) - cwd = os.getcwd() - sources = map(os.path.abspath, ext.sources) - os.chdir(bdir) - archs = 'x86_64 ppc64' if leopard_build else 'x86 ppc' - try: - headers = set([f for f in sources if f.endswith('.h')]) - sources = set(sources) - headers - name = ext.name.rpartition('.')[-1] - pro = '''\ -TARGET = %s -TEMPLATE = lib -HEADERS = %s -SOURCES = %s -VERSION = 1.0.0 -CONFIG += %s -'''%(name, ' '.join(headers), ' '.join(sources), archs) - open(name+'.pro', 'wb').write(pro) - self.spawn([QMAKE, '-o', 'Makefile.qt', name+'.pro']) - if leopard_build: - raw = open('Makefile.qt', 'rb').read() - open('Makefile.qt', 'wb').write(raw.replace('ppc64', 'x86_64')) - self.make('Makefile.qt') - pat = 'release\\*.obj' if iswindows else '*.o' - return map(os.path.abspath, glob.glob(pat)) - finally: - os.chdir(cwd) - - def build_sbf(self, sip, sbf, bdir): - print '\tBuilding sbf...' - sip_bin = self.sipcfg.sip_bin - pyqt_sip_flags = [] - if hasattr(self, 'pyqtcfg'): - pyqt_sip_flags += ['-I', self.pyqtcfg.pyqt_sip_dir] - pyqt_sip_flags += self.pyqtcfg.pyqt_sip_flags.split() - self.spawn([sip_bin, - "-c", bdir, - "-b", sbf, - ] + pyqt_sip_flags + - [sip]) - - def build_pyqt(self, bdir, sbf, ext, qtobjs, headers): - makefile = ext.module_makefile(configuration=self.pyqtcfg, - build_file=sbf, dir=bdir, - makefile='Makefile.pyqt', - universal=OSX_SDK, qt=1) - makefile.extra_libs = ext.libraries - makefile.extra_lib_dirs = ext.library_dirs - makefile.extra_cxxflags = ext.extra_compile_args - - if 'win32' in sys.platform: - makefile.extra_lib_dirs += WINDOWS_PYTHON - makefile.extra_include_dirs = list(set(map(os.path.dirname, headers))) - makefile.extra_include_dirs += ext.include_dirs - makefile.extra_lflags += qtobjs - makefile.generate() - cwd = os.getcwd() - os.chdir(bdir) - if leopard_build: - mf = 'Makefile.pyqt' - raw = open(mf, 'rb').read() - raw = raw.replace('ppc64 x86_64', 'x86_64') - for x in ('ppc64', 'ppc', 'i386'): - raw = raw.replace(x, 'x86_64') - open(mf, 'wb').write(raw) - try: - self.make('Makefile.pyqt') - finally: - os.chdir(cwd) - - - - def build_extension(self, ext): - self.inplace = True # Causes extensions to be built in the source tree - - fullname = self.get_ext_fullname(ext.name) - if self.inplace: - # ignore build-lib -- put the compiled extension into - # the source tree along with pure Python modules - - modpath = string.split(fullname, '.') - package = string.join(modpath[0:-1], '.') - base = modpath[-1] - - build_py = self.get_finalized_command('build_py') - package_dir = build_py.get_package_dir(package) - ext_filename = os.path.join(package_dir, - self.get_ext_filename(base)) - else: - ext_filename = os.path.join(self.build_lib, - self.get_ext_filename(fullname)) - bdir = os.path.abspath(os.path.join(self.build_temp, fullname)) - if not os.path.exists(bdir): - os.makedirs(bdir) - - if not isinstance(ext, PyQtExtension): - if not iswindows: - return _build_ext.build_extension(self, ext) - - c_sources = [f for f in ext.sources if os.path.splitext(f)[1].lower() in ('.c', '.cpp', '.cxx')] - compile_args = '/c /nologo /Ox /MD /W3 /EHsc /DNDEBUG'.split() - compile_args += ext.extra_compile_args - self.swig_opts = '' - inc_dirs = self.include_dirs + [x.replace('/', '\\') for x in ext.include_dirs] - cc = [msvc.cc] + compile_args + ['-I%s'%x for x in list(set(inc_dirs))] - objects = [] - for f in c_sources: - o = os.path.join(bdir, os.path.basename(f)+'.obj') - objects.append(o) - inf = '/Tp' if f.endswith('.cpp') else '/Tc' - compiler = cc + [inf+f, '/Fo'+o] - self.spawn(compiler) - out = os.path.join(bdir, base+'.pyd') - linker = [msvc.linker] + '/DLL /nologo /INCREMENTAL:NO'.split() - linker += ['/LIBPATH:'+x for x in self.library_dirs+ext.library_dirs] - linker += [x+'.lib' for x in ext.libraries] - linker += ['/EXPORT:init'+base] + objects + ['/OUT:'+out] - self.spawn(linker) - for src in (out, out+'.manifest'): - shutil.copyfile(src, os.path.join('src', 'calibre', 'plugins', os.path.basename(src))) - return - - - - if not os.path.exists(bdir): - os.makedirs(bdir) - ext.sources2 = map(os.path.abspath, ext.sources) - qt_dir = 'qt\\release' if iswindows else 'qt' - objects = set(map(lambda x: os.path.join(bdir, qt_dir, replace_suffix(os.path.basename(x), '.o')), - [s for s in ext.sources2 if not s.endswith('.h')])) - newer = False - for object in objects: - if newer_group(ext.sources2, object, missing='newer'): - newer = True - break - headers = [f for f in ext.sources2 if f.endswith('.h')] - if self.force or newer: - log.info('building \'%s\' extension', ext.name) - objects = self.build_qt_objects(ext, bdir) - - self.sipcfg = sipconfig.Configuration() - self.pyqtcfg = pyqtconfig.Configuration() - sbf_sources = [] - for sip in ext.sip_sources: - sipbasename = os.path.basename(sip) - sbf = os.path.join(bdir, replace_suffix(sipbasename, ".sbf")) - sbf_sources.append(sbf) - if self.force or newer_group(ext.sip_sources, sbf, 'newer'): - self.build_sbf(sip, sbf, bdir) - generated_sources = [] - for sbf in sbf_sources: - generated_sources += self.get_sip_output_list(sbf, bdir) - - depends = generated_sources + list(objects) - mod = os.path.join(bdir, os.path.basename(ext_filename)) - - if self.force or newer_group(depends, mod, 'newer'): - self.build_pyqt(bdir, sbf_sources[0], ext, list(objects), headers) - - if self.force or newer_group([mod], ext_filename, 'newer'): - if os.path.exists(ext_filename): - os.unlink(ext_filename) - shutil.copyfile(mod, ext_filename) - shutil.copymode(mod, ext_filename) - - - if self.force or newer_group([mod], ext_filename, 'newer'): - if os.path.exists(ext_filename): - os.unlink(ext_filename) - shutil.copyfile(mod, ext_filename) - shutil.copymode(mod, ext_filename) - - - def get_sip_output_list(self, sbf, bdir): - """ - Parse the sbf file specified to extract the name of the generated source - files. Make them absolute assuming they reside in the temp directory. - """ - for L in file(sbf): - key, value = L.split("=", 1) - if key.strip() == "sources": - out = [] - for o in value.split(): - out.append(os.path.join(bdir, o)) - return out - - raise RuntimeError, "cannot parse SIP-generated '%s'" % sbf - - def run_sip(self, sip_files): - sip_bin = self.sipcfg.sip_bin - sip_sources = [i[0] for i in sip_files] - generated_sources = [] - for sip, sbf in sip_files: - if not (self.force or newer_group(sip_sources, sbf, 'newer')): - log.info(sbf + ' is up to date') - continue - self.spawn([sip_bin, - "-c", self.build_temp, - "-b", sbf, - '-I', self.pyqtcfg.pyqt_sip_dir, - ] + self.pyqtcfg.pyqt_sip_flags.split()+ - [sip]) - generated_sources += self.get_sip_output_list(sbf) - return generated_sources - diff --git a/session.vim b/session.vim index f33590c838..50c2d5285e 100644 --- a/session.vim +++ b/session.vim @@ -1,5 +1,5 @@ " Project wide builtins -let g:pyflakes_builtins += ["dynamic_property", "__"] +let g:pyflakes_builtins += ["dynamic_property", "__", "P"] python << EOFPY import os diff --git a/setup.py b/setup.py index d8072af50d..76ddd8b43a 100644 --- a/setup.py +++ b/setup.py @@ -1,298 +1,64 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai from __future__ import with_statement + __license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal ' - -import sys, re, os, subprocess -sys.path.append('src') -iswindows = re.search('win(32|64)', sys.platform) -isosx = 'darwin' in sys.platform -islinux = not isosx and not iswindows -src = open('src/calibre/constants.py', 'rb').read() -VERSION = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1) -APPNAME = re.search(r'__appname__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1) -print 'Setup', APPNAME, 'version:', VERSION - -epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).search(open('src/%s/linux.py'%APPNAME, 'rb').read()).group(1) -entry_points = eval(epsrc, {'__appname__': APPNAME}) - -def _ep_to_script(ep, base='src'): - return (base+os.path.sep+re.search(r'.*=\s*(.*?):', ep).group(1).replace('.', '/')+'.py').strip() +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' -scripts = { - 'console' : [_ep_to_script(i) for i in entry_points['console_scripts']], - 'gui' : [_ep_to_script(i) for i in entry_points['gui_scripts']], - } +import sys, os, optparse -def _ep_to_basename(ep): - return re.search(r'\s*(.*?)\s*=', ep).group(1).strip() -basenames = { - 'console' : [_ep_to_basename(i) for i in entry_points['console_scripts']], - 'gui' : [_ep_to_basename(i) for i in entry_points['gui_scripts']], - } +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) -def _ep_to_module(ep): - return re.search(r'.*=\s*(.*?)\s*:', ep).group(1).strip() -main_modules = { - 'console' : [_ep_to_module(i) for i in entry_points['console_scripts']], - 'gui' : [_ep_to_module(i) for i in entry_points['gui_scripts']], - } +import setup.commands as commands +from setup import prints, get_warnings -def _ep_to_function(ep): - return ep[ep.rindex(':')+1:].strip() -main_functions = { - 'console' : [_ep_to_function(i) for i in entry_points['console_scripts']], - 'gui' : [_ep_to_function(i) for i in entry_points['gui_scripts']], - } +def check_version_info(): + vi = sys.version_info + if vi[0] == 2 and vi[1] > 5: + return None + return 'calibre requires python >= 2.6' -def setup_mount_helper(): - def warn(): - print 'WARNING: Failed to compile mount helper. Auto mounting of', - print 'devices will not work' +def option_parser(): + parser = optparse.OptionParser() + return parser - if os.geteuid() != 0: - return warn() - import stat - src = os.path.join('src', 'calibre', 'devices', 'linux_mount_helper.c') - dest = '/usr/bin/calibre-mount-helper' - p = subprocess.Popen(['gcc', '-Wall', src, '-o', dest]) - ret = p.wait() - if ret != 0: - return warn() - os.chown(dest, 0, 0) - os.chmod(dest, - stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) +def main(args=sys.argv): + if len(args) == 1 or args[1] in ('-h', '--help'): + print 'Usage: python', args[0], 'command', '[options]' + print '\nWhere command is one of:', ', '.join(commands.__all__) + print '\nTo get help on a particular command, run:' + print '\tpython', args[0], 'command -h' + return 1 + + command = args[1] + if command not in commands.__all__: + print command, 'is not a recognized command.' + print 'Valid commands:', ', '.join(commands.__all__) + return 1 + + command = getattr(commands, command) + + parser = option_parser() + command.add_all_options(parser) + lines = parser.usage.splitlines() + lines[0] = 'Usage: python setup.py %s [options]'%args[1] + parser.set_usage('\n'.join(lines)) + + opts, args = parser.parse_args(args) + command.run_all(opts) + + warnings = get_warnings() + if warnings: + print + prints('There were', len(warnings), 'warning(s):') + print + for args, kwargs in warnings: + prints(*args, **kwargs) + print + + return 0 if __name__ == '__main__': - from setuptools import setup, find_packages - from pyqtdistutils import PyQtExtension, build_ext, Extension, QMAKE - from upload import sdist, pot, build, build_py, manual, \ - resources, clean, gui, translations, update, \ - tag_release, upload_demo, build_linux, build_windows, \ - build_osx, upload_installers, upload_user_manual, \ - upload_to_pypi, stage3, stage2, stage1, upload, \ - upload_rss, betas, build_linux32, build_linux64, \ - build_osx64, get_translations - resources.SCRIPTS = {} - for x in ('console', 'gui'): - for name in basenames[x]: - resources.SCRIPTS[name] = x - - list(basenames['console']+basenames['gui']) - - entry_points['console_scripts'].append( - 'calibre_postinstall = calibre.linux:post_install') - optional = [] - def qmake_query(arg=''): - cmd = [QMAKE, '-query'] - if arg: - cmd += [arg] - return subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout.read() - qt_inc = qt_lib = None - qt_inc = qmake_query('QT_INSTALL_HEADERS').splitlines()[0] - qt_inc = qt_inc if qt_inc not in ('', '**Unknown**') and os.path.isdir(qt_inc) else None - qt_lib = qmake_query('QT_INSTALL_LIBS').splitlines()[0] - qt_lib = qt_lib if qt_lib not in ('', '**Unknown**') and os.path.isdir(qt_lib) else None - if qt_lib is None or qt_inc is None: - print '\n\nWARNING: Could not find QT librariers and headers.', - print 'Is qmake in your PATH?\n\n' - - - if iswindows: - optional.append(Extension('calibre.plugins.winutil', - sources=['src/calibre/utils/windows/winutil.c'], - libraries=['shell32', 'setupapi'], - include_dirs=os.environ.get('INCLUDE', - 'C:/WinDDK/6001.18001/inc/api/;' - 'C:/WinDDK/6001.18001/inc/crt/').split(';'), - extra_compile_args=['/X'] - )) - - poppler_inc = '/usr/include/poppler/qt4' - poppler_lib = '/usr/lib' - poppler_libs = [] - if iswindows: - poppler_inc = r'C:\cygwin\home\kovid\poppler\include\poppler\qt4' - poppler_lib = r'C:\cygwin\home\kovid\poppler\lib' - poppler_libs = ['QtCore4', 'QtGui4'] - if isosx: - poppler_inc = '/Volumes/sw/build/poppler-0.10.7/qt4/src' - poppler_lib = '/Users/kovid/poppler/lib' - poppler_inc = os.environ.get('POPPLER_INC_DIR', poppler_inc) - if os.path.exists(os.path.join(poppler_inc, 'poppler-qt4.h'))\ - and qt_lib is not None and qt_inc is not None: - optional.append(Extension('calibre.plugins.calibre_poppler', - sources=['src/calibre/utils/poppler/poppler.cpp'], - libraries=(['poppler', 'poppler-qt4']+poppler_libs), - library_dirs=[os.environ.get('POPPLER_LIB_DIR', - poppler_lib), qt_lib], - include_dirs=[poppler_inc, qt_inc])) - else: - print '\n\nWARNING: Poppler not found on your system. Various PDF related', - print 'functionality will not work. Use the POPPLER_INC_DIR and', - print 'POPPLER_LIB_DIR environment variables.\n\n' - - podofo_inc = '/usr/include/podofo' if islinux else \ - 'C:\\podofo\\include\\podofo' if iswindows else \ - '/usr/local/include/podofo' - podofo_lib = '/usr/lib' if islinux else r'C:\podofo' if iswindows else \ - '/usr/local/lib' - podofo_inc = os.environ.get('PODOFO_INC_DIR', podofo_inc) - if os.path.exists(os.path.join(podofo_inc, 'podofo.h')): - optional.append(Extension('calibre.plugins.podofo', - sources=['src/calibre/utils/podofo/podofo.cpp'], - libraries=['podofo'], - library_dirs=[os.environ.get('PODOFO_LIB_DIR', podofo_lib)], - include_dirs=[podofo_inc])) - else: - print '\n\nWARNING: PoDoFo not found on your system. Various PDF related', - print 'functionality will not work. Use the PODOFO_INC_DIR and', - print 'PODOFO_LIB_DIR environment variables.\n\n' - - fc_inc = '/usr/include/fontconfig' if islinux else \ - r'C:\cygwin\home\kovid\fontconfig\include\fontconfig' if iswindows else \ - '/Users/kovid/fontconfig/include/fontconfig' - fc_lib = '/usr/lib' if islinux else \ - r'C:\cygwin\home\kovid\fontconfig\lib' if iswindows else \ - '/Users/kovid/fontconfig/lib' - - fc_inc = os.environ.get('FC_INC_DIR', fc_inc) - fc_lib = os.environ.get('FC_LIB_DIR', fc_lib) - if not os.path.exists(os.path.join(fc_inc, 'fontconfig.h')): - print '\n\nERROR: fontconfig not found on your system.', - print 'Use the FC_INC_DIR and FC_LIB_DIR environment variables.\n\n' - raise SystemExit(1) - ext_modules = optional + [ - - Extension('calibre.plugins.fontconfig', - sources = ['src/calibre/utils/fonts/fontconfig.c'], - include_dirs = [fc_inc], - libraries=['fontconfig'], - library_dirs=[fc_lib]), - - Extension('calibre.plugins.lzx', - sources=['src/calibre/utils/lzx/lzxmodule.c', - 'src/calibre/utils/lzx/compressor.c', - 'src/calibre/utils/lzx/lzxd.c', - 'src/calibre/utils/lzx/lzc.c', - 'src/calibre/utils/lzx/lzxc.c'], - include_dirs=['src/calibre/utils/lzx']), - - Extension('calibre.plugins.msdes', - sources=['src/calibre/utils/msdes/msdesmodule.c', - 'src/calibre/utils/msdes/des.c'], - include_dirs=['src/calibre/utils/msdes']), - - Extension('calibre.plugins.cPalmdoc', - sources=['src/calibre/ebooks/compression/palmdoc.c']), - - PyQtExtension('calibre.plugins.pictureflow', - ['src/calibre/gui2/pictureflow/pictureflow.cpp', - 'src/calibre/gui2/pictureflow/pictureflow.h'], - ['src/calibre/gui2/pictureflow/pictureflow.sip'] - ) - ] - if isosx: - ext_modules.append(Extension('calibre.plugins.usbobserver', - sources=['src/calibre/devices/usbobserver/usbobserver.c'], - extra_link_args=['-framework', 'IOKit']) - ) - - if not iswindows: - plugins = ['plugins/%s.so'%(x.name.rpartition('.')[-1]) for x in ext_modules] - else: - plugins = ['plugins/%s.pyd'%(x.name.rpartition('.')[-1]) for x in ext_modules] + \ - ['plugins/%s.pyd.manifest'%(x.name.rpartition('.')[-1]) \ - for x in ext_modules if 'pictureflow' not in x.name] - - - setup( - name = APPNAME, - packages = find_packages('src'), - package_dir = { '' : 'src' }, - version = VERSION, - author = 'Kovid Goyal', - author_email = 'kovid@kovidgoyal.net', - url = 'http://%s.kovidgoyal.net'%APPNAME, - package_data = {'calibre':plugins}, - entry_points = entry_points, - zip_safe = False, - options = { 'bdist_egg' : {'exclude_source_files': True,}, }, - ext_modules = ext_modules, - description = - ''' - E-book management application. - ''', - long_description = - ''' - %s is an e-book library manager. It can view, convert and catalog e-books \ - in most of the major e-book formats. It can also talk to e-book reader \ - devices. It can go out to the internet and fetch metadata for your books. \ - It can download newspapers and convert them into e-books for convenient \ - reading. It is cross platform, running on Linux, Windows and OS X. - - For screenshots: https://%s.kovidgoyal.net/wiki/Screenshots - - For installation/usage instructions please see - http://%s.kovidgoyal.net - - For source code access: - bzr branch lp:%s - - To update your copy of the source code: - bzr merge - - '''%(APPNAME, APPNAME, APPNAME, APPNAME), - license = 'GPL', - classifiers = [ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Environment :: X11 Applications :: Qt', - 'Intended Audience :: Developers', - 'Intended Audience :: End Users/Desktop', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Natural Language :: English', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python', - 'Topic :: Software Development :: Libraries :: Python Modules', - 'Topic :: System :: Hardware :: Hardware Drivers' - ], - cmdclass = { - 'build_ext' : build_ext, - 'build' : build, - 'build_py' : build_py, - 'pot' : pot, - 'manual' : manual, - 'resources' : resources, - 'translations' : translations, - 'get_translations': get_translations, - 'gui' : gui, - 'clean' : clean, - 'sdist' : sdist, - 'update' : update, - 'tag_release' : tag_release, - 'upload_demo' : upload_demo, - 'build_linux' : build_linux, - 'build_linux32' : build_linux32, - 'build_linux64' : build_linux64, - 'build_windows' : build_windows, - 'build_osx' : build_osx, - 'build_osx64' : build_osx64, - 'upload_installers': upload_installers, - 'upload_user_manual': upload_user_manual, - 'upload_to_pypi': upload_to_pypi, - 'upload_rss' : upload_rss, - 'stage3' : stage3, - 'stage2' : stage2, - 'stage1' : stage1, - 'publish' : upload, - 'betas' : betas, - }, - ) - - if 'develop' in ' '.join(sys.argv) and islinux: - subprocess.check_call('calibre_postinstall --do-not-reload-udev-hal', shell=True) - setup_mount_helper() - if 'install' in sys.argv and islinux: - subprocess.check_call('calibre_postinstall', shell=True) - setup_mount_helper() + sys.exit(main()) diff --git a/setup/__init__.py b/setup/__init__.py new file mode 100644 index 0000000000..3be9df08c2 --- /dev/null +++ b/setup/__init__.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import sys, re, os + +iswindows = re.search('win(32|64)', sys.platform) +isosx = 'darwin' in sys.platform +islinux = not isosx and not iswindows +SRC = os.path.abspath('src') + +__version__ = __appname__ = modules = functions = basenames = scripts = None + +def initialize_constants(): + global __version__, __appname__, modules, functions, basenames, scripts + + src = open('src/calibre/constants.py', 'rb').read() + __version__ = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1) + __appname__ = re.search(r'__appname__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1) + epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).\ + search(open('src/calibre/linux.py', 'rb').read()).group(1) + entry_points = eval(epsrc, {'__appname__': __appname__}) + + def e2b(ep): + return re.search(r'\s*(.*?)\s*=', ep).group(1).strip() + + def e2s(ep, base='src'): + return (base+os.path.sep+re.search(r'.*=\s*(.*?):', ep).group(1).replace('.', '/')+'.py').strip() + + def e2m(ep): + return re.search(r'.*=\s*(.*?)\s*:', ep).group(1).strip() + + def e2f(ep): + return ep[ep.rindex(':')+1:].strip() + + basenames, functions, modules, scripts = {}, {}, {}, {} + for x in ('console', 'gui'): + y = x + '_scripts' + basenames[x] = list(map(e2b, entry_points[y])) + functions[x] = list(map(e2f, entry_points[y])) + modules[x] = list(map(e2m, entry_points[y])) + scripts[x] = list(map(e2s, entry_points[y])) + +initialize_constants() + +preferred_encoding = 'utf-8' + +def prints(*args, **kwargs): + ''' + Print unicode arguments safely by encoding them to preferred_encoding + Has the same signature as the print function from Python 3, except for the + additional keyword argument safe_encode, which if set to True will cause the + function to use repr when encoding fails. + ''' + file = kwargs.get('file', sys.stdout) + sep = kwargs.get('sep', ' ') + end = kwargs.get('end', '\n') + enc = preferred_encoding + safe_encode = kwargs.get('safe_encode', False) + for i, arg in enumerate(args): + if isinstance(arg, unicode): + try: + arg = arg.encode(enc) + except UnicodeEncodeError: + if not safe_encode: + raise + arg = repr(arg) + if not isinstance(arg, str): + try: + arg = str(arg) + except ValueError: + arg = unicode(arg) + if isinstance(arg, unicode): + try: + arg = arg.encode(enc) + except UnicodeEncodeError: + if not safe_encode: + raise + arg = repr(arg) + + file.write(arg) + if i != len(args)-1: + file.write(sep) + file.write(end) + +warnings = [] + +def get_warnings(): + return list(warnings) + +class Command(object): + + SRC = SRC + RESOURCES = os.path.join(os.path.dirname(SRC), 'resources') + description = '' + + sub_commands = [] + + def __init__(self): + self.d = os.path.dirname + self.j = os.path.join + self.a = os.path.abspath + self.b = os.path.basename + self.s = os.path.splitext + self.e = os.path.exists + self.real_uid = os.environ.get('SUDO_UID', None) + self.real_gid = os.environ.get('SUDO_GID', None) + self.real_user = os.environ.get('SUDO_USER', None) + + def drop_privileges(self): + if not islinux or isosx: + return + if self.real_user is not None: + self.info('Dropping privileges to those of', self.real_user+':', + self.real_uid) + if self.real_uid is not None: + os.seteuid(int(self.real_uid)) + #if self.real_gid is not None: + # os.setegid(int(self.real_gid)) + + def regain_privileges(self): + if not islinux or isosx: + return + if os.geteuid() != 0: + self.info('Trying to get root privileges') + os.seteuid(0) + #if os.getegid() != 0: + # os.setegid(0) + + def pre_sub_commands(self, opts): + pass + + def run_all(self, opts): + self.pre_sub_commands(opts) + for cmd in self.sub_commands: + self.info('Running', cmd.__class__.__name__) + cmd.run(opts) + + self.info('Running', self.__class__.__name__) + self.run(opts) + + def add_all_options(self, parser): + import setup.commands as commands + self.sub_commands = [getattr(commands, cmd) for cmd in + self.sub_commands] + for cmd in self.sub_commands: + cmd.add_options(parser) + + self.add_options(parser) + + + def run(self, opts): + raise NotImplementedError + + def add_options(self, parser): + pass + + def clean(self): + pass + + @classmethod + def newer(cls, targets, sources): + ''' + Return True if sources is newer that targets or if targets + does not exist. + ''' + if isinstance(targets, basestring): + targets = [targets] + if isinstance(sources, basestring): + sources = [sources] + for f in targets: + if not os.path.exists(f): + return True + ttimes = map(lambda x: os.stat(x).st_mtime, targets) + stimes = map(lambda x: os.stat(x).st_mtime, sources) + newest_source, oldest_target = max(stimes), min(ttimes) + return newest_source > oldest_target + + def info(self, *args, **kwargs): + prints(*args, **kwargs) + sys.stdout.flush() + + def warn(self, *args, **kwargs): + print '\n'+'_'*20, 'WARNING','_'*20 + prints(*args, **kwargs) + print '_'*50 + warnings.append((args, kwargs)) + sys.stdout.flush() + diff --git a/setup/build_environment.py b/setup/build_environment.py new file mode 100644 index 0000000000..29a4577072 --- /dev/null +++ b/setup/build_environment.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os +from distutils.spawn import find_executable + +from PyQt4 import pyqtconfig + +from setup import isosx, iswindows + +OSX_SDK = '/Developer/SDKs/MacOSX10.5.sdk' +if not os.path.exists(OSX_SDK): + OSX_SDK = '/Developer/SDKs/MacOSX10.4u.sdk' +leopard_build = '10.5' in OSX_SDK + +os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.5' if leopard_build else '10.4' + +NMAKE = RC = msvc = MT = win_inc = win_lib = None +if iswindows: + from distutils import msvc9compiler + msvc = msvc9compiler.MSVCCompiler() + msvc.initialize() + NMAKE = msvc.find_exe('nmake.exe') + RC = msvc.find_exe('rc.exe') + SDK = os.environ.get('WINSDK', r'C:\Program Files\Microsoft SDKs\Windows\v6.0A') + win_inc = os.environ['include'].split(';') + win_lib = os.environ['lib'].split(';') + for p in win_inc: + if 'SDK' in p: + MT = os.path.join(os.path.dirname(p), 'bin', 'mt.exe') + MT = os.path.join(SDK, 'bin', 'mt.exe') + +QMAKE = '/Volumes/sw/qt/bin/qmake' if isosx else 'qmake' +if find_executable('qmake-qt4'): + QMAKE = find_executable('qmake-qt4') +elif find_executable('qmake'): + QMAKE = find_executable('qmake') +QMAKE = os.environ.get('QMAKE', QMAKE) + + +pyqt = pyqtconfig.Configuration() + +qt_inc = pyqt.qt_inc_dir +qt_lib = pyqt.qt_lib_dir + +fc_inc = '/usr/include/fontconfig' +fc_lib = '/usr/lib' +poppler_inc = '/usr/include/poppler/qt4' +poppler_lib = '/usr/lib' +poppler_libs = [] +podofo_inc = '/usr/include/podofo' +podofo_lib = '/usr/lib' + +if iswindows: + fc_inc = r'C:\cygwin\home\kovid\fontconfig\include\fontconfig' + fc_lib = r'C:\cygwin\home\kovid\fontconfig\lib' + poppler_inc = r'C:\cygwin\home\kovid\poppler\include\poppler\qt4' + poppler_lib = r'C:\cygwin\home\kovid\poppler\lib' + poppler_libs = ['QtCore4', 'QtGui4'] + podofo_inc = 'C:\\podofo\\include\\podofo' + podofo_lib = r'C:\podofo' + +if isosx: + fc_inc = '/Users/kovid/fontconfig/include/fontconfig' + fc_lib = '/Users/kovid/fontconfig/lib' + poppler_inc = '/Volumes/sw/build/poppler-0.10.7/qt4/src' + poppler_lib = '/Users/kovid/poppler/lib' + podofo_inc = '/usr/local/include/podofo' + podofo_lib = '/usr/local/lib' + + +fc_inc = os.environ.get('FC_INC_DIR', fc_inc) +fc_lib = os.environ.get('FC_LIB_DIR', fc_lib) +fc_error = None if os.path.exists(os.path.join(fc_inc, 'fontconfig.h')) else \ + ('fontconfig header files not found on your system. ' + 'Try setting the FC_INC_DIR and FC_LIB_DIR environment ' + 'variables.') + + +poppler_inc = os.environ.get('POPPLER_INC_DIR', poppler_inc) +poppler_lib = os.environ.get('POPPLER_LIB_DIR', poppler_lib) +poppler_error = None if os.path.exists(os.path.join(poppler_inc, + 'poppler-qt4.h')) else \ + ('Poppler not found on your system. Various PDF related', + ' functionality will not work. Use the POPPLER_INC_DIR and', + ' POPPLER_LIB_DIR environment variables.') + + +podofo_lib = os.environ.get('PODOFO_LIB_DIR', podofo_lib) +podofo_inc = os.environ.get('PODOFO_INC_DIR', podofo_inc) +podofo_error = None if os.path.exists(os.path.join(podofo_inc, 'podofo.h')) else \ + ('PoDoFo not found on your system. Various PDF related', + ' functionality will not work. Use the PODOFO_INC_DIR and', + ' PODOFO_LIB_DIR environment variables.') + + diff --git a/setup/commands.py b/setup/commands.py new file mode 100644 index 0000000000..721caa165c --- /dev/null +++ b/setup/commands.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +__all__ = [ + 'pot', 'translations', 'get_translations', 'iso639', + 'build', + 'develop', + 'clean' + ] + +import os, shutil + +from setup.translations import POT, GetTranslations, Translations, ISO639 +from setup import Command +pot = POT() +translations = Translations() +get_translations = GetTranslations() +iso639 = ISO639() + +from setup.extensions import Build +build = Build() + +from setup.install import Develop +develop = Develop() + +class Clean(Command): + + description='''Delete all computer generated files in the source tree''' + + sub_commands = __all__ + + def add_options(self, parser): + opt = parser.remove_option('--only') + help = 'Only run clean for the specified command. Choices: '+\ + ', '.join(__all__) + parser.add_option('-1', '--only', default='all', + choices=__all__+['all'], help=help) + + def run_all(self, opts): + self.info('Cleaning...') + only = None if opts.only == 'all' else commands[opts.only] + for cmd in self.sub_commands: + if only is not None and only is not cmd: + continue + self.info('\tCleaning', command_names[cmd]) + cmd.clean() + + def clean(self): + for root, _, files in os.walk(self.d(self.SRC)): + for name in files: + for t in ('.pyc', '.pyo', '~', '.swp', '.swo'): + if name.endswith(t): + os.remove(os.path.join(root, name)) + break + + for dir in ('dist', os.path.join('src', 'calibre.egg-info')): + shutil.rmtree(dir, ignore_errors=True) + +clean = Clean() + + +commands = {} +for x in __all__: + commands[x] = locals()[x] + +command_names = dict(zip(commands.values(), commands.keys())) diff --git a/setup/extensions.py b/setup/extensions.py new file mode 100644 index 0000000000..25b4979022 --- /dev/null +++ b/setup/extensions.py @@ -0,0 +1,348 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import textwrap, os, shlex, subprocess, glob, shutil +from distutils import sysconfig + +from PyQt4.pyqtconfig import QtGuiModuleMakefile + +from setup import Command, islinux, isosx, SRC, iswindows +from setup.build_environment import fc_inc, fc_lib, qt_inc, qt_lib, \ + fc_error, poppler_libs, poppler_lib, poppler_inc, podofo_inc, \ + podofo_lib, podofo_error, poppler_error, pyqt, OSX_SDK, NMAKE, \ + leopard_build, QMAKE, msvc, MT, win_inc, win_lib + +isunix = islinux or isosx + +make = 'make' if isunix else NMAKE + +class Extension(object): + + def absolutize(self, paths): + return [x if os.path.isabs(x) else os.path.join(SRC, x.replace('/', + os.sep)) for x in paths] + + + def __init__(self, name, sources, **kwargs): + self.name = name + self.needs_cxx = bool([1 for x in sources if os.path.splitext(x)[1] in + ('.cpp', '.c++', '.cxx')]) + self.sources = self.absolutize(sources) + self.headers = self.absolutize(kwargs.get('headers', [])) + self.sip_files = self.absolutize(kwargs.get('sip_files', [])) + self.inc_dirs = self.absolutize(kwargs.get('inc_dirs', [])) + self.lib_dirs = self.absolutize(kwargs.get('lib_dirs', [])) + self.error = kwargs.get('error', None) + self.libraries = kwargs.get('libraries', []) + self.cflags = kwargs.get('cflags', []) + self.ldflags = kwargs.get('ldflags', []) + self.optional = kwargs.get('optional', False) + +extensions = [ + Extension('lzx', + ['calibre/utils/lzx/lzxmodule.c', + 'calibre/utils/lzx/compressor.c', + 'calibre/utils/lzx/lzxd.c', + 'calibre/utils/lzx/lzc.c', + 'calibre/utils/lzx/lzxc.c'], + headers=['calibre/utils/lzx/msstdint.h', + 'calibre/utils/lzx/lzc.h', + 'calibre/utils/lzx/lzxmodule.h', + 'calibre/utils/lzx/system.h', + 'calibre/utils/lzx/lzxc.h', + 'calibre/utils/lzx/lzxd.h', + 'calibre/utils/lzx/mspack.h'], + inc_dirs=['calibre/utils/lzx']), + + Extension('fontconfig', + ['calibre/utils/fonts/fontconfig.c'], + inc_dirs = [fc_inc], + libraries=['fontconfig'], + lib_dirs=[fc_lib], + error=fc_error), + + Extension('msdes', + ['calibre/utils/msdes/msdesmodule.c', + 'calibre/utils/msdes/des.c'], + headers=['calibre/utils/msdes/spr.h', + 'calibre/utils/msdes/d3des.h'], + inc_dirs=['calibre/utils/msdes']), + + Extension('cPalmdoc', + ['calibre/ebooks/compression/palmdoc.c']), + + Extension('calibre_poppler', + ['calibre/utils/poppler/poppler.cpp'], + libraries=(['poppler', 'poppler-qt4']+poppler_libs), + lib_dirs=[os.environ.get('POPPLER_LIB_DIR', + poppler_lib), qt_lib], + inc_dirs=[poppler_inc, qt_inc], + error=poppler_error, + optional=True), + + Extension('podofo', + ['calibre/utils/podofo/podofo.cpp'], + libraries=['podofo'], + lib_dirs=[podofo_lib], + inc_dirs=[podofo_inc], + error=podofo_error), + + Extension('pictureflow', + ['calibre/gui2/pictureflow/pictureflow.cpp'], + inc_dirs = ['calibre/gui2/pictureflow'], + headers = ['calibre/gui2/pictureflow/pictureflow.h'], + sip_files = ['calibre/gui2/pictureflow/pictureflow.sip'] + ) + + ] + +if iswindows: + extensions.append(Extension('winutil', + ['calibre/utils/windows/winutil.c'], + libraries=['shell32', 'setupapi'], + include_dirs=os.environ.get('INCLUDE', + 'C:/WinDDK/6001.18001/inc/api/;' + 'C:/WinDDK/6001.18001/inc/crt/').split(';'), + cflags=['/X'] + )) +if isosx: + extensions.append(Extension('usbobserver', + ['calibre/devices/usbobserver/usbobserver.c'], + ldflags=['-framework', 'IOKit']) + ) + + +if isunix: + cc = os.environ.get('CC', 'gcc') + cxx = os.environ.get('CXX', 'g++') + cflags = '-O3 -Wall -DNDEBUG -fPIC -fno-strict-aliasing -pipe'.split() + ldflags = ['-Wall'] + cflags += shlex.split(os.environ.get('CFLAGS', '')) + ldflags += shlex.split(os.environ.get('LDFLAGS', '')) + +if islinux: + cflags.append('-pthread') + ldflags.append('-shared') + cflags.append('-I'+sysconfig.get_python_inc()) + ldflags.append('-lpython'+sysconfig.get_python_version()) + + +if isosx: + x, p = ('x86_64', 'ppc64') if leopard_build else ('i386', 'ppc') + archs = ['-arch', x, '-arch', p, '-isysroot', + OSX_SDK] + cflags.extend(archs) + ldflags.extend(archs) + ldflags.extend('-bundle -undefined dynamic_lookup'.split()) + cflags.extend(['-fno-common', '-dynamic']) + cflags.append('-I'+sysconfig.get_python_inc()) + + +if iswindows: + cc = cxx = msvc.cc + cflags = '/c /nologo /Ox /MD /W3 /EHsc /DNDEBUG'.split() + ldflags = '/DLL /nologo /INCREMENTAL:NO'.split() + for p in win_inc: + cflags.append('-I'+p) + for p in win_lib: + ldflags.append('/LIBPATH:'+p) + cflags.append('-I%s'%sysconfig.get_python_inc()) + ldflags.append('/LIBPATH:'+os.path.join(sysconfig.PREFIX, 'libs')) + + +class Build(Command): + + def add_options(self, parser): + parser.set_usage(parser.usage + textwrap.dedent(''' + + calibre depends on several python extensions written in C/C++. + This command will compile them. You can influence the compile + process by several environment variables, listed below: + + CC - C Compiler defaults to gcc + CXX - C++ Compiler, defaults to g++ + CFLAGS - Extra compiler flags + LDFLAGS - Extra linker flags + + FC_INC_DIR - fontconfig header files + FC_LIB_DIR - fontconfig library + + POPPLER_INC_DIR - poppler header files + POPPLER_LIB_DIR - poppler-qt4 library + + PODOFO_INC_DIR - podofo header files + PODOFO_LIB_DIR - podofo library files + + QMAKE - Path to qmake + VS90COMNTOOLS - Location of Microsoft Visual Studio 9 Tools + + ''')) + choices = [e.name for e in extensions]+['all'] + parser.add_option('-1', '--only', choices=choices, default='all', + help=('Build only the named extension. Available: '+ + ', '.join(choices)+'. Default:%default')) + + def run(self, opts): + self.obj_dir = os.path.join(os.path.dirname(SRC), 'build', 'objects') + if not os.path.exists(self.obj_dir): + os.makedirs(self.obj_dir) + for ext in extensions: + if opts.only != 'all' and opts.only != ext.name: + continue + if ext.error is not None: + if ext.optional: + self.warn(ext.error) + continue + else: + raise Exception(ext.error) + dest = self.dest(ext) + self.info('\n####### Building extension', ext.name, '#'*7) + self.build(ext, dest) + + def dest(self, ext): + ex = '.pyd' if iswindows else '.so' + return os.path.join(SRC, 'calibre', 'plugins', ext.name)+ex + + def inc_dirs_to_cflags(self, dirs): + return ['-I'+x for x in dirs] + + def lib_dirs_to_ldflags(self, dirs): + pref = '/LIBPATH:' if iswindows else '-L' + return [pref+x for x in dirs] + + def libraries_to_ldflags(self, dirs): + pref = '' if iswindows else '-l' + suff = '.lib' if iswindows else '' + return [pref+x+suff for x in dirs] + + def build(self, ext, dest): + if ext.sip_files: + return self.build_pyqt_extension(ext, dest) + compiler = cxx if ext.needs_cxx else cc + linker = msvc.linker if iswindows else compiler + objects = [] + einc = self.inc_dirs_to_cflags(ext.inc_dirs) + obj_dir = self.j(self.obj_dir, ext.name) + if not os.path.exists(obj_dir): + os.makedirs(obj_dir) + for src in ext.sources: + obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o') + objects.append(obj) + if self.newer(obj, [src]+ext.headers): + inf = '/Tp' if src.endswith('.cpp') else '/Tc' + sinc = [inf+src] if iswindows else ['-c', src] + oinc = ['/Fo'+obj] if iswindows else ['-o', obj] + cmd = [compiler] + cflags + ext.cflags + einc + sinc + oinc + self.info(' '.join(cmd)) + subprocess.check_call(cmd) + + dest = self.dest(ext) + elib = self.lib_dirs_to_ldflags(ext.lib_dirs) + xlib = self.libraries_to_ldflags(ext.libraries) + if self.newer(dest, objects): + print 'Linking', ext.name + cmd = [linker] + if iswindows: + cmd += ldflags + ext.ldflags + elib + xlib + \ + ['/EXPORT:init'+ext.name] + objects + ['/OUT:'+dest] + else: + cmd += objects + ['-o', dest] + ldflags + ext.ldflags + elib + xlib + print ' '.join(cmd) + subprocess.check_call(cmd) + if iswindows: + manifest = dest+'.manifest' + cmd = [MT, '-manifest', manifest, '-outputresource:%s;2'%dest] + self.info(*cmd) + subprocess.check_call(cmd) + os.remove(manifest) + for x in ('.exp', '.lib'): + x = os.path.splitext(dest)[0]+x + if os.path.exists(x): + os.remove(x) + + def build_qt_objects(self, ext): + obj_pat = 'release\\*.obj' if iswindows else '*.o' + objects = glob.glob(obj_pat) + if not objects or self.newer(objects, ext.sources+ext.headers): + archs = 'x86_64 ppc64' if leopard_build else 'x86 ppc' + pro = textwrap.dedent('''\ + TARGET = %s + TEMPLATE = lib + HEADERS = %s + SOURCES = %s + VERSION = 1.0.0 + CONFIG += %s + ''')%(ext.name, ' '.join(ext.headers), ' '.join(ext.sources), archs) + open(ext.name+'.pro', 'wb').write(pro) + subprocess.check_call([QMAKE, '-o', 'Makefile', ext.name+'.pro']) + if leopard_build: + raw = open('Makefile', 'rb').read() + open('Makefile', 'wb').write(raw.replace('ppc64', 'x86_64')) + subprocess.check_call([make, '-f', 'Makefile']) + objects = glob.glob(obj_pat) + return list(map(self.a, objects)) + + def build_pyqt_extension(self, ext, dest): + pyqt_dir = self.j(self.d(self.SRC), 'build', 'pyqt') + src_dir = self.j(pyqt_dir, ext.name) + qt_dir = self.j(src_dir, 'qt') + if not self.e(qt_dir): + os.makedirs(qt_dir) + cwd = os.getcwd() + try: + os.chdir(qt_dir) + qt_objects = self.build_qt_objects(ext) + finally: + os.chdir(cwd) + + sip_files = ext.sip_files + ext.sip_files = [] + sipf = sip_files[0] + sbf = self.j(src_dir, self.b(sipf)+'.sbf') + if self.newer(sbf, [sipf]+ext.headers): + exe = '.exe' if iswindows else '' + cmd = [pyqt.sip_bin+exe, '-w', '-c', src_dir, '-b', sbf, '-I'+\ + pyqt.pyqt_sip_dir] + shlex.split(pyqt.pyqt_sip_flags) + [sipf] + self.info(' '.join(cmd)) + subprocess.check_call(cmd) + module = self.j(src_dir, self.b(dest)) + if self.newer(dest, [sbf]+qt_objects): + mf = self.j(src_dir, 'Makefile') + makefile = QtGuiModuleMakefile(configuration=pyqt, build_file=sbf, + makefile=mf, universal=OSX_SDK, qt=1) + makefile.extra_lflags = qt_objects + makefile.extra_include_dirs = ext.inc_dirs + makefile.generate() + if leopard_build: + raw = open(mf, 'rb').read() + raw = raw.replace('ppc64 x86_64', 'x86_64') + for x in ('ppc64', 'ppc', 'i386'): + raw = raw.replace(x, 'x86_64') + open(mf, 'wb').write(raw) + + subprocess.check_call([make, '-f', mf], cwd=src_dir) + shutil.copy2(module, dest) + + def clean(self): + for ext in extensions: + dest = self.dest(ext) + for x in (dest, dest+'.manifest'): + if os.path.exists(x): + os.remove(x) + shutil.rmtree(self.j(self.d(self.SRC), 'build')) + + + + + + + + + + + diff --git a/setup/install.py b/setup/install.py new file mode 100644 index 0000000000..3d82cb3183 --- /dev/null +++ b/setup/install.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import sys, os, textwrap + +from setup import Command, islinux, basenames, modules, functions + +TEMPLATE = '''\ +#!/usr/bin/env python + +""" +This is the standard runscript for all of calibre's tools. +Do not modify it unless you know what you are doing. +""" + +import sys +sys.path.insert(0, {path!r}) + +sys.resources_location = {resources!r} +sys.extensions_location = {extensions!r} + +from {module} import {func!s} +sys.exit({func!s}()) +''' + +class Develop(Command): + + description = 'Setup a development environment' + MODE = 0755 + + sub_commands = ['build'] + + def add_options(self, parser): + parser.set_usage(textwrap.dedent('''\ + *** + + Setup a development environment for calibre. + This allows you to run calibre directly from the source tree. + Binaries will be installed in /bin where is + the prefix of your python installation. This can be controlled + via the --prefix option. + ''')) + parser.add_option('--prefix', + help='Binaries will be installed in /bin') + + def pre_sub_commands(self, opts): + if not islinux: + self.info('\nSetting up a development environment is only ' + 'supported on linux. On other platforms, install the calibre ' + 'binary and use the calibre-debug command.') + raise SystemExit(1) + + if not os.geteuid() == 0: + self.info('\nError: This command must be run as root.') + raise SystemExit(1) + self.drop_privileges() + + def run(self, opts): + self.regain_privileges() + self.find_locations(opts) + self.write_templates(opts) + self.success() + + def success(self): + self.info('\nDevelopment environment successfully setup') + + def find_locations(self, opts): + self.path = self.SRC + self.resources = self.j(self.d(self.SRC), 'resources') + self.extensions = self.j(self.SRC, 'calibre', 'plugins') + + def write_templates(self, opts): + for typ in ('console', 'gui'): + for name, mod, func in zip(basenames[typ], modules[typ], + functions[typ]): + script = TEMPLATE.format( + module=mod, func=func, + path=self.path, resources=self.resources, + extensions=self.extensions) + prefix = opts.prefix + if prefix is None: + prefix = sys.prefix + path = self.j(prefix, 'bin', name) + self.info('Installing binary:', path) + open(path, 'wb').write(script) + os.chmod(path, self.MODE) + diff --git a/src/calibre/translations/pygettext.py b/setup/pygettext.py similarity index 99% rename from src/calibre/translations/pygettext.py rename to setup/pygettext.py index 9578ef2d51..9ef2b7aa1b 100644 --- a/src/calibre/translations/pygettext.py +++ b/setup/pygettext.py @@ -164,8 +164,7 @@ DEFAULTKEYWORDS = ', '.join(default_keywords) EMPTYSTRING = '' -from calibre.constants import __appname__ -from calibre.constants import __version__ as version +from setup import __appname__, __version__ as version # The normal pot-file header. msgmerge and Emacs's po-mode work better if it's # there. @@ -638,7 +637,7 @@ def main(outfile, args=sys.argv[1:]): fp.close() # write the output - eater.write(outfile) + eater.write(outfile) if __name__ == '__main__': main(sys.stdout) diff --git a/setup/translations.py b/setup/translations.py new file mode 100644 index 0000000000..fb38c52504 --- /dev/null +++ b/setup/translations.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, cStringIO, tempfile, shutil, atexit, subprocess, glob, re +from distutils import sysconfig + +from setup import Command, __appname__ +from setup.pygettext import main as pygettext + +class POT(Command): + + description = 'Update the .pot translation template' + PATH = os.path.join(Command.SRC, __appname__, 'translations') + + def source_files(self): + ans = [] + for root, _, files in os.walk(os.path.dirname(self.PATH)): + for name in files: + if name.endswith('.py'): + ans.append(os.path.abspath(os.path.join(root, name))) + return ans + + + def run(self, opts): + files = self.source_files() + buf = cStringIO.StringIO() + self.info('Creating translations template...') + tempdir = tempfile.mkdtemp() + atexit.register(shutil.rmtree, tempdir) + pygettext(buf, ['-k', '__', '-p', tempdir]+files) + src = buf.getvalue() + pot = os.path.join(self.PATH, __appname__+'.pot') + f = open(pot, 'wb') + f.write(src) + f.close() + self.info('Translations template:', os.path.abspath(pot)) + return pot + + +class Translations(POT): + description='''Compile the translations''' + DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization', + 'locales') + + def po_files(self): + return glob.glob(os.path.join(self.PATH, '*.po')) + + def mo_file(self, po_file): + locale = os.path.splitext(os.path.basename(po_file))[0] + return locale, os.path.join(self.DEST, locale, 'LC_MESSAGES', 'messages.mo') + + + def run(self, opts): + for f in self.po_files(): + locale, dest = self.mo_file(f) + base = os.path.dirname(dest) + if not os.path.exists(base): + os.makedirs(base) + if self.newer(dest, f): + self.info('\tCompiling translations for', locale) + subprocess.check_call(['msgfmt', '-o', dest, f]) + if locale in ('en_GB', 'nds', 'te', 'yi'): + continue + pycountry = self.j(sysconfig.get_python_lib(), 'pycountry', + 'locales', locale, 'LC_MESSAGES') + if os.path.exists(pycountry): + iso639 = self.j(pycountry, 'iso639.mo') + dest = self.j(self.d(dest), self.b(iso639)) + if self.newer(dest, iso639): + self.info('\tCopying ISO 639 translations') + shutil.copy2(iso639, dest) + else: + self.warn('No ISO 639 translations for locale:', locale) + + self.write_stats() + + @property + def stats(self): + return self.j(self.d(self.DEST), 'stats.pickle') + + def get_stats(self, path): + return subprocess.Popen(['msgfmt', '--statistics', '-o', '/dev/null', + path], + stderr=subprocess.PIPE).stderr.read() + + def write_stats(self): + files = self.po_files() + dest = self.stats + if not self.newer(dest, files): + return + self.info('Calculating translation statistics...') + raw = self.get_stats(self.j(self.PATH, 'calibre.pot')) + total = int(raw.split(',')[-1].strip().split()[0]) + stats = {} + for f in files: + raw = self.get_stats(f) + trans = int(raw.split()[0]) + locale = self.mo_file(f)[0] + stats[locale] = min(1.0, float(trans)/total) + + + import cPickle + cPickle.dump(stats, open(dest, 'wb'), -1) + + def clean(self): + if os.path.exists(self.stats): + os.remove(self.stats) + for f in self.po_files(): + l, d = self.mo_file(f) + i = self.j(self.d(d), 'iso639.mo') + for x in (i, d): + if os.path.exists(x): + os.remove(x) + + +class GetTranslations(Translations): + + description = 'Get updated translations from Launchpad' + BRANCH = 'lp:~kovid/calibre/translations' + + @classmethod + def modified_translations(cls): + raw = subprocess.Popen(['bzr', 'status'], + stdout=subprocess.PIPE).stdout.read().strip() + for line in raw.splitlines(): + line = line.strip() + if line.startswith(cls.PATH) and line.endswith('.po'): + yield line + + def run(self, opts): + if len(list(self.modified_translations())) == 0: + subprocess.check_call(['bzr', 'merge', self.BRANCH]) + if len(list(self.modified_translations())) == 0: + print 'No updated translations available' + else: + subprocess.check_call(['bzr', 'commit', '-m', + 'IGN:Updated translations', self.PATH]) + self.check_for_errors() + + @classmethod + def check_for_errors(cls): + errors = os.path.join(tempfile.gettempdir(), 'calibre-translation-errors') + if os.path.exists(errors): + shutil.rmtree(errors) + os.mkdir(errors) + pofilter = ('pofilter', '-i', cls.PATH, '-o', errors, + '-t', 'accelerators', '-t', 'escapes', '-t', 'variables', + #'-t', 'xmltags', + #'-t', 'brackets', + #'-t', 'emails', + #'-t', 'doublequoting', + #'-t', 'filepaths', + #'-t', 'numbers', + '-t', 'options', + #'-t', 'urls', + '-t', 'printf') + subprocess.check_call(pofilter) + errfiles = glob.glob(errors+os.sep+'*.po') + subprocess.check_call(['gvim', '-f', '-p', '--']+errfiles) + for f in errfiles: + with open(f, 'r+b') as f: + raw = f.read() + raw = re.sub(r'# \(pofilter\).*', '', raw) + f.seek(0) + f.truncate() + f.write(raw) + + subprocess.check_call(['pomerge', '-t', cls.PATH, '-i', errors, '-o', + cls.PATH]) + if len(list(cls.modified_translations())) > 0: + subprocess.call(['bzr', 'diff', cls.PATH]) + yes = raw_input('Merge corrections? [y/n]: ').strip() + if yes in ['', 'y']: + subprocess.check_call(['bzr', 'commit', '-m', + 'IGN:Translation corrections', cls.PATH]) + + +class ISO639(Command): + + XML = '/usr/lib/python2.6/site-packages/pycountry/databases/iso639.xml' + + def run(self, opts): + src = self.XML + if not os.path.exists(src): + raise Exception(src + ' does not exist') + dest = self.j(self.d(self.SRC), 'resources', 'localization', + 'iso639.pickle') + if not self.newer(dest, src): + self.info('Pickled code is up to date') + return + self.info('Pickling ISO-639 codes to', dest) + from lxml import etree + root = etree.fromstring(open(src, 'rb').read()) + by_2 = {} + by_3b = {} + by_3t = {} + codes2, codes3t, codes3b = set([]), set([]), set([]) + for x in root.xpath('//iso_639_entry'): + name = x.get('name') + two = x.get('iso_639_1_code', None) + if two is not None: + by_2[two] = name + codes2.add(two) + by_3b[x.get('iso_639_2B_code')] = name + by_3t[x.get('iso_639_2T_code')] = name + codes3b.add(x.get('iso_639_2B_code')) + codes3t.add(x.get('iso_639_2T_code')) + + from cPickle import dump + x = {'by_2':by_2, 'by_3b':by_3b, 'by_3t':by_3t, 'codes2':codes2, + 'codes3b':codes3b, 'codes3t':codes3t} + dump(x, open(dest, 'wb'), -1) + diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 6f1f895ab2..ef8882943c 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -14,13 +14,14 @@ numeric_version = tuple(_ver) Various run time constants. ''' -import sys, locale, codecs, os +import sys, locale, codecs from calibre.utils.terminfo import TerminalController terminal_controller = TerminalController(sys.stdout) iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower() isosx = 'darwin' in sys.platform.lower() +isnewosx = isosx and getattr(sys, 'new_app_bundle', False) islinux = not(iswindows or isosx) isfrozen = hasattr(sys, 'frozen') @@ -50,19 +51,8 @@ if plugins is None: # Load plugins def load_plugins(): plugins = {} - if isfrozen: - if iswindows: - plugin_path = os.path.join(os.path.dirname(sys.executable), 'plugins') - sys.path.insert(1, os.path.dirname(sys.executable)) - elif isosx: - plugin_path = os.path.join(getattr(sys, 'frameworks_dir'), 'plugins') - elif islinux: - plugin_path = os.path.join(getattr(sys, 'frozen_path'), 'plugins') - sys.path.insert(0, plugin_path) - else: - import pkg_resources - plugin_path = getattr(pkg_resources, 'resource_filename')('calibre', 'plugins') - sys.path.insert(0, plugin_path) + plugin_path = sys.extensions_location + sys.path.insert(0, plugin_path) for plugin in ['pictureflow', 'lzx', 'msdes', 'podofo', 'cPalmdoc', 'fontconfig', 'calibre_poppler'] + \ @@ -74,6 +64,7 @@ if plugins is None: p = None err = str(err) plugins[plugin] = (p, err) + sys.path.remove(plugin_path) return plugins plugins = load_plugins() diff --git a/src/calibre/startup.py b/src/calibre/startup.py index 52648a20a0..6cdb0062e9 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -25,6 +25,12 @@ _run_once = False if not _run_once: _run_once = True + ################################################################################ + # Setup resources + import calibre.utils.resources as resources + resources + + ################################################################################ # Setup translations diff --git a/src/calibre/utils/ipc/launch.py b/src/calibre/utils/ipc/launch.py index dd7cd356f2..46100b5071 100644 --- a/src/calibre/utils/ipc/launch.py +++ b/src/calibre/utils/ipc/launch.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' import subprocess, os, sys, time -from calibre.constants import iswindows, isosx, isfrozen +from calibre.constants import iswindows, isosx, isfrozen, isnewosx from calibre.utils.config import prefs from calibre.ptempfile import PersistentTemporaryFile @@ -16,8 +16,6 @@ if iswindows: import win32process _windows_null_file = open(os.devnull, 'wb') -isnewosx = isosx and getattr(sys, 'new_app_bundle', False) - class Worker(object): ''' Platform independent object for launching child processes. All processes diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py new file mode 100644 index 0000000000..aaac62ea1e --- /dev/null +++ b/src/calibre/utils/localization.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os + +_available_translations = None + +def available_translations(): + global _available_translations + if _available_translations is None: + base = P('resources/localization/locales') + _available_translations = os.listdir(base) + return _available_translations diff --git a/src/calibre/utils/resources.py b/src/calibre/utils/resources.py new file mode 100644 index 0000000000..98f056e065 --- /dev/null +++ b/src/calibre/utils/resources.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +import __builtin__, sys, os + +def get_path(path): + path = path.replace(os.sep, '/') + return os.path.join(sys.resources_location, *path.split('/')) + +__builtin__.__dict__['P'] = get_path + diff --git a/src/calibre/web/feeds/recipes/recipe_ars_technica.py b/src/calibre/web/feeds/recipes/recipe_ars_technica.py index 943889e223..d390f006ec 100644 --- a/src/calibre/web/feeds/recipes/recipe_ars_technica.py +++ b/src/calibre/web/feeds/recipes/recipe_ars_technica.py @@ -1,78 +1,87 @@ -#!/usr/bin/env python - -__license__ = 'GPL v3' -__copyright__ = '2008-2009, Darko Miletic ' -''' -arstechnica.com -''' - -from calibre.web.feeds.news import BasicNewsRecipe - -class ArsTechnica2(BasicNewsRecipe): - title = u'Ars Technica' - language = 'en' - - __author__ = 'Darko Miletic' - description = 'The art of technology' - publisher = 'Ars Technica' - category = 'news, IT, technology' - oldest_article = 2 - max_articles_per_feed = 100 - no_stylesheets = True - encoding = 'utf8' - remove_javascript = True - use_embedded_content = False - - html2lrf_options = [ - '--comment', description - , '--category', category - , '--publisher', publisher - ] - - html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' - - keep_only_tags = [dict(name='div', attrs={'id':['news-item-info','news-item']})] - - remove_tags = [ - dict(name=['object','link','embed']) - ,dict(name='div', attrs={'class':'related-stories'}) - ] - - - feeds = [ - (u'Infinite Loop (Apple content)' , u'http://feeds.arstechnica.com/arstechnica/apple/' ) - ,(u'Opposable Thumbs (Gaming content)' , u'http://feeds.arstechnica.com/arstechnica/gaming/' ) - ,(u'Gear and Gadgets' , u'http://feeds.arstechnica.com/arstechnica/gadgets/' ) - ,(u'Chipster (Hardware content)' , u'http://feeds.arstechnica.com/arstechnica/hardware/' ) - ,(u'Uptime (IT content)' , u'http://feeds.arstechnica.com/arstechnica/business/' ) - ,(u'Open Ended (Open Source content)' , u'http://feeds.arstechnica.com/arstechnica/open-source/') - ,(u'One Microsoft Way' , u'http://feeds.arstechnica.com/arstechnica/microsoft/' ) - ,(u'Nobel Intent (Science content)' , u'http://feeds.arstechnica.com/arstechnica/science/' ) - ,(u'Law & Disorder (Tech policy content)' , u'http://feeds.arstechnica.com/arstechnica/tech-policy/') - ] - - def append_page(self, soup, appendtag, position): - pager = soup.find('div',attrs={'id':'pager'}) - if pager: - for atag in pager.findAll('a',href=True): - str = self.tag_to_string(atag) - if str.startswith('Next'): - soup2 = self.index_to_soup(atag['href']) - texttag = soup2.find('div', attrs={'class':'news-item-text'}) - for it in texttag.findAll(style=True): - del it['style'] - newpos = len(texttag.contents) - self.append_page(soup2,texttag,newpos) - texttag.extract() - pager.extract() - appendtag.insert(position,texttag) - - - def preprocess_html(self, soup): - ftag = soup.find('div', attrs={'class':'news-item-byline'}) - if ftag: - ftag.insert(4,'

') - for item in soup.findAll(style=True): - del item['style'] - self.append_page(soup, soup.body, 3) - return soup +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2008-2009, Darko Miletic ' +''' +arstechnica.com +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class ArsTechnica2(BasicNewsRecipe): + title = u'Ars Technica' + language = _('English') + __author__ = 'Darko Miletic and Sujata Raman' + description = 'The art of technology' + publisher = 'Ars Technica' + category = 'news, IT, technology' + oldest_article = 2 + max_articles_per_feed = 100 + no_stylesheets = True + encoding = 'utf8' + remove_javascript = True + use_embedded_content = False + + extra_css = ''' + .news-item-title{font-size: medium ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;} + .news-item-teaser{font-size: small ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;} + .news-item-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} + .news-item-text{font-size:x-small;font-family:Arial,Helvetica,sans-serif;} + .news-item-figure-caption-text{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:bold;} + .news-item-figure-caption-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;} + ''' + + keep_only_tags = [dict(name='div', attrs={'id':['news-item-info','news-item']})] + + remove_tags = [ + dict(name=['object','link','embed']) + ,dict(name='div', attrs={'class':'related-stories'}) + ] + + + feeds = [ + (u'Infinite Loop (Apple content)' , u'http://feeds.arstechnica.com/arstechnica/apple/' ) + ,(u'Opposable Thumbs (Gaming content)' , u'http://feeds.arstechnica.com/arstechnica/gaming/' ) + ,(u'Gear and Gadgets' , u'http://feeds.arstechnica.com/arstechnica/gadgets/' ) + ,(u'Chipster (Hardware content)' , u'http://feeds.arstechnica.com/arstechnica/hardware/' ) + ,(u'Uptime (IT content)' , u'http://feeds.arstechnica.com/arstechnica/business/' ) + ,(u'Open Ended (Open Source content)' , u'http://feeds.arstechnica.com/arstechnica/open-source/') + ,(u'One Microsoft Way' , u'http://feeds.arstechnica.com/arstechnica/microsoft/' ) + ,(u'Nobel Intent (Science content)' , u'http://feeds.arstechnica.com/arstechnica/science/' ) + ,(u'Law & Disorder (Tech policy content)' , u'http://feeds.arstechnica.com/arstechnica/tech-policy/') + ] + + def append_page(self, soup, appendtag, position): + pager = soup.find('div',attrs={'id':'pager'}) + if pager: + for atag in pager.findAll('a',href=True): + str = self.tag_to_string(atag) + if str.startswith('Next'): + soup2 = self.index_to_soup(atag['href']) + + texttag = soup2.find('div', attrs={'class':'news-item-text'}) + for it in texttag.findAll(style=True): + del it['style'] + + newpos = len(texttag.contents) + self.append_page(soup2,texttag,newpos) + texttag.extract() + pager.extract() + appendtag.insert(position,texttag) + + + def preprocess_html(self, soup): + + ftag = soup.find('div', attrs={'class':'news-item-byline'}) + if ftag: + ftag.insert(4,'

') + + for item in soup.findAll(style=True): + del item['style'] + + self.append_page(soup, soup.body, 3) + + return soup + + + diff --git a/upload.py b/upload.py index 492c80e68d..6f3c5eea9e 100644 --- a/upload.py +++ b/upload.py @@ -6,10 +6,14 @@ __docformat__ = 'restructuredtext en' import shutil, os, glob, re, cStringIO, sys, tempfile, time, textwrap, socket, \ struct, subprocess, platform from datetime import datetime -from setuptools.command.build_py import build_py as _build_py, convert_path +from stat import ST_MODE +from distutils.command.build_py import build_py as _build_py, convert_path +from distutils.command.install_scripts import install_scripts as _install_scripts +from distutils.command.install import install as _install from distutils.core import Command from subprocess import check_call, call, Popen from distutils.command.build import build as _build +from distutils import log raw = open(os.path.join('src', 'calibre', 'constants.py'), 'rb').read() __version__ = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', raw).group(1) @@ -25,6 +29,9 @@ TXT2LRF = "src/calibre/ebooks/lrf/txt/demo" MOBILEREAD = 'ftp://dev.mobileread.com/calibre/' is64bit = platform.architecture()[0] == '64bit' +iswindows = re.search('win(32|64)', sys.platform) +isosx = 'darwin' in sys.platform +islinux = not isosx and not iswindows def get_ip_address(ifname): import fcntl @@ -66,6 +73,59 @@ class OptionlessCommand(Command): for cmd_name in self.get_sub_commands(): self.run_command(cmd_name) +def setup_mount_helper(tdir): + def warn(): + print 'WARNING: Failed to compile mount helper. Auto mounting of', + print 'devices will not work' + + if os.geteuid() != 0: + return warn() + import stat + src = os.path.join('src', 'calibre', 'devices', 'linux_mount_helper.c') + dest = os.path.join(tdir, 'calibre-mount-helper') + log.info('Installing mount helper to '+ tdir) + p = subprocess.Popen(['gcc', '-Wall', src, '-o', dest]) + ret = p.wait() + if ret != 0: + return warn() + os.chown(dest, 0, 0) + os.chmod(dest, + stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH) + return dest + + +class develop(_install_scripts): + + def run(self): + if not iswindows and os.geteuid() != 0: + raise Exception('Must be root to run this command.') + if not self.skip_build: + self.run_command('build_ext') + self.run_command('build_scripts') + + for script in os.listdir(self.build_dir): + script = os.path.join(self.build_dir, script) + raw = open(script, 'rb').read() + raw = re.sub(r'"""##DEVELOP_HOOK##([^#]+)##END_DEVELOP_HOOK##"""', + r'\1', raw) + raw = raw.replace('#!python', '#!'+sys.executable) + f = os.path.join(self.install_dir, os.path.basename(script)) + open(f, 'wb').write(raw) + mode = ((os.stat(f)[ST_MODE]) | 0555) & 07777 + log.info('changing mode of %s to %o'%(f, mode)) + os.chmod(f, mode) + + if islinux: + setup_mount_helper(self.install_dir) + subprocess.check_call('calibre_postinstall') + +class install(_install): + + def run(self): + _install.run(self) + if islinux: + setup_mount_helper(self.install_dir) + subprocess.check_call('calibre_postinstall') class sdist(OptionlessCommand): @@ -77,39 +137,6 @@ class sdist(OptionlessCommand): self.distribution.dist_files.append(('sdist', '', name)) print 'Source distribution created in', os.path.abspath(name) -class pot(OptionlessCommand): - description = '''Create the .pot template for all translatable strings''' - - PATH = os.path.join('src', __appname__, 'translations') - - def source_files(self): - ans = [] - for root, _, files in os.walk(os.path.dirname(self.PATH)): - for name in files: - if name.endswith('.py'): - ans.append(os.path.abspath(os.path.join(root, name))) - return ans - - - def run(self): - sys.path.insert(0, os.path.abspath(self.PATH)) - try: - pygettext = __import__('pygettext', fromlist=['main']).main - files = self.source_files() - buf = cStringIO.StringIO() - print 'Creating translations template' - tempdir = tempfile.mkdtemp() - pygettext(buf, ['-k', '__', '-p', tempdir]+files) - src = buf.getvalue() - pot = os.path.join(self.PATH, __appname__+'.pot') - f = open(pot, 'wb') - f.write(src) - f.close() - print 'Translations template:', os.path.abspath(pot) - return pot - finally: - sys.path.remove(os.path.abspath(self.PATH)) - class manual(OptionlessCommand): description='''Build the User Manual ''' @@ -249,99 +276,6 @@ class resources(OptionlessCommand): if os.path.exists(path): os.remove(path) -class translations(OptionlessCommand): - description='''Compile the translations''' - PATH = os.path.join('src', __appname__, 'translations') - DEST = os.path.join(PATH, 'compiled.py') - - def run(self): - sys.path.insert(0, os.path.abspath(self.PATH)) - try: - files = glob.glob(os.path.join(self.PATH, '*.po')) - if newer([self.DEST], files): - msgfmt = __import__('msgfmt', fromlist=['main']).main - translations = {} - print 'Compiling translations...' - for po in files: - lang = os.path.basename(po).partition('.')[0] - buf = cStringIO.StringIO() - print 'Compiling', lang - msgfmt(buf, [po]) - translations[lang] = buf.getvalue() - open(self.DEST, 'wb').write('translations = '+repr(translations)) - else: - print 'Translations up to date' - finally: - sys.path.remove(os.path.abspath(self.PATH)) - - - @classmethod - def clean(cls): - path = cls.DEST - if os.path.exists(path): - os.remove(path) - -class get_translations(translations): - - description = 'Get updated translations from Launchpad' - BRANCH = 'lp:~kovid/calibre/translations' - - @classmethod - def modified_translations(cls): - raw = subprocess.Popen(['bzr', 'status'], - stdout=subprocess.PIPE).stdout.read().strip() - for line in raw.splitlines(): - line = line.strip() - if line.startswith(cls.PATH) and line.endswith('.po'): - yield line - - def run(self): - if len(list(self.modified_translations())) == 0: - subprocess.check_call(['bzr', 'merge', self.BRANCH]) - if len(list(self.modified_translations())) == 0: - print 'No updated translations available' - else: - subprocess.check_call(['bzr', 'commit', '-m', - 'IGN:Updated translations', self.PATH]) - self.check_for_errors() - - @classmethod - def check_for_errors(cls): - errors = os.path.join(tempfile.gettempdir(), 'calibre-translation-errors') - if os.path.exists(errors): - shutil.rmtree(errors) - os.mkdir(errors) - pofilter = ('pofilter', '-i', cls.PATH, '-o', errors, - '-t', 'accelerators', '-t', 'escapes', '-t', 'variables', - #'-t', 'xmltags', - #'-t', 'brackets', - #'-t', 'emails', - #'-t', 'doublequoting', - #'-t', 'filepaths', - #'-t', 'numbers', - '-t', 'options', - #'-t', 'urls', - '-t', 'printf') - subprocess.check_call(pofilter) - errfiles = glob.glob(errors+os.sep+'*.po') - subprocess.check_call(['gvim', '-f', '-p', '--']+errfiles) - for f in errfiles: - with open(f, 'r+b') as f: - raw = f.read() - raw = re.sub(r'# \(pofilter\).*', '', raw) - f.seek(0) - f.truncate() - f.write(raw) - - subprocess.check_call(['pomerge', '-t', cls.PATH, '-i', errors, '-o', - cls.PATH]) - if len(list(cls.modified_translations())) > 0: - subprocess.call(['bzr', 'diff', cls.PATH]) - yes = raw_input('Merge corrections? [y/n]: ').strip() - if yes in ['', 'y']: - subprocess.check_call(['bzr', 'commit', '-m', - 'IGN:Translation corrections', cls.PATH]) - class gui(OptionlessCommand): description='''Compile all GUI forms and images''' PATH = os.path.join('src', __appname__, 'gui2') @@ -439,29 +373,6 @@ class gui(OptionlessCommand): if os.path.exists(x): os.remove(x) -class clean(OptionlessCommand): - - description='''Delete all computer generated files in the source tree''' - - def run(self): - print 'Cleaning...' - manual.clean() - gui.clean() - translations.clean() - resources.clean() - - for f in glob.glob(os.path.join('src', 'calibre', 'plugins', '*')): - os.remove(f) - for root, _, files in os.walk('.'): - for name in files: - for t in ('.pyc', '.pyo', '~'): - if name.endswith(t): - os.remove(os.path.join(root, name)) - break - - for dir in ('build', 'dist', os.path.join('src', 'calibre.egg-info')): - shutil.rmtree(dir, ignore_errors=True) - class build_py(_build_py): def find_data_files(self, package, src_dir):