build and develop commands done

.bzrignore

@@ -13,6 +13,7 @@ src/calibre/manual/cli/
 build
 dist
 docs
+resources
 nbproject/
 src/calibre/gui2/pictureflow/Makefile.Debug
 src/calibre/gui2/pictureflow/Makefile.Release

INSTALL

@@ -0,0 +1,34 @@
+calibre supports installation from source only on Linux. On Windows and OS X use the provided installers and usethe facilities provided by calibre-debug to hack on the calibre source. 
+
+On Linux, there are two kinds of installation from source possible. Note that both kinds require lots of dependencies as well as a full development environment (compilers, headers files, etc.) 
+
+All installation related functions are accessed by the command::
+
+    python setup.py
+
+Install
+==========
+
+The first type of install will actually "install" calibre to your computer by putting its files into the system in the following locations:
+
+  - Binaries (actually python wrapper scripts) in <prefix>/bin
+  - Python and C modules in <prefix>/lib/calibre
+  - Resources like icons, etc. in <prefix>/share/calibre
+
+This type of install can be run by the command::
+
+    sudo python setup.py install
+
+<prefix> is normally the installation prefix of python, usually /usr. It can be controlled by the --prefix 
+option.
+
+Develop
+=============
+
+This type of install is designed to let you run calibre from your home directory, making it easy to hack on it. It will only install binaries into /usr/bin, but all the actual code and resource files will be read from the calibre source tree in your home drectory (or wherever you choose to put it).
+
+This type of install can be run with the command::
+
+    sudo python setup.py develop
+
+

README

@@ -0,0 +1,17 @@
+calibre is an e-book library manager. It can view, convert and catalog e-books \
+in most of the major e-book formats. It can also talk to e-book reader \
+devices. It can go out to the internet and fetch metadata for your books. \
+It can download newspapers and convert them into e-books for convenient \
+reading. It is cross platform, running on Linux, Windows and OS X.
+
+For screenshots: https://calibre.kovidgoyal.net/wiki/Screenshots
+
+For installation/usage instructions please see
+http://calibre.kovidgoyal.net
+
+For source code access:
+bzr branch lp:calibre
+
+To update your copy of the source code:
+bzr merge
+

pyqtdistutils.py

@@ -1,279 +0,0 @@
-#!/usr/bin/env  python
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
-__docformat__ = 'restructuredtext en'
-
-'''
-Build PyQt extensions. Integrates with distutils (but uses the PyQt build system).
-'''
-from distutils.core import Extension as _Extension
-from distutils.command.build_ext import build_ext as _build_ext
-from distutils.dep_util import newer_group
-from distutils import log
-from distutils.spawn import find_executable
-
-import sipconfig, os, sys, string, glob, shutil
-from PyQt4 import pyqtconfig
-iswindows = 'win32' in sys.platform
-isosx = 'darwin' in sys.platform
-QMAKE = '/Volumes/sw/qt/bin/qmake' if isosx else 'qmake'
-if find_executable('qmake-qt4'):
-    QMAKE = find_executable('qmake-qt4')
-elif find_executable('qmake'):
-    QMAKE = find_executable('qmake')
-QMAKE = os.environ.get('QMAKE', QMAKE)
-WINDOWS_PYTHON = ['C:/Python26/libs']
-OSX_SDK = '/Developer/SDKs/MacOSX10.5.sdk'
-if not os.path.exists(OSX_SDK):
-    OSX_SDK = '/Developer/SDKs/MacOSX10.4u.sdk'
-
-leopard_build = '10.5' in OSX_SDK
-
-def replace_suffix(path, new_suffix):
-    return os.path.splitext(path)[0] + new_suffix
-
-class Extension(_Extension):
-
-    def __init__(self, *args, **kwargs):
-        if leopard_build:
-            prev = kwargs.get('extra_compile_args', [])
-            prev.extend(['-arch', 'ppc64', '-arch', 'x86_64'])
-            kwargs['extra_compile_args'] = prev
-        _Extension.__init__(self, *args, **kwargs)
-
-
-if iswindows:
-    from distutils import msvc9compiler
-    msvc = msvc9compiler.MSVCCompiler()
-    msvc.initialize()
-    nmake = msvc.find_exe('nmake.exe')
-    rc = msvc.find_exe('rc.exe')
-
-class PyQtExtension(Extension):
-
-    def __init__(self, name, sources, sip_sources, **kw):
-        '''
-        :param sources: Qt .cpp and .h files needed for this extension
-        :param sip_sources: List of .sip files this extension depends on. The
-                            first .sip file will be used toactually build the extension.
-        '''
-        self.module_makefile = pyqtconfig.QtGuiModuleMakefile
-        self.sip_sources = map(lambda x: x.replace('/', os.sep), sip_sources)
-        Extension.__init__(self, name, sources, **kw)
-
-
-class build_ext(_build_ext):
-
-    def make(self, makefile):
-        make = nmake if iswindows else 'make'
-        self.spawn([make, '-f', makefile])
-
-    def build_qt_objects(self, ext, bdir):
-        if not iswindows:
-            bdir = os.path.join(bdir, 'qt')
-        if not os.path.exists(bdir):
-            os.makedirs(bdir)
-        cwd = os.getcwd()
-        sources = map(os.path.abspath, ext.sources)
-        os.chdir(bdir)
-        archs = 'x86_64 ppc64' if leopard_build else 'x86 ppc'
-        try:
-            headers = set([f for f in sources if f.endswith('.h')])
-            sources = set(sources) - headers
-            name = ext.name.rpartition('.')[-1]
-            pro = '''\
-TARGET   = %s
-TEMPLATE = lib
-HEADERS  = %s
-SOURCES  = %s
-VERSION  = 1.0.0
-CONFIG   += %s
-'''%(name, ' '.join(headers), ' '.join(sources), archs)
-            open(name+'.pro', 'wb').write(pro)
-            self.spawn([QMAKE, '-o', 'Makefile.qt', name+'.pro'])
-            if leopard_build:
-                raw = open('Makefile.qt', 'rb').read()
-                open('Makefile.qt', 'wb').write(raw.replace('ppc64', 'x86_64'))
-            self.make('Makefile.qt')
-            pat = 'release\\*.obj' if iswindows else '*.o'
-            return map(os.path.abspath, glob.glob(pat))
-        finally:
-            os.chdir(cwd)
-
-    def build_sbf(self, sip, sbf, bdir):
-        print '\tBuilding sbf...'
-        sip_bin = self.sipcfg.sip_bin
-        pyqt_sip_flags = []
-        if hasattr(self, 'pyqtcfg'):
-            pyqt_sip_flags += ['-I', self.pyqtcfg.pyqt_sip_dir]
-            pyqt_sip_flags += self.pyqtcfg.pyqt_sip_flags.split()
-        self.spawn([sip_bin,
-                    "-c", bdir,
-                    "-b", sbf,
-                    ] + pyqt_sip_flags +
-                    [sip])
-
-    def build_pyqt(self, bdir, sbf, ext, qtobjs, headers):
-        makefile = ext.module_makefile(configuration=self.pyqtcfg,
-                                       build_file=sbf, dir=bdir,
-                                       makefile='Makefile.pyqt',
-                                       universal=OSX_SDK, qt=1)
-        makefile.extra_libs = ext.libraries
-        makefile.extra_lib_dirs = ext.library_dirs
-        makefile.extra_cxxflags = ext.extra_compile_args
-
-        if 'win32' in sys.platform:
-            makefile.extra_lib_dirs += WINDOWS_PYTHON
-        makefile.extra_include_dirs = list(set(map(os.path.dirname, headers)))
-        makefile.extra_include_dirs += ext.include_dirs
-        makefile.extra_lflags += qtobjs
-        makefile.generate()
-        cwd = os.getcwd()
-        os.chdir(bdir)
-        if leopard_build:
-            mf = 'Makefile.pyqt'
-            raw = open(mf, 'rb').read()
-            raw = raw.replace('ppc64 x86_64', 'x86_64')
-            for x in ('ppc64', 'ppc', 'i386'):
-                raw = raw.replace(x, 'x86_64')
-            open(mf, 'wb').write(raw)
-        try:
-            self.make('Makefile.pyqt')
-        finally:
-            os.chdir(cwd)
-
-
-
-    def build_extension(self, ext):
-        self.inplace = True # Causes extensions to be built in the source tree
-
-        fullname = self.get_ext_fullname(ext.name)
-        if self.inplace:
-            # ignore build-lib -- put the compiled extension into
-            # the source tree along with pure Python modules
-
-            modpath = string.split(fullname, '.')
-            package = string.join(modpath[0:-1], '.')
-            base = modpath[-1]
-
-            build_py = self.get_finalized_command('build_py')
-            package_dir = build_py.get_package_dir(package)
-            ext_filename = os.path.join(package_dir,
-                                        self.get_ext_filename(base))
-        else:
-            ext_filename = os.path.join(self.build_lib,
-                                        self.get_ext_filename(fullname))
-        bdir = os.path.abspath(os.path.join(self.build_temp, fullname))
-        if not os.path.exists(bdir):
-            os.makedirs(bdir)
-
-        if not isinstance(ext, PyQtExtension):
-            if not iswindows:
-                return _build_ext.build_extension(self, ext)
-
-            c_sources = [f for f in ext.sources if os.path.splitext(f)[1].lower() in ('.c', '.cpp', '.cxx')]
-            compile_args = '/c /nologo /Ox /MD /W3 /EHsc /DNDEBUG'.split()
-            compile_args += ext.extra_compile_args
-            self.swig_opts = ''
-            inc_dirs = self.include_dirs + [x.replace('/', '\\') for x in ext.include_dirs]
-            cc = [msvc.cc] + compile_args + ['-I%s'%x for x in list(set(inc_dirs))]
-            objects = []
-            for f in c_sources:
-                o = os.path.join(bdir, os.path.basename(f)+'.obj')
-                objects.append(o)
-                inf = '/Tp' if f.endswith('.cpp') else '/Tc'
-                compiler =  cc + [inf+f, '/Fo'+o]
-                self.spawn(compiler)
-            out = os.path.join(bdir, base+'.pyd')
-            linker = [msvc.linker] + '/DLL /nologo /INCREMENTAL:NO'.split()
-            linker += ['/LIBPATH:'+x for x in self.library_dirs+ext.library_dirs]
-            linker += [x+'.lib' for x in ext.libraries]
-            linker += ['/EXPORT:init'+base] + objects + ['/OUT:'+out]
-            self.spawn(linker)
-            for src in (out, out+'.manifest'):
-                shutil.copyfile(src, os.path.join('src', 'calibre', 'plugins', os.path.basename(src)))
-            return
-
-
-
-        if not os.path.exists(bdir):
-            os.makedirs(bdir)
-        ext.sources2 = map(os.path.abspath, ext.sources)
-        qt_dir = 'qt\\release' if iswindows else 'qt'
-        objects = set(map(lambda x: os.path.join(bdir, qt_dir, replace_suffix(os.path.basename(x), '.o')),
-                      [s for s in ext.sources2 if not s.endswith('.h')]))
-        newer = False
-        for object in objects:
-            if newer_group(ext.sources2, object, missing='newer'):
-                newer = True
-                break
-        headers = [f for f in ext.sources2 if f.endswith('.h')]
-        if self.force or newer:
-            log.info('building \'%s\' extension', ext.name)
-            objects = self.build_qt_objects(ext, bdir)
-
-        self.sipcfg  = sipconfig.Configuration()
-        self.pyqtcfg = pyqtconfig.Configuration()
-        sbf_sources = []
-        for sip in ext.sip_sources:
-            sipbasename = os.path.basename(sip)
-            sbf = os.path.join(bdir, replace_suffix(sipbasename, ".sbf"))
-            sbf_sources.append(sbf)
-            if self.force or newer_group(ext.sip_sources, sbf, 'newer'):
-                self.build_sbf(sip, sbf, bdir)
-        generated_sources = []
-        for sbf in sbf_sources:
-            generated_sources += self.get_sip_output_list(sbf, bdir)
-
-        depends = generated_sources + list(objects)
-        mod = os.path.join(bdir, os.path.basename(ext_filename))
-
-        if self.force or newer_group(depends, mod, 'newer'):
-            self.build_pyqt(bdir, sbf_sources[0], ext, list(objects), headers)
-
-        if self.force or newer_group([mod], ext_filename, 'newer'):
-            if os.path.exists(ext_filename):
-                os.unlink(ext_filename)
-            shutil.copyfile(mod, ext_filename)
-            shutil.copymode(mod, ext_filename)
-
-
-        if self.force or newer_group([mod], ext_filename, 'newer'):
-            if os.path.exists(ext_filename):
-                os.unlink(ext_filename)
-            shutil.copyfile(mod, ext_filename)
-            shutil.copymode(mod, ext_filename)
-
-
-    def get_sip_output_list(self, sbf, bdir):
-        """
-        Parse the sbf file specified to extract the name of the generated source
-        files. Make them absolute assuming they reside in the temp directory.
-        """
-        for L in file(sbf):
-            key, value = L.split("=", 1)
-            if key.strip() == "sources":
-                out = []
-                for o in value.split():
-                    out.append(os.path.join(bdir, o))
-                return out
-
-        raise RuntimeError, "cannot parse SIP-generated '%s'" % sbf
-
-    def run_sip(self, sip_files):
-        sip_bin = self.sipcfg.sip_bin
-        sip_sources = [i[0] for i in sip_files]
-        generated_sources = []
-        for sip, sbf in sip_files:
-            if not (self.force or newer_group(sip_sources, sbf, 'newer')):
-                log.info(sbf + ' is up to date')
-                continue
-            self.spawn([sip_bin,
-                    "-c", self.build_temp,
-                    "-b", sbf,
-                    '-I', self.pyqtcfg.pyqt_sip_dir,
-                    ] + self.pyqtcfg.pyqt_sip_flags.split()+
-                    [sip])
-            generated_sources += self.get_sip_output_list(sbf)
-        return generated_sources
-

session.vim

@@ -1,5 +1,5 @@
 " Project wide builtins
-let g:pyflakes_builtins += ["dynamic_property", "__"]
+let g:pyflakes_builtins += ["dynamic_property", "__", "P"]
 
 python << EOFPY
 import os

setup.py

@@ -1,298 +1,64 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
 from __future__ import with_statement
+
 __license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
-
+__docformat__ = 'restructuredtext en'
-import sys, re, os, subprocess
-sys.path.append('src')
-iswindows = re.search('win(32|64)', sys.platform)
-isosx = 'darwin' in sys.platform
-islinux = not isosx and not iswindows
-src = open('src/calibre/constants.py', 'rb').read()
-VERSION = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1)
-APPNAME = re.search(r'__appname__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1)
-print 'Setup', APPNAME, 'version:', VERSION
-
-epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).search(open('src/%s/linux.py'%APPNAME, 'rb').read()).group(1)
-entry_points = eval(epsrc, {'__appname__': APPNAME})
-
-def _ep_to_script(ep, base='src'):
-    return (base+os.path.sep+re.search(r'.*=\s*(.*?):', ep).group(1).replace('.', '/')+'.py').strip()
 
 
-scripts = {
+import sys, os, optparse
-           'console' : [_ep_to_script(i) for i in entry_points['console_scripts']],
-           'gui' : [_ep_to_script(i) for i in entry_points['gui_scripts']],
-          }
 
-def _ep_to_basename(ep):
+sys.path.insert(0, os.path.abspath(os.path.dirname(__file__)))
-    return re.search(r'\s*(.*?)\s*=', ep).group(1).strip()
-basenames = {
-             'console' : [_ep_to_basename(i) for i in entry_points['console_scripts']],
-             'gui' : [_ep_to_basename(i) for i in entry_points['gui_scripts']],
-            }
 
-def _ep_to_module(ep):
+import setup.commands as commands
-    return re.search(r'.*=\s*(.*?)\s*:', ep).group(1).strip()
+from setup import prints, get_warnings
-main_modules = {
-                'console' : [_ep_to_module(i) for i in entry_points['console_scripts']],
-                'gui' : [_ep_to_module(i) for i in entry_points['gui_scripts']],
-               }
 
-def _ep_to_function(ep):
+def check_version_info():
-    return ep[ep.rindex(':')+1:].strip()
+    vi = sys.version_info
-main_functions = {
+    if vi[0] == 2 and vi[1] > 5:
-                'console' : [_ep_to_function(i) for i in entry_points['console_scripts']],
+        return None
-                'gui' : [_ep_to_function(i) for i in entry_points['gui_scripts']],
+    return 'calibre requires python >= 2.6'
-               }
 
-def setup_mount_helper():
+def option_parser():
-    def warn():
+    parser = optparse.OptionParser()
-        print 'WARNING: Failed to compile mount helper. Auto mounting of',
+    return parser
-        print 'devices will not work'
 
-    if os.geteuid() != 0:
+def main(args=sys.argv):
-        return warn()
+    if len(args) == 1 or args[1] in ('-h', '--help'):
-    import stat
+        print 'Usage: python', args[0], 'command', '[options]'
-    src = os.path.join('src', 'calibre', 'devices', 'linux_mount_helper.c')
+        print '\nWhere command is one of:', ', '.join(commands.__all__)
-    dest = '/usr/bin/calibre-mount-helper'
+        print '\nTo get help on a particular command, run:'
-    p = subprocess.Popen(['gcc', '-Wall', src, '-o', dest])
+        print '\tpython', args[0], 'command -h'
-    ret = p.wait()
+        return 1
-    if ret != 0:
+
-        return warn()
+    command = args[1]
-    os.chown(dest, 0, 0)
+    if command not in commands.__all__:
-    os.chmod(dest,
+        print command, 'is not a recognized command.'
-       stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH)
+        print 'Valid commands:', ', '.join(commands.__all__)
+        return 1
+
+    command = getattr(commands, command)
+
+    parser = option_parser()
+    command.add_all_options(parser)
+    lines = parser.usage.splitlines()
+    lines[0] = 'Usage: python setup.py %s [options]'%args[1]
+    parser.set_usage('\n'.join(lines))
+
+    opts, args = parser.parse_args(args)
+    command.run_all(opts)
+
+    warnings = get_warnings()
+    if warnings:
+        print
+        prints('There were', len(warnings), 'warning(s):')
+        print
+        for args, kwargs in warnings:
+            prints(*args, **kwargs)
+            print
+
+    return 0
 
 if __name__ == '__main__':
-    from setuptools import setup, find_packages
+    sys.exit(main())
-    from pyqtdistutils import PyQtExtension, build_ext, Extension, QMAKE
-    from upload import sdist, pot, build, build_py, manual, \
-                        resources, clean, gui, translations, update, \
-                        tag_release, upload_demo, build_linux, build_windows, \
-                        build_osx, upload_installers, upload_user_manual, \
-                        upload_to_pypi, stage3, stage2, stage1, upload, \
-                        upload_rss, betas, build_linux32, build_linux64, \
-                        build_osx64, get_translations
-    resources.SCRIPTS = {}
-    for x in ('console', 'gui'):
-        for name in basenames[x]:
-            resources.SCRIPTS[name] = x
-
-    list(basenames['console']+basenames['gui'])
-
-    entry_points['console_scripts'].append(
-                            'calibre_postinstall = calibre.linux:post_install')
-    optional = []
-    def qmake_query(arg=''):
-        cmd = [QMAKE, '-query']
-        if arg:
-            cmd += [arg]
-        return subprocess.Popen(cmd, stdout=subprocess.PIPE).stdout.read()
-    qt_inc = qt_lib = None
-    qt_inc = qmake_query('QT_INSTALL_HEADERS').splitlines()[0]
-    qt_inc = qt_inc if qt_inc not in ('', '**Unknown**') and os.path.isdir(qt_inc) else None
-    qt_lib = qmake_query('QT_INSTALL_LIBS').splitlines()[0]
-    qt_lib = qt_lib if qt_lib not in ('', '**Unknown**') and os.path.isdir(qt_lib) else None
-    if qt_lib is None or qt_inc is None:
-        print '\n\nWARNING: Could not find QT librariers and headers.',
-        print 'Is qmake in your PATH?\n\n'
-
-
-    if iswindows:
-        optional.append(Extension('calibre.plugins.winutil',
-                sources=['src/calibre/utils/windows/winutil.c'],
-                libraries=['shell32', 'setupapi'],
-                include_dirs=os.environ.get('INCLUDE',
-                        'C:/WinDDK/6001.18001/inc/api/;'
-                        'C:/WinDDK/6001.18001/inc/crt/').split(';'),
-                extra_compile_args=['/X']
-                ))
-
-    poppler_inc = '/usr/include/poppler/qt4'
-    poppler_lib = '/usr/lib'
-    poppler_libs = []
-    if iswindows:
-        poppler_inc = r'C:\cygwin\home\kovid\poppler\include\poppler\qt4'
-        poppler_lib = r'C:\cygwin\home\kovid\poppler\lib'
-        poppler_libs = ['QtCore4', 'QtGui4']
-    if isosx:
-        poppler_inc = '/Volumes/sw/build/poppler-0.10.7/qt4/src'
-        poppler_lib = '/Users/kovid/poppler/lib'
-    poppler_inc = os.environ.get('POPPLER_INC_DIR', poppler_inc)
-    if os.path.exists(os.path.join(poppler_inc, 'poppler-qt4.h'))\
-            and qt_lib is not None and qt_inc is not None:
-        optional.append(Extension('calibre.plugins.calibre_poppler',
-                        sources=['src/calibre/utils/poppler/poppler.cpp'],
-                        libraries=(['poppler', 'poppler-qt4']+poppler_libs),
-                        library_dirs=[os.environ.get('POPPLER_LIB_DIR',
-                            poppler_lib), qt_lib],
-                        include_dirs=[poppler_inc, qt_inc]))
-    else:
-        print '\n\nWARNING: Poppler not found on your system. Various PDF related',
-        print 'functionality will not work. Use the POPPLER_INC_DIR and',
-        print 'POPPLER_LIB_DIR environment variables.\n\n'
-
-    podofo_inc = '/usr/include/podofo' if islinux else \
-    'C:\\podofo\\include\\podofo' if iswindows else \
-    '/usr/local/include/podofo'
-    podofo_lib = '/usr/lib' if islinux else r'C:\podofo' if iswindows else \
-            '/usr/local/lib'
-    podofo_inc = os.environ.get('PODOFO_INC_DIR', podofo_inc)
-    if os.path.exists(os.path.join(podofo_inc, 'podofo.h')):
-        optional.append(Extension('calibre.plugins.podofo',
-                        sources=['src/calibre/utils/podofo/podofo.cpp'],
-                        libraries=['podofo'],
-                        library_dirs=[os.environ.get('PODOFO_LIB_DIR', podofo_lib)],
-                        include_dirs=[podofo_inc]))
-    else:
-        print '\n\nWARNING: PoDoFo not found on your system. Various PDF related',
-        print 'functionality will not work. Use the PODOFO_INC_DIR and',
-        print 'PODOFO_LIB_DIR environment variables.\n\n'
-
-    fc_inc = '/usr/include/fontconfig' if islinux else \
-            r'C:\cygwin\home\kovid\fontconfig\include\fontconfig' if iswindows else \
-            '/Users/kovid/fontconfig/include/fontconfig'
-    fc_lib = '/usr/lib' if islinux else \
-            r'C:\cygwin\home\kovid\fontconfig\lib' if iswindows else \
-            '/Users/kovid/fontconfig/lib'
-
-    fc_inc = os.environ.get('FC_INC_DIR', fc_inc)
-    fc_lib = os.environ.get('FC_LIB_DIR', fc_lib)
-    if not os.path.exists(os.path.join(fc_inc, 'fontconfig.h')):
-        print '\n\nERROR: fontconfig not found on your system.',
-        print 'Use the FC_INC_DIR and FC_LIB_DIR environment variables.\n\n'
-        raise SystemExit(1)
-    ext_modules = optional + [
-
-                   Extension('calibre.plugins.fontconfig',
-                       sources = ['src/calibre/utils/fonts/fontconfig.c'],
-                       include_dirs = [fc_inc],
-                       libraries=['fontconfig'],
-                       library_dirs=[fc_lib]),
-
-                   Extension('calibre.plugins.lzx',
-                             sources=['src/calibre/utils/lzx/lzxmodule.c',
-                                      'src/calibre/utils/lzx/compressor.c',
-                                      'src/calibre/utils/lzx/lzxd.c',
-                                      'src/calibre/utils/lzx/lzc.c',
-                                      'src/calibre/utils/lzx/lzxc.c'],
-                             include_dirs=['src/calibre/utils/lzx']),
-
-                   Extension('calibre.plugins.msdes',
-                             sources=['src/calibre/utils/msdes/msdesmodule.c',
-                                      'src/calibre/utils/msdes/des.c'],
-                             include_dirs=['src/calibre/utils/msdes']),
-
-                    Extension('calibre.plugins.cPalmdoc',
-                        sources=['src/calibre/ebooks/compression/palmdoc.c']),
-
-                    PyQtExtension('calibre.plugins.pictureflow',
-                                  ['src/calibre/gui2/pictureflow/pictureflow.cpp',
-                                   'src/calibre/gui2/pictureflow/pictureflow.h'],
-                                   ['src/calibre/gui2/pictureflow/pictureflow.sip']
-                                  )
-                 ]
-    if isosx:
-        ext_modules.append(Extension('calibre.plugins.usbobserver',
-                sources=['src/calibre/devices/usbobserver/usbobserver.c'],
-                extra_link_args=['-framework', 'IOKit'])
-                           )
-
-    if not iswindows:
-        plugins = ['plugins/%s.so'%(x.name.rpartition('.')[-1]) for x in ext_modules]
-    else:
-        plugins = ['plugins/%s.pyd'%(x.name.rpartition('.')[-1]) for x in ext_modules] + \
-                  ['plugins/%s.pyd.manifest'%(x.name.rpartition('.')[-1]) \
-                        for x in ext_modules if 'pictureflow' not in x.name]
-
-
-    setup(
-          name           = APPNAME,
-          packages       = find_packages('src'),
-          package_dir    = { '' : 'src' },
-          version        = VERSION,
-          author         = 'Kovid Goyal',
-          author_email   = 'kovid@kovidgoyal.net',
-          url            = 'http://%s.kovidgoyal.net'%APPNAME,
-          package_data   = {'calibre':plugins},
-          entry_points   = entry_points,
-          zip_safe       = False,
-          options        = { 'bdist_egg' : {'exclude_source_files': True,}, },
-          ext_modules    = ext_modules,
-          description    =
-                      '''
-                      E-book management application.
-                      ''',
-          long_description =
-          '''
-  %s is an e-book library manager. It can view, convert and catalog e-books \
-  in most of the major e-book formats. It can also talk to e-book reader \
-  devices. It can go out to the internet and fetch metadata for your books. \
-  It can download newspapers and convert them into e-books for convenient \
-  reading. It is cross platform, running on Linux, Windows and OS X.
-
-  For screenshots: https://%s.kovidgoyal.net/wiki/Screenshots
-
-  For installation/usage instructions please see
-  http://%s.kovidgoyal.net
-
-  For source code access:
-  bzr branch lp:%s
-
-  To update your copy of the source code:
-  bzr merge
-
-          '''%(APPNAME, APPNAME, APPNAME, APPNAME),
-          license = 'GPL',
-          classifiers    = [
-            'Development Status :: 4 - Beta',
-            'Environment :: Console',
-            'Environment :: X11 Applications :: Qt',
-            'Intended Audience :: Developers',
-            'Intended Audience :: End Users/Desktop',
-            'License :: OSI Approved :: GNU General Public License (GPL)',
-            'Natural Language :: English',
-            'Operating System :: POSIX :: Linux',
-            'Programming Language :: Python',
-            'Topic :: Software Development :: Libraries :: Python Modules',
-            'Topic :: System :: Hardware :: Hardware Drivers'
-            ],
-          cmdclass       = {
-                      'build_ext'     : build_ext,
-                      'build'         : build,
-                      'build_py'      : build_py,
-                      'pot'           : pot,
-                      'manual'        : manual,
-                      'resources'     : resources,
-                      'translations'  : translations,
-                      'get_translations': get_translations,
-                      'gui'           : gui,
-                      'clean'         : clean,
-                      'sdist'         : sdist,
-                      'update'        : update,
-                      'tag_release'   : tag_release,
-                      'upload_demo'   : upload_demo,
-                      'build_linux'   : build_linux,
-                      'build_linux32'  : build_linux32,
-                      'build_linux64' : build_linux64,
-                      'build_windows' : build_windows,
-                      'build_osx'     : build_osx,
-                      'build_osx64'   : build_osx64,
-                      'upload_installers': upload_installers,
-                      'upload_user_manual': upload_user_manual,
-                      'upload_to_pypi': upload_to_pypi,
-                      'upload_rss'    : upload_rss,
-                      'stage3' : stage3,
-                      'stage2' : stage2,
-                      'stage1' : stage1,
-                      'publish' : upload,
-                      'betas'  : betas,
-                      },
-         )
-
-    if 'develop' in ' '.join(sys.argv) and islinux:
-        subprocess.check_call('calibre_postinstall --do-not-reload-udev-hal', shell=True)
-        setup_mount_helper()
-    if 'install' in sys.argv and islinux:
-        subprocess.check_call('calibre_postinstall', shell=True)
-        setup_mount_helper()

setup/__init__.py

@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import sys, re, os
+
+iswindows = re.search('win(32|64)', sys.platform)
+isosx = 'darwin' in sys.platform
+islinux = not isosx and not iswindows
+SRC = os.path.abspath('src')
+
+__version__ = __appname__ = modules = functions = basenames = scripts = None
+
+def initialize_constants():
+    global __version__, __appname__, modules, functions, basenames, scripts
+
+    src = open('src/calibre/constants.py', 'rb').read()
+    __version__ = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1)
+    __appname__ = re.search(r'__appname__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1)
+    epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).\
+            search(open('src/calibre/linux.py', 'rb').read()).group(1)
+    entry_points = eval(epsrc, {'__appname__': __appname__})
+
+    def e2b(ep):
+        return re.search(r'\s*(.*?)\s*=', ep).group(1).strip()
+
+    def e2s(ep, base='src'):
+        return (base+os.path.sep+re.search(r'.*=\s*(.*?):', ep).group(1).replace('.', '/')+'.py').strip()
+
+    def e2m(ep):
+        return re.search(r'.*=\s*(.*?)\s*:', ep).group(1).strip()
+
+    def e2f(ep):
+        return ep[ep.rindex(':')+1:].strip()
+
+    basenames, functions, modules, scripts = {}, {}, {}, {}
+    for x in ('console', 'gui'):
+        y = x + '_scripts'
+        basenames[x] = list(map(e2b, entry_points[y]))
+        functions[x] = list(map(e2f, entry_points[y]))
+        modules[x] = list(map(e2m, entry_points[y]))
+        scripts[x] = list(map(e2s, entry_points[y]))
+
+initialize_constants()
+
+preferred_encoding = 'utf-8'
+
+def prints(*args, **kwargs):
+    '''
+    Print unicode arguments safely by encoding them to preferred_encoding
+    Has the same signature as the print function from Python 3, except for the
+    additional keyword argument safe_encode, which if set to True will cause the
+    function to use repr when encoding fails.
+    '''
+    file = kwargs.get('file', sys.stdout)
+    sep  = kwargs.get('sep', ' ')
+    end  = kwargs.get('end', '\n')
+    enc = preferred_encoding
+    safe_encode = kwargs.get('safe_encode', False)
+    for i, arg in enumerate(args):
+        if isinstance(arg, unicode):
+            try:
+                arg = arg.encode(enc)
+            except UnicodeEncodeError:
+                if not safe_encode:
+                    raise
+                arg = repr(arg)
+        if not isinstance(arg, str):
+            try:
+                arg = str(arg)
+            except ValueError:
+                arg = unicode(arg)
+            if isinstance(arg, unicode):
+                try:
+                    arg = arg.encode(enc)
+                except UnicodeEncodeError:
+                    if not safe_encode:
+                        raise
+                    arg = repr(arg)
+
+        file.write(arg)
+        if i != len(args)-1:
+            file.write(sep)
+    file.write(end)
+
+warnings = []
+
+def get_warnings():
+    return list(warnings)
+
+class Command(object):
+
+    SRC = SRC
+    RESOURCES = os.path.join(os.path.dirname(SRC), 'resources')
+    description = ''
+
+    sub_commands = []
+
+    def __init__(self):
+        self.d = os.path.dirname
+        self.j = os.path.join
+        self.a = os.path.abspath
+        self.b = os.path.basename
+        self.s = os.path.splitext
+        self.e = os.path.exists
+        self.real_uid = os.environ.get('SUDO_UID', None)
+        self.real_gid = os.environ.get('SUDO_GID', None)
+        self.real_user = os.environ.get('SUDO_USER', None)
+
+    def drop_privileges(self):
+        if not islinux or isosx:
+            return
+        if self.real_user is not None:
+            self.info('Dropping privileges to those of', self.real_user+':',
+                    self.real_uid)
+        if self.real_uid is not None:
+            os.seteuid(int(self.real_uid))
+        #if self.real_gid is not None:
+        #    os.setegid(int(self.real_gid))
+
+    def regain_privileges(self):
+        if not islinux or isosx:
+            return
+        if os.geteuid() != 0:
+            self.info('Trying to get root privileges')
+            os.seteuid(0)
+        #if os.getegid() != 0:
+        #    os.setegid(0)
+
+    def pre_sub_commands(self, opts):
+        pass
+
+    def run_all(self, opts):
+        self.pre_sub_commands(opts)
+        for cmd in self.sub_commands:
+            self.info('Running', cmd.__class__.__name__)
+            cmd.run(opts)
+
+        self.info('Running', self.__class__.__name__)
+        self.run(opts)
+
+    def add_all_options(self, parser):
+        import setup.commands as commands
+        self.sub_commands = [getattr(commands, cmd) for cmd in
+                self.sub_commands]
+        for cmd in self.sub_commands:
+            cmd.add_options(parser)
+
+        self.add_options(parser)
+
+
+    def run(self, opts):
+        raise NotImplementedError
+
+    def add_options(self, parser):
+        pass
+
+    def clean(self):
+        pass
+
+    @classmethod
+    def newer(cls, targets, sources):
+        '''
+        Return True if sources is newer that targets or if targets
+        does not exist.
+        '''
+        if isinstance(targets, basestring):
+            targets = [targets]
+        if isinstance(sources, basestring):
+            sources = [sources]
+        for f in targets:
+            if not os.path.exists(f):
+                return True
+        ttimes = map(lambda x: os.stat(x).st_mtime, targets)
+        stimes = map(lambda x: os.stat(x).st_mtime, sources)
+        newest_source, oldest_target = max(stimes), min(ttimes)
+        return newest_source > oldest_target
+
+    def info(self, *args, **kwargs):
+        prints(*args, **kwargs)
+        sys.stdout.flush()
+
+    def warn(self, *args, **kwargs):
+        print '\n'+'_'*20, 'WARNING','_'*20
+        prints(*args, **kwargs)
+        print '_'*50
+        warnings.append((args, kwargs))
+        sys.stdout.flush()
+

setup/build_environment.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+from distutils.spawn import find_executable
+
+from PyQt4 import pyqtconfig
+
+from setup import isosx, iswindows
+
+OSX_SDK = '/Developer/SDKs/MacOSX10.5.sdk'
+if not os.path.exists(OSX_SDK):
+    OSX_SDK = '/Developer/SDKs/MacOSX10.4u.sdk'
+leopard_build = '10.5' in OSX_SDK
+
+os.environ['MACOSX_DEPLOYMENT_TARGET'] = '10.5' if leopard_build else '10.4'
+
+NMAKE = RC = msvc = MT = win_inc = win_lib = None
+if iswindows:
+    from distutils import msvc9compiler
+    msvc = msvc9compiler.MSVCCompiler()
+    msvc.initialize()
+    NMAKE = msvc.find_exe('nmake.exe')
+    RC = msvc.find_exe('rc.exe')
+    SDK = os.environ.get('WINSDK', r'C:\Program Files\Microsoft SDKs\Windows\v6.0A')
+    win_inc = os.environ['include'].split(';')
+    win_lib = os.environ['lib'].split(';')
+    for p in win_inc:
+        if 'SDK' in p:
+            MT = os.path.join(os.path.dirname(p), 'bin', 'mt.exe')
+    MT = os.path.join(SDK, 'bin', 'mt.exe')
+
+QMAKE = '/Volumes/sw/qt/bin/qmake' if isosx else 'qmake'
+if find_executable('qmake-qt4'):
+    QMAKE = find_executable('qmake-qt4')
+elif find_executable('qmake'):
+    QMAKE = find_executable('qmake')
+QMAKE = os.environ.get('QMAKE', QMAKE)
+
+
+pyqt = pyqtconfig.Configuration()
+
+qt_inc = pyqt.qt_inc_dir
+qt_lib = pyqt.qt_lib_dir
+
+fc_inc = '/usr/include/fontconfig'
+fc_lib = '/usr/lib'
+poppler_inc = '/usr/include/poppler/qt4'
+poppler_lib = '/usr/lib'
+poppler_libs = []
+podofo_inc = '/usr/include/podofo'
+podofo_lib = '/usr/lib'
+
+if iswindows:
+    fc_inc = r'C:\cygwin\home\kovid\fontconfig\include\fontconfig'
+    fc_lib = r'C:\cygwin\home\kovid\fontconfig\lib'
+    poppler_inc = r'C:\cygwin\home\kovid\poppler\include\poppler\qt4'
+    poppler_lib = r'C:\cygwin\home\kovid\poppler\lib'
+    poppler_libs = ['QtCore4', 'QtGui4']
+    podofo_inc = 'C:\\podofo\\include\\podofo'
+    podofo_lib = r'C:\podofo'
+
+if isosx:
+    fc_inc = '/Users/kovid/fontconfig/include/fontconfig'
+    fc_lib = '/Users/kovid/fontconfig/lib'
+    poppler_inc = '/Volumes/sw/build/poppler-0.10.7/qt4/src'
+    poppler_lib = '/Users/kovid/poppler/lib'
+    podofo_inc = '/usr/local/include/podofo'
+    podofo_lib = '/usr/local/lib'
+
+
+fc_inc = os.environ.get('FC_INC_DIR', fc_inc)
+fc_lib = os.environ.get('FC_LIB_DIR', fc_lib)
+fc_error = None if os.path.exists(os.path.join(fc_inc, 'fontconfig.h')) else \
+    ('fontconfig header files not found on your system. '
+            'Try setting the FC_INC_DIR and FC_LIB_DIR environment '
+            'variables.')
+
+
+poppler_inc = os.environ.get('POPPLER_INC_DIR', poppler_inc)
+poppler_lib = os.environ.get('POPPLER_LIB_DIR', poppler_lib)
+poppler_error = None if os.path.exists(os.path.join(poppler_inc,
+    'poppler-qt4.h'))  else \
+    ('Poppler not found on your system. Various PDF related',
+    ' functionality will not work. Use the POPPLER_INC_DIR and',
+    ' POPPLER_LIB_DIR environment variables.')
+
+
+podofo_lib = os.environ.get('PODOFO_LIB_DIR', podofo_lib)
+podofo_inc = os.environ.get('PODOFO_INC_DIR', podofo_inc)
+podofo_error = None if os.path.exists(os.path.join(podofo_inc, 'podofo.h')) else \
+        ('PoDoFo not found on your system. Various PDF related',
+    ' functionality will not work. Use the PODOFO_INC_DIR and',
+    ' PODOFO_LIB_DIR environment variables.')
+
+

setup/commands.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+__all__ = [
+        'pot', 'translations', 'get_translations', 'iso639',
+        'build',
+        'develop',
+        'clean'
+        ]
+
+import os, shutil
+
+from setup.translations import POT, GetTranslations, Translations, ISO639
+from setup import Command
+pot = POT()
+translations = Translations()
+get_translations = GetTranslations()
+iso639 = ISO639()
+
+from setup.extensions import Build
+build = Build()
+
+from setup.install import Develop
+develop = Develop()
+
+class Clean(Command):
+
+    description='''Delete all computer generated files in the source tree'''
+
+    sub_commands = __all__
+
+    def add_options(self, parser):
+        opt = parser.remove_option('--only')
+        help = 'Only run clean for the specified command. Choices: '+\
+                ', '.join(__all__)
+        parser.add_option('-1', '--only', default='all',
+        choices=__all__+['all'], help=help)
+
+    def run_all(self, opts):
+        self.info('Cleaning...')
+        only = None if opts.only == 'all' else commands[opts.only]
+        for cmd in self.sub_commands:
+            if only is not None and only is not cmd:
+                continue
+            self.info('\tCleaning', command_names[cmd])
+            cmd.clean()
+
+    def clean(self):
+        for root, _, files in os.walk(self.d(self.SRC)):
+            for name in files:
+                for t in ('.pyc', '.pyo', '~', '.swp', '.swo'):
+                    if name.endswith(t):
+                        os.remove(os.path.join(root, name))
+                        break
+
+        for dir in ('dist', os.path.join('src', 'calibre.egg-info')):
+            shutil.rmtree(dir, ignore_errors=True)
+
+clean = Clean()
+
+
+commands = {}
+for x in __all__:
+    commands[x] = locals()[x]
+
+command_names = dict(zip(commands.values(), commands.keys()))

setup/extensions.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import textwrap, os, shlex, subprocess, glob, shutil
+from distutils import sysconfig
+
+from PyQt4.pyqtconfig import QtGuiModuleMakefile
+
+from setup import Command, islinux, isosx, SRC, iswindows
+from setup.build_environment import fc_inc, fc_lib, qt_inc, qt_lib, \
+        fc_error, poppler_libs, poppler_lib, poppler_inc, podofo_inc, \
+        podofo_lib, podofo_error, poppler_error, pyqt, OSX_SDK, NMAKE, \
+        leopard_build, QMAKE, msvc, MT, win_inc, win_lib
+
+isunix = islinux or isosx
+
+make = 'make' if isunix else NMAKE
+
+class Extension(object):
+
+    def absolutize(self, paths):
+        return [x if os.path.isabs(x) else os.path.join(SRC, x.replace('/',
+            os.sep)) for x in paths]
+
+
+    def __init__(self, name, sources, **kwargs):
+        self.name = name
+        self.needs_cxx = bool([1 for x in sources if os.path.splitext(x)[1] in
+            ('.cpp', '.c++', '.cxx')])
+        self.sources = self.absolutize(sources)
+        self.headers = self.absolutize(kwargs.get('headers', []))
+        self.sip_files = self.absolutize(kwargs.get('sip_files', []))
+        self.inc_dirs = self.absolutize(kwargs.get('inc_dirs', []))
+        self.lib_dirs = self.absolutize(kwargs.get('lib_dirs', []))
+        self.error = kwargs.get('error', None)
+        self.libraries = kwargs.get('libraries', [])
+        self.cflags = kwargs.get('cflags', [])
+        self.ldflags = kwargs.get('ldflags', [])
+        self.optional = kwargs.get('optional', False)
+
+extensions = [
+    Extension('lzx',
+            ['calibre/utils/lzx/lzxmodule.c',
+                    'calibre/utils/lzx/compressor.c',
+                    'calibre/utils/lzx/lzxd.c',
+                    'calibre/utils/lzx/lzc.c',
+                    'calibre/utils/lzx/lzxc.c'],
+            headers=['calibre/utils/lzx/msstdint.h',
+                    'calibre/utils/lzx/lzc.h',
+                    'calibre/utils/lzx/lzxmodule.h',
+                    'calibre/utils/lzx/system.h',
+                    'calibre/utils/lzx/lzxc.h',
+                    'calibre/utils/lzx/lzxd.h',
+                    'calibre/utils/lzx/mspack.h'],
+            inc_dirs=['calibre/utils/lzx']),
+
+    Extension('fontconfig',
+        ['calibre/utils/fonts/fontconfig.c'],
+        inc_dirs = [fc_inc],
+        libraries=['fontconfig'],
+        lib_dirs=[fc_lib],
+        error=fc_error),
+
+    Extension('msdes',
+                ['calibre/utils/msdes/msdesmodule.c',
+                        'calibre/utils/msdes/des.c'],
+                headers=['calibre/utils/msdes/spr.h',
+                        'calibre/utils/msdes/d3des.h'],
+                inc_dirs=['calibre/utils/msdes']),
+
+    Extension('cPalmdoc',
+        ['calibre/ebooks/compression/palmdoc.c']),
+
+    Extension('calibre_poppler',
+                ['calibre/utils/poppler/poppler.cpp'],
+                libraries=(['poppler', 'poppler-qt4']+poppler_libs),
+                lib_dirs=[os.environ.get('POPPLER_LIB_DIR',
+                    poppler_lib), qt_lib],
+                inc_dirs=[poppler_inc, qt_inc],
+                error=poppler_error,
+                optional=True),
+
+    Extension('podofo',
+                    ['calibre/utils/podofo/podofo.cpp'],
+                    libraries=['podofo'],
+                    lib_dirs=[podofo_lib],
+                    inc_dirs=[podofo_inc],
+                    error=podofo_error),
+
+    Extension('pictureflow',
+                ['calibre/gui2/pictureflow/pictureflow.cpp'],
+                inc_dirs = ['calibre/gui2/pictureflow'],
+                headers = ['calibre/gui2/pictureflow/pictureflow.h'],
+                sip_files = ['calibre/gui2/pictureflow/pictureflow.sip']
+                )
+
+    ]
+
+if iswindows:
+    extensions.append(Extension('winutil',
+                ['calibre/utils/windows/winutil.c'],
+                libraries=['shell32', 'setupapi'],
+                include_dirs=os.environ.get('INCLUDE',
+                        'C:/WinDDK/6001.18001/inc/api/;'
+                        'C:/WinDDK/6001.18001/inc/crt/').split(';'),
+                cflags=['/X']
+                ))
+if isosx:
+    extensions.append(Extension('usbobserver',
+                ['calibre/devices/usbobserver/usbobserver.c'],
+                ldflags=['-framework', 'IOKit'])
+            )
+
+
+if isunix:
+    cc = os.environ.get('CC', 'gcc')
+    cxx = os.environ.get('CXX', 'g++')
+    cflags = '-O3 -Wall -DNDEBUG -fPIC -fno-strict-aliasing -pipe'.split()
+    ldflags = ['-Wall']
+    cflags += shlex.split(os.environ.get('CFLAGS', ''))
+    ldflags += shlex.split(os.environ.get('LDFLAGS', ''))
+
+if islinux:
+    cflags.append('-pthread')
+    ldflags.append('-shared')
+    cflags.append('-I'+sysconfig.get_python_inc())
+    ldflags.append('-lpython'+sysconfig.get_python_version())
+
+
+if isosx:
+    x, p = ('x86_64', 'ppc64') if leopard_build else ('i386', 'ppc')
+    archs = ['-arch', x, '-arch', p, '-isysroot',
+                OSX_SDK]
+    cflags.extend(archs)
+    ldflags.extend(archs)
+    ldflags.extend('-bundle -undefined dynamic_lookup'.split())
+    cflags.extend(['-fno-common', '-dynamic'])
+    cflags.append('-I'+sysconfig.get_python_inc())
+
+
+if iswindows:
+    cc = cxx = msvc.cc
+    cflags = '/c /nologo /Ox /MD /W3 /EHsc /DNDEBUG'.split()
+    ldflags = '/DLL /nologo /INCREMENTAL:NO'.split()
+    for p in win_inc:
+        cflags.append('-I'+p)
+    for p in win_lib:
+        ldflags.append('/LIBPATH:'+p)
+    cflags.append('-I%s'%sysconfig.get_python_inc())
+    ldflags.append('/LIBPATH:'+os.path.join(sysconfig.PREFIX, 'libs'))
+
+
+class Build(Command):
+
+    def add_options(self, parser):
+        parser.set_usage(parser.usage + textwrap.dedent('''
+
+        calibre depends on several python extensions written in C/C++.
+        This command will compile them. You can influence the compile
+        process by several environment variables, listed below:
+
+           CC      - C Compiler defaults to gcc
+           CXX     - C++ Compiler, defaults to g++
+           CFLAGS  - Extra compiler flags
+           LDFLAGS - Extra linker flags
+
+           FC_INC_DIR - fontconfig header files
+           FC_LIB_DIR - fontconfig library
+
+           POPPLER_INC_DIR - poppler header files
+           POPPLER_LIB_DIR - poppler-qt4 library
+
+           PODOFO_INC_DIR - podofo header files
+           PODOFO_LIB_DIR - podofo library files
+
+           QMAKE          - Path to qmake
+           VS90COMNTOOLS  - Location of Microsoft Visual Studio 9 Tools
+
+        '''))
+        choices = [e.name for e in extensions]+['all']
+        parser.add_option('-1', '--only', choices=choices, default='all',
+                help=('Build only the named extension. Available: '+
+                    ', '.join(choices)+'. Default:%default'))
+
+    def run(self, opts):
+        self.obj_dir = os.path.join(os.path.dirname(SRC), 'build', 'objects')
+        if not os.path.exists(self.obj_dir):
+            os.makedirs(self.obj_dir)
+        for ext in extensions:
+            if opts.only != 'all' and opts.only != ext.name:
+                continue
+            if ext.error is not None:
+                if ext.optional:
+                    self.warn(ext.error)
+                    continue
+                else:
+                    raise Exception(ext.error)
+            dest = self.dest(ext)
+            self.info('\n####### Building extension', ext.name, '#'*7)
+            self.build(ext, dest)
+
+    def dest(self, ext):
+        ex = '.pyd' if iswindows else '.so'
+        return os.path.join(SRC, 'calibre', 'plugins', ext.name)+ex
+
+    def inc_dirs_to_cflags(self, dirs):
+        return ['-I'+x for x in dirs]
+
+    def lib_dirs_to_ldflags(self, dirs):
+        pref = '/LIBPATH:' if iswindows else '-L'
+        return [pref+x for x in dirs]
+
+    def libraries_to_ldflags(self, dirs):
+        pref = '' if iswindows else '-l'
+        suff = '.lib' if iswindows else ''
+        return [pref+x+suff for x in dirs]
+
+    def build(self, ext, dest):
+        if ext.sip_files:
+            return self.build_pyqt_extension(ext, dest)
+        compiler = cxx if ext.needs_cxx else cc
+        linker = msvc.linker if iswindows else compiler
+        objects = []
+        einc = self.inc_dirs_to_cflags(ext.inc_dirs)
+        obj_dir = self.j(self.obj_dir, ext.name)
+        if not os.path.exists(obj_dir):
+            os.makedirs(obj_dir)
+        for src in ext.sources:
+            obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o')
+            objects.append(obj)
+            if self.newer(obj, [src]+ext.headers):
+                inf = '/Tp' if src.endswith('.cpp') else '/Tc'
+                sinc = [inf+src] if iswindows else ['-c', src]
+                oinc = ['/Fo'+obj] if iswindows else ['-o', obj]
+                cmd = [compiler] + cflags + ext.cflags + einc + sinc + oinc
+                self.info(' '.join(cmd))
+                subprocess.check_call(cmd)
+
+        dest = self.dest(ext)
+        elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
+        xlib = self.libraries_to_ldflags(ext.libraries)
+        if self.newer(dest, objects):
+            print 'Linking', ext.name
+            cmd = [linker]
+            if iswindows:
+                cmd += ldflags + ext.ldflags + elib + xlib + \
+                    ['/EXPORT:init'+ext.name] + objects + ['/OUT:'+dest]
+            else:
+                cmd += objects + ['-o', dest] + ldflags + ext.ldflags + elib + xlib
+            print ' '.join(cmd)
+            subprocess.check_call(cmd)
+            if iswindows:
+                manifest = dest+'.manifest'
+                cmd = [MT, '-manifest', manifest, '-outputresource:%s;2'%dest]
+                self.info(*cmd)
+                subprocess.check_call(cmd)
+                os.remove(manifest)
+                for x in ('.exp', '.lib'):
+                    x = os.path.splitext(dest)[0]+x
+                    if os.path.exists(x):
+                        os.remove(x)
+
+    def build_qt_objects(self, ext):
+        obj_pat = 'release\\*.obj' if iswindows else '*.o'
+        objects = glob.glob(obj_pat)
+        if not objects or self.newer(objects, ext.sources+ext.headers):
+            archs = 'x86_64 ppc64' if leopard_build else 'x86 ppc'
+            pro = textwrap.dedent('''\
+                TARGET   = %s
+                TEMPLATE = lib
+                HEADERS  = %s
+                SOURCES  = %s
+                VERSION  = 1.0.0
+                CONFIG   += %s
+            ''')%(ext.name, ' '.join(ext.headers), ' '.join(ext.sources), archs)
+            open(ext.name+'.pro', 'wb').write(pro)
+            subprocess.check_call([QMAKE, '-o', 'Makefile', ext.name+'.pro'])
+            if leopard_build:
+                raw = open('Makefile', 'rb').read()
+                open('Makefile', 'wb').write(raw.replace('ppc64', 'x86_64'))
+            subprocess.check_call([make, '-f', 'Makefile'])
+            objects = glob.glob(obj_pat)
+        return list(map(self.a, objects))
+
+    def build_pyqt_extension(self, ext, dest):
+        pyqt_dir = self.j(self.d(self.SRC), 'build', 'pyqt')
+        src_dir = self.j(pyqt_dir, ext.name)
+        qt_dir  = self.j(src_dir, 'qt')
+        if not self.e(qt_dir):
+            os.makedirs(qt_dir)
+        cwd = os.getcwd()
+        try:
+            os.chdir(qt_dir)
+            qt_objects = self.build_qt_objects(ext)
+        finally:
+            os.chdir(cwd)
+
+        sip_files = ext.sip_files
+        ext.sip_files = []
+        sipf = sip_files[0]
+        sbf = self.j(src_dir, self.b(sipf)+'.sbf')
+        if self.newer(sbf, [sipf]+ext.headers):
+            exe = '.exe' if iswindows else ''
+            cmd = [pyqt.sip_bin+exe, '-w', '-c', src_dir, '-b', sbf, '-I'+\
+                    pyqt.pyqt_sip_dir] + shlex.split(pyqt.pyqt_sip_flags) + [sipf]
+            self.info(' '.join(cmd))
+            subprocess.check_call(cmd)
+        module = self.j(src_dir, self.b(dest))
+        if self.newer(dest, [sbf]+qt_objects):
+            mf = self.j(src_dir, 'Makefile')
+            makefile = QtGuiModuleMakefile(configuration=pyqt, build_file=sbf,
+                    makefile=mf, universal=OSX_SDK, qt=1)
+            makefile.extra_lflags = qt_objects
+            makefile.extra_include_dirs = ext.inc_dirs
+            makefile.generate()
+            if leopard_build:
+                raw = open(mf, 'rb').read()
+                raw = raw.replace('ppc64 x86_64', 'x86_64')
+                for x in ('ppc64', 'ppc', 'i386'):
+                    raw = raw.replace(x, 'x86_64')
+                open(mf, 'wb').write(raw)
+
+            subprocess.check_call([make, '-f', mf], cwd=src_dir)
+            shutil.copy2(module, dest)
+
+    def clean(self):
+        for ext in extensions:
+            dest = self.dest(ext)
+            for x in (dest, dest+'.manifest'):
+                if os.path.exists(x):
+                    os.remove(x)
+        shutil.rmtree(self.j(self.d(self.SRC), 'build'))
+
+
+
+
+
+
+
+
+
+
+

setup/install.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import sys, os, textwrap
+
+from setup import Command, islinux, basenames, modules, functions
+
+TEMPLATE = '''\
+#!/usr/bin/env python
+
+"""
+This is the standard runscript for all of calibre's tools.
+Do not modify it unless you know what you are doing.
+"""
+
+import sys
+sys.path.insert(0, {path!r})
+
+sys.resources_location = {resources!r}
+sys.extensions_location = {extensions!r}
+
+from {module} import {func!s}
+sys.exit({func!s}())
+'''
+
+class Develop(Command):
+
+    description = 'Setup a development environment'
+    MODE = 0755
+
+    sub_commands = ['build']
+
+    def add_options(self, parser):
+        parser.set_usage(textwrap.dedent('''\
+                ***
+
+                Setup a development environment for calibre.
+                This allows you to run calibre directly from the source tree.
+                Binaries will be installed in <prefix>/bin where <prefix> is
+                the prefix of your python installation. This can be controlled
+                via the --prefix option.
+                '''))
+        parser.add_option('--prefix',
+            help='Binaries will be installed in <prefix>/bin')
+
+    def pre_sub_commands(self, opts):
+        if not islinux:
+            self.info('\nSetting up a development environment is only '
+                    'supported on linux. On other platforms, install the calibre '
+                    'binary and use the calibre-debug command.')
+            raise SystemExit(1)
+
+        if not os.geteuid() == 0:
+            self.info('\nError: This command must be run as root.')
+            raise SystemExit(1)
+        self.drop_privileges()
+
+    def run(self, opts):
+        self.regain_privileges()
+        self.find_locations(opts)
+        self.write_templates(opts)
+        self.success()
+
+    def success(self):
+        self.info('\nDevelopment environment successfully setup')
+
+    def find_locations(self, opts):
+        self.path = self.SRC
+        self.resources = self.j(self.d(self.SRC), 'resources')
+        self.extensions = self.j(self.SRC, 'calibre', 'plugins')
+
+    def write_templates(self, opts):
+        for typ in ('console', 'gui'):
+            for name, mod, func in zip(basenames[typ], modules[typ],
+                    functions[typ]):
+                script = TEMPLATE.format(
+                        module=mod, func=func,
+                        path=self.path, resources=self.resources,
+                        extensions=self.extensions)
+                prefix = opts.prefix
+                if prefix is None:
+                    prefix = sys.prefix
+                path = self.j(prefix, 'bin', name)
+                self.info('Installing binary:', path)
+                open(path, 'wb').write(script)
+                os.chmod(path, self.MODE)
+

setup/pygettext.py

@@ -164,8 +164,7 @@ DEFAULTKEYWORDS = ', '.join(default_keywords)
 
 EMPTYSTRING = ''
 
-from calibre.constants import __appname__
+from setup import __appname__, __version__ as version
-from calibre.constants import __version__ as version
 
 # The normal pot-file header. msgmerge and Emacs's po-mode work better if it's
 # there.

setup/translations.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os, cStringIO, tempfile, shutil, atexit, subprocess, glob, re
+from distutils import sysconfig
+
+from setup import Command, __appname__
+from setup.pygettext import main as pygettext
+
+class POT(Command):
+
+    description = 'Update the .pot translation template'
+    PATH = os.path.join(Command.SRC, __appname__, 'translations')
+
+    def source_files(self):
+        ans = []
+        for root, _, files in os.walk(os.path.dirname(self.PATH)):
+            for name in files:
+                if name.endswith('.py'):
+                    ans.append(os.path.abspath(os.path.join(root, name)))
+        return ans
+
+
+    def run(self, opts):
+        files = self.source_files()
+        buf = cStringIO.StringIO()
+        self.info('Creating translations template...')
+        tempdir = tempfile.mkdtemp()
+        atexit.register(shutil.rmtree, tempdir)
+        pygettext(buf, ['-k', '__', '-p', tempdir]+files)
+        src = buf.getvalue()
+        pot = os.path.join(self.PATH, __appname__+'.pot')
+        f = open(pot, 'wb')
+        f.write(src)
+        f.close()
+        self.info('Translations template:', os.path.abspath(pot))
+        return pot
+
+
+class Translations(POT):
+    description='''Compile the translations'''
+    DEST = os.path.join(os.path.dirname(POT.SRC), 'resources', 'localization',
+            'locales')
+
+    def po_files(self):
+        return glob.glob(os.path.join(self.PATH, '*.po'))
+
+    def mo_file(self, po_file):
+        locale = os.path.splitext(os.path.basename(po_file))[0]
+        return locale, os.path.join(self.DEST, locale, 'LC_MESSAGES', 'messages.mo')
+
+
+    def run(self, opts):
+        for f in self.po_files():
+            locale, dest = self.mo_file(f)
+            base = os.path.dirname(dest)
+            if not os.path.exists(base):
+                os.makedirs(base)
+            if self.newer(dest, f):
+                self.info('\tCompiling translations for', locale)
+                subprocess.check_call(['msgfmt', '-o', dest, f])
+            if locale in ('en_GB', 'nds', 'te', 'yi'):
+                continue
+            pycountry = self.j(sysconfig.get_python_lib(), 'pycountry',
+                    'locales', locale, 'LC_MESSAGES')
+            if os.path.exists(pycountry):
+                iso639 = self.j(pycountry, 'iso639.mo')
+                dest = self.j(self.d(dest), self.b(iso639))
+                if self.newer(dest, iso639):
+                    self.info('\tCopying ISO 639 translations')
+                    shutil.copy2(iso639, dest)
+            else:
+                self.warn('No ISO 639 translations for locale:', locale)
+
+        self.write_stats()
+
+    @property
+    def stats(self):
+        return self.j(self.d(self.DEST), 'stats.pickle')
+
+    def get_stats(self, path):
+        return subprocess.Popen(['msgfmt', '--statistics', '-o', '/dev/null',
+            path],
+            stderr=subprocess.PIPE).stderr.read()
+
+    def write_stats(self):
+        files = self.po_files()
+        dest = self.stats
+        if not self.newer(dest, files):
+            return
+        self.info('Calculating translation statistics...')
+        raw = self.get_stats(self.j(self.PATH, 'calibre.pot'))
+        total = int(raw.split(',')[-1].strip().split()[0])
+        stats = {}
+        for f in files:
+            raw = self.get_stats(f)
+            trans = int(raw.split()[0])
+            locale = self.mo_file(f)[0]
+            stats[locale] = min(1.0, float(trans)/total)
+
+
+        import cPickle
+        cPickle.dump(stats, open(dest, 'wb'), -1)
+
+    def clean(self):
+        if os.path.exists(self.stats):
+            os.remove(self.stats)
+        for f in self.po_files():
+            l, d = self.mo_file(f)
+            i = self.j(self.d(d), 'iso639.mo')
+            for x in (i, d):
+                if os.path.exists(x):
+                    os.remove(x)
+
+
+class GetTranslations(Translations):
+
+    description = 'Get updated translations from Launchpad'
+    BRANCH = 'lp:~kovid/calibre/translations'
+
+    @classmethod
+    def modified_translations(cls):
+        raw = subprocess.Popen(['bzr', 'status'],
+                stdout=subprocess.PIPE).stdout.read().strip()
+        for line in raw.splitlines():
+            line = line.strip()
+            if line.startswith(cls.PATH) and line.endswith('.po'):
+                yield line
+
+    def run(self, opts):
+        if len(list(self.modified_translations())) == 0:
+            subprocess.check_call(['bzr', 'merge', self.BRANCH])
+        if len(list(self.modified_translations())) == 0:
+            print 'No updated translations available'
+        else:
+            subprocess.check_call(['bzr', 'commit', '-m',
+                'IGN:Updated translations', self.PATH])
+        self.check_for_errors()
+
+    @classmethod
+    def check_for_errors(cls):
+        errors = os.path.join(tempfile.gettempdir(), 'calibre-translation-errors')
+        if os.path.exists(errors):
+            shutil.rmtree(errors)
+        os.mkdir(errors)
+        pofilter = ('pofilter', '-i', cls.PATH, '-o', errors,
+                '-t', 'accelerators', '-t', 'escapes', '-t', 'variables',
+                #'-t', 'xmltags',
+                #'-t', 'brackets',
+                #'-t', 'emails',
+                #'-t', 'doublequoting',
+                #'-t', 'filepaths',
+                #'-t', 'numbers',
+                '-t', 'options',
+                #'-t', 'urls',
+                '-t', 'printf')
+        subprocess.check_call(pofilter)
+        errfiles = glob.glob(errors+os.sep+'*.po')
+        subprocess.check_call(['gvim', '-f', '-p', '--']+errfiles)
+        for f in errfiles:
+            with open(f, 'r+b') as f:
+                raw = f.read()
+                raw = re.sub(r'# \(pofilter\).*', '', raw)
+                f.seek(0)
+                f.truncate()
+                f.write(raw)
+
+        subprocess.check_call(['pomerge', '-t', cls.PATH, '-i', errors, '-o',
+            cls.PATH])
+        if len(list(cls.modified_translations())) > 0:
+            subprocess.call(['bzr', 'diff', cls.PATH])
+            yes = raw_input('Merge corrections? [y/n]: ').strip()
+            if yes in ['', 'y']:
+                subprocess.check_call(['bzr', 'commit', '-m',
+                    'IGN:Translation corrections', cls.PATH])
+
+
+class ISO639(Command):
+
+    XML = '/usr/lib/python2.6/site-packages/pycountry/databases/iso639.xml'
+
+    def run(self, opts):
+        src = self.XML
+        if not os.path.exists(src):
+            raise Exception(src + ' does not exist')
+        dest = self.j(self.d(self.SRC), 'resources', 'localization',
+                'iso639.pickle')
+        if not self.newer(dest, src):
+            self.info('Pickled code is up to date')
+            return
+        self.info('Pickling ISO-639 codes to', dest)
+        from lxml import etree
+        root = etree.fromstring(open(src, 'rb').read())
+        by_2 = {}
+        by_3b = {}
+        by_3t = {}
+        codes2, codes3t, codes3b = set([]), set([]), set([])
+        for x in root.xpath('//iso_639_entry'):
+            name = x.get('name')
+            two = x.get('iso_639_1_code', None)
+            if two is not None:
+                by_2[two] = name
+                codes2.add(two)
+            by_3b[x.get('iso_639_2B_code')] = name
+            by_3t[x.get('iso_639_2T_code')] = name
+            codes3b.add(x.get('iso_639_2B_code'))
+            codes3t.add(x.get('iso_639_2T_code'))
+
+        from cPickle import dump
+        x = {'by_2':by_2, 'by_3b':by_3b, 'by_3t':by_3t, 'codes2':codes2,
+                'codes3b':codes3b, 'codes3t':codes3t}
+        dump(x, open(dest, 'wb'), -1)
+

src/calibre/constants.py

@@ -14,13 +14,14 @@ numeric_version = tuple(_ver)
 Various run time constants.
 '''
 
-import sys, locale, codecs, os
+import sys, locale, codecs
 from calibre.utils.terminfo import TerminalController
 
 terminal_controller = TerminalController(sys.stdout)
 
 iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower()
 isosx     = 'darwin' in sys.platform.lower()
+isnewosx = isosx and getattr(sys, 'new_app_bundle', False)
 islinux   = not(iswindows or isosx)
 isfrozen  = hasattr(sys, 'frozen')
 
@@ -50,19 +51,8 @@ if plugins is None:
     # Load plugins
     def load_plugins():
         plugins = {}
-        if isfrozen:
+        plugin_path = sys.extensions_location
-            if iswindows:
+        sys.path.insert(0, plugin_path)
-                plugin_path = os.path.join(os.path.dirname(sys.executable), 'plugins')
-                sys.path.insert(1, os.path.dirname(sys.executable))
-            elif isosx:
-                plugin_path = os.path.join(getattr(sys, 'frameworks_dir'), 'plugins')
-            elif islinux:
-                plugin_path = os.path.join(getattr(sys, 'frozen_path'), 'plugins')
-            sys.path.insert(0, plugin_path)
-        else:
-            import pkg_resources
-            plugin_path = getattr(pkg_resources, 'resource_filename')('calibre', 'plugins')
-            sys.path.insert(0, plugin_path)
 
         for plugin in ['pictureflow', 'lzx', 'msdes', 'podofo', 'cPalmdoc',
             'fontconfig', 'calibre_poppler'] + \
@@ -74,6 +64,7 @@ if plugins is None:
                 p = None
                 err = str(err)
             plugins[plugin] = (p, err)
+        sys.path.remove(plugin_path)
         return plugins
 
     plugins = load_plugins()

src/calibre/startup.py

@@ -25,6 +25,12 @@ _run_once = False
 if not _run_once:
     _run_once = True
 
+    ################################################################################
+    # Setup resources
+    import calibre.utils.resources as resources
+    resources
+
+
     ################################################################################
     # Setup translations
 

src/calibre/utils/ipc/launch.py

@@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en'
 
 import subprocess, os, sys, time
 
-from calibre.constants import iswindows, isosx, isfrozen
+from calibre.constants import iswindows, isosx, isfrozen, isnewosx
 from calibre.utils.config import prefs
 from calibre.ptempfile import PersistentTemporaryFile
 
@@ -16,8 +16,6 @@ if iswindows:
     import win32process
     _windows_null_file = open(os.devnull, 'wb')
 
-isnewosx = isosx and getattr(sys, 'new_app_bundle', False)
-
 class Worker(object):
     '''
     Platform independent object for launching child processes. All processes

src/calibre/utils/localization.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import os
+
+_available_translations = None
+
+def available_translations():
+    global _available_translations
+    if _available_translations is None:
+        base = P('resources/localization/locales')
+        _available_translations = os.listdir(base)
+    return _available_translations

src/calibre/utils/resources.py

@@ -0,0 +1,17 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+import __builtin__, sys, os
+
+def get_path(path):
+    path = path.replace(os.sep, '/')
+    return os.path.join(sys.resources_location, *path.split('/'))
+
+__builtin__.__dict__['P'] = get_path
+

src/calibre/web/feeds/recipes/recipe_ars_technica.py

@@ -10,9 +10,8 @@ from calibre.web.feeds.news import BasicNewsRecipe
 
 class ArsTechnica2(BasicNewsRecipe):
     title                 = u'Ars Technica'
-    language = 'en'
+    language              = _('English')
-
+    __author__            = 'Darko Miletic and Sujata Raman'
-    __author__            = 'Darko Miletic'
     description           = 'The art of technology'
     publisher             = 'Ars Technica'
     category              = 'news, IT, technology'
@@ -23,13 +22,14 @@ class ArsTechnica2(BasicNewsRecipe):
     remove_javascript     = True
     use_embedded_content  = False
 
-    html2lrf_options = [
+    extra_css = '''
-                          '--comment', description
+                    .news-item-title{font-size: medium ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;}
-                        , '--category', category
+                    .news-item-teaser{font-size: small ;font-family:Arial,Helvetica,sans-serif; font-weight:bold;}
-                        , '--publisher', publisher
+                    .news-item-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
-                        ]
+                    .news-item-text{font-size:x-small;font-family:Arial,Helvetica,sans-serif;}
-    
+                    .news-item-figure-caption-text{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:bold;}
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+                    .news-item-figure-caption-byline{font-size:xx-small; font-family:Arial,Helvetica,sans-serif;font-weight:normal;}
+                '''
 
     keep_only_tags = [dict(name='div', attrs={'id':['news-item-info','news-item']})]
 
@@ -58,9 +58,11 @@ class ArsTechnica2(BasicNewsRecipe):
                str = self.tag_to_string(atag)
                if str.startswith('Next'):
                   soup2 = self.index_to_soup(atag['href'])
+
                   texttag = soup2.find('div', attrs={'class':'news-item-text'})
                   for it in texttag.findAll(style=True):
                       del it['style']
+
                   newpos = len(texttag.contents)
                   self.append_page(soup2,texttag,newpos)
                   texttag.extract()
@@ -69,10 +71,17 @@ class ArsTechnica2(BasicNewsRecipe):
 
 
     def preprocess_html(self, soup):
+
         ftag = soup.find('div', attrs={'class':'news-item-byline'})
         if ftag:
            ftag.insert(4,'<br /><br />')
+
         for item in soup.findAll(style=True):
-            del item['style']
+           del item['style']
+
         self.append_page(soup, soup.body, 3)
+
         return soup
+
+
+

upload.py

@@ -6,10 +6,14 @@ __docformat__ = 'restructuredtext en'
 import shutil, os, glob, re, cStringIO, sys, tempfile, time, textwrap, socket, \
        struct, subprocess, platform
 from datetime import datetime
-from setuptools.command.build_py import build_py as _build_py, convert_path
+from stat import ST_MODE
+from distutils.command.build_py import build_py as _build_py, convert_path
+from distutils.command.install_scripts import install_scripts as _install_scripts
+from distutils.command.install import install as _install
 from distutils.core import Command
 from subprocess import check_call, call, Popen
 from distutils.command.build import build as _build
+from distutils import log
 
 raw = open(os.path.join('src', 'calibre', 'constants.py'), 'rb').read()
 __version__ = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', raw).group(1)
@@ -25,6 +29,9 @@ TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
 MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
 
 is64bit = platform.architecture()[0] == '64bit'
+iswindows = re.search('win(32|64)', sys.platform)
+isosx = 'darwin' in sys.platform
+islinux = not isosx and not iswindows
 
 def get_ip_address(ifname):
     import fcntl
@@ -66,6 +73,59 @@ class OptionlessCommand(Command):
         for cmd_name in self.get_sub_commands():
             self.run_command(cmd_name)
 
+def setup_mount_helper(tdir):
+    def warn():
+        print 'WARNING: Failed to compile mount helper. Auto mounting of',
+        print 'devices will not work'
+
+    if os.geteuid() != 0:
+        return warn()
+    import stat
+    src = os.path.join('src', 'calibre', 'devices', 'linux_mount_helper.c')
+    dest = os.path.join(tdir, 'calibre-mount-helper')
+    log.info('Installing mount helper to '+ tdir)
+    p = subprocess.Popen(['gcc', '-Wall', src, '-o', dest])
+    ret = p.wait()
+    if ret != 0:
+        return warn()
+    os.chown(dest, 0, 0)
+    os.chmod(dest,
+       stat.S_ISUID|stat.S_ISGID|stat.S_IRUSR|stat.S_IWUSR|stat.S_IXUSR|stat.S_IXGRP|stat.S_IXOTH)
+    return dest
+
+
+class develop(_install_scripts):
+
+    def run(self):
+        if not iswindows and os.geteuid() != 0:
+            raise Exception('Must be root to run this command.')
+        if not self.skip_build:
+            self.run_command('build_ext')
+            self.run_command('build_scripts')
+
+        for script in os.listdir(self.build_dir):
+            script = os.path.join(self.build_dir, script)
+            raw = open(script, 'rb').read()
+            raw = re.sub(r'"""##DEVELOP_HOOK##([^#]+)##END_DEVELOP_HOOK##"""',
+                    r'\1', raw)
+            raw = raw.replace('#!python', '#!'+sys.executable)
+            f = os.path.join(self.install_dir, os.path.basename(script))
+            open(f, 'wb').write(raw)
+            mode = ((os.stat(f)[ST_MODE]) | 0555) & 07777
+            log.info('changing mode of %s to %o'%(f, mode))
+            os.chmod(f, mode)
+
+        if islinux:
+            setup_mount_helper(self.install_dir)
+            subprocess.check_call('calibre_postinstall')
+
+class install(_install):
+
+    def run(self):
+        _install.run(self)
+        if islinux:
+            setup_mount_helper(self.install_dir)
+            subprocess.check_call('calibre_postinstall')
 
 class sdist(OptionlessCommand):
 
@@ -77,39 +137,6 @@ class sdist(OptionlessCommand):
         self.distribution.dist_files.append(('sdist', '', name))
         print 'Source distribution created in', os.path.abspath(name)
 
-class pot(OptionlessCommand):
-    description = '''Create the .pot template for all translatable strings'''
-
-    PATH = os.path.join('src', __appname__, 'translations')
-
-    def source_files(self):
-        ans = []
-        for root, _, files in os.walk(os.path.dirname(self.PATH)):
-            for name in files:
-                if name.endswith('.py'):
-                    ans.append(os.path.abspath(os.path.join(root, name)))
-        return ans
-
-
-    def run(self):
-        sys.path.insert(0, os.path.abspath(self.PATH))
-        try:
-            pygettext = __import__('pygettext', fromlist=['main']).main
-            files = self.source_files()
-            buf = cStringIO.StringIO()
-            print 'Creating translations template'
-            tempdir = tempfile.mkdtemp()
-            pygettext(buf, ['-k', '__', '-p', tempdir]+files)
-            src = buf.getvalue()
-            pot = os.path.join(self.PATH, __appname__+'.pot')
-            f = open(pot, 'wb')
-            f.write(src)
-            f.close()
-            print 'Translations template:', os.path.abspath(pot)
-            return pot
-        finally:
-            sys.path.remove(os.path.abspath(self.PATH))
-
 class manual(OptionlessCommand):
 
     description='''Build the User Manual '''
@@ -249,99 +276,6 @@ class resources(OptionlessCommand):
             if os.path.exists(path):
                 os.remove(path)
 
-class translations(OptionlessCommand):
-    description='''Compile the translations'''
-    PATH = os.path.join('src', __appname__, 'translations')
-    DEST = os.path.join(PATH, 'compiled.py')
-
-    def run(self):
-        sys.path.insert(0, os.path.abspath(self.PATH))
-        try:
-            files = glob.glob(os.path.join(self.PATH, '*.po'))
-            if newer([self.DEST], files):
-                msgfmt = __import__('msgfmt', fromlist=['main']).main
-                translations = {}
-                print 'Compiling translations...'
-                for po in files:
-                    lang = os.path.basename(po).partition('.')[0]
-                    buf = cStringIO.StringIO()
-                    print 'Compiling', lang
-                    msgfmt(buf, [po])
-                    translations[lang] = buf.getvalue()
-                open(self.DEST, 'wb').write('translations = '+repr(translations))
-            else:
-                print 'Translations up to date'
-        finally:
-            sys.path.remove(os.path.abspath(self.PATH))
-
-
-    @classmethod
-    def clean(cls):
-        path = cls.DEST
-        if os.path.exists(path):
-            os.remove(path)
-
-class get_translations(translations):
-
-    description = 'Get updated translations from Launchpad'
-    BRANCH = 'lp:~kovid/calibre/translations'
-
-    @classmethod
-    def modified_translations(cls):
-        raw = subprocess.Popen(['bzr', 'status'],
-                stdout=subprocess.PIPE).stdout.read().strip()
-        for line in raw.splitlines():
-            line = line.strip()
-            if line.startswith(cls.PATH) and line.endswith('.po'):
-                yield line
-
-    def run(self):
-        if len(list(self.modified_translations())) == 0:
-            subprocess.check_call(['bzr', 'merge', self.BRANCH])
-        if len(list(self.modified_translations())) == 0:
-            print 'No updated translations available'
-        else:
-            subprocess.check_call(['bzr', 'commit', '-m',
-                'IGN:Updated translations', self.PATH])
-        self.check_for_errors()
-
-    @classmethod
-    def check_for_errors(cls):
-        errors = os.path.join(tempfile.gettempdir(), 'calibre-translation-errors')
-        if os.path.exists(errors):
-            shutil.rmtree(errors)
-        os.mkdir(errors)
-        pofilter = ('pofilter', '-i', cls.PATH, '-o', errors,
-                '-t', 'accelerators', '-t', 'escapes', '-t', 'variables',
-                #'-t', 'xmltags',
-                #'-t', 'brackets',
-                #'-t', 'emails',
-                #'-t', 'doublequoting',
-                #'-t', 'filepaths',
-                #'-t', 'numbers',
-                '-t', 'options',
-                #'-t', 'urls',
-                '-t', 'printf')
-        subprocess.check_call(pofilter)
-        errfiles = glob.glob(errors+os.sep+'*.po')
-        subprocess.check_call(['gvim', '-f', '-p', '--']+errfiles)
-        for f in errfiles:
-            with open(f, 'r+b') as f:
-                raw = f.read()
-                raw = re.sub(r'# \(pofilter\).*', '', raw)
-                f.seek(0)
-                f.truncate()
-                f.write(raw)
-
-        subprocess.check_call(['pomerge', '-t', cls.PATH, '-i', errors, '-o',
-            cls.PATH])
-        if len(list(cls.modified_translations())) > 0:
-            subprocess.call(['bzr', 'diff', cls.PATH])
-            yes = raw_input('Merge corrections? [y/n]: ').strip()
-            if yes in ['', 'y']:
-                subprocess.check_call(['bzr', 'commit', '-m',
-                    'IGN:Translation corrections', cls.PATH])
-
 class gui(OptionlessCommand):
     description='''Compile all GUI forms and images'''
     PATH  = os.path.join('src', __appname__, 'gui2')
@@ -439,29 +373,6 @@ class gui(OptionlessCommand):
             if os.path.exists(x):
                 os.remove(x)
 
-class clean(OptionlessCommand):
-
-    description='''Delete all computer generated files in the source tree'''
-
-    def run(self):
-        print 'Cleaning...'
-        manual.clean()
-        gui.clean()
-        translations.clean()
-        resources.clean()
-
-        for f in glob.glob(os.path.join('src', 'calibre', 'plugins', '*')):
-            os.remove(f)
-        for root, _, files in os.walk('.'):
-            for name in files:
-                for t in ('.pyc', '.pyo', '~'):
-                    if name.endswith(t):
-                        os.remove(os.path.join(root, name))
-                        break
-
-        for dir in ('build', 'dist', os.path.join('src', 'calibre.egg-info')):
-            shutil.rmtree(dir, ignore_errors=True)
-
 class build_py(_build_py):
 
     def find_data_files(self, package, src_dir):