IGN:Fix various minor bugs and rationalize the build system

pyqtdistutils.py

@@ -122,16 +122,16 @@ CONFIG   += x86 ppc
         bdir = os.path.abspath(os.path.join(self.build_temp, fullname))
         if not os.path.exists(bdir):
             os.makedirs(bdir)
-        ext.sources = map(os.path.abspath, ext.sources)
+        ext.sources2 = map(os.path.abspath, ext.sources)
         qt_dir = 'qt\\release' if iswindows else 'qt'
         objects = set(map(lambda x: os.path.join(bdir, qt_dir, replace_suffix(os.path.basename(x), '.o')),
-                      [s for s in ext.sources if not s.endswith('.h')]))
+                      [s for s in ext.sources2 if not s.endswith('.h')]))
         newer = False
         for object in objects:
-            if newer_group(ext.sources, object, missing='newer'):
+            if newer_group(ext.sources2, object, missing='newer'):
                 newer = True
                 break
-        headers = [f for f in ext.sources if f.endswith('.h')]
+        headers = [f for f in ext.sources2 if f.endswith('.h')]
         if self.force or newer:
             log.info('building \'%s\' extension', ext.name)
             objects = self.build_qt_objects(ext, bdir)
@@ -193,4 +193,3 @@ CONFIG   += x86 ppc
             generated_sources += self.get_sip_output_list(sbf)
         return generated_sources
 
-        

resources.py

@@ -1,45 +0,0 @@
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-'''
-Compile resource files.
-'''
-import os, sys, glob
-sys.path.insert(1, os.path.join(os.getcwd(), 'src'))
-
-RESOURCES = dict(
-    opf_template    = '%p/ebooks/metadata/opf.xml',
-    ncx_template    = '%p/ebooks/metadata/ncx.xml',
-    fb2_xsl         = '%p/ebooks/lrf/fb2/fb2.xsl',
-    metadata_sqlite = '%p/library/metadata_sqlite.sql',
-                 )
-
-def main(args=sys.argv):
-    data = ''
-    for key, value in RESOURCES.items():
-        path = value.replace('%p', 'src'+os.sep+'calibre')
-        bytes = repr(open(path, 'rb').read())
-        data += key + ' = ' + bytes + '\n\n'
-    
-    translations_found = False
-    for TPATH in ('/usr/share/qt4/translations', '/usr/lib/qt4/translations'):
-        if os.path.exists(TPATH):
-             files = glob.glob(TPATH + '/qt_??.qm')
-        
-             for f in files:
-                 key = os.path.basename(f).partition('.')[0]
-                 bytes = repr(open(f, 'rb').read())
-                 data += key + ' = ' + bytes + '\n\n'
-             translations_found = True
-             break
-    if not translations_found:
-        print 'WARNING: Could not find Qt transations'
-    
-    dest = os.path.abspath(os.path.join('src', 'calibre', 'resources.py'))
-    print 'Writing resources to', dest 
-    open(dest, 'wb').write(data) 
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main())

setup.py

@@ -1,8 +1,8 @@
-#!/usr/bin/env python
+from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
-import sys, re, os, shutil
+import sys, re, os, shutil, cStringIO, tempfile, subprocess
 sys.path.append('src')
 iswindows = re.search('win(32|64)', sys.platform)
 isosx = 'darwin' in sys.platform
@@ -48,66 +48,291 @@ main_functions = {
 if __name__ == '__main__':
     from setuptools import setup, find_packages, Extension
     from distutils.command.build import build as _build
-    from distutils.core import Command
+    from distutils.core import Command as _Command
     from pyqtdistutils import PyQtExtension, build_ext
     import subprocess, glob
     
-    class pot(Command):
+    def newer(targets, sources):
+        '''
+        Return True is sources is newer that targets or if targets
+        does not exist. 
+        '''
+        for f in targets:
+            if not os.path.exists(f):
+                return True
+        ttimes = map(lambda x: os.stat(x).st_mtime, targets)
+        stimes = map(lambda x: os.stat(x).st_mtime, sources)
+        newest_source, oldest_target = max(stimes), min(ttimes)
+        return newest_source > oldest_target
+    
+    class Command(_Command):
         user_options = []
         def initialize_options(self): pass
         def finalize_options(self): pass
     
-        def run(self):
+    class pot(Command):
-            from calibre.translations import create_pot
+        ''' Create the .pot template for all translatable strings '''
-            create_pot()
         
-    def build_manual():
+        PATH = os.path.join('src', APPNAME, 'translations')
-        cwd = os.path.abspath(os.getcwd())
+        
-        os.chdir(os.path.join('src', 'calibre', 'manual'))
+        def source_files(self):
-        try:
+            ans = []
-            for d in ('.build', 'cli'):
+            for root, dirs, files in os.walk(os.path.dirname(self.PATH)):
-                if os.path.exists(d):
+                for name in files:
-                    shutil.rmtree(d)
+                    if name.endswith('.py'):
-                os.makedirs(d)
+                        ans.append(os.path.abspath(os.path.join(root, name)))
-            if not os.path.exists('.build'+os.sep+'html'):
+            return ans
-                os.makedirs('.build'+os.sep+'html')
+
-            subprocess.check_call(['sphinx-build', '-b', 'custom', '-d', 
+        
-                                   '.build/doctrees', '.', '.build/html'])
+        def run(self):
-        finally:
+            sys.path.insert(0, os.path.abspath(self.PATH))
-            os.chdir(cwd)
+            try:
+                from pygettext import main as pygettext
+                files = self.source_files()
+                buf = cStringIO.StringIO()
+                print 'Creating translations template'
+                tempdir = tempfile.mkdtemp()
+                pygettext(buf, ['-p', tempdir]+files)
+                src = buf.getvalue()
+                pot = os.path.join(tempdir, 'calibre.pot')
+                f = open(pot, 'wb')
+                f.write(src)
+                f.close()
+                print 'Translations template:', pot
+                return pot
+            finally:
+                sys.path.remove(os.path.abspath(self.PATH))
             
     class manual(Command):
-        user_options = []
+        ''' Build the User Manual '''
-        def initialize_options(self): pass
+        def run(self):
-        def finalize_options(self): pass
+            cwd = os.path.abspath(os.getcwd())
+            os.chdir(os.path.join('src', 'calibre', 'manual'))
+            try:
+                for d in ('.build', 'cli'):
+                    if os.path.exists(d):
+                        shutil.rmtree(d)
+                    os.makedirs(d)
+                if not os.path.exists('.build'+os.sep+'html'):
+                    os.makedirs('.build'+os.sep+'html')
+                subprocess.check_call(['sphinx-build', '-b', 'custom', '-d', 
+                                       '.build/doctrees', '.', '.build/html'])
+            finally:
+                os.chdir(cwd)
+            
+        @classmethod
+        def clean(cls):
+            path = os.path.join('src', 'calibre', 'manual', '.build')
+            if os.path.exists(path):
+                shutil.rmtree(path)
+            
+    class resources(Command):
+        '''
+        Compile various resource files used in calibre. 
+        '''
+        
+        RESOURCES = dict(
+            opf_template    = 'ebooks/metadata/opf.xml',
+            ncx_template    = 'ebooks/metadata/ncx.xml',
+            fb2_xsl         = 'ebooks/lrf/fb2/fb2.xsl',
+            metadata_sqlite = 'library/metadata_sqlite.sql',
+         )
+        
+        DEST = os.path.join('src', APPNAME, 'resources.py')
+        
+        def get_qt_translations(self):
+            data = {}
+            translations_found = False
+            for TPATH in ('/usr/share/qt4/translations', '/usr/lib/qt4/translations'):
+                if os.path.exists(TPATH):
+                     files = glob.glob(TPATH + '/qt_??.qm')
+                     for f in files:
+                         key = os.path.basename(f).partition('.')[0]
+                         data[key] = f
+                     translations_found = True
+                     break
+            if not translations_found:
+                print 'WARNING: Could not find Qt transations'
+            return data
         
         def run(self):
-            build_manual()
+            data, dest, RESOURCES = {}, self.DEST, self.RESOURCES
+            for key in RESOURCES:
+                path = RESOURCES[key]
+                if not os.path.isabs(path):
+                    RESOURCES[key] = os.path.join('src', APPNAME, path)
+            translations = self.get_qt_translations()
+            RESOURCES.update(translations)
+            if newer([dest], RESOURCES.values()):
+                print 'Compiling resources...'
+                with open(dest, 'wb') as f:
+                    for key in RESOURCES:
+                        data = open(RESOURCES[key], 'rb').read()
+                        f.write(key + ' = ' + repr(data)+'\n\n')
+            else:
+                print 'Resources are up to date'
         
+        @classmethod
+        def clean(cls):
+            path = cls.DEST
+            for path in glob.glob(path+'*'):
+                if os.path.exists(path):
+                    os.remove(path)
+    
+    class translations(Command):
+        '''
+        Compile the translations
+        '''
+        PATH = os.path.join('src', APPNAME, 'translations')
+        DEST = os.path.join(PATH, 'compiled.py')
+        
+        def run(self):
+            sys.path.insert(0, os.path.abspath(self.PATH))
+            try:
+                files = glob.glob(os.path.join(self.PATH, '*.po'))
+                if newer([self.DEST], files):
+                    from msgfmt import main as msgfmt
+                    translations = {}
+                    print 'Compiling translations...'
+                    for po in files:
+                        lang = os.path.basename(po).partition('.')[0]
+                        buf = cStringIO.StringIO()
+                        print 'Compiling', lang
+                        msgfmt(buf, [po])
+                        translations[lang] = buf.getvalue()
+                    open(self.DEST, 'wb').write('translations = '+repr(translations))
+                else:
+                    print 'Translations up to date'
+            finally:
+                sys.path.remove(os.path.abspath(self.PATH))
+        
+                
+        @classmethod
+        def clean(cls):
+            path = cls.DEST
+            if os.path.exists(path):
+                os.remove(path)
+            
+    
+    class gui(Command):
+        '''
+        Compile all GUI forms and image related resources.
+        '''
+        PATH  = os.path.join('src', APPNAME, 'gui2')
+        IMAGES_DEST = os.path.join(PATH, 'images_rc.py')
+        
+        @classmethod
+        def find_forms(cls):
+            forms = []
+            for root, dirs, files in os.walk(cls.PATH):
+                for name in files:
+                    if name.endswith('.ui'):
+                        forms.append(os.path.abspath(os.path.join(root, name)))
+                
+            return forms
+        
+        @classmethod
+        def form_to_compiled_form(cls, form):
+            return form.rpartition('.')[0]+'_ui.py'
+        
+        def run(self):
+            self.build_forms()
+            self.build_images()
+        
+        def build_images(self):
+            cwd, images = os.getcwd(), os.path.basename(self.IMAGES_DEST)
+            try:
+                os.chdir(self.PATH)
+                sources, files = [], []
+                for root, dirs, files in os.walk('images'):
+                    for name in files:
+                        sources.append(os.path.join(root, name))
+                if newer([images], sources):
+                    print 'Compiling images...'
+                    for s in sources:
+                        alias = ' alias="library"' if s.endswith('images'+os.sep+'library.png') else ''
+                        files.append('<file%s>%s</file>'%(alias, s))
+                    manifest = '<RCC>\n<qresource prefix="/">\n%s\n</qresource>\n</RCC>'%'\n'.join(files)
+                    with open('images.qrc', 'wb') as f:
+                        f.write(manifest)
+                    subprocess.check_call(['pyrcc4', '-o', images, 'images.qrc'])
+                    os.remove('images.qrc')
+                else:
+                    print 'Images are up to date'
+            finally:
+                os.chdir(cwd)
+            
+        
+        def build_forms(self):
+            from PyQt4.uic import compileUi
+            forms = self.find_forms()
+            for form in forms:
+                compiled_form = self.form_to_compiled_form(form) 
+                if not os.path.exists(compiled_form) or os.stat(form).st_mtime > os.stat(compiled_form).st_mtime:
+                    print 'Compiling form', form
+                    buf = cStringIO.StringIO()
+                    compileUi(form, buf)
+                    dat = buf.getvalue()
+                    dat = dat.replace('__appname__', APPNAME)
+                    dat = dat.replace('import images_rc', 'from calibre.gui2 import images_rc')
+                    dat = dat.replace('from library import', 'from calibre.gui2.library import')
+                    dat = dat.replace('from widgets import', 'from calibre.gui2.widgets import')
+                    dat = re.compile(r'QtGui.QApplication.translate\(.+?,\s+"(.+?)(?<!\\)",.+?\)', re.DOTALL).sub(r'_("\1")', dat)
+                    
+                    # Workaround bug in Qt 4.4 on Windows
+                    if form.endswith('dialogs%sconfig.ui'%os.sep) or form.endswith('dialogs%slrf_single.ui'%os.sep):
+                        print 'Implementing Workaround for buggy pyuic in form', form
+                        dat = re.sub(r'= QtGui\.QTextEdit\(self\..*?\)', '= QtGui.QTextEdit()', dat) 
+                        dat = re.sub(r'= QtGui\.QListWidget\(self\..*?\)', '= QtGui.QListWidget()', dat)
+                    
+                    open(compiled_form, 'wb').write(dat)
+
+                
+        @classmethod
+        def clean(cls):
+            forms = cls.find_forms()
+            for form in forms:
+                c = cls.form_to_compiled_form(form)
+                if os.path.exists(c):
+                    os.remove(c)
+            images = cls.IMAGES_DEST
+            if os.path.exists(images):
+                os.remove(images)
+    
+    class clean(Command):
+        ''' Delete all computer generated files in the source tree'''
+        
+        def run(self):
+            print 'Cleaning...'
+            manual.clean()
+            gui.clean()
+            translations.clean()
+            resources.clean()
+            
+            for f in glob.glob(os.path.join('src', 'calibre', 'plugins', '*')):
+                os.remove(f)
+            for root, dirs, files in os.walk('.'):
+                for name in files:
+                    if name.endswith('~') or \
+                       name.endswith('.pyc') or \
+                       name.endswith('.pyo'):
+                        os.remove(os.path.join(root, name))
+                        
+            for dir in 'build', 'dist':
+                for f in os.listdir(dir):
+                    if os.path.isdir(dir + os.sep + f):
+                        shutil.rmtree(dir + os.sep + f)
+                    else:
+                        os.remove(dir + os.sep + f)
     
     class build(_build):
         
-        def run(self):
+        sub_commands = \
-            # Build resources
+                        [
-            resources = __import__('resources')
+                         ('resources',    lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
-            resources.main([sys.executable, 'resources.py'])
+                         ('translations', lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
-            from calibre.translations import main as translations
+                         ('gui',          lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
-            cwd = os.path.abspath(os.getcwd())
+                         ] + _build.sub_commands
-            # Build translations
-            try:
-                os.chdir(os.path.join('src', 'calibre', 'translations'))
-                translations([sys.executable])
-            finally:
-                os.chdir(cwd)
-            # Build GUI
-            from calibre.gui2.make import main as gui2
-            try:
-                os.chdir(os.path.join('src', 'calibre', 'gui2'))
-                print 'Compiling GUI resources...'
-                gui2([sys.executable])
-            finally:
-                os.chdir(cwd)
-            _build.run(self)
     
     entry_points['console_scripts'].append('calibre_postinstall = calibre.linux:post_install')
     ext_modules = [
@@ -115,10 +340,12 @@ if __name__ == '__main__':
                              sources=['src/calibre/utils/lzx/lzxmodule.c',
                                       'src/calibre/utils/lzx/lzxd.c'],
                              include_dirs=['src/calibre/utils/lzx']),
+                   
                    Extension('calibre.plugins.msdes',
                              sources=['src/calibre/utils/msdes/msdesmodule.c',
                                       'src/calibre/utils/msdes/des.c'],
                              include_dirs=['src/calibre/utils/msdes']),
+                   
                     PyQtExtension('calibre.plugins.pictureflow',
                                   ['src/calibre/gui2/pictureflow/pictureflow.cpp',
                                    'src/calibre/gui2/pictureflow/pictureflow.h'],
@@ -137,20 +364,20 @@ if __name__ == '__main__':
                            )
     
     setup(
-          name=APPNAME,
+          name           = APPNAME,
-          packages = find_packages('src'),
+          packages       = find_packages('src'),
-          package_dir = { '' : 'src' },
+          package_dir    = { '' : 'src' },
-          version=VERSION,
+          version        = VERSION,
-          author='Kovid Goyal',
+          author         = 'Kovid Goyal',
-          author_email='kovid@kovidgoyal.net',
+          author_email   = 'kovid@kovidgoyal.net',
-          url = 'http://%s.kovidgoyal.net'%APPNAME,
+          url            = 'http://%s.kovidgoyal.net'%APPNAME,
-          package_data = {'calibre':['plugins/*']},
+          package_data   = {'calibre':['plugins/*']},
-          include_package_data=True,
+          include_package_data = True,
-          entry_points = entry_points,
+          entry_points   = entry_points,
-          zip_safe = False,
+          zip_safe       = False,
-          options = { 'bdist_egg' : {'exclude_source_files': True,}, },
+          options        = { 'bdist_egg' : {'exclude_source_files': True,}, },
-          ext_modules=ext_modules,
+          ext_modules    = ext_modules,
-          description =
+          description    =
                       '''
                       E-book management application.
                       ''',
@@ -171,7 +398,7 @@ if __name__ == '__main__':
 
           '''%(APPNAME, APPNAME, APPNAME, APPNAME, APPNAME),
           license = 'GPL',
-          classifiers = [
+          classifiers    = [
             'Development Status :: 4 - Beta',
             'Environment :: Console',
             'Environment :: X11 Applications :: Qt',
@@ -184,9 +411,17 @@ if __name__ == '__main__':
             'Topic :: Software Development :: Libraries :: Python Modules',
             'Topic :: System :: Hardware :: Hardware Drivers'
             ],
-          cmdclass = {'build_ext': build_ext, 'build' : build, 'pot' : pot,
+          cmdclass       = {
-                      'manual' : manual},
+                      'build_ext'     : build_ext, 
+                      'build'         : build, 
+                      'pot'           : pot,
+                      'manual'        : manual,
+                      'resources'     : resources,
+                      'translations'  : translations,
+                      'gui'           : gui,
+                      'clean'         : clean,
+                      },
          )
 
     if 'develop' in ' '.join(sys.argv) and islinux:
-        subprocess.check_call('calibre_postinstall', shell=True)
+        subprocess.check_call('calibre_postinstall --do-not-reload-udev-hal', shell=True)

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.84b2'
+__version__   = '0.4.84b3'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/gui2/make.py

@@ -1,116 +0,0 @@
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-'''
-Manage the PyQt build system pyrcc4, pylupdate4, lrelease and friends.
-'''
-
-import sys, os, subprocess, cStringIO, compiler, re
-from functools import partial
-
-from PyQt4.uic import compileUi
-
-check_call = partial(subprocess.check_call, shell=True)
-sys.path.insert(1, os.path.abspath('..%s..'%os.sep))
-
-from calibre import __appname__
-from calibre.path import path
-
-def find_forms():
-    forms = []
-    for root, dirs, files in os.walk('.'):
-        for name in files:
-            if name.endswith('.ui'):
-                forms.append(os.path.abspath(os.path.join(root, name)))
-        
-    return forms
-
-def form_to_compiled_form(form):
-    return form.rpartition('.')[0]+'_ui.py'
-
-def build_forms(forms):
-    for form in forms:
-        compiled_form = form_to_compiled_form(form) 
-        if not os.path.exists(compiled_form) or os.stat(form).st_mtime > os.stat(compiled_form).st_mtime:
-            print 'Compiling form', form
-            buf = cStringIO.StringIO()
-            compileUi(form, buf)
-            dat = buf.getvalue()
-            dat = dat.replace('__appname__', __appname__)
-            dat = dat.replace('import images_rc', 'from calibre.gui2 import images_rc')
-            dat = dat.replace('from library import', 'from calibre.gui2.library import')
-            dat = dat.replace('from widgets import', 'from calibre.gui2.widgets import')
-            dat = re.compile(r'QtGui.QApplication.translate\(.+?,\s+"(.+?)(?<!\\)",.+?\)', re.DOTALL).sub(r'_("\1")', dat)
-            
-            # Workaround bug in Qt 4.4 on Windows
-            if form.endswith('dialogs%sconfig.ui'%os.sep) or form.endswith('dialogs%slrf_single.ui'%os.sep):
-                print 'Implementing Workaround for buggy pyuic in form', form
-                dat = re.sub(r'= QtGui\.QTextEdit\(self\..*?\)', '= QtGui.QTextEdit()', dat) 
-                dat = re.sub(r'= QtGui\.QListWidget\(self\..*?\)', '= QtGui.QListWidget()', dat)
-            
-            open(compiled_form, 'wb').write(dat)
-            
-                
-def build_images():
-    p = path('images')
-    mtime = p.mtime
-    for x in p.walk():
-        mtime = max(x.mtime, mtime)
-    images = path('images_rc.py')
-    if not images.exists() or mtime > images.mtime:
-        print 'Compiling images...'
-        files = []
-        for x in p.walk():
-            if '.svn' in x or '.bzr' in x or x.isdir():
-                continue
-            alias = ' alias="library"' if x == p/'library.png' else ''
-            files.append('<file%s>%s</file>'%(alias, x))
-        qrc = '<RCC>\n<qresource prefix="/">\n%s\n</qresource>\n</RCC>'%'\n'.join(files)
-        f = open('images.qrc', 'wb')
-        f.write(qrc)
-        f.close()
-        check_call(' '.join(['pyrcc4', '-o', images, 'images.qrc']))
-        compiler.compileFile(images)
-        os.utime(images, None)
-        os.utime(images, None)
-        print 'Size of images:', '%.2f MB'%(path(images+'c').size/(1024*1024.))
-        
-            
-def build(forms):
-    build_forms(forms)
-    build_images()
-
-def clean(forms):
-    for form in forms:
-        compiled_form = form_to_compiled_form(form)
-        if os.path.exists(compiled_form):
-            print 'Removing compiled form', compiled_form
-            os.unlink(compiled_form)
-    print 'Removing compiled images'
-    os.unlink('images_rc.py')
-    os.unlink('images_rc.pyc')
-
-def main(args=sys.argv):
-    
-    if not os.getcwd().endswith('gui2'):
-        raise Exception('Must be run from the gui2 directory')
-    
-    forms = find_forms()
-    if len(args) == 1:
-        args.append('all')
-    
-    if   args[1] == 'all':
-        build(forms)
-    elif args[1] == 'clean':
-        clean(forms)
-    elif args[1] == 'test':
-        build(forms)
-        print 'Running main.py'
-        subprocess.call('python main.py', shell=True)
-    else:
-        print 'Usage: %s [all|clean|test]'%(args[0])
-        return 1 
-    
-    return 0
-
-if __name__ == '__main__':
-    sys.exit(main())

src/calibre/library/database2.py

@@ -246,11 +246,20 @@ class LibraryDatabase2(LibraryDatabase):
         spath = os.path.join(self.library_path, *current_path.split('/'))
         if current_path and os.path.exists(spath):
             for f in os.listdir(spath):
-                copyfile(os.path.join(spath, f), os.path.join(tpath, f))
+                try:
+                    copyfile(os.path.join(spath, f), os.path.join(tpath, f))
+                except OSError, err:
+                    if err.errno == 78: # Happens if database is mounted via sshfs 
+                        shutil.copyfile(os.path.join(spath, f), os.path.join(tpath, f))
+                    else:
+                        raise
         self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
         self.conn.commit()
         if current_path and os.path.exists(spath):
             shutil.rmtree(spath)
+            parent  = os.path.dirname(spath)
+            if len(os.listdir(parent)) == 0:
+                shutil.rmtree(parent)
             
     def cover(self, index, index_is_id=False, as_file=False, as_image=False):
         '''
@@ -495,6 +504,7 @@ class LibraryDatabase2(LibraryDatabase):
         QCoreApplication.processEvents()
         db.conn.row_factory = lambda cursor, row : tuple(row)
         books = db.conn.execute('SELECT id, title, sort, timestamp, uri, series_index, author_sort, isbn FROM books ORDER BY id ASC').fetchall()
+        progress.setAutoReset(False)
         progress.setRange(0, len(books))
         
         for book in books:
@@ -533,4 +543,5 @@ books_series_link      feeds
         self.conn.commit()
         progress.setLabelText(_('Compacting database'))
         self.vacuum()
+        progress.reset()
         

src/calibre/startup.py

@@ -72,30 +72,35 @@ if not _run_once:
 
     ################################################################################
     # Load plugins
-    if isfrozen:
+    def load_plugins():
-        if iswindows:
+        plugins = {}
-            plugin_path = os.path.join(os.path.dirname(sys.executable), 'plugins')
+        if isfrozen:
-            sys.path.insert(1, os.path.dirname(sys.executable))
+            if iswindows:
-        elif isosx:
+                plugin_path = os.path.join(os.path.dirname(sys.executable), 'plugins')
-            plugin_path = os.path.join(getattr(sys, 'frameworks_dir'), 'plugins')
+                sys.path.insert(1, os.path.dirname(sys.executable))
-        elif islinux:
+            elif isosx:
-            plugin_path = os.path.join(getattr(sys, 'frozen_path'), 'plugins')
+                plugin_path = os.path.join(getattr(sys, 'frameworks_dir'), 'plugins')
-        sys.path.insert(0, plugin_path)
+            elif islinux:
-    else:
+                plugin_path = os.path.join(getattr(sys, 'frozen_path'), 'plugins')
-        import pkg_resources
+            sys.path.insert(0, plugin_path)
-        plugins = getattr(pkg_resources, 'resource_filename')('calibre', 'plugins')
+        else:
-        sys.path.insert(0, plugins)
+            import pkg_resources
+            plugin_path = getattr(pkg_resources, 'resource_filename')('calibre', 'plugins')
+            sys.path.insert(0, plugin_path)
+
+        for plugin in ['pictureflow', 'lzx', 'msdes'] + \
+                    (['winutil'] if iswindows else []) + \
+                    (['usbobserver'] if isosx else []):
+            try:
+                p, err = __import__(plugin), ''
+            except Exception, err:
+                p = None
+                err = str(err)
+            plugins[plugin] = (p, err)
+        return plugins
+
+    plugins = load_plugins()
 
-    plugins = {}
-    for plugin in ['pictureflow', 'lzx', 'msdes'] + \
-                (['winutil'] if iswindows else []) + \
-                (['usbobserver'] if isosx else []):
-        try:
-            p, err = __import__(plugin), ''
-        except Exception, err:
-            p = None
-            err = str(err)
-        plugins[plugin] = (p, err)
 
     ################################################################################
     # Improve builtin path functions to handle unicode sensibly
@@ -151,4 +156,3 @@ if not _run_once:
     for i in range(1, len(sys.argv)):
         if not isinstance(sys.argv[i], unicode):
             sys.argv[i] = sys.argv[i].decode(preferred_encoding, 'replace')
-    

src/calibre/translations/__init__.py

@@ -3,102 +3,3 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
 Manage translation of user visible strings.
 '''
-import sys, os, cStringIO, tempfile, subprocess, functools, tarfile, re, time, \
-       glob, urllib2, shutil
-check_call = functools.partial(subprocess.check_call, shell=True)
-
-try:
-    from calibre.translations.pygettext import main as pygettext
-    from calibre.translations.msgfmt import main as msgfmt
-except ImportError:
-    cwd = os.getcwd()
-    sys.path.insert(0, os.path.dirname(os.path.dirname(cwd)))
-    from calibre.translations.pygettext import main as pygettext
-    from calibre.translations.msgfmt import main as msgfmt
-
-
-def source_files():
-    ans = []
-    for root, dirs, files in os.walk(os.path.dirname(os.getcwdu())):
-        for name in files:
-            if name.endswith('.py'):
-                ans.append(os.path.abspath(os.path.join(root, name)))
-    return ans
-                
-
-def create_pot():
-    files = source_files()
-    buf = cStringIO.StringIO()
-    print 'Creating translations template'
-    tempdir = tempfile.mkdtemp()
-    pygettext(buf, ['-p', tempdir]+files)
-    src = buf.getvalue()
-    pot = os.path.join(tempdir, 'calibre.pot')
-    f = open(pot, 'wb')
-    f.write(src)
-    f.close()
-    print 'Translations template:', pot
-    return pot
-    
-
-def compile_translations():
-    translations = {}
-    print 'Compiling translations...'
-    for po in glob.glob('*.po'):
-        lang = os.path.basename(po).partition('.')[0]
-        buf = cStringIO.StringIO()
-        print 'Compiling', lang
-        msgfmt(buf, [po])
-        translations[lang] = buf.getvalue()
-        open('compiled.py', 'wb').write('translations = '+repr(translations))
-
-def import_from_launchpad(url):
-    f = open('/tmp/launchpad_export.tar.gz', 'wb')
-    shutil.copyfileobj(urllib2.urlopen(url), f)
-    f.close()
-    tf = tarfile.open('/tmp/launchpad_export.tar.gz', 'r:gz')
-    next = tf.next()
-    while next is not None:
-        if next.isfile() and next.name.endswith('.po'):
-            try:
-                po = re.search(r'-([a-z]{2,3}\.po)', next.name).group(1)
-            except:
-                next = tf.next()
-                continue
-            out = os.path.abspath(os.path.join('.', os.path.basename(po)))
-            print 'Updating', '%6s'%po, '-->', out
-            open(out, 'wb').write(tf.extractfile(next).read())
-        next = tf.next()
-    
-    check_for_critical_bugs()
-    return 0
-
-def check_for_critical_bugs():
-    if os.path.exists('.errors'):
-        shutil.rmtree('.errors')
-    pofilter = ('pofilter', '-i', '.', '-o', '.errors',
-                '-t', 'accelerators', '-t', 'escapes', '-t', 'variables', 
-                '-t', 'xmltags')
-    subprocess.check_call(pofilter)
-    errs = os.listdir('.errors')
-    if errs:
-        print 'WARNING: Translation errors detected'
-        print 'See the .errors directory and http://translate.sourceforge.net/wiki/toolkit/using_pofilter'
-
-
-def main(args=sys.argv):
-    if len(args) > 1:
-        if args[1] == 'pot':
-            create_pot()
-        else:
-            import_from_launchpad(args[1])
-    else:
-        compile_translations()
-    return 0
-        
-if __name__ == '__main__':
-    cwd = os.getcwd()
-    sys.path.insert(0, os.path.dirname(os.path.dirname(cwd)))
-
-    sys.exit(main())
-

src/calibre/utils/config.py

@@ -284,6 +284,8 @@ class OptionSet(object):
     
     def parse_string(self, src):
         options = {'cPickle':cPickle}
+        if not isinstance(src, unicode):
+            src = src.decode('utf-8')
         if src is not None:
             exec src in options
         opts = OptionValues()
@@ -352,7 +354,7 @@ class Config(ConfigInterface):
         if os.path.exists(self.config_file_path):
             try:
                 with ExclusiveFile(self.config_file_path) as f:
-                    src = f.read()
+                    src = f.read().decode('utf-8')
             except LockError:
                 raise IOError('Could not lock config file: %s'%self.config_file_path)
         return self.option_set.parse_string(src)
@@ -362,7 +364,7 @@ class Config(ConfigInterface):
             return ''
         try:
             with ExclusiveFile(self.config_file_path) as f:
-                return f.read()
+                return f.read().decode('utf-8')
         except LockError:
             raise IOError('Could not lock config file: %s'%self.config_file_path)
     
@@ -380,6 +382,8 @@ class Config(ConfigInterface):
                 src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n'
                 f.seek(0)
                 f.truncate()
+                if isinstance(src, unicode):
+                    src = src.encode('utf-8')
                 f.write(src)
         except LockError:
             raise IOError('Could not lock config file: %s'%self.config_file_path)

src/calibre/utils/windows/winutil.c

@@ -118,7 +118,7 @@ winutil_argv(PyObject *self, PyObject *args) {
     LPSTR buf;
     int argc, i, bytes;
     if (!PyArg_ParseTuple(args, "")) return NULL;
-    _argv = CommandLineToArgvW(GetCommandLine(), &argc);
+    _argv = CommandLineToArgvW(GetCommandLineW(), &argc);
     if (_argv == NULL) { PyErr_NoMemory(); return NULL; }
     argv = PyList_New(argc);
     if (argv != NULL) {

upload.py

@@ -27,6 +27,7 @@ TXT2LRF  = "src/calibre/ebooks/lrf/txt/demo"
 MOBILEREAD = 'ftp://dev.mobileread.com/calibre/'
 BUILD_SCRIPT ='''\
 #!/bin/bash
+export CALIBRE_BUILDBOT=1
 cd ~/build && \
 rsync -avz --exclude src/calibre/plugins --exclude calibre/src/calibre.egg-info --exclude docs --exclude .bzr --exclude .build --exclude build --exclude dist --exclude "*.pyc" --exclude "*.pyo" rsync://%(host)s/work/%(project)s . && \
 cd %(project)s && \