Pull from trunk

.pydevproject

@@ -2,7 +2,7 @@
 <?eclipse-pydev version="1.0"?>
 
 <pydev_project>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.5</pydev_property>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/calibre-pluginize/src</path>
 </pydev_pathproperty>

installer/linux/freeze.py

@@ -38,6 +38,10 @@ def freeze():
                        '/usr/lib/libxml2.so.2',
                        '/usr/lib/libxslt.so.1',
                        '/usr/lib/libxslt.so.1',
+                       '/usr/lib/libgthread-2.0.so.0',
+                       '/usr/lib/libglib-2.0.so.0',
+                       '/usr/lib/gcc/i686-pc-linux-gnu/4.3.3/libstdc++.so.6',
+                       '/usr/lib/libpng12.so.0',
                        '/usr/lib/libexslt.so.0',
                        '/usr/lib/libMagickWand.so',
                        '/usr/lib/libMagickCore.so',

installer/windows/freeze.py

@@ -28,33 +28,12 @@ for icon in ICONS:
         raise Exception('No icon at '+icon)
 
 VERSION = re.sub('[a-z]\d+', '', VERSION)
+WINVER = VERSION+'.0'
 
 PY2EXE_DIR = os.path.join(BASE_DIR, 'build','py2exe')
 
 class BuildEXE(py2exe.build_exe.py2exe):
-    manifest_resource_id = 0
     
-    MANIFEST_TEMPLATE = '''
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0"> 
-  <assemblyIdentity version="%(version)s"
-     processorArchitecture="x86"
-     name="net.kovidgoyal.%(prog)s"
-     type="win32"
-     /> 
-  <description>Ebook management application</description> 
-  <!-- Identify the application security requirements. -->
-  <trustInfo xmlns="urn:schemas-microsoft-com:asm.v2">
-    <security>
-      <requestedPrivileges>
-        <requestedExecutionLevel
-          level="asInvoker"
-          uiAccess="false"/>
-        </requestedPrivileges>
-       </security>
-  </trustInfo>
-</assembly>
-'''
     def run(self):
         py2exe.build_exe.py2exe.run(self)
         print 'Adding plugins...'
@@ -129,39 +108,35 @@ class BuildEXE(py2exe.build_exe.py2exe):
             shutil.copyfile(f, os.path.join(PY2EXE_DIR, os.path.basename(f)))
         
     
-    @classmethod
+def exe_factory(dest_base, script, icon_resources=None):
-    def manifest(cls, prog):
+    exe = {
-        cls.manifest_resource_id += 1
+           'dest_base'       : dest_base,
-        return (24, cls.manifest_resource_id,
+           'script'          : script,
-                cls.MANIFEST_TEMPLATE % dict(prog=prog, version=(VERSION+'.0')))
+           'name'            : dest_base,
-
+           'version'         : WINVER,
+           'description'     : 'calibre - E-book library management',
+           'author'          : 'Kovid Goyal',
+           'copyright'       : '(c) Kovid Goyal, 2008',
+           'company'         : 'kovidgoyal.net',
+           }
+    if icon_resources is not None:
+        exe['icon_resources'] = icon_resources
+    return exe
 
 def main(args=sys.argv):
     sys.argv[1:2] = ['py2exe']
     if os.path.exists(PY2EXE_DIR):
         shutil.rmtree(PY2EXE_DIR)
 
-    console = [dict(dest_base=basenames['console'][i], script=scripts['console'][i])
+    console = [exe_factory(basenames['console'][i], scripts['console'][i])
                for i in range(len(scripts['console']))]
     setup(
           cmdclass = {'py2exe': BuildEXE},
           windows = [
-                     {'script'          : scripts['gui'][0],
+                     exe_factory(APPNAME, scripts['gui'][0], [(1, ICONS[0])]),
-                      'dest_base'       : APPNAME,
+                     exe_factory('lrfviewer', scripts['gui'][1], [(1, ICONS[1])]),
-                      'icon_resources'  : [(1, ICONS[0])],
+                     exe_factory('ebook-viewer', scripts['gui'][2], [(1, ICONS[1])]),
-                      #'other_resources' : [BuildEXE.manifest(APPNAME)],
+                    ],
-                      },
-                      {'script'         : scripts['gui'][1],
-                      'dest_base'       : 'lrfviewer',
-                      'icon_resources'  : [(1, ICONS[1])],
-                      #'other_resources' : [BuildEXE.manifest('lrfviewer')],
-                      },
-                      {'script'         : scripts['gui'][2],
-                      'dest_base'       : 'ebook-viewer',
-                      'icon_resources'  : [(1, ICONS[1])],
-                      #'other_resources' : [BuildEXE.manifest('ebook-viewer')],
-                      },
-                      ],
           console = console,
           options = { 'py2exe' : {'compressed': 1,
                                   'optimize'  : 2,

setup.py

@@ -47,350 +47,15 @@ main_functions = {
 
 if __name__ == '__main__':
     from setuptools import setup, find_packages
-    from setuptools.command.build_py import build_py as _build_py, convert_path
-    from distutils.command.build import build as _build
-    from distutils.core import Command as _Command
     from pyqtdistutils import PyQtExtension, build_ext, Extension
-    import subprocess, glob
+    from upload import sdist, pot, build, build_py, manual, \
+                        resources, clean, gui, translations, update, \
+                        tag_release, upload_demo, build_linux, build_windows, \
+                        build_osx, upload_installers, upload_user_manual, \
+                        upload_to_pypi, stage3, stage2, stage1, upload
     
-    def newer(targets, sources):
+    entry_points['console_scripts'].append(
-        '''
+                            'calibre_postinstall = calibre.linux:post_install')
-        Return True is sources is newer that targets or if targets
-        does not exist. 
-        '''
-        for f in targets:
-            if not os.path.exists(f):
-                return True
-        ttimes = map(lambda x: os.stat(x).st_mtime, targets)
-        stimes = map(lambda x: os.stat(x).st_mtime, sources)
-        newest_source, oldest_target = max(stimes), min(ttimes)
-        return newest_source > oldest_target
-    
-    class build_py(_build_py):
-        
-        def find_data_files(self, package, src_dir):
-            """
-            Return filenames for package's data files in 'src_dir'
-            Modified to treat data file specs as paths not globs
-            """
-            globs = (self.package_data.get('', [])
-                     + self.package_data.get(package, []))
-            files = self.manifest_files.get(package, [])[:]
-            for pattern in globs:
-                # Each pattern has to be converted to a platform-specific path
-                pattern = os.path.join(src_dir, convert_path(pattern))
-                next = glob.glob(pattern)
-                files.extend(next if next else [pattern])
-                
-            return self.exclude_data_files(package, src_dir, files)
-
-    
-    class Command(_Command):
-        user_options = []
-        def initialize_options(self): pass
-        def finalize_options(self): pass
-    
-    class sdist(Command):
-        
-        description = "create a source distribution using bzr"
-        
-        def run(self):
-            name = 'dist/calibre-%s.tar.gz'%VERSION
-            subprocess.check_call(('bzr export '+name).split())
-            self.distribution.dist_files.append(('sdist', '', name))
-    
-    class pot(Command):
-        description = '''Create the .pot template for all translatable strings'''
-        
-        PATH = os.path.join('src', APPNAME, 'translations')
-        
-        def source_files(self):
-            ans = []
-            for root, dirs, files in os.walk(os.path.dirname(self.PATH)):
-                for name in files:
-                    if name.endswith('.py'):
-                        ans.append(os.path.abspath(os.path.join(root, name)))
-            return ans
-
-        
-        def run(self):
-            sys.path.insert(0, os.path.abspath(self.PATH))
-            try:
-                from pygettext import main as pygettext
-                files = self.source_files()
-                buf = cStringIO.StringIO()
-                print 'Creating translations template'
-                tempdir = tempfile.mkdtemp()
-                pygettext(buf, ['-k', '__', '-p', tempdir]+files)
-                src = buf.getvalue()
-                pot = os.path.join(tempdir, 'calibre.pot')
-                f = open(pot, 'wb')
-                f.write(src)
-                f.close()
-                print 'Translations template:', pot
-                return pot
-            finally:
-                sys.path.remove(os.path.abspath(self.PATH))
-            
-    class manual(Command):
-        description='''Build the User Manual '''
-
-        def run(self):
-            cwd = os.path.abspath(os.getcwd())
-            os.chdir(os.path.join('src', 'calibre', 'manual'))
-            try:
-                for d in ('.build', 'cli'):
-                    if os.path.exists(d):
-                        shutil.rmtree(d)
-                    os.makedirs(d)
-                if not os.path.exists('.build'+os.sep+'html'):
-                    os.makedirs('.build'+os.sep+'html')
-                subprocess.check_call(['sphinx-build', '-b', 'custom', '-d', 
-                                       '.build/doctrees', '.', '.build/html'])
-            finally:
-                os.chdir(cwd)
-            
-        @classmethod
-        def clean(cls):
-            path = os.path.join('src', 'calibre', 'manual', '.build')
-            if os.path.exists(path):
-                shutil.rmtree(path)
-            
-    class resources(Command):
-        description='''Compile various resource files used in calibre. '''
-        
-        RESOURCES = dict(
-            opf_template    = 'ebooks/metadata/opf.xml',
-            ncx_template    = 'ebooks/metadata/ncx.xml',
-            fb2_xsl         = 'ebooks/lrf/fb2/fb2.xsl',
-            metadata_sqlite = 'library/metadata_sqlite.sql',
-            jquery          = 'gui2/viewer/jquery.js',
-            jquery_scrollTo = 'gui2/viewer/jquery_scrollTo.js',
-            html_css        = 'ebooks/oeb/html.css',
-        )
-        
-        DEST = os.path.join('src', APPNAME, 'resources.py')
-        
-        def get_qt_translations(self):
-            data = {}
-            translations_found = False
-            for TPATH in ('/usr/share/qt4/translations', '/usr/lib/qt4/translations'):
-                if os.path.exists(TPATH):
-                     files = glob.glob(TPATH + '/qt_??.qm')
-                     for f in files:
-                         key = os.path.basename(f).partition('.')[0]
-                         data[key] = f
-                     translations_found = True
-                     break
-            if not translations_found:
-                print 'WARNING: Could not find Qt transations'
-            return data
-        
-        def get_static_resources(self):
-            sdir = os.path.join('src', 'calibre', 'library', 'static')
-            resources, max = {}, 0
-            for f in os.listdir(sdir):
-                resources[f] = open(os.path.join(sdir, f), 'rb').read()
-                mtime = os.stat(os.path.join(sdir, f)).st_mtime
-                max = mtime if mtime > max else max
-            return resources, max
-        
-        def get_recipes(self):
-            sdir = os.path.join('src', 'calibre', 'web', 'feeds', 'recipes')
-            resources, max = {}, 0
-            for f in os.listdir(sdir):
-                if f.endswith('.py') and f != '__init__.py':
-                    resources[f.replace('.py', '')] = open(os.path.join(sdir, f), 'rb').read()
-                    mtime = os.stat(os.path.join(sdir, f)).st_mtime
-                    max = mtime if mtime > max else max
-            return resources, max
-        
-        def run(self):
-            data, dest, RESOURCES = {}, self.DEST, self.RESOURCES
-            for key in RESOURCES:
-                path = RESOURCES[key]
-                if not os.path.isabs(path):
-                    RESOURCES[key] = os.path.join('src', APPNAME, path)
-            translations = self.get_qt_translations()
-            RESOURCES.update(translations)
-            static, smax = self.get_static_resources()
-            recipes, rmax = self.get_recipes()
-            amax = max(rmax, smax)
-            if newer([dest], RESOURCES.values()) or os.stat(dest).st_mtime < amax:
-                print 'Compiling resources...'
-                with open(dest, 'wb') as f:
-                    for key in RESOURCES:
-                        data = open(RESOURCES[key], 'rb').read()
-                        f.write(key + ' = ' + repr(data)+'\n\n')
-                    f.write('server_resources = %s\n\n'%repr(static))
-                    f.write('recipes = %s\n\n'%repr(recipes))
-                    f.write('build_time = "%s"\n\n'%time.strftime('%d %m %Y %H%M%S'))
-            else:
-                print 'Resources are up to date'
-        
-        @classmethod
-        def clean(cls):
-            path = cls.DEST
-            for path in glob.glob(path+'*'):
-                if os.path.exists(path):
-                    os.remove(path)
-    
-    class translations(Command):
-        description='''Compile the translations'''
-        PATH = os.path.join('src', APPNAME, 'translations')
-        DEST = os.path.join(PATH, 'compiled.py')
-        
-        def run(self):
-            sys.path.insert(0, os.path.abspath(self.PATH))
-            try:
-                files = glob.glob(os.path.join(self.PATH, '*.po'))
-                if newer([self.DEST], files):
-                    from msgfmt import main as msgfmt
-                    translations = {}
-                    print 'Compiling translations...'
-                    for po in files:
-                        lang = os.path.basename(po).partition('.')[0]
-                        buf = cStringIO.StringIO()
-                        print 'Compiling', lang
-                        msgfmt(buf, [po])
-                        translations[lang] = buf.getvalue()
-                    open(self.DEST, 'wb').write('translations = '+repr(translations))
-                else:
-                    print 'Translations up to date'
-            finally:
-                sys.path.remove(os.path.abspath(self.PATH))
-        
-                
-        @classmethod
-        def clean(cls):
-            path = cls.DEST
-            if os.path.exists(path):
-                os.remove(path)
-            
-    
-    class gui(Command):
-        description='''Compile all GUI forms and images'''
-        PATH  = os.path.join('src', APPNAME, 'gui2')
-        IMAGES_DEST = os.path.join(PATH, 'images_rc.py')
-        QRC = os.path.join(PATH, 'images.qrc')
-        
-        @classmethod
-        def find_forms(cls):
-            forms = []
-            for root, dirs, files in os.walk(cls.PATH):
-                for name in files:
-                    if name.endswith('.ui'):
-                        forms.append(os.path.abspath(os.path.join(root, name)))
-                
-            return forms
-        
-        @classmethod
-        def form_to_compiled_form(cls, form):
-            return form.rpartition('.')[0]+'_ui.py'
-        
-        def run(self):
-            self.build_forms()
-            self.build_images()
-        
-        def build_images(self):
-            cwd, images = os.getcwd(), os.path.basename(self.IMAGES_DEST)
-            try:
-                os.chdir(self.PATH)
-                sources, files = [], []
-                for root, dirs, files in os.walk('images'):
-                    for name in files:
-                        sources.append(os.path.join(root, name))
-                if newer([images], sources):
-                    print 'Compiling images...'
-                    for s in sources:
-                        alias = ' alias="library"' if s.endswith('images'+os.sep+'library.png') else ''
-                        files.append('<file%s>%s</file>'%(alias, s))
-                    manifest = '<RCC>\n<qresource prefix="/">\n%s\n</qresource>\n</RCC>'%'\n'.join(files)
-                    with open('images.qrc', 'wb') as f:
-                        f.write(manifest)
-                    subprocess.check_call(['pyrcc4', '-o', images, 'images.qrc'])
-                else:
-                    print 'Images are up to date'
-            finally:
-                os.chdir(cwd)
-            
-        
-        def build_forms(self):
-            from PyQt4.uic import compileUi
-            forms = self.find_forms()
-            for form in forms:
-                compiled_form = self.form_to_compiled_form(form) 
-                if not os.path.exists(compiled_form) or os.stat(form).st_mtime > os.stat(compiled_form).st_mtime:
-                    print 'Compiling form', form
-                    buf = cStringIO.StringIO()
-                    compileUi(form, buf)
-                    dat = buf.getvalue()
-                    dat = dat.replace('__appname__', APPNAME)
-                    dat = dat.replace('import images_rc', 'from calibre.gui2 import images_rc')
-                    dat = dat.replace('from library import', 'from calibre.gui2.library import')
-                    dat = dat.replace('from widgets import', 'from calibre.gui2.widgets import')
-                    dat = re.compile(r'QtGui.QApplication.translate\(.+?,\s+"(.+?)(?<!\\)",.+?\)', re.DOTALL).sub(r'_("\1")', dat)
-                    
-                    # Workaround bug in Qt 4.4 on Windows
-                    if form.endswith('dialogs%sconfig.ui'%os.sep) or form.endswith('dialogs%slrf_single.ui'%os.sep):
-                        print 'Implementing Workaround for buggy pyuic in form', form
-                        dat = re.sub(r'= QtGui\.QTextEdit\(self\..*?\)', '= QtGui.QTextEdit()', dat) 
-                        dat = re.sub(r'= QtGui\.QListWidget\(self\..*?\)', '= QtGui.QListWidget()', dat)
-                    
-                    if form.endswith('viewer%smain.ui'%os.sep):
-                        print 'Promoting WebView'
-                        dat = dat.replace('self.view = QtWebKit.QWebView(', 'self.view = DocumentView(')
-                        dat += '\n\nfrom calibre.gui2.viewer.documentview import DocumentView'
-                    
-                    open(compiled_form, 'wb').write(dat)
-
-                
-        @classmethod
-        def clean(cls):
-            forms = cls.find_forms()
-            for form in forms:
-                c = cls.form_to_compiled_form(form)
-                if os.path.exists(c):
-                    os.remove(c)
-            for x in (cls.IMAGES_DEST, cls.QRC):
-                if os.path.exists(x):
-                    os.remove(x)
-    
-    class clean(Command):
-        description='''Delete all computer generated files in the source tree'''
-        
-        def run(self):
-            print 'Cleaning...'
-            manual.clean()
-            gui.clean()
-            translations.clean()
-            resources.clean()
-            
-            for f in glob.glob(os.path.join('src', 'calibre', 'plugins', '*')):
-                os.remove(f)
-            for root, dirs, files in os.walk('.'):
-                for name in files:
-                    for t in ('.pyc', '.pyo', '~'):
-                        if name.endswith(t):
-                            os.remove(os.path.join(root, name))
-                            break
-                        
-            for dir in ('build', 'dist', os.path.join('src', 'calibre.egg-info')):
-                shutil.rmtree(dir, ignore_errors=True)
-    
-    class build(_build):
-        
-        sub_commands = [
-                         ('resources',    lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
-                         ('translations', lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
-                         ('gui',          lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
-                         ('build_ext',    lambda self: True),
-                         ('build_py',     lambda self: True),
-                         ('build_clib',    _build.has_c_libraries),
-                         ('build_scripts', _build.has_scripts),
-                       ]
-        
-    entry_points['console_scripts'].append('calibre_postinstall = calibre.linux:post_install')
     ext_modules = [
                    Extension('calibre.plugins.lzx',
                              sources=['src/calibre/utils/lzx/lzxmodule.c',
@@ -430,7 +95,9 @@ if __name__ == '__main__':
         plugins = ['plugins/%s.so'%(x.name.rpartition('.')[-1]) for x in ext_modules]
     else:
         plugins = ['plugins/%s.pyd'%(x.name.rpartition('.')[-1]) for x in ext_modules] + \
-                  ['plugins/%s.pyd.manifest'%(x.name.rpartition('.')[-1]) for x in ext_modules if 'pictureflow' not in x.name]
+                  ['plugins/%s.pyd.manifest'%(x.name.rpartition('.')[-1]) \
+                        for x in ext_modules if 'pictureflow' not in x.name]
+
     
     setup(
           name           = APPNAME,
@@ -451,7 +118,11 @@ if __name__ == '__main__':
                       ''',
           long_description =
           '''
-  %s is an e-book library manager. It can view, convert and catalog e-books in most of the major e-book formats. It can also talk to a few e-book reader devices. It can go out to the internet and fetch metadata for your books. It can download newspapers and convert them into e-books for convenient reading. It is cross platform, running on Linux, Windows and OS X.
+  %s is an e-book library manager. It can view, convert and catalog e-books \
+  in most of the major e-book formats. It can also talk to e-book reader \
+  devices. It can go out to the internet and fetch metadata for your books. \
+  It can download newspapers and convert them into e-books for convenient \
+  reading. It is cross platform, running on Linux, Windows and OS X.
 
   For screenshots: https://%s.kovidgoyal.net/wiki/Screenshots
 
@@ -490,6 +161,19 @@ if __name__ == '__main__':
                       'gui'           : gui,
                       'clean'         : clean,
                       'sdist'         : sdist,
+                      'update'        : update,
+                      'tag_release'   : tag_release,
+                      'upload_demo'   : upload_demo,
+                      'build_linux'   : build_linux,
+                      'build_windows' : build_windows,
+                      'build_osx'     : build_osx,
+                      'upload_installers': upload_installers,
+                      'upload_user_manual': upload_user_manual,
+                      'upload_to_pypi': upload_to_pypi,
+                      'stage3' : stage3,
+                      'stage2' : stage2,
+                      'stage1' : stage1,
+                      'publish' : upload,
                       },
          )
 

src/calibre/__init__.py

@@ -29,7 +29,8 @@ mimetypes.add_type('application/adobe-page-template+xml', '.xpgt')
 mimetypes.add_type('application/x-font-opentype',         '.otf')
 mimetypes.add_type('application/x-font-truetype',         '.ttf')
 mimetypes.add_type('application/oebps-package+xml',       '.opf')
-
+import cssutils
+cssutils.log.setLevel(logging.WARN)
 
 def to_unicode(raw, encoding='utf-8', errors='strict'):
     if isinstance(raw, unicode):
@@ -71,7 +72,7 @@ def sanitize_file_name(name, substitute='_', as_unicode=False):
     one = re.sub(r'^\.+$', '_', one)
     if as_unicode:
         one = one.decode(filesystem_encoding)
-    return one
+    return one.replace('..', '_')
 
 
 def prints(*args, **kwargs):
@@ -401,8 +402,10 @@ def walk(dir):
         for f in record[-1]:
             yield os.path.join(record[0], f)
 
-def strftime(fmt, t=time.localtime()):
+def strftime(fmt, t=None):
     ''' A version of strtime that returns unicode strings. '''
+    if t is None:
+        t = time.localtime()
     if iswindows:
         if isinstance(fmt, unicode):
             fmt = fmt.encode('mbcs')

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.136'
+__version__   = '0.4.138'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/builtins.py

@@ -132,7 +132,7 @@ class HTMLMetadataReader(MetadataReaderPlugin):
 class MOBIMetadataReader(MetadataReaderPlugin):
     
     name        = 'Read MOBI metadata'
-    file_types  = set(['mobi', 'prc'])
+    file_types  = set(['mobi', 'prc', '.azw'])
     description = _('Read metadata from %s files')%'MOBI'
     
     def get_metadata(self, stream, ftype):

src/calibre/devices/prs505/books.py

@@ -186,7 +186,10 @@ class BookList(_BookList):
         node = self.document.createElement(self.prefix + "text")
         mime = MIME_MAP[name.rpartition('.')[-1].lower()]
         cid = self.max_id()+1
-        sourceid = str(self[0].sourceid) if len(self) else "1"
+        try:
+            sourceid = str(self[0].sourceid) if len(self) else '1'
+        except:
+            sourceid = '1'
         attrs = {
                  "title"  : info["title"], 
                  'titleSorter' : sortable_title(info['title']),

src/calibre/devices/prs505/driver.py

@@ -32,7 +32,7 @@ class PRS505(Device):
     BCD          = [0x229]  #: Needed to disambiguate 505 and 700 on linux
     PRODUCT_NAME = 'PRS-505'
     VENDOR_NAME  = 'SONY'
-    FORMATS      = ['lrf', 'epub', 'lrx', 'rtf', 'pdf', 'txt']
+    FORMATS      = ['epub', 'lrf', 'lrx', 'rtf', 'pdf', 'txt']
     
     MEDIA_XML    = 'database/cache/media.xml'
     CACHE_XML    = 'Sony Reader/database/cache.xml'

src/calibre/ebooks/epub/from_any.py

@@ -124,6 +124,7 @@ MAP = {
        'lit'  : lit2opf,
        'mobi' : mobi2opf,
        'prc'  : mobi2opf,
+       'azw'  : mobi2opf,
        'fb2'  : fb22opf,
        'rtf'  : rtf2opf,
        'txt'  : txt2opf,
@@ -131,7 +132,8 @@ MAP = {
        'epub' : epub2opf,
        'odt'  : odt2epub,
        }
-SOURCE_FORMATS = ['lit', 'mobi', 'prc', 'fb2', 'odt', 'rtf', 'txt', 'pdf', 'rar', 'zip', 'oebzip', 'htm', 'html', 'epub']
+SOURCE_FORMATS = ['lit', 'mobi', 'prc', 'azw', 'fb2', 'odt', 'rtf', 
+                  'txt', 'pdf', 'rar', 'zip', 'oebzip', 'htm', 'html', 'epub']
 
 def unarchive(path, tdir):
     extract(path, tdir)

src/calibre/ebooks/epub/from_html.py

@@ -310,6 +310,8 @@ def create_cover_image(src, dest, screen_size, rescale_cover=True):
 def process_title_page(mi, filelist, htmlfilemap, opts, tdir):
     old_title_page = None
     f = lambda x : os.path.normcase(os.path.normpath(x))
+    if not isinstance(mi.cover, basestring):
+        mi.cover = None
     if mi.cover:
         if f(filelist[0].path) == f(mi.cover):
             old_title_page = htmlfilemap[filelist[0].path]

src/calibre/ebooks/html.py

@@ -332,6 +332,8 @@ class PreProcessor(object):
                   (re.compile(r'&(\S+?);'), convert_entities),
                   # Remove the <![if/endif tags inserted by everybody's darling, MS Word
                   (re.compile(r'(?i)<{0,1}!\[(end){0,1}if[^>]*>'), lambda match: ''),
+                  # Strip all comments since Adobe DE is petrified of them
+                  (re.compile(r'<!--[^>]*>'), lambda match : ''),
                   ]
                      
     # Fix pdftohtml markup
@@ -491,9 +493,25 @@ class Parser(PreProcessor, LoggingInterface):
             self.root.insert(0, head)
 
         self.head = head 
-        self.body = self.root.body
+        try:
+            self.body = self.root.body
+        except:
+            import traceback
+            err = traceback.format_exc()
+            self.root = fromstring(u'<html><head/><body><p>This page was too '
+                                   'severely malformed for calibre to handle. '
+                                   'It has been replaced by this error message.'
+                                   '</p><pre>%s</pre></body></html>'%err)
+            self.head = self.root.xpath('./head')[0]
+            self.body = self.root.body
+        invalid_counter = 0
         for a in self.root.xpath('//a[@name]'):
-            a.set('id', a.get('name'))
+            try:
+                a.set('id', a.get('name'))
+            except:
+                invalid_counter += 1
+                for x in ('id', 'name'):
+                    a.set(x, 'calibre_invalid_id_%d'%invalid_counter)
         if not self.head.xpath('./title'):
             title = etree.SubElement(self.head, 'title')
             title.text = _('Unknown')

src/calibre/ebooks/lit/reader.py

@@ -780,7 +780,7 @@ class LitReader(object):
                 if u != 0:
                     raise LitError("Reset table entry greater than 32 bits")
                 if size >= len(content):
-                    raise LitError("Reset table entry out of bounds")
+                    self._warn("LZX reset table entry out of bounds")
                 if bytes_remaining >= window_bytes:
                     lzx.reset()
                     try:

src/calibre/ebooks/lrf/__init__.py

@@ -29,6 +29,7 @@ preferred_source_formats = [
                             'XHTM',
                             'XHTML',
                             'PRC',
+                            'AZW',
                             'RTF',
                             'PDF',
                             'TXT',

src/calibre/ebooks/lrf/any/convert_from.py

@@ -154,7 +154,7 @@ def process_file(path, options, logger=None):
             convertor = txt2lrf
         elif 'epub' == ext:
             convertor = epub2lrf
-        elif ext in ['mobi', 'prc']:
+        elif ext in ['mobi', 'prc', 'azw']:
             convertor = mobi2lrf
         elif ext == 'fb2':
             convertor = fb22lrf

src/calibre/ebooks/metadata/ncx.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0"  encoding="UTF-8"?>
 <?python
 from uuid import uuid4
+import re
 ?>
 <ncx version="2005-1" 
      xml:lang="en" 
@@ -19,7 +20,7 @@ from uuid import uuid4
     <py:def function="navpoint(np, level)">
         ${'%*s'%(4*level,'')}<navPoint id="${str(uuid4())}" playOrder="${str(np.play_order)}">
             ${'%*s'%(4*level,'')}<navLabel>
-                ${'%*s'%(4*level,'')}<text>${np.text}</text>
+                ${'%*s'%(4*level,'')}<text>${re.sub(r'\s+', ' ', np.text)}</text>
             ${'%*s'%(4*level,'')}</navLabel>
             ${'%*s'%(4*level,'')}<content src="${unicode(np.href)+(('#' + unicode(np.fragment)) if np.fragment else '')}" />
             <py:for each="np2 in np">${navpoint(np2, level+1)}</py:for>

src/calibre/ebooks/mobi/reader.py

@@ -186,6 +186,8 @@ class MobiReader(object):
         self.processed_html = self.processed_html.decode(self.book_header.codec, 'ignore')
         for pat in ENCODING_PATS:
             self.processed_html = pat.sub('', self.processed_html)
+        self.processed_html = re.sub(r'&(\S+?);', entity_to_unicode,
+                                     self.processed_html)
         self.extract_images(processed_records, output_dir)
         self.replace_page_breaks()
         self.cleanup_html()
@@ -247,9 +249,6 @@ class MobiReader(object):
             self.processed_html = '<html><p>'+self.processed_html.replace('\n\n', '<p>')+'</html>'
         self.processed_html = self.processed_html.replace('\r\n', '\n')
         self.processed_html = self.processed_html.replace('> <', '>\n<')
-        for t, c in [('b', 'bold'), ('i', 'italic')]:
-            self.processed_html = re.sub(r'(?i)<%s>'%t, r'<span class="%s">'%c, self.processed_html)
-            self.processed_html = re.sub(r'(?i)</%s>'%t, r'</span>', self.processed_html)
         
     def upshift_markup(self, root):
         if self.verbose:
@@ -290,35 +289,44 @@ class MobiReader(object):
                 align = attrib.pop('align').strip()
                 if align:
                     styles.append('text-align: %s' % align)
-            if mobi_version == 1 and tag.tag == 'hr':
+            if tag.tag == 'hr':
-                tag.tag = 'div'
+                if mobi_version == 1:
-                styles.append('page-break-before: always')
+                    tag.tag = 'div'
-                styles.append('display: block')
+                    styles.append('page-break-before: always')
-                styles.append('margin: 0')
+                    styles.append('display: block')
-            if styles:
+                    styles.append('margin: 0')
-                attrib['style'] = '; '.join(styles)
+            elif tag.tag == 'i':
-                
+                tag.tag = 'span'
-            if tag.tag.lower() == 'font':
+                tag.attrib['class'] = 'italic'
+            elif tag.tag == 'b':
+                tag.tag = 'span'
+                tag.attrib['class'] = 'bold'
+            elif tag.tag == 'font':
                 sz = tag.get('size', '').lower()
                 try:
                     float(sz)
                 except ValueError:
                     if sz in size_map.keys():
                         attrib['size'] = size_map[sz]
+            elif tag.tag == 'img':
+                recindex = None
+                for attr in self.IMAGE_ATTRS:
+                    recindex = attrib.pop(attr, None) or recindex
+                if recindex is not None:
+                    attrib['src'] = 'images/%s.jpg' % recindex
+            elif tag.tag == 'pre':
+                if not tag.text:
+                    tag.tag = 'div'
+            if styles:
+                attrib['style'] = '; '.join(styles)
             if 'filepos-id' in attrib:
                 attrib['id'] = attrib.pop('filepos-id')
             if 'filepos' in attrib:
                 filepos = attrib.pop('filepos')
                 try:
                     attrib['href'] = "#filepos%d" % int(filepos)
-                except:
+                except ValueError:
-                    attrib['href'] = filepos
+                    pass
-            if tag.tag == 'img':
-                recindex = None
-                for attr in self.IMAGE_ATTRS:
-                    recindex = attrib.pop(attr, None) or recindex
-                if recindex is not None:
-                    attrib['src'] = 'images/%s.jpg' % recindex
     
     def create_opf(self, htmlfile, guide=None):
         mi = self.book_header.exth.mi
@@ -328,7 +336,7 @@ class MobiReader(object):
         manifest = [(htmlfile, 'text/x-oeb1-document')]
         bp = os.path.dirname(htmlfile)
         for i in getattr(self, 'image_names', []):
-            manifest.append((os.path.join(bp, 'images/', i), 'image/jpg'))
+            manifest.append((os.path.join(bp, 'images/', i), 'image/jpeg'))
         
         opf.create_manifest(manifest)
         opf.create_spine([os.path.basename(htmlfile)])
@@ -452,6 +460,7 @@ class MobiReader(object):
             pos = end
         self.processed_html += self.mobi_html[pos:]
         
+    
     def extract_images(self, processed_records, output_dir):
         if self.verbose:
             print 'Extracting images...'

src/calibre/ebooks/mobi/writer.py

@@ -416,7 +416,11 @@ class MobiWriter(object):
         coverid = metadata.cover[0] if metadata.cover else None
         for _, href in images:
             item = self._oeb.manifest.hrefs[href]
-            data = rescale_image(item.data, self._imagemax)
+            try:
+                data = rescale_image(item.data, self._imagemax)
+            except IOError:
+                self._oeb.logger.warn('Bad image file %r' % item.href)
+                continue
             self._records.append(data)
     
     def _generate_record0(self):
@@ -486,9 +490,11 @@ class MobiWriter(object):
             index = self._images[href] - 1
             exth.write(pack('>III', 0xc9, 0x0c, index))
             exth.write(pack('>III', 0xcb, 0x0c, 0))
-            index = self._add_thumbnail(item) - 1
+            nrecs += 2
-            exth.write(pack('>III', 0xca, 0x0c, index))
+            index = self._add_thumbnail(item)
-            nrecs += 3
+            if index is not None:
+                exth.write(pack('>III', 0xca, 0x0c, index - 1))
+                nrecs += 1
         exth = exth.getvalue()
         trail = len(exth) % 4
         pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte
@@ -496,7 +502,11 @@ class MobiWriter(object):
         return ''.join(exth)
 
     def _add_thumbnail(self, item):
-        data = rescale_image(item.data, MAX_THUMB_SIZE, MAX_THUMB_DIMEN)
+        try:
+            data = rescale_image(item.data, MAX_THUMB_SIZE, MAX_THUMB_DIMEN)
+        except IOError:
+            self._oeb.logger.warn('Bad image file %r' % item.href)
+            return None
         manifest = self._oeb.manifest
         id, href = manifest.generate('thumbnail', 'thumbnail.jpeg')
         manifest.add(id, href, 'image/jpeg', data=data)

src/calibre/ebooks/oeb/stylizer.py

@@ -8,24 +8,20 @@ from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys
 import os
-import locale
-import codecs
 import itertools
-import types
 import re
 import copy
-from itertools import izip
 from weakref import WeakKeyDictionary
 from xml.dom import SyntaxErr as CSSSyntaxError
 import cssutils
 from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \
     CSSValueList, cssproperties
+from cssutils.profiles import profiles as cssprofiles
 from lxml import etree
 from lxml.cssselect import css_to_xpath, ExpressionError
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
-from calibre.ebooks.oeb.base import XPNSMAP, xpath, barename, urlnormalize
+from calibre.ebooks.oeb.base import XPNSMAP, xpath, urlnormalize
 from calibre.ebooks.oeb.profile import PROFILES
 from calibre.resources import html_css
 
@@ -163,7 +159,7 @@ class Stylizer(object):
         for _, _, cssdict, text, _ in rules:
             try:
                 selector = CSSSelector(text)
-            except ExpressionError, e:
+            except ExpressionError:
                 continue
             for elem in selector(tree):
                 self.style(elem)._update_cssdict(cssdict)
@@ -246,7 +242,7 @@ class Stylizer(object):
             primitives.reverse()
             value = primitives.pop()
             for key in composition:
-                if cssproperties.cssvalues[key](value):
+                if cssprofiles.validate(key, value):
                     style[key] = value
                     if not primitives: break
                     value = primitives.pop()

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -6,13 +6,9 @@ from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys
-import os
 from itertools import chain
 from urlparse import urldefrag
-from lxml import etree
+from calibre.ebooks.oeb.base import CSS_MIME, OEB_DOCS
-import cssutils
-from calibre.ebooks.oeb.base import XPNSMAP, CSS_MIME, OEB_DOCS
 from calibre.ebooks.oeb.base import LINK_SELECTORS, CSSURL_RE
 from calibre.ebooks.oeb.base import urlnormalize
 

src/calibre/gui2/dialogs/metadata_single.py

@@ -113,6 +113,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
     def set_cover(self):
         row = self.formats.currentRow()
         fmt = self.formats.item(row)
+        if fmt is None:
+            error_dialog(self, _('No format selected'),
+                    _('No format selected')).exec_()
+            return
         ext = fmt.ext.lower()
         if fmt.path is None:
             stream = self.db.format(self.row, ext, as_file=True)
@@ -265,7 +269,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
     def cover_dropped(self):
         self.cover_changed = True
     
-    def initialize_series_and_publisher(self):
+    def initialize_series(self):
         all_series = self.db.all_series()
         all_series.sort(cmp=lambda x, y : cmp(x[1], y[1]))
         series_id = self.db.series_id(self.row)
@@ -289,6 +293,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                 l.invalidate()
                 l.activate()
         
+    def initialize_series_and_publisher(self):
+        self.initialize_series()
         all_publishers = self.db.all_publishers()
         all_publishers.sort(cmp=lambda x, y : cmp(x[1], y[1]))
         publisher_id = self.db.publisher_id(self.row)

src/calibre/gui2/library.py

@@ -216,12 +216,9 @@ class BooksModel(QAbstractTableModel):
 
 
     def delete_books(self, indices):
-        ids = [ self.id(i) for i in indices ]
+        ids = map(self.id, indices)
         for id in ids:
-            row = self.db.index(id)
+            self.db.delete_book(id, notify=False)
-            self.beginRemoveRows(QModelIndex(), row, row)
-            self.db.delete_book(id)
-            self.endRemoveRows()
         self.count_changed()
         self.clear_caches()
         self.reset()
@@ -245,7 +242,7 @@ class BooksModel(QAbstractTableModel):
         if num > 0:
             self.beginInsertRows(QModelIndex(), 0, num-1)
             self.endInsertRows()
-        self.count_changed()
+            self.count_changed()
 
     def search(self, text, refinement, reset=True):
         self.db.search(text)
@@ -414,8 +411,11 @@ class BooksModel(QAbstractTableModel):
                      
             
     
-    def get_preferred_formats(self, rows, formats, paths=False, set_metadata=False):
+    def get_preferred_formats(self, rows, formats, paths=False, 
+                              set_metadata=False, specific_format=None):
         ans = []
+        if specific_format is not None:
+            formats = [specific_format.lower()]
         for row in (row.row() for row in rows):
             format = None
             fmts = self.db.formats(row)

src/calibre/gui2/main.py

@@ -156,6 +156,9 @@ class Main(MainWindow, Ui_MainWindow):
         sm.addAction(QIcon(':/images/sd.svg'), _('Send to storage card'))
         sm.addAction(QIcon(':/images/reader.svg'), _('Send to main memory')+' '+_('and delete from library'))
         sm.addAction(QIcon(':/images/sd.svg'), _('Send to storage card')+' '+_('and delete from library'))
+        sm.addAction(self.action_send_specific_format_to_device)
+        self.connect(self.action_send_specific_format_to_device, 
+                     SIGNAL('triggered()'), self.send_specific_format_to_device)
         sm.addSeparator()
         sm.addAction(_('Send to storage card by default'))
         sm.actions()[-1].setCheckable(True)
@@ -330,12 +333,17 @@ class Main(MainWindow, Ui_MainWindow):
             self.cover_flow.setVisible(False)
             if not config['separate_cover_flow']:
                 self.library.layout().addWidget(self.cover_flow)
-            self.connect(self.cover_flow, SIGNAL('currentChanged(int)'), self.sync_cf_to_listview)
+            self.connect(self.cover_flow, SIGNAL('currentChanged(int)'), 
-            self.connect(self.cover_flow, SIGNAL('itemActivated(int)'), self.show_book_info)
+                         self.sync_cf_to_listview)
-            self.connect(self.status_bar.cover_flow_button, SIGNAL('toggled(bool)'), self.toggle_cover_flow)
+            self.connect(self.cover_flow, SIGNAL('itemActivated(int)'), 
-            self.connect(self.cover_flow, SIGNAL('stop()'), self.status_bar.cover_flow_button.toggle)
+                         self.show_book_info)
-            QObject.connect(self.library_view.selectionModel(), SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
+            self.connect(self.status_bar.cover_flow_button, 
-                        self.sync_cf_to_listview)
+                         SIGNAL('toggled(bool)'), self.toggle_cover_flow)
+            self.connect(self.cover_flow, SIGNAL('stop()'), 
+                         self.status_bar.cover_flow_button.toggle)
+            QObject.connect(self.library_view.selectionModel(), 
+                            SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
+                            self.sync_cf_to_listview)
             self.db_images = DatabaseImages(self.library_view.model())
             self.cover_flow.setImages(self.db_images)
         else:
@@ -482,7 +490,8 @@ class Main(MainWindow, Ui_MainWindow):
         if not hasattr(index, 'row') and self.library_view.currentIndex().row() != index:
             index = self.library_view.model().index(index, 0)
             self.library_view.setCurrentIndex(index)
-        if hasattr(index, 'row') and self.cover_flow.isVisible() and self.cover_flow.currentSlide() != index.row():
+        if hasattr(index, 'row') and self.cover_flow.isVisible() and \
+                self.cover_flow.currentSlide() != index.row():
             self.cover_flow.setCurrentSlide(index.row())
 
     def another_instance_wants_to_talk(self, msg):
@@ -709,6 +718,7 @@ class Main(MainWindow, Ui_MainWindow):
                 t.process_duplicates()
         if t.number_of_books_added > 0:
             self.library_view.model().books_added(t.number_of_books_added)
+            self.db_images.reset()
         
     def upload_books(self, files, names, metadata, on_card=False, memory=None):
         '''
@@ -886,8 +896,16 @@ class Main(MainWindow, Ui_MainWindow):
                 self.upload_books(files, names, metadata, on_card=on_card, memory=[[f.name for f in files], remove])
                 self.status_bar.showMessage(_('Sending news to device.'), 5000)
                 
+    def send_specific_format_to_device(self):
+        d = ChooseFormatDialog(self, _('Choose format to send to device'), 
+                               self.device_manager.device_class.FORMATS)
+        d.exec_()
+        fmt = d.format().lower()
+        on_card = config['send_to_storage_card_by_default']
+        self.sync_to_device(on_card, False, specific_format=fmt)
         
-    def sync_to_device(self, on_card, delete_from_library):
+    
+    def sync_to_device(self, on_card, delete_from_library, specific_format=None):
         rows = self.library_view.selectionModel().selectedRows()
         if not self.device_manager or not rows or len(rows) == 0:
             return
@@ -900,7 +918,8 @@ class Main(MainWindow, Ui_MainWindow):
         metadata = iter(metadata)
         _files   = self.library_view.model().get_preferred_formats(rows,
                                     self.device_manager.device_class.FORMATS, 
-                                    paths=True, set_metadata=True)
+                                    paths=True, set_metadata=True,
+                                    specific_format=specific_format)
         files = [getattr(f, 'name', None) for f in _files]
         bad, good, gf, names, remove_ids = [], [], [], [], []
         for f in files:
@@ -1336,10 +1355,13 @@ class Main(MainWindow, Ui_MainWindow):
         '''
         Handle exceptions in threaded device jobs.
         '''
-        if 'Could not read 32 bytes on the control bus.' in str(job.exception):
+        try:
-            error_dialog(self, _('Error talking to device'), 
+            if 'Could not read 32 bytes on the control bus.' in unicode(job.exception):
-                         _('There was a temporary error talking to the device. Please unplug and reconnect the device and or reboot.')).show()
+                error_dialog(self, _('Error talking to device'), 
-            return
+                             _('There was a temporary error talking to the device. Please unplug and reconnect the device and or reboot.')).show()
+                return
+        except:
+            pass
         try:
             print >>sys.stderr, job.console_text()
         except:

src/calibre/gui2/main.ui

@@ -656,6 +656,15 @@
     <string>Books with the same tags</string>
    </property>
   </action>
+  <action name="action_send_specific_format_to_device" >
+   <property name="icon" >
+    <iconset resource="images.qrc" >
+     <normaloff>:/images/book.svg</normaloff>:/images/book.svg</iconset>
+   </property>
+   <property name="text" >
+    <string>Send specific format to device</string>
+   </property>
+  </action>
  </widget>
  <customwidgets>
   <customwidget>

src/calibre/library/database2.py

@@ -19,7 +19,8 @@ from calibre.library import title_sort
 from calibre.library.database import LibraryDatabase
 from calibre.library.sqlite import connect, IntegrityError
 from calibre.utils.search_query_parser import SearchQueryParser
-from calibre.ebooks.metadata import string_to_authors, authors_to_string, MetaInformation
+from calibre.ebooks.metadata import string_to_authors, authors_to_string, \
+                                    MetaInformation, authors_to_sort_string
 from calibre.ebooks.metadata.meta import get_metadata, set_metadata, \
     metadata_from_formats
 from calibre.ebooks.metadata.opf2 import OPFCreator
@@ -728,7 +729,7 @@ class LibraryDatabase2(LibraryDatabase):
         if notify:
             self.notify('metadata', [id])
         
-    def delete_book(self, id):
+    def delete_book(self, id, notify=True):
         '''
         Removes book from the result cache and the underlying database.
         '''
@@ -743,7 +744,8 @@ class LibraryDatabase2(LibraryDatabase):
         self.conn.commit()
         self.clean()
         self.data.books_deleted([id])
-        self.notify('delete', [id])
+        if notify:
+            self.notify('delete', [id])
     
     def remove_format(self, index, format, index_is_id=False, notify=True):
         id = index if index_is_id else self.id(index)
@@ -1197,11 +1199,17 @@ class LibraryDatabase2(LibraryDatabase):
 
     def import_book(self, mi, formats, notify=True):
         series_index = 1 if mi.series_index is None else mi.series_index
+        if not mi.title:
+            mi.title = _('Unknown')
         if not mi.authors:
             mi.authors = [_('Unknown')]
-        aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
+        aus = mi.author_sort if mi.author_sort else authors_to_sort_string(mi.authors)
+        if isinstance(aus, str):
+            aus = aus.decode(preferred_encoding, 'replace')
+        title = mi.title if isinstance(mi.title, unicode) else \
+                mi.title.decode(preferred_encoding, 'replace')
         obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)', 
-                          (mi.title, None, series_index, aus))
+                          (title, None, series_index, aus))
         id = obj.lastrowid
         self.data.books_added([id], self.conn)
         self.set_path(id, True)
@@ -1210,8 +1218,7 @@ class LibraryDatabase2(LibraryDatabase):
             ext = os.path.splitext(path)[1][1:].lower()
             if ext == 'opf':
                 continue
-            stream = open(path, 'rb')
+            self.add_format_with_hooks(id, ext, path, index_is_id=True)
-            self.add_format(id, ext, stream, index_is_id=True)
         self.conn.commit()
         self.data.refresh_ids(self.conn, [id]) # Needed to update format list and size
         if notify:

src/calibre/library/sqlite.py

@@ -12,11 +12,50 @@ from sqlite3 import IntegrityError
 from threading import Thread
 from Queue import Queue
 from threading import RLock
+from datetime import tzinfo, datetime, timedelta
 
 from calibre.ebooks.metadata import title_sort
 
 global_lock = RLock()
 
+def convert_timestamp(val):
+    datepart, timepart = val.split(' ')
+    tz, mult = None, 1
+    x = timepart.split('+')
+    if len(x) > 1:
+        timepart, tz = x
+    else:
+        x = timepart.split('-')
+        if len(x) > 1:
+            timepart, tz = x
+            mult = -1
+
+    year, month, day = map(int, datepart.split("-"))
+    timepart_full = timepart.split(".")
+    hours, minutes, seconds = map(int, timepart_full[0].split(":"))
+    if len(timepart_full) == 2:
+        microseconds = int(timepart_full[1])
+    else:
+        microseconds = 0
+    if tz is not None:
+        h, m = map(int, tz.split(':'))
+        delta = timedelta(minutes=mult*(60*h + m))
+        tz = type('CustomTZ', (tzinfo,), {'utcoffset':lambda self, dt:delta, 
+                                          'dst':lambda self,dt:timedelta(0)})()
+
+    val = datetime(year, month, day, hours, minutes, seconds, microseconds, 
+                   tzinfo=tz)
+    if tz is not None:
+        val = datetime(*(val.utctimetuple()[:6]))
+    return val
+
+def adapt_datetime(dt):
+    dt = datetime(*(dt.utctimetuple()[:6]))
+    return dt.isoformat(' ')
+
+sqlite.register_adapter(datetime, adapt_datetime)
+sqlite.register_converter('timestamp', convert_timestamp)
+
 class Concatenate(object):
     '''String concatenation aggregator for sqlite'''
     def __init__(self, sep=','):

src/calibre/linux.py

@@ -480,6 +480,7 @@ VIEWER = '''\
 Version=%s
 Type=Application
 Name=LRF Viewer
+GenericName=Viewer for LRF files
 Comment=Viewer for LRF files (SONY ebook format files)
 TryExec=lrfviewer
 Exec=lrfviewer %%F
@@ -492,8 +493,9 @@ EVIEWER = '''\
 [Desktop Entry]
 Version=%s
 Type=Application
-Name=Ebook Viewer
+Name=E-book Viewer
-Comment=Viewer for Ebooks
+GenericName=Viewer for E-books
+Comment=Viewer for E-books
 TryExec=ebook-viewer
 Exec=ebook-viewer %%F
 Icon=calibre-viewer
@@ -506,7 +508,8 @@ GUI = '''\
 [Desktop Entry]
 Version=%s
 Type=Application
-Name=calibre - Ebook library management
+Name=calibre
+GenericName=E-book library management
 Comment=E-book library management
 TryExec=calibre
 Exec=calibre

src/calibre/manual/faq.rst

@@ -28,7 +28,7 @@ What formats does |app| support conversion to/from?
 |                   |        |                  |                       |                       |
 |                   |  LIT   |       ✔          |          ✔            |          ✔            |
 |                   |        |                  |                       |                       |
-|                   |  PRC   |       ✔          |          ✔            |          ✔            |
+|                   |  PRC** |       ✔          |          ✔            |          ✔            |
 |                   |        |                  |                       |                       |
 |                   |  EPUB  |       ✔          |          ✔            |          ✔            |
 |                   |        |                  |                       |                       |
@@ -49,6 +49,7 @@ What formats does |app| support conversion to/from?
 |                   |  LRS   |                  |          ✔            |                       |
 +-------------------+--------+------------------+-----------------------+-----------------------+
 
+** PRC is a generic format, |app| supports PRC files with TextRead and MOBIBook headers
 
 
 What are the best source formats to convert?
@@ -146,7 +147,7 @@ When you first run |app|, it will ask you for a folder in which to store your bo
 Why doesn't |app| let me store books in my own directory structure?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The whole point if |app|'s library management features is that they provide an interface for locating books that is *much* more efficient than any possible directory scheme you could come up with for your collection. Indeed, once you become comfortable using |app|'s interface to find, sort and browse your collection, you wont ever feel the need to hunt through the files on your disk to find a book again. By managing books in its own directory struture of Author -> Title -> Book files, |app| is able to achieve a high level of reliability and standardization.  
+The whole point of |app|'s library management features is that they provide an interface for locating books that is *much* more efficient than any possible directory scheme you could come up with for your collection. Indeed, once you become comfortable using |app|'s interface to find, sort and browse your collection, you wont ever feel the need to hunt through the files on your disk to find a book again. By managing books in its own directory struture of Author -> Title -> Book files, |app| is able to achieve a high level of reliability and standardization.  
 
 Why doesn't |app| have a column for foo?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/templates/layout.html

@@ -0,0 +1,12 @@
+{% extends "!layout.html" %}
+{% block sidebarlogo %}
+{{ super() }}
+         <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
+               <input type="hidden" name="cmd" value="_s-xclick" />
+               <input type="hidden" name="hosted_button_id" value="3028915" />
+               <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donate_LG.gif" border="0" name="submit" alt="Donate to support calibre development" style="border:0pt" />
+               <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1"/>
+           </form>
+           <hr/>
+{% endblock %}
+

src/calibre/trac/donations/server.py

@@ -197,6 +197,7 @@ class Server(object):
     def calculate_month_trend(self, days=31):
         stats = self.get_slice(date.today()-timedelta(days=days-1), date.today())
         fig = plt.figure(2, (12, 4), 96)#, facecolor, edgecolor, frameon, FigureClass)
+        fig.clear()
         ax = fig.add_subplot(111)
         x = list(range(days-1, -1, -1))
         y = stats.daily_totals
@@ -235,6 +236,7 @@ Donors per day: %(dpd).2f
         y = [m.total for m in _months]
         ml   = mdates.MonthLocator() # every month
         fig = plt.figure(1, (8, 4), 96)#, facecolor, edgecolor, frameon, FigureClass)
+        fig.clear()
         ax = fig.add_subplot(111)
         average = sum(y)/len(y)
         ax.bar(x, y, align='center', width=20, color='g')

src/calibre/trac/plugins/download.py

@@ -1,419 +1,386 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import re
-from pkg_resources import resource_filename
-
-from trac.core import Component, implements
-from trac.web.chrome import INavigationContributor, ITemplateProvider, add_stylesheet
-from trac.web.main import IRequestHandler
-from trac.util import Markup
-
-
 __appname__ = 'calibre'
-DOWNLOAD_DIR = '/var/www/calibre.kovidgoyal.net/htdocs/downloads'
+import re, textwrap
-MOBILEREAD = 'https://dev.mobileread.com/dist/kovid/calibre/'
 
-class OS(dict):
+DEPENDENCIES = [
-    """Dictionary with a default value for unknown keys."""
+            #(Generic, version, gentoo, ubuntu, fedora)
-    def __init__(self, dict):
+            ('python', '2.5', None, None, None),
-        self.update(dict)
+            ('setuptools', '0.6c5', 'setuptools', 'python-setuptools', 'python-setuptools-devel'),
-        if not dict.has_key('img'):
+            ('Python Imaging Library', '1.1.6', 'imaging', 'python-imaging', 'python-imaging'),
-            self['img'] = self['name']
+            ('libusb', '0.1.12', None, None, None),
+            ('Qt', '4.4.0', 'qt', 'libqt4-core libqt4-gui', 'qt4'),
+            ('PyQt', '4.4.2', 'PyQt4', 'python-qt4', 'PyQt4'),
+            ('python-mechanize', '0.1.11', 'dev-python/mechanize', 'python-mechanize', 'python-mechanize'),
+            ('ImageMagick', '6.3.5', 'imagemagick', 'imagemagick', 'ImageMagick'),
+            ('xdg-utils', '1.0.2', 'xdg-utils', 'xdg-utils', 'xdg-utils'),
+            ('dbus-python', '0.82.2', 'dbus-python', 'python-dbus', 'dbus-python'),
+            ('lxml', '2.0.5', 'lxml', 'python-lxml', 'python-lxml'),
+            ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil'),
+            ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'),
+            ('help2man', '1.36.4', 'help2man', 'help2man', 'help2man'),
+            ]
         
-class Distribution(object):
 
-    DEPENDENCIES = [
+class CoolDistro:
-        #(Generic, version, gentoo, ubuntu, fedora)
-        ('python', '2.5', None, None, None),
-        ('setuptools', '0.6c5', 'setuptools', 'python-setuptools', 'python-setuptools-devel'),
-        ('Python Imaging Library', '1.1.6', 'imaging', 'python-imaging', 'python-imaging'),
-        ('libusb', '0.1.12', None, None, None),
-        ('Qt', '4.4.0', 'qt', 'libqt4-core libqt4-gui', 'qt4'),
-        ('PyQt', '4.4.2', 'PyQt4', 'python-qt4', 'PyQt4'),
-        ('mechanize for python', '0.1.11', 'dev-python/mechanize', 'python-mechanize', 'python-mechanize'),
-        ('ImageMagick', '6.3.5', 'imagemagick', 'imagemagick', 'ImageMagick'),
-        ('xdg-utils', '1.0.2', 'xdg-utils', 'xdg-utils', 'xdg-utils'),
-        ('dbus-python', '0.82.2', 'dbus-python', 'python-dbus', 'dbus-python'),
-        ('lxml', '2.0.5', 'lxml', 'python-lxml', 'python-lxml'),
-        ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil'),
-        ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'),
-        ('help2man', '1.36.4', 'help2man', 'help2man', 'help2man'),
-        ]
     
-    DISTRO_MAP = {'gentoo':2, 'ubuntu':3, 'fedora':4, 'debian':3}
+    def __init__(self, name, title, prefix=''):
+        self.title = title
+        url = prefix + '/chrome/dl/images/%s_logo.png'
+        self.img = url%name
 
-    INSTALLERS = ('emerge -avn', 'apt-get install', 'yum install')
+def get_linux_data(version='1.0.0'):
-    AS_ROOT    = (True, False, True)
+    data = {'version':version, 'app':__appname__}
+    data['title'] = 'Download calibre for linux'
+    data['supported'] = []
+    for name, title in [
+                        ('ubuntu', 'Ubuntu Jaunty Jackalope'),
+                        ('debian', 'Debian Sid'),
+                        ('exherbo', 'Exherbo'),
+                        ]:
+        data['supported'].append(CoolDistro(name, title, 
+                                        prefix='http://calibre.kovidgoyal.net'))
+        data['dependencies'] = DEPENDENCIES
+    return data
 
-    TITLEMAP = {'gentoo':'Gentoo', 'ubuntu':'Ubuntu Intrepid Ibex',
+if __name__ == '__main__':
-                'fedora':'Fedora 10', 'debian':'Debian sid', 'generic': 'Install from source'}
+    import os
+    from calibre.utils.genshi.template import MarkupTemplate
+    import cherrypy
+    class Test:
+        def index(self):
+            raw = open(os.path.dirname(os.path.abspath(__file__))+'/templates/linux.html').read()
+            return MarkupTemplate(raw).generate(**get_linux_data()).render('xhtml')
+        index.exposed = True
+    t = Test()
+    t.index()    
+    cherrypy.quickstart(t)
+else:
+    from pkg_resources import resource_filename
     
-    MANUAL_MAP = {
+    from trac.core import Component, implements
-                  'fedora' : '''<li>You have to upgrade Qt to at least 4.4.0 and PyQt to at least 4.4.2</li>''',
+    from trac.web.chrome import INavigationContributor, ITemplateProvider, add_stylesheet
-                  }
+    from trac.web.main import IRequestHandler
+    from trac.util import Markup
     
-    def __init__(self, os):
+    
-        self.os = os
+    
-        self.img = os
+    DOWNLOAD_DIR = '/var/www/calibre.kovidgoyal.net/htdocs/downloads'
-        self.title = self.TITLEMAP[os]
+    MOBILEREAD = 'https://dev.mobileread.com/dist/kovid/calibre/'
-        self.app = __appname__
+    
-        self.is_generic = os == 'generic'
+    class OS(dict):
-        offset = 0
+        """Dictionary with a default value for unknown keys."""
-        if not self.is_generic:
+        def __init__(self, dict):
-            index = self.DISTRO_MAP[self.os]
+            self.update(dict)
-            if os == 'debian':
+            if not dict.has_key('img'):
-                self.as_root = True  
+                self['img'] = self['name']
-            else: self.as_root = self.AS_ROOT[index-2]
+    
-            prefix = '' 
+        
-            if not self.as_root: prefix =  'sudo '
+    class Download(Component):
-            cmd = prefix + self.INSTALLERS[index-2]
+        implements(INavigationContributor, IRequestHandler, ITemplateProvider)
-            pre = ' \\\n '.ljust(len(cmd)+4)
+        
-            for dep in self.DEPENDENCIES:
+        request_pat = re.compile(r'\/download$|\/download_\S+')
-                if len(cmd) > 70+offset:
+        
-                    offset += 70
+        # INavigationContributor methods
-                    cmd += pre
+        def get_active_navigation_item(self, req):
-                cmd += ' ' 
+            return 'download'
-                if dep[index]: cmd += dep[index]
+    
-            self.command = cmd.strip()
+        def get_navigation_items(self, req):
-            easy_install = 'easy_install'
+            yield 'mainnav', 'download', Markup('<a href="/download">Get %s</a>'%(__appname__,))
-            if os == 'debian':
+        
-                easy_install = 'easy_install-2.5'
+        def get_templates_dirs(self):
-            self.command += '\n'+prefix+easy_install+' -U calibre \n'+prefix+'calibre_postinstall'
+            return [resource_filename(__name__, 'templates')]
+        
+        def get_htdocs_dirs(self):
+            return [('dl', resource_filename(__name__, 'htdocs'))]
+    
+        # IRequestHandler methods
+        def match_request(self, req):
+            return self.__class__.request_pat.match(req.path_info)
+    
+        def process_request(self, req):
+            add_stylesheet(req, 'dl/css/download.css')
+            if req.path_info == '/download':
+                return self.top_level(req)
+            elif req.path_info == '/download_linux_binary_installer':
+                req.send(LINUX_INSTALLER.replace('%version', self.version_from_filename()), 'text/x-python')
+            else:
+                match = re.match(r'\/download_(\S+)', req.path_info)
+                if match:
+                    os = match.group(1)
+                    if os == 'windows':
+                        return self.windows(req)
+                    elif os == 'osx':
+                        return self.osx(req)
+                    elif os == 'linux':
+                        return self.linux(req)
+        
+        def top_level(self, req):
+            operating_systems = [
+                OS({'name' : 'windows', 'title' : 'Windows'}),
+                OS({'name' : 'osx', 'title' : 'OS X'}),
+                OS({'name' : 'linux', 'title' : 'Linux'}),
+            ]
+            data = dict(title='Get ' + __appname__, 
+                        operating_systems=operating_systems, width=200,
+                        font_size='xx-large', top_level=True)
+            return 'download.html', data, None
+        
+        def version_from_filename(self):
             try:
-                self.manual = Markup(self.MANUAL_MAP[os])
+                return open(DOWNLOAD_DIR+'/latest_version', 'rb').read().strip()
-            except KeyError:
+            except:
-                self.manual = None
+                return '0.0.0'
-        else:
+        
-            self.img = 'linux'
+        def windows(self, req):
+            version = self.version_from_filename()
+            file = '%s-%s.exe'%(__appname__, version,)         
+            data = dict(version = version, name='windows',
+                installer_name='Windows installer', 
+                title='Download %s for windows'%(__appname__),
+                compatibility='%s works on Windows XP and Windows Vista.'%(__appname__,),
+                path=MOBILEREAD+file, app=__appname__,
+                note=Markup(\
+    '''
+    <p>If you are using the <b>SONY PRS-500</b> and %(appname)s does not detect your reader, read on:</p>
+    <blockquote>
+    <p>
+    If you are using 64-bit windows, you're out of luck. 
+    </p>
+    <p>
+    There may be a conflict with the USB driver from SONY. In windows, you cannot install two drivers
+    for one device. In order to resolve the conflict:
+    <ol>
+    <li>Start Device Manager by clicking Start->Run, typing devmgmt.msc and pressing enter.</li>
+    <li>Uninstall all PRS500 related drivers. You will find them in two locations:
+    <ul>
+    <li>Under "Libusb-Win32"</li>
+    <li>Under "Universal Serial ..." (... depends on the version of windows)</li>
+    </ul>
+    You can uninstall a driver by right clicking on it and selecting uninstall.
+    </li>
+    <li>Once the drivers have been uninstalled, find the file prs500.inf (it will be in the 
+    driver folder in the folder in which you installed %(appname)s. Right click on it and
+    select Install.</li>
+    </ol>
+    </p>
+    </blockquote>
+    '''%dict(appname=__appname__)))
+            return 'binary.html', data, None
+        
+        def osx(self, req):
+            version = self.version_from_filename()
+            file = 'calibre-%s.dmg'%(version,) 
+            data = dict(version = version, name='osx',
+                installer_name='OS X universal dmg', 
+                title='Download %s for OS X'%(__appname__),
+                compatibility='%s works on OS X Tiger and above.'%(__appname__,),
+                path=MOBILEREAD+file, app=__appname__,
+                note=Markup(\
+    '''
+    <ol>
+    <li>Before trying to use the command line tools, you must run the app at least once. This will ask you for you password and then setup the symbolic links for the command line tools.</li>
+    <li>The app cannot be run from within the dmg. You must drag it to a folder on your filesystem (The Desktop, Applications, wherever).</li> 
+    <li>In order for localization of the user interface in your language, select your language in the configuration dialog (by clicking the hammer icon next to the search bar) and select your language.</li>
+    </ol>
+    '''))
+            return 'binary.html', data, None
+        
+        def linux(self, req):
+            data = get_linux_data(version=self.version_from_filename())
+            return 'linux.html', data, None
                                 
     
-class Download(Component):
+    LINUX_INSTALLER = textwrap.dedent(r'''
-    implements(INavigationContributor, IRequestHandler, ITemplateProvider)
+    import sys, os, shutil, tarfile, subprocess, tempfile, urllib2, re, stat
     
-    request_pat = re.compile(r'\/download$|\/download_\S+')
+    MOBILEREAD='https://dev.mobileread.com/dist/kovid/calibre/'
     
-    # INavigationContributor methods
+    class TerminalController:
-    def get_active_navigation_item(self, req):
+        BOL = ''             #: Move the cursor to the beginning of the line
-        return 'download'
+        UP = ''              #: Move the cursor up one line
+        DOWN = ''            #: Move the cursor down one line
+        LEFT = ''            #: Move the cursor left one char
+        RIGHT = ''           #: Move the cursor right one char
         
-    def get_navigation_items(self, req):
+        # Deletion:
-        yield 'mainnav', 'download', Markup('<a href="/download">Get %s</a>'%(__appname__,))
+        CLEAR_SCREEN = ''    #: Clear the screen and move to home position
+        CLEAR_EOL = ''       #: Clear to the end of the line.
+        CLEAR_BOL = ''       #: Clear to the beginning of the line.
+        CLEAR_EOS = ''       #: Clear to the end of the screen
         
-    def get_templates_dirs(self):
+        # Output modes:
-        return [resource_filename(__name__, 'templates')]
+        BOLD = ''            #: Turn on bold mode
+        BLINK = ''           #: Turn on blink mode
+        DIM = ''             #: Turn on half-bright mode
+        REVERSE = ''         #: Turn on reverse-video mode
+        NORMAL = ''          #: Turn off all modes
         
-    def get_htdocs_dirs(self):
+        # Cursor display:
-        return [('dl', resource_filename(__name__, 'htdocs'))]
+        HIDE_CURSOR = ''     #: Make the cursor invisible
+        SHOW_CURSOR = ''     #: Make the cursor visible
         
-    # IRequestHandler methods
+        # Terminal size:
-    def match_request(self, req):
+        COLS = None          #: Width of the terminal (None for unknown)
-        return self.__class__.request_pat.match(req.path_info)
+        LINES = None         #: Height of the terminal (None for unknown)
         
-    def process_request(self, req):
+        # Foreground colors:
-        add_stylesheet(req, 'dl/css/download.css')
+        BLACK = BLUE = GREEN = CYAN = RED = MAGENTA = YELLOW = WHITE = ''
-        if req.path_info == '/download':
-            return self.top_level(req)
-        elif req.path_info == '/download_linux_binary_installer':
-            req.send(LINUX_INSTALLER.replace('%version', self.version_from_filename()), 'text/x-python')
-        else:
-            match = re.match(r'\/download_(\S+)', req.path_info)
-            if match:
-                os = match.group(1)
-                if os == 'windows':
-                    return self.windows(req)
-                elif os == 'osx':
-                    return self.osx(req)
-                elif os == 'linux':
-                    return self.linux(req)
-                elif 'binary' in os:
-                    return self.linux_binary(req)
-                else:
-                    return self.linux_distro(req, os)       
         
-    def linux_distro(self, req, os):
+        # Background colors:
-        version = self.version_from_filename()
+        BG_BLACK = BG_BLUE = BG_GREEN = BG_CYAN = ''
-        distro = Distribution(os)
+        BG_RED = BG_MAGENTA = BG_YELLOW = BG_WHITE = ''
-        data = dict(distro=distro,title=distro.title, version=version)
-        return 'distro.html', data, None
         
-    def top_level(self, req):
+        _STRING_CAPABILITIES = """
-        operating_systems = [
+        BOL=cr UP=cuu1 DOWN=cud1 LEFT=cub1 RIGHT=cuf1
-            OS({'name' : 'windows', 'title' : 'Windows'}),
+        CLEAR_SCREEN=clear CLEAR_EOL=el CLEAR_BOL=el1 CLEAR_EOS=ed BOLD=bold
-            OS({'name' : 'osx', 'title' : 'OS X'}),
+        BLINK=blink DIM=dim REVERSE=rev UNDERLINE=smul NORMAL=sgr0
-            OS({'name' : 'linux', 'title' : 'Linux'}),
+        HIDE_CURSOR=cinvis SHOW_CURSOR=cnorm""".split()
-        ]
+        _COLORS = """BLACK BLUE GREEN CYAN RED MAGENTA YELLOW WHITE""".split()
-        data = dict(title='Get ' + __appname__, 
+        _ANSICOLORS = "BLACK RED GREEN YELLOW BLUE MAGENTA CYAN WHITE".split()
-                    operating_systems=operating_systems, width=200,
-                    font_size='xx-large', top_level=True)
-        return 'download.html', data, None
         
-    def version_from_filename(self):
+        def __init__(self, term_stream=sys.stdout):
+            # Curses isn't available on all platforms
+            try: import curses
+            except: return
+            
+            # If the stream isn't a tty, then assume it has no capabilities.
+            if not hasattr(term_stream, 'isatty') or not term_stream.isatty(): return
+            
+            # Check the terminal type.  If we fail, then assume that the
+            # terminal has no capabilities.
+            try: curses.setupterm()
+            except: return
+            
+            # Look up numeric capabilities.
+            self.COLS = curses.tigetnum('cols')
+            self.LINES = curses.tigetnum('lines')
+            
+            # Look up string capabilities.
+            for capability in self._STRING_CAPABILITIES:
+                (attrib, cap_name) = capability.split('=')
+                setattr(self, attrib, self._tigetstr(cap_name) or '')
+            
+            # Colors
+            set_fg = self._tigetstr('setf')
+            if set_fg:
+                for i,color in zip(range(len(self._COLORS)), self._COLORS):
+                    setattr(self, color, curses.tparm(set_fg, i) or '')
+            set_fg_ansi = self._tigetstr('setaf')
+            if set_fg_ansi:
+                for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
+                    setattr(self, color, curses.tparm(set_fg_ansi, i) or '')
+            set_bg = self._tigetstr('setb')
+            if set_bg:
+                for i,color in zip(range(len(self._COLORS)), self._COLORS):
+                    setattr(self, 'BG_'+color, curses.tparm(set_bg, i) or '')
+            set_bg_ansi = self._tigetstr('setab')
+            if set_bg_ansi:
+                for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
+                    setattr(self, 'BG_'+color, curses.tparm(set_bg_ansi, i) or '')
+        
+        def _tigetstr(self, cap_name):
+            # String capabilities can include "delays" of the form "$<2>".
+            # For any modern terminal, we should be able to just ignore
+            # these, so strip them out.
+            import curses
+            cap = curses.tigetstr(cap_name) or ''
+            return re.sub(r'\$<\d+>[/*]?', '', cap)
+        
+        def render(self, template):
+            return re.sub(r'\$\$|\${\w+}', self._render_sub, template)
+        
+        def _render_sub(self, match):
+            s = match.group()
+            if s == '$$': return s
+            else: return getattr(self, s[2:-1])
+    
+    class ProgressBar:
+        BAR = '%3d%% ${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}\n'
+        HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n'
+        
+        def __init__(self, term, header):
+            self.term = term
+            if not (self.term.CLEAR_EOL and self.term.UP and self.term.BOL):
+                raise ValueError("Terminal isn't capable enough -- you "
+                "should use a simpler progress dispaly.")
+            self.width = self.term.COLS or 75
+            self.bar = term.render(self.BAR)
+            self.header = self.term.render(self.HEADER % header.center(self.width))
+            self.cleared = 1 #: true if we haven't drawn the bar yet.
+            
+        def update(self, percent, message=''):
+            if isinstance(message, unicode):
+                message = message.encode('utf-8', 'ignore')
+            if self.cleared:
+                sys.stdout.write(self.header)
+                self.cleared = 0
+            n = int((self.width-10)*percent)
+            msg = message.center(self.width)
+            sys.stdout.write(
+            self.term.BOL + self.term.UP + self.term.CLEAR_EOL +
+            (self.bar % (100*percent, '='*n, '-'*(self.width-10-n))) +
+            self.term.CLEAR_EOL + msg)
+            sys.stdout.flush()
+        
+        def clear(self):
+            if not self.cleared:
+                sys.stdout.write(self.term.BOL + self.term.CLEAR_EOL +
+                self.term.UP + self.term.CLEAR_EOL +
+                self.term.UP + self.term.CLEAR_EOL)
+                self.cleared = 1
+    
+    def download_tarball():
         try:
-            return open(DOWNLOAD_DIR+'/latest_version', 'rb').read().strip()
+            pb  = ProgressBar(TerminalController(sys.stdout), 'Downloading calibre...')
-        except:
+        except ValueError:
-            return '0.0.0'
+            print 'Downloading calibre...'
-    
+            pb = None
-    def windows(self, req):
+        local = 'calibre-test.tar.bz2'
-        version = self.version_from_filename()
+        src = open(local) if os.access(local, os.R_OK) else urllib2.urlopen(MOBILEREAD+'calibre-%version-i686.tar.bz2')
-        file = '%s-%s.exe'%(__appname__, version,)         
+        if hasattr(src, 'info'):
-        data = dict(version = version, name='windows',
+                size = int(src.info()['content-length'])
-            installer_name='Windows installer', 
-            title='Download %s for windows'%(__appname__),
-            compatibility='%s works on Windows XP and Windows Vista.'%(__appname__,),
-            path=MOBILEREAD+file, app=__appname__,
-            note=Markup(\
-'''
-<p>If you are using the <b>SONY PRS-500</b> and %(appname)s does not detect your reader, read on:</p>
-<blockquote>
-<p>
-If you are using 64-bit windows, you're out of luck. 
-</p>
-<p>
-There may be a conflict with the USB driver from SONY. In windows, you cannot install two drivers
-for one device. In order to resolve the conflict:
-<ol>
-<li>Start Device Manager by clicking Start->Run, typing devmgmt.msc and pressing enter.</li>
-<li>Uninstall all PRS500 related drivers. You will find them in two locations:
-<ul>
-<li>Under "Libusb-Win32"</li>
-<li>Under "Universal Serial ..." (... depends on the version of windows)</li>
-</ul>
-You can uninstall a driver by right clicking on it and selecting uninstall.
-</li>
-<li>Once the drivers have been uninstalled, find the file prs500.inf (it will be in the 
-driver folder in the folder in which you installed %(appname)s. Right click on it and
-select Install.</li>
-</ol>
-</p>
-</blockquote>
-'''%dict(appname=__appname__)))
-        return 'binary.html', data, None
-    
-    def linux_binary(self, req):
-        version = self.version_from_filename()
-        return 'pyinstaller.html', {'app':__appname__, 'version':version}, None
-    
-    def osx(self, req):
-        version = self.version_from_filename()
-        file = 'calibre-%s.dmg'%(version,) 
-        data = dict(version = version, name='osx',
-            installer_name='OS X universal dmg', 
-            title='Download %s for OS X'%(__appname__),
-            compatibility='%s works on OS X Tiger and above.'%(__appname__,),
-            path=MOBILEREAD+file, app=__appname__,
-            note=Markup(\
-'''
-<ol>
-<li>Before trying to use the command line tools, you must run the app at least once. This will ask you for you password and then setup the symbolic links for the command line tools.</li>
-<li>The app cannot be run from within the dmg. You must drag it to a folder on your filesystem (The Desktop, Applications, wherever).</li> 
-<li>In order for localization of the user interface in your language, select your language in the configuration dialog (by clicking the hammer icon next to the search bar) and select your language.</li>
-</ol>
-'''))
-        return 'binary.html', data, None
-    
-    def linux(self, req):
-        operating_systems = [
-            OS({'name' : 'binary', 'title': 'Binary Installer'}),
-            OS({'name' : 'gentoo', 'title': 'Gentoo'}),
-            OS({'name' : 'ubuntu', 'title': 'Ubuntu'}),
-            OS({'name' : 'fedora', 'title': 'Fedora'}),
-            OS({'name' : 'debian', 'title': 'Debian'}),
-            OS({'name' : 'generic','title': 'Install from source', 'img':'linux'}),
-                             ]
-        data = dict(title='Choose linux distribution', width=100,
-                    operating_systems=operating_systems, font_size='x-large', top_level=False)
-        return 'download.html', data, None
-                            
-
-LINUX_INSTALLER = r'''
-import sys, os, shutil, tarfile, subprocess, tempfile, urllib2, re, stat
-
-MOBILEREAD='https://dev.mobileread.com/dist/kovid/calibre/'
-
-class TerminalController:
-    BOL = ''             #: Move the cursor to the beginning of the line
-    UP = ''              #: Move the cursor up one line
-    DOWN = ''            #: Move the cursor down one line
-    LEFT = ''            #: Move the cursor left one char
-    RIGHT = ''           #: Move the cursor right one char
-    
-    # Deletion:
-    CLEAR_SCREEN = ''    #: Clear the screen and move to home position
-    CLEAR_EOL = ''       #: Clear to the end of the line.
-    CLEAR_BOL = ''       #: Clear to the beginning of the line.
-    CLEAR_EOS = ''       #: Clear to the end of the screen
-    
-    # Output modes:
-    BOLD = ''            #: Turn on bold mode
-    BLINK = ''           #: Turn on blink mode
-    DIM = ''             #: Turn on half-bright mode
-    REVERSE = ''         #: Turn on reverse-video mode
-    NORMAL = ''          #: Turn off all modes
-    
-    # Cursor display:
-    HIDE_CURSOR = ''     #: Make the cursor invisible
-    SHOW_CURSOR = ''     #: Make the cursor visible
-    
-    # Terminal size:
-    COLS = None          #: Width of the terminal (None for unknown)
-    LINES = None         #: Height of the terminal (None for unknown)
-    
-    # Foreground colors:
-    BLACK = BLUE = GREEN = CYAN = RED = MAGENTA = YELLOW = WHITE = ''
-    
-    # Background colors:
-    BG_BLACK = BG_BLUE = BG_GREEN = BG_CYAN = ''
-    BG_RED = BG_MAGENTA = BG_YELLOW = BG_WHITE = ''
-    
-    _STRING_CAPABILITIES = """
-    BOL=cr UP=cuu1 DOWN=cud1 LEFT=cub1 RIGHT=cuf1
-    CLEAR_SCREEN=clear CLEAR_EOL=el CLEAR_BOL=el1 CLEAR_EOS=ed BOLD=bold
-    BLINK=blink DIM=dim REVERSE=rev UNDERLINE=smul NORMAL=sgr0
-    HIDE_CURSOR=cinvis SHOW_CURSOR=cnorm""".split()
-    _COLORS = """BLACK BLUE GREEN CYAN RED MAGENTA YELLOW WHITE""".split()
-    _ANSICOLORS = "BLACK RED GREEN YELLOW BLUE MAGENTA CYAN WHITE".split()
-    
-    def __init__(self, term_stream=sys.stdout):
-        # Curses isn't available on all platforms
-        try: import curses
-        except: return
-        
-        # If the stream isn't a tty, then assume it has no capabilities.
-        if not hasattr(term_stream, 'isatty') or not term_stream.isatty(): return
-        
-        # Check the terminal type.  If we fail, then assume that the
-        # terminal has no capabilities.
-        try: curses.setupterm()
-        except: return
-        
-        # Look up numeric capabilities.
-        self.COLS = curses.tigetnum('cols')
-        self.LINES = curses.tigetnum('lines')
-        
-        # Look up string capabilities.
-        for capability in self._STRING_CAPABILITIES:
-            (attrib, cap_name) = capability.split('=')
-            setattr(self, attrib, self._tigetstr(cap_name) or '')
-        
-        # Colors
-        set_fg = self._tigetstr('setf')
-        if set_fg:
-            for i,color in zip(range(len(self._COLORS)), self._COLORS):
-                setattr(self, color, curses.tparm(set_fg, i) or '')
-        set_fg_ansi = self._tigetstr('setaf')
-        if set_fg_ansi:
-            for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
-                setattr(self, color, curses.tparm(set_fg_ansi, i) or '')
-        set_bg = self._tigetstr('setb')
-        if set_bg:
-            for i,color in zip(range(len(self._COLORS)), self._COLORS):
-                setattr(self, 'BG_'+color, curses.tparm(set_bg, i) or '')
-        set_bg_ansi = self._tigetstr('setab')
-        if set_bg_ansi:
-            for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS):
-                setattr(self, 'BG_'+color, curses.tparm(set_bg_ansi, i) or '')
-    
-    def _tigetstr(self, cap_name):
-        # String capabilities can include "delays" of the form "$<2>".
-        # For any modern terminal, we should be able to just ignore
-        # these, so strip them out.
-        import curses
-        cap = curses.tigetstr(cap_name) or ''
-        return re.sub(r'\$<\d+>[/*]?', '', cap)
-    
-    def render(self, template):
-        return re.sub(r'\$\$|\${\w+}', self._render_sub, template)
-    
-    def _render_sub(self, match):
-        s = match.group()
-        if s == '$$': return s
-        else: return getattr(self, s[2:-1])
-
-class ProgressBar:
-    BAR = '%3d%% ${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}\n'
-    HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n'
-    
-    def __init__(self, term, header):
-        self.term = term
-        if not (self.term.CLEAR_EOL and self.term.UP and self.term.BOL):
-            raise ValueError("Terminal isn't capable enough -- you "
-            "should use a simpler progress dispaly.")
-        self.width = self.term.COLS or 75
-        self.bar = term.render(self.BAR)
-        self.header = self.term.render(self.HEADER % header.center(self.width))
-        self.cleared = 1 #: true if we haven't drawn the bar yet.
-        
-    def update(self, percent, message=''):
-        if isinstance(message, unicode):
-            message = message.encode('utf-8', 'ignore')
-        if self.cleared:
-            sys.stdout.write(self.header)
-            self.cleared = 0
-        n = int((self.width-10)*percent)
-        msg = message.center(self.width)
-        sys.stdout.write(
-        self.term.BOL + self.term.UP + self.term.CLEAR_EOL +
-        (self.bar % (100*percent, '='*n, '-'*(self.width-10-n))) +
-        self.term.CLEAR_EOL + msg)
-        sys.stdout.flush()
-    
-    def clear(self):
-        if not self.cleared:
-            sys.stdout.write(self.term.BOL + self.term.CLEAR_EOL +
-            self.term.UP + self.term.CLEAR_EOL +
-            self.term.UP + self.term.CLEAR_EOL)
-            self.cleared = 1
-
-def download_tarball():
-    try:
-        pb  = ProgressBar(TerminalController(sys.stdout), 'Downloading calibre...')
-    except ValueError:
-        print 'Downloading calibre...'
-        pb = None
-    local = 'calibre-test.tar.bz2'
-    src = open(local) if os.access(local, os.R_OK) else urllib2.urlopen(MOBILEREAD+'calibre-%version-i686.tar.bz2')
-    if hasattr(src, 'info'):
-            size = int(src.info()['content-length'])
-    else:
-        src.seek(0, 2)
-        size = src.tell()
-        src.seek(0)
-    f = tempfile.NamedTemporaryFile()
-    while f.tell() < size:
-        f.write(src.read(4*1024))
-        percent = f.tell()/float(size)
-        if pb is not None:
-            pb.update(percent)
         else:
-            print '%d%%, '%int(percent*100),
+            src.seek(0, 2)
-    f.seek(0)
+            size = src.tell()
-    return f
+            src.seek(0)
+        f = tempfile.NamedTemporaryFile()
+        while f.tell() < size:
+            f.write(src.read(4*1024))
+            percent = f.tell()/float(size)
+            if pb is not None:
+                pb.update(percent)
+            else:
+                print '%d%%, '%int(percent*100),
+        f.seek(0)
+        return f
     
-def extract_tarball(tar, destdir):
+    def extract_tarball(tar, destdir):
-    print 'Extracting application files...'
+        print 'Extracting application files...'
-    if hasattr(tar, 'read'):
+        if hasattr(tar, 'read'):
-        try:
+            try:
-            tarfile.open(fileobj=tar, mode='r').extractall(destdir)
+                tarfile.open(fileobj=tar, mode='r').extractall(destdir)
-        except: # tarfile.py on Fedora 9 is buggy
+            except: # tarfile.py on Fedora 9 is buggy
-            subprocess.check_call(['tar', 'xjf', tar.name, '-C', destdir])
+                subprocess.check_call(['tar', 'xjf', tar.name, '-C', destdir])
-    else:
+        else:
-        tarfile.open(tar, 'r').extractall(destdir)
+            tarfile.open(tar, 'r').extractall(destdir)
     
-def main():
+    def main():
-    defdir = '/opt/calibre'
+        defdir = '/opt/calibre'
-    destdir = raw_input('Enter the installation directory for calibre (Its contents will be deleted!)[%s]: '%defdir).strip()
+        destdir = raw_input('Enter the installation directory for calibre (Its contents will be deleted!)[%s]: '%defdir).strip()
-    if not destdir:
+        if not destdir:
-        destdir = defdir
+            destdir = defdir
-    destdir = os.path.abspath(destdir)
+        destdir = os.path.abspath(destdir)
-    if os.path.exists(destdir):
+        if os.path.exists(destdir):
-        shutil.rmtree(destdir)
+            shutil.rmtree(destdir)
-    os.makedirs(destdir)
+        os.makedirs(destdir)
         
-    f = download_tarball()
+        f = download_tarball()
+        
+        print 'Extracting files to %s ...'%destdir
+        extract_tarball(f, destdir)
+        pi = os.path.join(destdir, 'calibre_postinstall')
+        subprocess.call(pi, shell=True)
+        return 0
+    ''')
     
-    print 'Extracting files to %s ...'%destdir
-    extract_tarball(f, destdir)
-    pi = os.path.join(destdir, 'calibre_postinstall')
-    subprocess.call(pi, shell=True)
-    return 0
-'''

src/calibre/trac/plugins/templates/distro.html

@@ -1,68 +0,0 @@
-<!DOCTYPE html
-    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:py="http://genshi.edgewall.org/"
-      xmlns:xi="http://www.w3.org/2001/XInclude">
-  <xi:include href="layout.html" />
-  <head>
-    <title>$title</title>
-  </head>
-
-  <body>
-    <div id="ctxtnav" class="nav"></div>
-
-    <div id="content" class="download">
-      <h1><img src="${href.chrome('/dl/images/%s_logo.png'%(distro.img,))}" valign="middle" width="60" height="80"/> $title</h1>
-      See the <a href="/wiki/Changelog">Changelog</a> for the changes in the latest version. <span py:if="not distro.is_generic"><b>Note:</b> As of 0.4.80 this install
-      will not work on 64-bit CPUs. Try the precompiled binary available <a href="/download_binary">here</a> instead.</span>
-      <div py:if="not distro.is_generic">
-        First verify that you have a sufficiently new installation of python
-        <pre class="wiki">python --version</pre> should return at least 2.5.1<br />
-        <p py:if="distro.os == 'gentoo'">
-            Make sure your python is compiled with the sqlite USE flag. 
-        </p>
-        <br />
-        Run the following commands in a terminal <span py:if="distro.as_root">as root</span>
-        <pre class="wiki">${distro.command}</pre>
-        <div py:if="distro.manual">
-            <h2>Manual steps</h2>
-            <ul>
-                ${distro.manual}
-            </ul>
-        </div>
-      </div>
-      <div py:if="distro.is_generic">
-        <ol>
-            <li>Make sure that your system has <code>python &gt;= 2.5</code></li>
-            <li>Install the various dependencies listed below: Make sure that any python packages are installed into python2.5 (e.g. setuptools, python-imaging, PyQt4, etc). You will also have to install the development versions of the packages (these packages usually have  -dev added to their names).</li>
-            <li>Run the following commands in a terminal:
-            <pre class="wiki">
-wget -O- http://calibre.kovidgoyal.net/downloads/calibre-${version}.tar.gz | tar xvz 
-cd calibre*
-python setup.py build &amp;&amp; sudo python setup.py install
-            </pre></li>
-        </ol>
-        <h2>Dependencies</h2>
-        <table border="1" cellpadding="10">
-            <tr><th style="font-weight:bold">Name</th><th style="font-weight:bold">Minimum version</th></tr>
-            <tr py:for="dep in distro.DEPENDENCIES">
-                <td>${dep[0]}</td><td>${dep[1]}</td>
-            </tr>
-            <tr>
-                <td>HAL</td><td>0.5.10</td>
-            </tr>
-        </table>
-      </div>
-      While you wait for the installation to complete, please consider donating to support the development of ${distro.app}.
-        <div>
-            <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
-                <input type="hidden" name="cmd" value="_s-xclick" />
-                <input type="hidden" name="hosted_button_id" value="3029289" />
-                <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="Donate to support calibre development" />
-                <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1" />
-          </form>
-        </div>
-    </div>
-  </body>
-</html>

src/calibre/trac/plugins/templates/linux.html

@@ -0,0 +1,155 @@
+<!DOCTYPE html
+    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml"
+      xmlns:py="http://genshi.edgewall.org/"
+      xmlns:xi="http://www.w3.org/2001/XInclude">
+  <xi:include href="layout.html" />
+  <head>
+    <title>${title}</title>
+    <style type="text/css">
+        table tr th.distro_type { 
+            font-family: monospace;
+            font-weight: bold;
+            font-size: larger;
+            vertical-align:middle;
+            text-align: center;
+            border-bottom: solid 1pt black;
+         }
+        
+        .right {
+            margin-left:2em;
+            border-left: solid 1pt black;
+            }
+        
+        table#install_info {
+            border-collapse: collapse;
+            border-width: 1pt;
+            border-style: solid;
+            table-layout: fixed;
+            width: 95%;
+            background: #F7F7F0 none repeat scroll 0 0;
+        }
+        table#dependencies {
+            border-collapse: collapse;
+            border-width: 0pt;
+            font-family:monospace;
+        }
+        .tdata {vertical-align: top;}
+    </style>
+  </head>
+
+  <body>
+    <div id="ctxtnav" class="nav"></div>
+    <div id="content" class="download">
+        <h1>${title}</h1>
+        <p>
+            The latest release of ${app} is ${version}. See the 
+            <a href="/wiki/Changelog">Changelog</a> for a list of new features.
+        </p>
+        <p>
+            ${app} is available in the software repositories of the following 
+            linux distributions:
+            <table id="install_info">
+                <col width="150" /><col width="*" />
+                <tr>
+                    <th class="left distro_type">Supported<br/>distributions</th>
+                    <th class="right distro_type">Unsupported distributions</th>
+                </tr>
+                <tr class="tdata">
+                    <td class="left" style="overflow-y:scroll">
+                        <div py:for="distro in supported" 
+                        style="text-align:center;margin-top:2ex;">
+                            <div>
+                                <img width="64px" height="64px" 
+                                    src="${distro.img}" alt="${distro.title}" />
+                            </div>
+                            ${distro.title}
+                        </div>
+                    </td>
+                    <td class="right">
+                        <div style="margin-left:2em">
+                            <h3>Binary install</h3>
+                            <p>
+                                ${app} has a binary installer that has been 
+                                tested on a number of distributions on both 
+                                32-bit and 64-bit x86 machines. To install, 
+                                copy paste the following command into a terminal 
+                                and press Enter: 
+                            </p>
+                            <pre class="wiki">
+sudo python -c "import urllib2; exec urllib2.urlopen('http://calibre.kovidgoyal.net/download_linux_binary_installer').read(); main()"
+                            </pre>
+                            <h4>Note</h4>
+                            <ul>
+                                <li>
+                                    When running the command line utilities,
+                                    they will segfault after completion. This can
+                                    be ignored.
+                                </li>
+                                <li>
+                                    You must have help2man and xdg-utils installed 
+                                    on your system before running the installer.
+                                </li>
+                                <li>
+                                    On a 64bit machine, you must have 32-bit versions
+                                    of common libraries like X11, freetype, fontconfig,
+                                    expat and their varios dependencies installed.
+                                </li>
+                            </ul>
+                            <h3>Source install</h3>
+                            <p>
+                                <ol>
+                                    <li>
+                                        Make sure your system has python &ge; ${dependencies[0][1]}
+                                    </li>
+                                    <li>
+                                        Install the various dependencies listed below
+                                    </li>
+                                    <li>
+                                        Run the following commands in a terminal:
+                                    </li>
+                                </ol>
+                                <pre class="wiki">
+wget -O- http://calibre.kovidgoyal.net/downloads/${app}-${version}.tar.gz | tar xvz 
+cd calibre*
+python setup.py build &amp;&amp; sudo python setup.py install
+                                </pre>
+                                Note that if your distribution does not have a
+                                correctly compiled libunrar.so, ${app} will not 
+                                support rar files.
+                            </p>
+                        </div>
+                    </td>
+                </tr>
+            </table>
+        </p>
+        <p>
+            While you wait for the download to complete, please consider 
+            donating to support the development of ${app}.
+        </p>
+      <div>
+        <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
+            <input type="hidden" name="cmd" value="_s-xclick" />
+            <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!" />
+            <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1" />
+            <input type="hidden" name="encrypted" value="-----BEGIN PKCS7-----MIIHbwYJKoZIhvcNAQcEoIIHYDCCB1wCAQExggEwMIIBLAIBADCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwDQYJKoZIhvcNAQEBBQAEgYBn7jneGiSLVO8rcDrBtOUXL+HftY+CiC47hTntwICio6qqpLKezIryyG8tKcjY58Rcocur/kDwljEutIafVG7XRA7BJL9eZdHAZsZdX04f4dApzkWwR9w6GQhj0kwmO2ZNE878UcgGZBve4qQKWM8bf2pMY7vJwCNoo6ozpIi3VTELMAkGBSsOAwIaBQAwgewGCSqGSIb3DQEHATAUBggqhkiG9w0DBwQIBTALt7s1gJmAgcjEAwUMRYeIdIOE/yi0Y5vrVKBFxOUCbqTx/lu3Rk4EHsODZXLHT+BDA5WSWYO3AXfv2Lmlv1kJ7jWrjUVirYoQ5M4qdIhY9DtvPioIMMRoTJmYM9JKH8n2TWcjJ1XIzIuDP4zn8/Ya9hap3RHOrj2RBj89g7iSuFRsjoA0PYZgtWAKwR7g3LLpjRachn041JO55BEd3YWUgorNQeo3WEHgowLFfTWgFFePkm8OoWA1klWkYp4S07IhX5NaRc8OegkdshpkiIHGAKCCA4cwggODMIIC7KADAgECAgEAMA0GCSqGSIb3DQEBBQUAMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbTAeFw0wNDAyMTMxMDEzMTVaFw0zNTAyMTMxMDEzMTVaMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwUdO3fxEzEtcnI7ZKZL412XvZPugoni7i7D7prCe0AtaHTc97CYgm7NsAtJyxNLixmhLV8pyIEaiHXWAh8fPKW+R017+EmXrr9EaquPmsVvTywAAE1PMNOKqo2kl4Gxiz9zZqIajOm1fZGWcGS0f5JQ2kBqNbvbg2/Za+GJ/qwUCAwEAAaOB7jCB6zAdBgNVHQ4EFgQUlp98u8ZvF71ZP1LXChvsENZklGswgbsGA1UdIwSBszCBsIAUlp98u8ZvF71ZP1LXChvsENZklGuhgZSkgZEwgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEUMBIGA1UEChMLUGF5UGFsIEluYy4xEzARBgNVBAsUCmxpdmVfY2VydHMxETAPBgNVBAMUCGxpdmVfYXBpMRwwGgYJKoZIhvcNAQkBFg1yZUBwYXlwYWwuY29tggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAgV86VpqAWuXvX6Oro4qJ1tYVIT5DgWpE692Ag422H7yRIr/9j/iKG4Thia/Oflx4TdL+IFJBAyPK9v6zZNZtBgPBynXb048hsP16l2vi0k5Q2JKiPDsEfBhGI+HnxLXEaUWAcVfCsQFvd2A1sxRr67ip5y2wwBelUecP3AjJ+YcxggGaMIIBlgIBATCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwCQYFKw4DAhoFAKBdMBgGCSqGSIb3DQEJAzELBgkqhkiG9w0BBwEwHAYJKoZIhvcNAQkFMQ8XDTA4MDQzMDE1MzkyMlowIwYJKoZIhvcNAQkEMRYEFJSI9/zWx7TUlKPY7kLjnvzB1h6sMA0GCSqGSIb3DQEBAQUABIGAikZNCmQdkWPdfmYnGqOb1f65ViaK0zjHf50azvsigWQLlhHqJ3PgB+jEJH3JU9Pm9M4wgiK23Bg2oIGuIsAfQkYO9mw/HjtDtOQHqXyZZbrM32YGtNWUD4ynakLYnaz7OnPl40aTPD4iDApgsGcj1oMdmw7KA2E9J0l2J9iJXF4=-----END PKCS7-----" />
+        </form>
+      </div>
+      <hr/>
+      <h3>Dependencies</h3>
+      ${app} has the following dependencies (the listed version is the minimum version)
+      <br/><br/>
+      <table id="dependencies">
+          <tr>
+              <th class="distro_type" style="margin-right:2em">Package</th>
+              <th class="distro_type">Version</th>
+          </tr>
+              
+          <tr class="dependency" py:for="dep in dependencies">
+              <td style="margin-right:2em">${dep[0]}</td><td>${dep[1]}</td>
+          </tr>
+      </table>    
+    </div>
+  </body>
+</html>

src/calibre/trac/plugins/templates/pyinstaller.html

@@ -1,47 +0,0 @@
-<!DOCTYPE html
-    PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
-    "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:py="http://genshi.edgewall.org/"
-      xmlns:xi="http://www.w3.org/2001/XInclude">
-  <xi:include href="layout.html" />
-  <head>
-    <title>Download $app for Linux</title>
-  </head>
-
-  <body>
-    <div id="ctxtnav" class="nav"></div>
-
-    <div id="content" class="binary">
-      <h1>Download $app for Linux</h1>
-      <p>This binary package is compatible with most recent linux distributions running on Intel 32 bit CPUs. It needs testing on 64 bit CPUs. There have been reports of its working on some 64bit machines. </p>
-      <p>
-      <img width="50" height="50" style="border:1px red solid" src="${href.chrome('/dl/images/binary_logo.png')}" /> 
-      (Version: $version <a href="/wiki/Changelog">Changelog</a>)
-      </p>
-      <p>To install, copy paste the following command into a terminal and press Enter:
-      </p>
-      <pre class="wiki">sudo python -c "import urllib2; exec urllib2.urlopen('http://calibre.kovidgoyal.net/download_linux_binary_installer').read(); main()"</pre>
-      <p>
-      While you wait for the download to complete, please consider donating to support the development
-      of ${app}.</p>
-      <div>
-        <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
-            <input type="hidden" name="cmd" value="_s-xclick" />
-            <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donateCC_LG.gif" border="0" name="submit" alt="PayPal - The safer, easier way to pay online!" />
-            <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1" />
-            <input type="hidden" name="encrypted" value="-----BEGIN PKCS7-----MIIHbwYJKoZIhvcNAQcEoIIHYDCCB1wCAQExggEwMIIBLAIBADCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwDQYJKoZIhvcNAQEBBQAEgYBn7jneGiSLVO8rcDrBtOUXL+HftY+CiC47hTntwICio6qqpLKezIryyG8tKcjY58Rcocur/kDwljEutIafVG7XRA7BJL9eZdHAZsZdX04f4dApzkWwR9w6GQhj0kwmO2ZNE878UcgGZBve4qQKWM8bf2pMY7vJwCNoo6ozpIi3VTELMAkGBSsOAwIaBQAwgewGCSqGSIb3DQEHATAUBggqhkiG9w0DBwQIBTALt7s1gJmAgcjEAwUMRYeIdIOE/yi0Y5vrVKBFxOUCbqTx/lu3Rk4EHsODZXLHT+BDA5WSWYO3AXfv2Lmlv1kJ7jWrjUVirYoQ5M4qdIhY9DtvPioIMMRoTJmYM9JKH8n2TWcjJ1XIzIuDP4zn8/Ya9hap3RHOrj2RBj89g7iSuFRsjoA0PYZgtWAKwR7g3LLpjRachn041JO55BEd3YWUgorNQeo3WEHgowLFfTWgFFePkm8OoWA1klWkYp4S07IhX5NaRc8OegkdshpkiIHGAKCCA4cwggODMIIC7KADAgECAgEAMA0GCSqGSIb3DQEBBQUAMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbTAeFw0wNDAyMTMxMDEzMTVaFw0zNTAyMTMxMDEzMTVaMIGOMQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDU1vdW50YWluIFZpZXcxFDASBgNVBAoTC1BheVBhbCBJbmMuMRMwEQYDVQQLFApsaXZlX2NlcnRzMREwDwYDVQQDFAhsaXZlX2FwaTEcMBoGCSqGSIb3DQEJARYNcmVAcGF5cGFsLmNvbTCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEAwUdO3fxEzEtcnI7ZKZL412XvZPugoni7i7D7prCe0AtaHTc97CYgm7NsAtJyxNLixmhLV8pyIEaiHXWAh8fPKW+R017+EmXrr9EaquPmsVvTywAAE1PMNOKqo2kl4Gxiz9zZqIajOm1fZGWcGS0f5JQ2kBqNbvbg2/Za+GJ/qwUCAwEAAaOB7jCB6zAdBgNVHQ4EFgQUlp98u8ZvF71ZP1LXChvsENZklGswgbsGA1UdIwSBszCBsIAUlp98u8ZvF71ZP1LXChvsENZklGuhgZSkgZEwgY4xCzAJBgNVBAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNTW91bnRhaW4gVmlldzEUMBIGA1UEChMLUGF5UGFsIEluYy4xEzARBgNVBAsUCmxpdmVfY2VydHMxETAPBgNVBAMUCGxpdmVfYXBpMRwwGgYJKoZIhvcNAQkBFg1yZUBwYXlwYWwuY29tggEAMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADgYEAgV86VpqAWuXvX6Oro4qJ1tYVIT5DgWpE692Ag422H7yRIr/9j/iKG4Thia/Oflx4TdL+IFJBAyPK9v6zZNZtBgPBynXb048hsP16l2vi0k5Q2JKiPDsEfBhGI+HnxLXEaUWAcVfCsQFvd2A1sxRr67ip5y2wwBelUecP3AjJ+YcxggGaMIIBlgIBATCBlDCBjjELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNBMRYwFAYDVQQHEw1Nb3VudGFpbiBWaWV3MRQwEgYDVQQKEwtQYXlQYWwgSW5jLjETMBEGA1UECxQKbGl2ZV9jZXJ0czERMA8GA1UEAxQIbGl2ZV9hcGkxHDAaBgkqhkiG9w0BCQEWDXJlQHBheXBhbC5jb20CAQAwCQYFKw4DAhoFAKBdMBgGCSqGSIb3DQEJAzELBgkqhkiG9w0BBwEwHAYJKoZIhvcNAQkFMQ8XDTA4MDQzMDE1MzkyMlowIwYJKoZIhvcNAQkEMRYEFJSI9/zWx7TUlKPY7kLjnvzB1h6sMA0GCSqGSIb3DQEBAQUABIGAikZNCmQdkWPdfmYnGqOb1f65ViaK0zjHf50azvsigWQLlhHqJ3PgB+jEJH3JU9Pm9M4wgiK23Bg2oIGuIsAfQkYO9mw/HjtDtOQHqXyZZbrM32YGtNWUD4ynakLYnaz7OnPl40aTPD4iDApgsGcj1oMdmw7KA2E9J0l2J9iJXF4=-----END PKCS7-----" />
-        </form>
-        
-      </div>        
-      <h2>Note</h2>
-      <div class="note">
-      <ul>
-      <li>This installer is very new and has only been tested on a couple of systems, so if you encounter
-      problems, please report them.</li>
-      <li>You shoud have help2man and xdg-utils installed on your system.</li>
-      </ul>
-      </div>
-    </div>
-  </body>
-</html>

src/calibre/utils/config.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 '''
 Manage application-wide preferences.
 '''
-import os, re, cPickle, textwrap
+import os, re, cPickle, textwrap, traceback
 from copy import deepcopy
 from functools import partial
 from optparse import OptionParser as _OptionParser
@@ -314,7 +314,12 @@ class OptionSet(object):
         if not isinstance(src, unicode):
             src = src.decode('utf-8')
         if src is not None:
-            exec src in options
+            try:
+                exec src in options
+            except:
+                print 'Failed to parse options string:'
+                print repr(src)
+                traceback.print_exc()
         opts = OptionValues()
         for pref in self.preferences:
             val = options.get(pref.name, pref.default)
@@ -539,7 +544,7 @@ def _prefs():
               help=_('Path to directory in which your library of books is stored'))
     c.add_opt('language', default=None,
               help=_('The language in which to display the user interface'))
-    c.add_opt('output_format', default='LRF', 
+    c.add_opt('output_format', default='EPUB', 
               help=_('The default output format for ebook conversions.'))
     c.add_opt('read_file_metadata', default=True,
               help=_('Read metadata from files'))

src/calibre/web/feeds/news.py

@@ -190,7 +190,7 @@ class BasicNewsRecipe(object, LoggingInterface):
     #: For the format for specifying a tag see :attr:`BasicNewsRecipe.remove_tags`.
     #: For example::
     #:
-    #:     remove_tags_before = [dict(id='content')]
+    #:     remove_tags_before = dict(id='content')
     #:
     #: will remove all
     #: tags before the first element with `id="content"`.

src/calibre/web/feeds/recipes/__init__.py

@@ -28,7 +28,9 @@ recipe_modules = ['recipe_' + r for r in (
            'la_tercera', 'el_mercurio_chile', 'la_cuarta', 'lanacion_chile', 'la_segunda',
            'jb_online', 'estadao', 'o_globo', 'vijesti', 'elmundo', 'the_oz',
            'honoluluadvertiser', 'starbulletin', 'exiled', 'indy_star', 'dna',
-           'pobjeda',
+           'pobjeda', 'chicago_breaking_news', 'glasgow_herald', 'linuxdevices',
+           'hindu', 'cincinnati_enquirer', 'physics_world', 'pressonline',
+           'la_republica', 'physics_today',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_ars_technica.py

@@ -1,50 +1,77 @@
 #!/usr/bin/env  python
-__license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
-__docformat__ = 'restructuredtext en'
 
+__license__   = 'GPL v3'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 arstechnica.com
 '''
 
 from calibre.web.feeds.news import BasicNewsRecipe
 
-class ArsTechnica(BasicNewsRecipe):
+class ArsTechnica2(BasicNewsRecipe):
-    title          = 'Ars Technica'
+    title                 = u'Ars Technica'
-    description    = 'The art of technology'
+    language              = _('English')
-    oldest_article = 7
+    __author__            = 'Darko Miletic'
-    language = _('English')
+    description           = 'The art of technology'    
-    no_stylesheets = True
+    publisher             = 'Ars Technica'
-    __author__     = 'Michael Warner'
+    category              = 'news, IT, technology'    
+    oldest_article        = 2
     max_articles_per_feed = 100
-    extra_css = """
+    no_stylesheets        = True
-body {
+    encoding              = 'utf8'
-   font: normal 19px/180% Times, serif;
+    remove_javascript     = True
-}
+    use_embedded_content  = False
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+
+    keep_only_tags = [dict(name='div', attrs={'id':['news-item-info','news-item']})]
 
-h1, h2, h3, h4 {
-   font: bold 28px/100% Verdana, Arial, Helvetica, sans-serif;
-   margin-top: 19px
-}
-"""
     remove_tags = [
-                   dict(id="Masthead"),
+                     dict(name=['object','link','embed'])
-                   dict(id="Banner"),
+                    ,dict(name='div', attrs={'class':'related-stories'})
-                   dict(id="Nav"),
+                  ]
-                   dict(name='div', attrs={'class':'ContentHeader'}), 
+
-                   dict(name='img'), 
+    
-                   dict(name='div', attrs={'class':'Inset RelatedStories'}), 
+    feeds = [
-                   dict(name='div', attrs={'class':'Tags'}), 
+              (u'Infinite Loop (Apple content)'        , u'http://feeds.arstechnica.com/arstechnica/apple/'      )
-                   dict(name='div', attrs={'class':'PostOptions flat'}),
+             ,(u'Opposable Thumbs (Gaming content)'    , u'http://feeds.arstechnica.com/arstechnica/gaming/'     )
-                   dict(name='div', attrs={'class':'ContentFooter'}),
+             ,(u'Gear and Gadgets'                     , u'http://feeds.arstechnica.com/arstechnica/gadgets/'    )
-                   dict(id="Sidebar"),
+             ,(u'Chipster (Hardware content)'          , u'http://feeds.arstechnica.com/arstechnica/hardware/'   )
-                   dict(id="LatestPosts"),
+             ,(u'Uptime (IT content)'                  , u'http://feeds.arstechnica.com/arstechnica/business/'   )
-                   dict(id="Footer")]
+             ,(u'Open Ended (Open Source content)'     , u'http://feeds.arstechnica.com/arstechnica/open-source/')
-    feeds       = [(u'News and Features', u'http://feeds.arstechnica.com/arstechnica/BAaf'),
+             ,(u'One Microsoft Way'                    , u'http://feeds.arstechnica.com/arstechnica/microsoft/'  )
-                   (u'Nobel Intent (Science)', u'http://arstechnica.com/journals/science.rssx'),
+             ,(u'Nobel Intent (Science content)'       , u'http://feeds.arstechnica.com/arstechnica/science/'    )
-                   (u'Infinite Loop (Apple)', u'http://arstechnica.com/journals/apple.rssx'),
+             ,(u'Law & Disorder (Tech policy content)' , u'http://feeds.arstechnica.com/arstechnica/tech-policy/')
-                   (u'M-Dollar (Microsoft)', u'http://arstechnica.com/journals/microsoft.rssx'),
+            ]
-                   (u'Open Ended (Linux)', u'http://arstechnica.com/journals/linux.rssx'),
+    
-                   (u'Opposable Thumbs (Games)', u'http://arstechnica.com/journals/thumbs.rssx'),
+    def append_page(self, soup, appendtag, position):
-                   (u'Kit (Hardware)', u'http://arstechnica.com/journals/hardware.rssx'),
+        pager = soup.find('div',attrs={'id':'pager'})
-                   (u'Journals', u'http://arstechnica.com/journals.rssx')]
+        if pager:
+           for atag in pager.findAll('a',href=True):
+               str = self.tag_to_string(atag)
+               if str.startswith('Next'):
+                  soup2 = self.index_to_soup(atag['href'])
+                  texttag = soup2.find('div', attrs={'class':'news-item-text'})
+                  for it in texttag.findAll(style=True):
+                      del it['style']
+                  newpos = len(texttag.contents)
+                  self.append_page(soup2,texttag,newpos)
+                  texttag.extract()
+                  pager.extract()
+                  appendtag.insert(position,texttag)
+        
+    
+    def preprocess_html(self, soup):
+        ftag = soup.find('div', attrs={'class':'news-item-byline'})
+        if ftag:
+           ftag.insert(4,'<br /><br />')
+        for item in soup.findAll(style=True):
+            del item['style']
+        self.append_page(soup, soup.body, 3)
+        return soup

src/calibre/web/feeds/recipes/recipe_b92.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 b92.net
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class B92(BasicNewsRecipe):
@@ -22,19 +21,22 @@ class B92(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     cover_url = 'http://static.b92.net/images/fp/logo.gif'
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
+    
+    html2lrf_options = [
+                          '--comment'  , description
+                        , '--category' , category
+                        , '--publisher', publisher
+                        , '--ignore-tables'
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     keep_only_tags = [ dict(name='div', attrs={'class':'sama_vest'}) ]
         
-    html2lrf_options = [
-                          '--comment', description
-                        , '--category', category
-                        , '--publisher', publisher
-                        ]
-    
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
-    
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
     feeds          = [
                         (u'Vesti', u'http://www.b92.net/info/rss/vesti.xml')
                        ,(u'Biz'  , u'http://www.b92.net/info/rss/biz.xml'  )
@@ -54,9 +56,10 @@ class B92(BasicNewsRecipe):
         return nurl
 
     def preprocess_html(self, soup):
-        soup.html['xml:lang'] = 'sr-Latn'
+        lng = 'sr-Latn-RS'
-        soup.html['lang']     = 'sr-Latn'
+        soup.html['xml:lang'] = lng
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        soup.html['lang']     = lng
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']
@@ -64,4 +67,3 @@ class B92(BasicNewsRecipe):
             del item['align']
             item.insert(0,'<br /><br />')
         return soup
-    language = _('Serbian')

src/calibre/web/feeds/recipes/recipe_blic.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 blic.rs
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Blic(BasicNewsRecipe):
@@ -21,15 +20,17 @@ class Blic(BasicNewsRecipe):
     remove_javascript     = True
     no_stylesheets        = True
     use_embedded_content  = False
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
@@ -44,10 +45,9 @@ class Blic(BasicNewsRecipe):
         return u'http://www.blic.rs/_print.php?' + rest_url
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']
         return soup
         
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_chicago_breaking_news.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+chicagobreakingnews.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class ChicagoBreakingNews(BasicNewsRecipe):
+    title                 = 'Chicago Breaking News'
+    __author__            = 'Darko Miletic'
+    description           = 'Breaking News from Chicago'    
+    oldest_article        = 1
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = True
+    publisher             = 'Chicago Breaking News'
+    category              = 'news, politics, USA, Chicago'
+    encoding              = 'utf8'
+    language              = _('English')
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+
+    feeds = [(u'Breaking news', u'http://feeds2.feedburner.com/ChicagoBreakingNews/')]
+
+    def preprocess_html(self, soup):
+        links = soup.findAll('a')
+        for item in soup.findAll('a'):
+            if item['href'].find('http://feedads.googleadservices.com') > -1:
+               item.extract()
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll(color=True):
+            del item['color']
+        for item in soup.findAll(size=True):
+            del item['size']
+        return soup

src/calibre/web/feeds/recipes/recipe_cincinnati_enquirer.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+__license__   = 'GPL v3'
+__copyright__ = '2009 Kovid Goyal <kovid at kovidgoyal.net>'
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1234144423(BasicNewsRecipe):
+    title          = u'Cincinnati Enquirer'
+    oldest_article = 7
+    language       = _('English')
+    __author__     = 'Joseph Kitzmiller'
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    remove_javascript     = True
+    encoding = 'cp1252'
+    extra_css = ' p {font-size: medium; font-weight: normal;} '
+    
+    keep_only_tags = [dict(name='div', attrs={'class':'padding'})]
+    
+    remove_tags = [
+                     dict(name=['object','link','table','embed'])
+                    ,dict(name='div',attrs={'id':'pluckcomments'})
+                    ,dict(name='div',attrs={'class':'articleflex-container'})
+                  ]
+   
+    feeds          = [(u'Cincinnati Enquirer', u'http://rss.cincinnati.com/apps/pbcs.dll/section?category=rssenq01&mime=xml')]
+
+    def preprocess_html(self, soup):
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll(face=True):
+            del item['face']
+        return soup

src/calibre/web/feeds/recipes/recipe_danas.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 danas.rs
 '''
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Danas(BasicNewsRecipe):
@@ -20,15 +19,17 @@ class Danas(BasicNewsRecipe):
     no_stylesheets        = False
     remove_javascript     = True
     use_embedded_content  = False
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
  
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
@@ -43,9 +44,8 @@ class Danas(BasicNewsRecipe):
     feeds          = [ (u'Vesti', u'http://www.danas.rs/rss/rss.asp')]
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_economist.py

@@ -60,8 +60,11 @@ class Economist(BasicNewsRecipe):
                 continue
             a = tag.find('a', href=True)
             if a is not None:
+                url=a['href'].replace('displaystory', 'PrinterFriendly') 
+                if url.startswith('/'):
+                    url = 'http://www.economist.com' + url
                 article = dict(title=text, 
-                    url='http://www.economist.com'+a['href'].replace('displaystory', 'PrinterFriendly'), 
+                    url = url,
                     description='', content='', date='')
                 feeds[key].append(article)
                 

src/calibre/web/feeds/recipes/recipe_elargentino.py

@@ -5,7 +5,6 @@ __copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 elargentino.com
 '''
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class ElArgentino(BasicNewsRecipe):
@@ -21,6 +20,7 @@ class ElArgentino(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'utf8'
     cover_url             = 'http://www.elargentino.com/TemplateWeb/MediosFooter/tapa_elargentino.png'
+    language              = _('Spanish')
 
     html2lrf_options = [
                           '--comment', description
@@ -59,5 +59,3 @@ class ElArgentino(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']        
         return soup
-    
-    language = _('Spanish')

src/calibre/web/feeds/recipes/recipe_glasgow_herald.py

@@ -0,0 +1,34 @@
+import re
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class GlasgowHerald(BasicNewsRecipe):
+    title          = u'Glasgow Herald'
+    oldest_article = 1
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    language       = _('English')
+    __author__     = 'McCande' 
+
+    preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in
+    [
+                (r'<center><h3>', lambda match : '<h3>'),
+                (r'Click here to comment on this story...', lambda match : ''),
+                (r'<h3>Related links</h3>.*?</head>', lambda match : '</head>'),
+        ]
+        ]
+
+
+
+
+    feeds          = [
+                        (u'News', u'http://www.theherald.co.uk/news/news/rss.xml'),
+                        (u'Politics', u'http://www.theherald.co.uk/politics/news/rss.xml'),
+                        (u'Features', u'http://www.theherald.co.uk/features/features/rss.xml'),
+                        (u'Business', u'http://www.theherald.co.uk/business/news/rss.xml')]
+
+    def print_version(self, url):
+        (beginning,end)=url.split(".var.")
+        num=end[0:7]
+        main="http://www.theherald.co.uk/misc/print.php?artid="+num
+        return main

src/calibre/web/feeds/recipes/recipe_granma.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 granma.cubaweb.cu
 '''
@@ -21,19 +21,22 @@ class Granma(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'cp1252'
     cover_url             = 'http://www.granma.cubaweb.cu/imagenes/granweb229d.jpg'
+    language              = _('Spanish')
     remove_javascript     = True
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
                         , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     keep_only_tags = [dict(name='table', attrs={'height':'466'})]
     
+    remove_tags = [dict(name=['embed','link','object'])]
+
     feeds = [(u'Noticias', u'http://www.granma.cubaweb.cu/noticias.xml' )]
 
     
@@ -49,4 +52,3 @@ class Granma(BasicNewsRecipe):
             del item['style']
         return soup
     
-    language = _('Spanish')

src/calibre/web/feeds/recipes/recipe_harpers.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 harpers.org
 '''
@@ -10,20 +10,31 @@ from calibre.web.feeds.news import BasicNewsRecipe
 class Harpers(BasicNewsRecipe):
     title                 = u"Harper's Magazine"
     __author__            = u'Darko Miletic'
-    language = _('English')
+    language              = _('English')
     description           = u"Harper's Magazine: Founded June 1850."
+    publisher             = "Harper's Magazine "
+    category              = 'news, politics, USA'
     oldest_article        = 30
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
-    timefmt               = ' [%A, %d %B, %Y]' 
+    remove_javascript     = True
+
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' 
+    
     
     keep_only_tags = [ dict(name='div', attrs={'id':'cached'}) ]
     remove_tags = [
                      dict(name='table', attrs={'class':'rcnt'})
                     ,dict(name='table', attrs={'class':'rcnt topline'})
+                    ,dict(name=['link','object','embed'])
                   ]
 
-    feeds       = [
+    feeds       = [(u"Harper's Magazine", u'http://www.harpers.org/rss/frontpage-rss20.xml')]
-                   (u"Harper's Magazine", u'http://www.harpers.org/rss/frontpage-rss20.xml')
+
-                   ]

src/calibre/web/feeds/recipes/recipe_harpers_full.py

@@ -23,7 +23,8 @@ class Harpers_full(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     simultaneous_downloads = 1
-    delay = 1
+    delay                  = 1
+    language               = _('English')
     needs_subscription = True
     INDEX = strftime('http://www.harpers.org/archive/%Y/%m')
     LOGIN = 'http://www.harpers.org'
@@ -36,7 +37,7 @@ class Harpers_full(BasicNewsRecipe):
                         , '--publisher', publisher
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' 
 
     keep_only_tags = [ dict(name='div', attrs={'id':'cached'}) ]
     remove_tags = [
@@ -72,9 +73,3 @@ class Harpers_full(BasicNewsRecipe):
                                 })
         return [(soup.head.title.string, articles)]
         
-    def preprocess_html(self, soup):
-        for item in soup.findAll(style=True):
-            del item['style']
-        return soup
-        
-    language = _('English')

src/calibre/web/feeds/recipes/recipe_hindu.py

@@ -0,0 +1,47 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TheHindu(BasicNewsRecipe):
+    title                 = u'The Hindu'
+    language              = _('English')
+    oldest_article        = 7
+    __author__            = _('Kovid Goyal')
+    max_articles_per_feed = 100
+    
+    remove_tags_before = {'name':'font', 'class':'storyhead'}
+    preprocess_regexps = [
+                (re.compile(r'<!-- story ends -->.*', re.DOTALL), 
+                 lambda match: '</body></html>'),                                                    
+                          ]
+    
+    feeds          = [
+      (u'Main - Font Page', u'http://www.hindu.com/rss/01hdline.xml'), 
+      (u'Main - National', u'http://www.hindu.com/rss/02hdline.xml'), 
+      (u'Main - International', u'http://www.hindu.com/rss/03hdline.xml'), 
+      (u'Main - Opinion', u'http://www.hindu.com/rss/05hdline.xml'), 
+      (u'Main - Business', u'http://www.hindu.com/rss/06hdline.xml'), 
+      (u'Main - Sport', u'http://www.hindu.com/rss/07hdline.xml'), 
+      (u'Main - Weather / Religion / Crossword / Cartoon', 
+       u'http://www.hindu.com/rss/10hdline.xml'), 
+      (u'Main - Engagements', u'http://www.hindu.com/rss/26hdline.xml'), 
+      (u'Supplement - Literary Review', 
+       u'http://www.hindu.com/rss/lrhdline.xml'), 
+      (u'Supplement - Sunday Magazine', 
+       u'http://www.hindu.com/rss/maghdline.xml'), 
+      (u'Supplement - Open Page', u'http://www.hindu.com/rss/ophdline.xml'), 
+      (u'Supplement - Business Review', 
+       u'http://www.hindu.com/rss/bizhdline.xml'), 
+      (u'Supplement - Book Review', 
+       u'http://www.hindu.com/rss/brhdline.xml'), 
+      (u'Supplement - Science & Technology', 
+       u'http://www.hindu.com/rss/setahdline.xml')
+      ]
+    
+    def postprocess_html(self, soup, first_fetch):
+        for t in soup.findAll(['table', 'tr', 'td']):
+            t.name = 'div'
+        return soup

src/calibre/web/feeds/recipes/recipe_infobae.py

@@ -14,21 +14,28 @@ class Infobae(BasicNewsRecipe):
     description           = 'Informacion Libre las 24 horas'
     publisher             = 'Infobae.com'
     category              = 'news, politics, Argentina'     
-    oldest_article        = 2
+    oldest_article        = 1
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
+    language              = _('Spanish')
     encoding              = 'iso-8859-1'
     cover_url             = 'http://www.infobae.com/imgs/header/header.gif'
     remove_javascript     = True
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
+
+    remove_tags = [
+                     dict(name=['embed','link','object'])
+                    ,dict(name='a', attrs={'onclick':'javascript:window.print()'})
+                  ]
     
     feeds = [ 
               (u'Noticias'  , u'http://www.infobae.com/adjuntos/html/RSS/hoy.xml'       )
@@ -48,5 +55,3 @@ class Infobae(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-
-    language = _('Spanish')

src/calibre/web/feeds/recipes/recipe_jutarnji.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 jutarnji.hr
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Jutarnji(BasicNewsRecipe):
@@ -16,32 +15,32 @@ class Jutarnji(BasicNewsRecipe):
     description           = u'Hrvatski portal'
     publisher             = 'Jutarnji.hr'
     category              = 'news, politics, Croatia'    
-    oldest_article        = 2
+    oldest_article        = 1
     max_articles_per_feed = 100
-    simultaneous_downloads = 1
+    simultaneous_downloads = 2
-    delay = 1    
+    delay                 = 1
     language              = _('Croatian')
     no_stylesheets        = True
     use_embedded_content  = False
     remove_javascript     = True
     encoding              = 'cp1250'
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
     remove_tags = [ 
-                    dict(name='embed')
+                    dict(name=['embed','hr','link','object'])
                    ,dict(name='a', attrs={'class':'a11'})
-                   ,dict(name='hr')
                   ]
     
     feeds = [
@@ -60,9 +59,7 @@ class Jutarnji(BasicNewsRecipe):
         return 'http://www.jutarnji.hr/ispis_clanka.jl?artid=' + rrest
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8">\n<meta http-equiv="Content-Language" content="hr-HR"/>'
-        soup.head.insert(0,mtag)
-        mtag = '<meta http-equiv="Content-Language" content="hr"/>'
         soup.head.insert(0,mtag)
         for item in soup.findAll(style=True):
             del item['style']        

src/calibre/web/feeds/recipes/recipe_juventudrebelde.py

@@ -1,12 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 juventudrebelde.cu
 '''
-from calibre import strftime
 
+from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Juventudrebelde(BasicNewsRecipe):
@@ -20,17 +20,18 @@ class Juventudrebelde(BasicNewsRecipe):
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'cp1252'
+    language              = _('Spanish')
     cover_url             = strftime('http://www.juventudrebelde.cu/UserFiles/File/impreso/iportada-%Y-%m-%d.jpg')
     remove_javascript     = True
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
                         , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     keep_only_tags = [dict(name='div', attrs={'id':'noticia'})]
 
@@ -51,4 +52,3 @@ class Juventudrebelde(BasicNewsRecipe):
             del item['style']
         return soup
              
-    language = _('Spanish')

src/calibre/web/feeds/recipes/recipe_la_republica.py

@@ -0,0 +1,28 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LaRepublica(BasicNewsRecipe):
+    title          = u'la Repubblica'
+    oldest_article = 1
+    language = _('Italian')
+    author = 'Darko Miletic'
+    max_articles_per_feed = 100
+    remove_javascript = True
+    no_stylesheets = True
+    
+    keep_only_tags     = [dict(name='div', attrs={'class':'articolo'})]
+
+
+    remove_tags        = [
+                            dict(name=['object','link'])
+                           ,dict(name='span',attrs={'class':'linkindice'})
+                           ,dict(name='div',attrs={'class':'bottom-mobile'})
+                           ,dict(name='div',attrs={'id':['rssdiv','blocco']})
+                         ]
+    
+    feeds          = [
+                       (u'Repubblica homepage', u'http://www.repubblica.it/rss/homepage/rss2.0.xml'),
+                       (u'Repubblica Scienze', u'http://www.repubblica.it/rss/scienze/rss2.0.xml'),
+                       (u'Repubblica Tecnologia', u'http://www.repubblica.it/rss/tecnologia/rss2.0.xml'),
+                       (u'Repubblica Esteri', u'http://www.repubblica.it/rss/esteri/rss2.0.xml')
+                     ]
+

src/calibre/web/feeds/recipes/recipe_linuxdevices.py

@@ -0,0 +1,80 @@
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
+'''
+Fetch Linuxdevices.
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+
+class Sueddeutsche(BasicNewsRecipe):
+    
+    title = u'Linuxdevices'
+    description = 'News about Linux driven Hardware'
+    __author__ = 'Oliver Niesner'
+    use_embedded_content   = False
+    timefmt = ' [%a, %d %b %Y]'
+    language = _('English')
+    max_articles_per_feed = 50
+    no_stylesheets = True
+    encoding = 'latin1'
+
+    remove_tags_after = [dict(id='nointelliTXT')]
+    filter_regexps = [r'ad\.doubleclick\.net']
+
+
+    remove_tags = [dict(name='div', attrs={'class':'bannerSuperBanner'}),
+                   dict(name='div', attrs={'class':'bannerSky'}),
+                   dict(name='div', attrs={'class':'footerLinks'}),
+                   dict(name='div', attrs={'class':'seitenanfang'}),
+                   dict(name='td', attrs={'class':'mar5'}),
+                   dict(name='td', attrs={'class':'mar5'}),
+                   dict(name='table', attrs={'class':'pageAktiv'}),
+                   dict(name='table', attrs={'class':'xartable'}),
+                   dict(name='table', attrs={'class':'wpnavi'}),
+                   dict(name='table', attrs={'class':'bgcontent absatz'}),
+                   dict(name='table', attrs={'class':'footer'}),
+                   dict(name='table', attrs={'class':'artikelBox'}),
+                   dict(name='table', attrs={'class':'kommentare'}),
+                   dict(name='table', attrs={'class':'pageBoxBot'}),
+                   #dict(name='table', attrs={'with':'100%'}),
+                   dict(name='td', attrs={'nowrap':'nowrap'}),
+                   dict(name='td', attrs={'valign':'middle'}),
+                   dict(name='td', attrs={'align':'left'}),
+                   dict(name='td', attrs={'align':'center'}),
+                   dict(name='td', attrs={'height':'5'}),
+                   dict(name='div', attrs={'class':'artikelBox navigatorBox'}),
+                   dict(name='div', attrs={'class':'similar-article-box'}),
+                   dict(name='div', attrs={'class':'videoBigHack'}),
+                   dict(name='td', attrs={'class':'artikelDruckenRight'}),
+                   dict(name='td', attrs={'class':'width="200"'}),
+                   dict(name='a', attrs={'href':'/news'}),
+                   dict(name='a', attrs={'href':'/'}),
+                   dict(name='a', attrs={'href':'/articles'}),
+                   dict(name='a', attrs={'href':'/cgi-bin/survey/survey.cgi'}),
+                   dict(name='a', attrs={'href':'/cgi-bin/board/UltraBoard.pl'}),
+                   dict(name='iframe'),
+                   dict(name='form'),
+                   #dict(name='tr', attrs={'td':'Click here to learn'}),
+                   dict(name='span', attrs={'class':'hidePrint'}),
+                   dict(id='headerLBox'),
+                   dict(id='nointelliTXT'),
+                   dict(id='rechteSpalte'),
+                   dict(id='newsticker-list-small'),
+                   dict(id='ntop5'),
+                   dict(id='ntop5send'),
+                   dict(id='ntop5commented'),
+                   dict(id='nnav-bgheader'),
+                   dict(id='nnav-headerteaser'),
+                   dict(id='nnav-head'),
+                   dict(id='nnav-top'),
+                   dict(id='nnav-logodiv'),
+                   dict(id='nnav-logo'),
+                   dict(id='nnav-oly'),
+                   dict(id='readcomment')]
+    
+
+
+    feeds =  [ (u'Linuxdevices', u'http://www.linuxdevices.com/backend/headlines.rss') ] 
+

src/calibre/web/feeds/recipes/recipe_nin.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 nin.co.yu
 '''
 
 import re, urllib
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Nin(BasicNewsRecipe):    
@@ -27,15 +26,17 @@ class Nin(BasicNewsRecipe):
     LOGIN                  = PREFIX + '/?logout=true'
     remove_javascript      = True
     use_embedded_content   = False
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
                           
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
@@ -69,5 +70,3 @@ class Nin(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_novosti.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 novosti.rs
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Novosti(BasicNewsRecipe):
@@ -22,15 +21,17 @@ class Novosti(BasicNewsRecipe):
     use_embedded_content  = False
     encoding              = 'utf8'
     remove_javascript     = True
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options  = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
@@ -40,10 +41,8 @@ class Novosti(BasicNewsRecipe):
     feeds              = [(u'Vesti', u'http://www.novosti.rs/php/vesti/rss.php')]
 
     def preprocess_html(self, soup):
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn"/>'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)    
         for item in soup.findAll(style=True):
             del item['style']        
         return soup
-
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_nspm.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 nspm.rs
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Nspm(BasicNewsRecipe):
@@ -16,26 +15,30 @@ class Nspm(BasicNewsRecipe):
     description           = 'Casopis za politicku teoriju i drustvena istrazivanja'    
     publisher             = 'NSPM'
     category              = 'news, politics, Serbia'    
-    oldest_article        = 7
+    oldest_article        = 2
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     INDEX                 = 'http://www.nspm.rs/?alphabet=l'
     encoding              = 'utf8'
     remove_javascript     = True
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
                         , '--ignore-tables'
                         ]
     
-    html2epub_options  = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
 
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
-    remove_tags        = [dict(name='a')]
+    remove_tags        = [
+                            dict(name=['a','img','link','object','embed'])
+                           ,dict(name='td', attrs={'class':'buttonheading'})
+                         ]
     
     def get_browser(self):
         br = BasicNewsRecipe.get_browser()
@@ -48,13 +51,12 @@ class Nspm(BasicNewsRecipe):
         return url.replace('.html','/stampa.html')
 
     def preprocess_html(self, soup):
-        soup.html['xml:lang'] = 'sr-Latn-RS'
+        lng = 'sr-Latn-RS'
-        soup.html['lang']     = 'sr-Latn-RS'
+        soup.html['xml:lang'] = lng
+        soup.html['lang']     = lng
         ftag = soup.find('meta',attrs={'http-equiv':'Content-Language'})
         if ftag:
-           ftag['content'] = 'sr-Latn-RS'
+           ftag['content'] = lng
         for item in soup.findAll(style=True):
             del item['style']     
         return soup
-
-    language = _('Serbian')

src/calibre/web/feeds/recipes/recipe_pescanik.py

@@ -1,13 +1,12 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+__copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 pescanik.net
 '''
 
 import re
-
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Pescanik(BasicNewsRecipe):
@@ -16,30 +15,32 @@ class Pescanik(BasicNewsRecipe):
     description           = 'Pescanik'
     publisher             = 'Pescanik'
     category              = 'news, politics, Serbia'    
-    oldest_article        = 7
+    oldest_article        = 5
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
     remove_javascript     = True
     encoding              = 'utf8'
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "monospace1";src:url(res:///opt/sony/ebook/FONT/tt0419m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{text-align: left; font-family: serif1, serif} .article_date{font-family: monospace1, monospace} .article_description{font-family: sans1, sans-serif} .navbar{font-family: monospace1, monospace}'
+    cover_url             = "http://pescanik.net/templates/ja_teline/images/logo.png"
+    language              = _('Serbian')
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
     
     html2lrf_options = [
-                          '--comment', description
+                          '--comment'  , description
-                        , '--category', category
+                        , '--category' , category
                         , '--publisher', publisher
+                        , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
     
-    cover_url = "http://pescanik.net/templates/ja_teline/images/logo.png"
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
     remove_tags = [
                      dict(name='td'  , attrs={'class':'buttonheading'})
                     ,dict(name='span', attrs={'class':'article_seperator'})
-                    ,dict(name=['object','link'])
+                    ,dict(name=['object','link','img','h4','ul'])
                   ]
 
     feeds       = [(u'Pescanik Online', u'http://pescanik.net/index.php?option=com_rd_rss&id=12')]
@@ -54,5 +55,3 @@ class Pescanik(BasicNewsRecipe):
         for item in soup.findAll(style=True):
             del item['style']
         return soup
-
-    language              = _('Serbian')

src/calibre/web/feeds/recipes/recipe_physics_today.py

@@ -0,0 +1,39 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre import strftime
+
+class Physicstoday(BasicNewsRecipe):
+    title          = u'Physicstoday'
+    __author__  = 'Hypernova'
+    description           = u'Physics Today magazine'
+    publisher             = 'American Institute of Physics'
+    category              = 'Physics'
+    language              = _('English')
+    cover_url = strftime('http://ptonline.aip.org/journals/doc/PHTOAD-home/jrnls/images/medcover%m_%Y.jpg')
+    oldest_article = 30
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    needs_subscription = True
+    remove_javascript     = True
+    remove_tags_before = dict(name='h1')
+    remove_tags =  [dict(name='div', attrs={'class':'highslide-footer'})]
+    remove_tags =  [dict(name='div', attrs={'class':'highslide-header'})]
+    #remove_tags =  [dict(name='a', attrs={'class':'highslide'})]
+    preprocess_regexps = [
+   #(re.compile(r'<!--start PHTOAD_tail.jsp -->.*</body>', re.DOTALL|re.IGNORECASE),
+   (re.compile(r'<!-- END ARTICLE and footer section -->.*</body>', re.DOTALL|re.IGNORECASE),
+    lambda match: '</body>'),
+]
+
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        if self.username is not None and self.password is not None:
+            br.open('http://www.physicstoday.org/pt/sso_login.jsp')
+            br.select_form(name='login')
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+
+    feeds          = [(u'All', u'http://www.physicstoday.org/feed.xml')]

src/calibre/web/feeds/recipes/recipe_physics_world.py

@@ -0,0 +1,35 @@
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class PhysicsWorld(BasicNewsRecipe):
+    title          = u'Physicsworld'
+    description    = 'News from the world of physics'
+    __author__     = 'Hypernova'
+    language       = _('English')
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    remove_javascript     = True
+    needs_subscription = True
+    remove_tags_before = dict(name='h1')
+    remove_tags_after = [dict(name='div', attrs={'id':'shareThis'})]
+    preprocess_regexps = [
+   (re.compile(r'<div id="shareThis">.*</body>', re.DOTALL|re.IGNORECASE),
+    lambda match: '</body>'),
+]    
+    feeds          = [
+                          (u'Headlines News', u'http://feeds.feedburner.com/PhysicsWorldNews')
+                      ]
+    
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser(self)
+        if self.username is not None and self.password is not None:
+            br.open('http://physicsworld.com/cws/sign-in')
+            br.select_form(nr=1)
+            br['username'] = self.username
+            br['password'] = self.password
+            br.submit()
+        return br
+        
+        

src/calibre/web/feeds/recipes/recipe_pobjeda.py

@@ -17,9 +17,6 @@ class Pobjeda(BasicNewsRecipe):
     description           = 'News from Montenegro'
     publisher             = 'Pobjeda a.d.'
     category              = 'news, politics, Montenegro'    
-    language              = _('Serbian')
-    oldest_article        = 2
-    max_articles_per_feed = 100
     no_stylesheets        = True
     remove_javascript     = True
     encoding              = 'utf8'
@@ -30,11 +27,13 @@ class Pobjeda(BasicNewsRecipe):
     
     html2lrf_options = [
                           '--comment', description
+                        , '--base-font-size', '10'
                         , '--category', category
                         , '--publisher', publisher
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"'
+
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
 
@@ -64,8 +63,6 @@ class Pobjeda(BasicNewsRecipe):
         soup.html['lang']     = 'sr-Latn-ME'
         mtag = '<meta http-equiv="Content-Language" content="sr-Latn-ME"/>'
         soup.head.insert(0,mtag)
-        for item in soup.findAll(style=True):
-            del item['style']
         return soup
 
     def get_cover_url(self):
@@ -87,7 +84,7 @@ class Pobjeda(BasicNewsRecipe):
             for item in soup.findAll('div', attrs={'class':'vijest'}):
                 description = self.tag_to_string(item.h2)
                 atag = item.h1.find('a')
-                if atag:
+                if atag and atag.has_key('href'):
                     url         = self.INDEX + '/' + atag['href']
                     title       = self.tag_to_string(atag)
                     date        = strftime(self.timefmt)