Merge upsteam changes.

setup.py

@@ -52,7 +52,8 @@ if __name__ == '__main__':
                         resources, clean, gui, translations, update, \
                         tag_release, upload_demo, build_linux, build_windows, \
                         build_osx, upload_installers, upload_user_manual, \
-                        upload_to_pypi, stage3, stage2, stage1, upload
+                        upload_to_pypi, stage3, stage2, stage1, upload, \
+                        upload_rss
     
     entry_points['console_scripts'].append(
                             'calibre_postinstall = calibre.linux:post_install')
@@ -170,6 +171,7 @@ if __name__ == '__main__':
                       'upload_installers': upload_installers,
                       'upload_user_manual': upload_user_manual,
                       'upload_to_pypi': upload_to_pypi,
+                      'upload_rss'    : upload_rss,
                       'stage3' : stage3,
                       'stage2' : stage2,
                       'stage1' : stage1,

src/calibre/__init__.py

@@ -3,11 +3,13 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 import sys, os, re, logging, time, subprocess, atexit, mimetypes, \
-       __builtin__
+       __builtin__, warnings
 __builtin__.__dict__['dynamic_property'] = lambda(func): func(None)
 from htmlentitydefs import name2codepoint
 from math import floor
-from logging import Formatter
+
+warnings.simplefilter('ignore', DeprecationWarning)
+
 
 from PyQt4.QtCore import QUrl
 from PyQt4.QtGui  import QDesktopServices
@@ -24,11 +26,17 @@ mimetypes.add_type('text/x-sony-bbeb+xml',                '.lrs')
 mimetypes.add_type('application/xhtml+xml',               '.xhtml')
 mimetypes.add_type('image/svg+xml',                       '.svg')
 mimetypes.add_type('application/x-sony-bbeb',             '.lrf')
+mimetypes.add_type('application/x-sony-bbeb',             '.lrx')
 mimetypes.add_type('application/x-dtbncx+xml',            '.ncx')
 mimetypes.add_type('application/adobe-page-template+xml', '.xpgt')
 mimetypes.add_type('application/x-font-opentype',         '.otf')
 mimetypes.add_type('application/x-font-truetype',         '.ttf')
 mimetypes.add_type('application/oebps-package+xml',       '.opf')
+mimetypes.add_type('application/ereader',                 '.pdb')
+mimetypes.add_type('application/mobi',                    '.mobi')
+mimetypes.add_type('application/mobi',                    '.prc')
+mimetypes.add_type('application/mobi',                    '.azw')
+guess_type = mimetypes.guess_type
 import cssutils
 cssutils.log.setLevel(logging.WARN)
 
@@ -86,6 +94,8 @@ def prints(*args, **kwargs):
     for i, arg in enumerate(args):
         if isinstance(arg, unicode):
             arg = arg.encode(preferred_encoding)
+        if not isinstance(arg, str):
+            arg = str(arg)
         file.write(arg)
         if i != len(args)-1:
             file.write(sep)
@@ -318,24 +328,6 @@ def english_sort(x, y):
     '''
     return cmp(_spat.sub('', x), _spat.sub('', y))
 
-class ColoredFormatter(Formatter):
-
-    def format(self, record):
-        ln = record.__dict__['levelname']
-        col = ''
-        if ln == 'CRITICAL':
-            col = terminal_controller.YELLOW
-        elif ln == 'ERROR':
-            col = terminal_controller.RED
-        elif ln in ['WARN', 'WARNING']:
-            col = terminal_controller.BLUE
-        elif ln == 'INFO':
-            col = terminal_controller.GREEN
-        elif ln == 'DEBUG':
-            col = terminal_controller.CYAN
-        record.__dict__['levelname'] = col + record.__dict__['levelname'] + terminal_controller.NORMAL
-        return Formatter.format(self, record)
-
 def walk(dir):
     ''' A nice interface to os.walk '''
     for record in os.walk(dir):

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.143'
+__version__   = '0.5.1'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/builtins.py

@@ -132,13 +132,24 @@ class HTMLMetadataReader(MetadataReaderPlugin):
 class MOBIMetadataReader(MetadataReaderPlugin):
     
     name        = 'Read MOBI metadata'
-    file_types  = set(['mobi', 'prc', '.azw'])
+    file_types  = set(['mobi', 'prc', 'azw'])
     description = _('Read metadata from %s files')%'MOBI'
     
     def get_metadata(self, stream, ftype):
         from calibre.ebooks.mobi.reader import get_metadata
         return get_metadata(stream)
 
+
+class TOPAZMetadataReader(MetadataReaderPlugin):
+    
+    name        = 'Read Topaz metadata'
+    file_types  = set(['tpz', 'azw1'])
+    description = _('Read metadata from %s files')%'MOBI'
+    
+    def get_metadata(self, stream, ftype):
+        from calibre.ebooks.metadata.topaz import get_metadata
+        return get_metadata(stream)
+
 class ODTMetadataReader(MetadataReaderPlugin):
     
     name        = 'Read ODT metadata'
@@ -244,11 +255,12 @@ class MOBIMetadataWriter(MetadataWriterPlugin):
 
 from calibre.ebooks.epub.input import EPUBInput
 from calibre.ebooks.mobi.input import MOBIInput
-from calibre.customize.profiles import input_profiles
+from calibre.ebooks.oeb.output import OEBOutput
+from calibre.customize.profiles import input_profiles, output_profiles
 
-plugins = [HTML2ZIP, EPUBInput, MOBIInput]
+plugins = [HTML2ZIP, EPUBInput, MOBIInput, OEBOutput]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataReader')]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataWriter')]
-plugins += input_profiles
+plugins += input_profiles + output_profiles

src/calibre/customize/conversion.py

@@ -24,7 +24,7 @@ class ConversionOption(object):
         self.choices = choices
         
         if self.long_switch is None:
-            self.long_switch = '--'+self.name.replace('_', '-')
+            self.long_switch = self.name.replace('_', '-')
         
         self.validate_parameters()
         
@@ -37,19 +37,24 @@ class ConversionOption(object):
         if not self.help:
             raise ValueError('You must set the help text')
         
+    def __hash__(self):
+        return hash(self.name)
+    
+    def __eq__(self, other):
+        return hash(self) == hash(other)
         
 class OptionRecommendation(object):
     LOW  = 1
     MED  = 2
     HIGH = 3
     
-    def __init__(self, recommeded_value, level=LOW, **kwargs):
+    def __init__(self, recommended_value=None, level=LOW, **kwargs):
         '''
         An option recommendation. That is, an option as well as its recommended 
         value and the level of the recommendation.
         '''
         self.level = level
-        self.recommended_value = recommeded_value
+        self.recommended_value = recommended_value
         self.option = kwargs.pop('option', None)
         if self.option is None:
             self.option = ConversionOption(**kwargs)
@@ -59,10 +64,12 @@ class OptionRecommendation(object):
     def validate_parameters(self):
         if self.option.choices and self.recommended_value not in \
                                                     self.option.choices:
-            raise ValueError('Recommended value not in choices')
+            raise ValueError('OpRec: %s: Recommended value not in choices'%
+                             self.option.name)
         if not (isinstance(self.recommended_value, (int, float, str, unicode))\
-            or self.default is None):
-            raise ValueError(unicode(self.default) + 
+            or self.recommended_value is None):
+            raise ValueError('OpRec: %s:'%self.option.name + 
+                             repr(self.recommended_value) + 
                              ' is not a string or a number')
          
 
@@ -110,7 +117,11 @@ class InputFormatPlugin(Plugin):
     #: instance of :class:`OptionRecommendation`.  
     options = set([])
     
-    def convert(self, stream, options, file_ext, parse_cache, log):
+    #: A set of 3-tuples of the form 
+    #: (option_name, recommended_value, recommendation_level)
+    recommendations = set([])
+    
+    def convert(self, stream, options, file_ext, parse_cache, log, accelerators):
         '''
         This method must be implemented in sub-classes. It must return
         the path to the created OPF file. All output should be contained in 
@@ -146,10 +157,16 @@ class InputFormatPlugin(Plugin):
         
         :param log: A :class:`calibre.utils.logging.Log` object. All output 
                     should use this object.
+                    
+        :param accelarators: A dictionary of various information that the input
+                             plugin can get easily that would speed up the
+                             subsequent stages of the conversion.
+                             
         '''
         raise NotImplementedError
     
-    def __call__(self, stream, options, file_ext, parse_cache, log, output_dir):
+    def __call__(self, stream, options, file_ext, parse_cache, log, 
+                 accelerators, output_dir):
         log('InputFormatPlugin: %s running'%self.name, end=' ')
         if hasattr(stream, 'name'):
             log('on', stream.name)
@@ -159,7 +176,8 @@ class InputFormatPlugin(Plugin):
                 shutil.rmtree(x) if os.path.isdir(x) else os.remove(x)
                     
                     
-            ret = self.convert(stream, options, file_ext, parse_cache, log)
+            ret = self.convert(stream, options, file_ext, parse_cache, 
+                               log, accelerators)
             for key in list(parse_cache.keys()):
                 if os.path.abspath(key) != key:
                     log.warn(('InputFormatPlugin: %s returned a '
@@ -187,3 +205,37 @@ class InputFormatPlugin(Plugin):
                     
         return ret
 
+
+class OutputFormatPlugin(Plugin):
+    '''
+    OutputFormatPlugins are responsible for converting an OEB document
+    (OPF+HTML) into an output ebook.
+    
+    The OEB document can be assumed to be encoded in UTF-8.
+    The main action happens in :method:`convert`.
+    '''
+    
+    type = _('Conversion Output')
+    can_be_disabled = False
+    supported_platforms = ['windows', 'osx', 'linux']
+    
+    #: The file type (extension without leading period) that this
+    #: plugin outputs
+    file_type     = None
+    
+    #: Options shared by all Input format plugins. Do not override
+    #: in sub-classes. Use :member:`options` instead. Every option must be an
+    #: instance of :class:`OptionRecommendation`. 
+    common_options = set([])
+    
+    #: Options to customize the behavior of this plugin. Every option must be an
+    #: instance of :class:`OptionRecommendation`.  
+    options = set([])
+    
+    #: A set of 3-tuples of the form 
+    #: (option_name, recommended_value, recommendation_level)
+    recommendations = set([])
+
+    def convert(self, oeb_book, input_plugin, options, parse_cache, log):
+        raise NotImplementedError
+ 

src/calibre/customize/profiles.py

@@ -3,6 +3,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import sys, re
 from calibre.customize import Plugin
 
 class InputProfile(Plugin):
@@ -16,12 +17,43 @@ class InputProfile(Plugin):
 #       inherit from this profile and override as needed
 
     name        = 'Default Input Profile'
-    short_name  = 'default' # Used in the CLI so dont spaces etc. in it
+    short_name  = 'default' # Used in the CLI so dont use spaces etc. in it
     description = _('This profile tries to provide sane defaults and is useful '
                     'if you know nothing about the input document.')
                   
 input_profiles = [InputProfile]
     
 
+class OutputProfile(Plugin):
+    
+    author = 'Kovid Goyal'
+    supported_platforms = set(['windows', 'osx', 'linux'])
+    can_be_disabled = False
+    type = _('Output profile')
+    
+    name        = 'Default Output Profile'
+    short_name  = 'default' # Used in the CLI so dont use spaces etc. in it
+    description = _('This profile tries to provide sane defaults and is useful '
+                    'if you want to produce a document intended to be read at a '
+                    'computer or on a range of devices.')
+    
+    epub_flow_size            = sys.maxint
+    screen_size               = None
+    remove_special_chars      = False
+    remove_object_tags        = False
+    
+class SonyReader(OutputProfile):
+    
+    name        = 'Sony Reader'
+    short_name  = 'sony'
+    description = _('This profile is intended for the SONY PRS line. '
+                    'The 500/505/700 etc.')
+     
+    epub_flow_size            = 270000
+    screen_size               = (590, 765)
+    remove_special_chars      = re.compile(u'[\u200b\u00ad]')
+    remove_object_tags        = True
     
     
+    
+output_profiles = [OutputProfile, SonyReader]

src/calibre/customize/ui.py

@@ -6,8 +6,8 @@ import os, shutil, traceback, functools, sys
 
 from calibre.customize import Plugin, FileTypePlugin, MetadataReaderPlugin, \
                               MetadataWriterPlugin
-from calibre.customize.conversion import InputFormatPlugin
-from calibre.customize.profiles import InputProfile
+from calibre.customize.conversion import InputFormatPlugin, OutputFormatPlugin
+from calibre.customize.profiles import InputProfile, OutputProfile
 from calibre.customize.builtins import plugins as builtin_plugins
 from calibre.constants import __version__, iswindows, isosx
 from calibre.ebooks.metadata import MetaInformation
@@ -76,6 +76,12 @@ def input_profiles():
         if isinstance(plugin, InputProfile):
             yield plugin
 
+def output_profiles():
+    for plugin in _initialized_plugins:
+        if isinstance(plugin, OutputProfile):
+            yield plugin
+
+
 def reread_filetype_plugins():
     global _on_import
     global _on_preprocess
@@ -245,7 +251,17 @@ def input_format_plugins():
         
 def plugin_for_input_format(fmt):
     for plugin in input_format_plugins():
-        if fmt in plugin.file_types:
+        if fmt.lower() in plugin.file_types:
+            return plugin
+
+def output_format_plugins():
+    for plugin in _initialized_plugins:
+        if isinstance(plugin, OutputFormatPlugin):
+            yield plugin
+
+def plugin_for_output_format(fmt):
+    for plugin in output_format_plugins():
+        if fmt.lower() == plugin.file_type:
             return plugin
         
 

src/calibre/devices/kindle/driver.py

@@ -4,9 +4,9 @@ __copyright__ = '2009, John Schember <john at nachtimwald.com>'
 Device driver for Amazon's Kindle
 '''
 
-import os
+import os, re
 
-from calibre.devices.usbms.driver import USBMS
+from calibre.devices.usbms.driver import USBMS, metadata_from_formats
 
 class KINDLE(USBMS):
     # Ordered list of supported formats
@@ -30,6 +30,9 @@ class KINDLE(USBMS):
     EBOOK_DIR_CARD = "documents"
     SUPPORTS_SUB_DIRS = True
     
+    WIRELESS_FILE_NAME_PATTERN = re.compile(
+    r'(?P<title>[^-]+)-asin_(?P<asin>[a-zA-Z\d]{10,})-type_(?P<type>\w{4})-v_(?P<index>\d+).*')
+
     def delete_books(self, paths, end_session=True):
         for path in paths:
             if os.path.exists(path):
@@ -41,6 +44,16 @@ class KINDLE(USBMS):
                 if os.path.exists(filepath + '.mbp'):
                     os.unlink(filepath + '.mbp')
                     
+    @classmethod
+    def metadata_from_path(cls, path):
+        mi = metadata_from_formats([path])
+        if mi.title == _('Unknown') or ('-asin' in mi.title and '-type' in mi.title):
+            match = cls.WIRELESS_FILE_NAME_PATTERN.match(os.path.basename(path))
+            if match is not None:
+                mi.title = match.group('title')
+        return mi
+
+
 class KINDLE2(KINDLE):
     
     PRODUCT_ID = [0x0002]

src/calibre/devices/mime.py

@@ -1,19 +1,20 @@
-__license__   = 'GPL v3'
-__copyright__ = '2009, John Schember <john at nachtimwald.com>'
-'''
-Global Mime mapping of ebook types.
-'''
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
 
-MIME_MAP   = {
-                'azw'  : 'application/azw',
-                'epub' : 'application/epub+zip',
-                'html' : 'text/html',
-                'lrf'  : 'application/x-sony-bbeb',
-                'lrx'  : 'application/x-sony-bbeb',
-                'mobi' : 'application/mobi',
-                'pdf'  : 'application/pdf',
-                'prc'  : 'application/prc',
-                'rtf'  : 'application/rtf',
-                'txt'  : 'text/plain',
-              }
+from calibre import guess_type
 
+def _mt(path):
+    mt = guess_type(path)[0]
+    if not mt:
+        mt = 'application/octet-stream'
+    return mt
+
+def mime_type_ext(ext):
+    if not ext.startswith('.'):
+        ext = '.'+ext
+    return _mt('a'+ext)
+
+def mime_type_path(path):
+    return _mt(path)

src/calibre/devices/usbms/driver.py

@@ -15,7 +15,7 @@ from calibre.ebooks.metadata import authors_to_string
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
 from calibre.devices.errors import FreeSpaceError, PathError
-from calibre.devices.mime import MIME_MAP
+from calibre.devices.mime import mime_type_ext
 
 class File(object):
     def __init__(self, path):
@@ -226,14 +226,17 @@ class USBMS(Device):
         if not os.path.isdir(path):
             os.utime(path, None)
 
+    @classmethod
+    def metadata_from_path(cls, path):
+        return metadata_from_formats([path])
+    
     @classmethod
     def book_from_path(cls, path):
         fileext = path_to_ext(path)
-    
-        mi = metadata_from_formats([path])
-        mime = MIME_MAP[fileext] if fileext in MIME_MAP.keys() else 'Unknown'
-        
+        mi = cls.metadata_from_path(path)
+        mime = mime_type_ext(fileext)
         authors = authors_to_string(mi.authors)
         
-        return Book(path, mi.title, authors, mime)
+        book = Book(path, mi.title, authors, mime)
+        return book
 

src/calibre/ebooks/BeautifulSoup.py

@@ -1306,7 +1306,10 @@ class BeautifulStoneSoup(Tag, SGMLParser):
         if self.convertEntities:
             if ref.lower().startswith('x'): #
                 ref = int(ref[1:], 16)      # Added by Kovid to handle hex numeric entities
+            try:
                 data = unichr(int(ref))
+            except ValueError: # Bad numerical entity. Added by Kovid
+                data = u'' 
         else:
             data = '&#%s;' % ref
         self.handle_data(data)

src/calibre/ebooks/chardet/__init__.py

@@ -97,9 +97,12 @@ def xml_to_unicode(raw, verbose=False, strip_encoding_pats=False,
         if encoding is None:
             encoding = force_encoding(raw, verbose)
         try:
+            if encoding.lower().strip() == 'macintosh':
+                encoding = 'mac-roman'
             raw = raw.decode(encoding, 'replace')
         except LookupError:
-            raw = raw.decode('utf-8', 'replace')
+            encoding = 'utf-8'
+            raw = raw.decode(encoding, 'replace')
     
     if strip_encoding_pats:
         raw = strip_encoding_declarations(raw)

src/calibre/ebooks/conversion/cli.py

@@ -0,0 +1,167 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+'''
+Command line interface to conversion sub-system
+'''
+
+USAGE = '%prog ' + _('''\
+input_file output_file [options]
+
+Convert an ebook from one format to another.
+
+input_file is the input and output_file is the output. Both must be \
+specified as the first two arguments to the command.
+
+The output ebook format is guessed from the file extension of \
+output_file. output_file can also be of the special format .EXT where \
+EXT is the output file extension. In this case, the name of the output \
+file is derived the name of the input file. Note that the filenames must \
+not start with a hyphen. Finally, if output_file has no extension, then \
+it is treated as a directory and an "open ebook" (OEB) consisting of HTML \
+files is written to that directory. These files are the files that would \
+normally have been passed to the output plugin.
+
+After specifying the input \
+and output file you can customize the conversion by specifying various \
+options. the available options depend on the input and output file types. \
+To get help on them specify the input and output file and then use the -h \
+option.
+
+For full documentation of the conversion system see 
+''') + 'http://calibre.kovidgoyal.net/user_manual/conversion.html'
+
+import sys, os
+from optparse import OptionGroup, Option
+
+from calibre.utils.config import OptionParser
+from calibre.utils.logging import Log
+from calibre.constants import preferred_encoding
+from calibre.customize.conversion import OptionRecommendation
+
+def print_help(parser, log):
+    help = parser.format_help().encode(preferred_encoding, 'replace')
+    log(help)
+
+def check_command_line_options(parser, args, log):
+    if len(args) < 3 or args[1].startswith('-') or args[2].startswith('-'):
+        print_help(parser)
+        log.error('\n\nYou must specify the input AND output files')
+        raise SystemExit(1)
+    
+    input = os.path.abspath(args[1])
+    if not os.access(input, os.R_OK):
+        log.error('Cannot read from', input)
+        raise SystemExit(1)
+    
+    output = args[2]
+    if output.startswith('.'):
+        output = os.path.splitext(os.path.basename(input))[0]+output
+    output = os.path.abspath(output)
+    
+    if '.' in output:
+        if os.path.exists(output):
+            log.warn('WARNING:', output, 'exists. Deleting.')
+            os.remove(output)
+        
+    return input, output
+
+def option_recommendation_to_cli_option(add_option, rec):
+    opt = rec.option
+    switches = ['-'+opt.short_switch] if opt.short_switch else []
+    switches.append('--'+opt.long_switch)
+    attrs = dict(dest=opt.name, help=opt.help,
+                     choices=opt.choices, default=rec.recommended_value)
+    add_option(Option(*switches, **attrs))
+
+def add_input_output_options(parser, plumber):
+    input_options, output_options = \
+                                plumber.input_options, plumber.output_options
+    
+    def add_options(group, options):
+        for opt in options:
+            option_recommendation_to_cli_option(group, opt)
+            
+    if input_options:
+        title = _('INPUT OPTIONS')
+        io = OptionGroup(parser, title, _('Options to control the processing'
+                          ' of the input %s file')%plumber.input_fmt)
+        add_options(io.add_option, input_options)
+        parser.add_option_group(io)
+        
+    if output_options:
+        title = plumber.output_fmt.upper() + ' ' + _('OPTIONS')
+        oo = OptionGroup(parser, title, _('Options to control the processing'
+                          ' of the output %s file')%plumber.input_fmt)
+        add_options(oo.add_option, output_options)
+        parser.add_option_group(oo)
+
+def add_pipeline_options(parser, plumber):
+    groups = {
+              '' : ('',
+                    [
+                     'input_profile',
+                     'output_profile',
+                     ]
+                    ),
+              
+              'METADATA' : (_('Options to set metadata in the output'),
+                            plumber.metadata_option_names,
+                            ),
+              'DEBUG': (_('Options to help with debugging the conversion'),
+                        [
+                         'verbose',
+                         ]),
+                         
+                
+              }
+    
+    group_order = ['', 'METADATA', 'DEBUG']
+    
+    for group in group_order:
+        desc, options = groups[group]
+        if group:
+            group = OptionGroup(parser, group, desc)
+            parser.add_option_group(group)
+        add_option = group.add_option if group != '' else parser.add_option
+        
+        for name in options:
+            rec = plumber.get_option_by_name(name)
+            if rec.level < rec.HIGH:
+                option_recommendation_to_cli_option(add_option, rec)
+
+def option_parser():
+    return OptionParser(usage=USAGE)
+
+def main(args=sys.argv):
+    log = Log()
+    parser = option_parser()
+    if len(args) < 3:
+        print_help(parser, log)
+        return 1
+    
+    input, output = check_command_line_options(parser, args, log)
+    
+    from calibre.ebooks.conversion.plumber import Plumber
+    
+    plumber = Plumber(input, output, log)
+    add_input_output_options(parser, plumber)
+    add_pipeline_options(parser, plumber)
+    
+    opts = parser.parse_args(args)[0]
+    recommendations = [(n.dest, getattr(opts, n.dest), 
+                        OptionRecommendation.HIGH) \
+                                        for n in parser.options_iter()
+                                        if n.dest]
+    plumber.merge_ui_recommendations(recommendations)
+    
+    plumber.run()
+    
+    log(_('Output saved to'), ' ', plumber.output)
+    
+    return 0
+    
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/ebooks/conversion/plumber.py

@@ -3,11 +3,29 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import os
 
 from calibre.customize.conversion import OptionRecommendation 
-from calibre.customize.ui import input_profiles
+from calibre.customize.ui import input_profiles, output_profiles, \
+        plugin_for_input_format, plugin_for_output_format
 
-pipeline_options = [
+class OptionValues(object):
+    pass
+
+class Plumber(object):
+    
+    metadata_option_names = [
+        'title', 'authors', 'title_sort', 'author_sort', 'cover', 'comments', 
+        'publisher', 'series', 'series_index', 'rating', 'isbn', 
+        'tags', 'book_producer', 'language'
+        ]
+    
+    def __init__(self, input, output, log):
+        self.input = input
+        self.output = output
+        self.log = log
+        
+        self.pipeline_options = [
 
 OptionRecommendation(name='verbose', 
             recommended_value=0, level=OptionRecommendation.LOW,
@@ -16,7 +34,6 @@ OptionRecommendation(name='verbose',
                    'verbosity.')
         ),
 
-
 OptionRecommendation(name='input_profile',
             recommended_value='default', level=OptionRecommendation.LOW,
             choices=[x.short_name for x in input_profiles()],
@@ -27,4 +44,193 @@ OptionRecommendation(name='input_profile',
                    'pixels).')
         ),
 
+OptionRecommendation(name='output_profile',
+            recommended_value='default', level=OptionRecommendation.LOW,
+            choices=[x.short_name for x in output_profiles()],
+            help=_('Specify the output profile. The output profile '
+                   'tells the conversion system how to optimize the '
+                   'created document for the specified device. In some cases, '
+                   'an output profile is required to produce documents that '
+                   'will work on a device. For example EPUB on the SONY reader.'
+                   )
+        ),
+        
+OptionRecommendation(name='read_metadata_from_opf', 
+            recommended_value=None, level=OptionRecommendation.LOW,
+            short_switch='m', 
+            help=_('Read metadata from the specified OPF file. Metadata read '
+                   'from this file will override any metadata in the source ' 
+                   'file.')
+        ),
+        
+OptionRecommendation(name='title',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the title.')),
+
+OptionRecommendation(name='authors',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the authors. Multiple authors should be separated ')),
+
+OptionRecommendation(name='title_sort',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('The version of the title to be used for sorting. ')),
+
+OptionRecommendation(name='author_sort',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('String to be used when sorting by author. ')),
+
+OptionRecommendation(name='cover',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the cover to the specified file.')),
+
+OptionRecommendation(name='comments',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the ebook description.')),
+
+OptionRecommendation(name='publisher',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the ebook publisher.')),
+
+OptionRecommendation(name='series',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the series this ebook belongs to.')),
+
+OptionRecommendation(name='series_index',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the index of the book in this series.')),
+
+OptionRecommendation(name='rating',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the rating. Should be a number between 1 and 5.')),
+
+OptionRecommendation(name='isbn',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the ISBN of the book.')),
+
+OptionRecommendation(name='tags',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the tags for the book. Should be a comma separated list.')),
+
+OptionRecommendation(name='book_producer',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the book producer.')),
+
+OptionRecommendation(name='language',
+    recommended_value=None, level=OptionRecommendation.LOW,
+    help=_('Set the language.')),
 ]
+
+        
+        input_fmt = os.path.splitext(input)[1]
+        if not input_fmt:
+            raise ValueError('Input file must have an extension')
+        input_fmt = input_fmt[1:].lower()
+        
+        output_fmt = os.path.splitext(output)[1]
+        if not output_fmt:
+            output_fmt = '.oeb'
+        output_fmt = output_fmt[1:].lower()
+        
+        self.input_plugin = plugin_for_input_format(input_fmt)
+        self.output_plugin = plugin_for_output_format(output_fmt)
+        
+        if self.input_plugin is None:
+            raise ValueError('No plugin to handle input format: '+input_fmt)
+        
+        if self.output_plugin is None:
+            raise ValueError('No plugin to handle output format: '+output_fmt)
+        
+        self.input_fmt = input_fmt
+        self.output_fmt = output_fmt
+        
+        self.input_options  = self.input_plugin.options.union(
+                                    self.input_plugin.common_options)
+        self.output_options = self.output_plugin.options.union(
+                                    self.output_plugin.common_options)  
+    
+        self.merge_plugin_recommendations()
+
+    def get_option_by_name(self, name):
+        for group in (self.input_options, self.pipeline_options, 
+                      self.output_options):
+            for rec in group:
+                if rec.option == name:
+                    return rec
+        
+    def merge_plugin_recommendations(self):
+        for source in (self.input_plugin, self.output_plugin):
+            for name, val, level in source.recommendations:
+                rec = self.get_option_by_name(name)
+                if rec is not None and rec.level <= level:
+                    rec.recommended_value = val
+    
+    def merge_ui_recommendations(self, recommendations):
+        for name, val, level in recommendations:
+            rec = self.get_option_by_name(name)
+            if rec is not None and rec.level <= level and rec.level < rec.HIGH:
+                rec.recommended_value = val
+    
+    def read_user_metadata(self):
+        from calibre.ebooks.metadata import MetaInformation, string_to_authors
+        from calibre.ebooks.metadata.opf2 import OPF
+        mi = MetaInformation(None, [])
+        if self.opts.read_metadata_from_opf is not None:
+            self.opts.read_metadata_from_opf = os.path.abspath(
+                                            self.opts.read_metadata_from_opf)
+            opf = OPF(open(self.opts.read_metadata_from_opf, 'rb'),
+                      os.path.dirname(self.opts.read_metadata_from_opf))
+            mi = MetaInformation(opf)
+        for x in self.metadata_option_names:
+            val = getattr(self.opts, x, None)
+            if val is not None:
+                if x == 'authors':
+                    val = string_to_authors(val)
+                elif x == 'tags':
+                    val = [i.strip() for i in val.split(',')]
+                elif x in ('rating', 'series_index'):
+                    val = float(val)
+                setattr(mi, x, val)
+        if mi.cover:
+            mi.cover_data = ('', open(mi.cover, 'rb').read())
+            mi.cover = None
+        self.user_metadata = mi
+            
+    
+    def setup_options(self):
+        self.opts = OptionValues()
+        for group in (self.input_options, self.pipeline_options, 
+                  self.output_options):
+            for rec in group:
+                setattr(self.opts, rec.option.name, rec.recommended_value)
+                
+        for x in input_profiles():
+            if x.short_name == self.opts.input_profile:
+                self.opts.input_profile = x
+                break
+            
+        for x in output_profiles():
+            if x.short_name == self.opts.output_profile:
+                self.opts.output_profile = x
+                break
+            
+        self.read_user_metadata()
+    
+    def run(self):
+        self.setup_options()
+        from calibre.customize.ui import run_plugins_on_preprocess
+        self.input = run_plugins_on_preprocess(self.input)
+        
+        from calibre.ebooks.oeb.reader import OEBReader
+        from calibre.ebooks.oeb.base import OEBBook
+        parse_cache, accelerators = {}, {}
+        
+        opfpath = self.input_plugin(open(self.input, 'rb'), self.opts, 
+                                    self.input_fmt, parse_cache, self.log,
+                                    accelerators)
+        
+        self.reader = OEBReader()
+        self.oeb = OEBBook(self.log, parse_cache=parse_cache) 
+        self.reader(self.oeb, opfpath)
+        
+    
+        

src/calibre/ebooks/epub/from_any.py

@@ -12,7 +12,7 @@ from contextlib import nested
 
 from calibre import extract, walk
 from calibre.ebooks import DRMError
-from calibre.ebooks.epub import config as common_config, process_encryption
+from calibre.ebooks.epub import config as common_config
 from calibre.ebooks.epub.from_html import convert as html2epub, find_html_index
 from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks.metadata import MetaInformation

src/calibre/ebooks/epub/from_html.py

@@ -197,6 +197,9 @@ class HTMLProcessor(Processor, Rationalizer):
             if not tag.text and not tag.get('src', False):
                 tag.getparent().remove(tag)
                 
+        for tag in self.root.xpath('//form'):
+            tag.getparent().remove(tag)
+                
         if self.opts.linearize_tables:
             for tag in self.root.xpath('//table | //tr | //th | //td'):
                 tag.tag = 'div'

src/calibre/ebooks/epub/input.py

@@ -51,7 +51,8 @@ class EPUBInput(InputFormatPlugin):
             traceback.print_exc()
         return False
 
-    def convert(self, stream, options, file_ext, parse_cache, log):
+    def convert(self, stream, options, file_ext, parse_cache, log, 
+                accelerators):
         from calibre.utils.zipfile import ZipFile
         from calibre import walk
         from calibre.ebooks import DRMError

src/calibre/ebooks/epub/iterator.py

@@ -118,11 +118,17 @@ class EbookIterator(object):
         self.spine = [SpineItem(i.path) for i in self.opf.spine]
         
         cover = self.opf.cover
-        if os.path.splitext(self.pathtoebook)[1].lower() in ('.lit', '.mobi', '.prc') and cover:
+        if os.path.splitext(self.pathtoebook)[1].lower() in \
+                                    ('.lit', '.mobi', '.prc') and cover:
             cfile = os.path.join(os.path.dirname(self.spine[0]), 'calibre_ei_cover.html')
             open(cfile, 'wb').write(TITLEPAGE%cover)
             self.spine[0:0] = [SpineItem(cfile)]
             
+        if self.opf.path_to_html_toc is not None and \
+           self.opf.path_to_html_toc not in self.spine:
+            self.spine.append(SpineItem(self.opf.path_to_html_toc))
+            
+        
         sizes = [i.character_count for i in self.spine]
         self.pages = [math.ceil(i/float(self.CHARACTERS_PER_PAGE)) for i in sizes]
         for p, s in zip(self.pages, self.spine):

src/calibre/ebooks/epub/pages.py

@@ -11,7 +11,7 @@ __docformat__ = 'restructuredtext en'
 import os, re
 from itertools import count, chain
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS
-from calibre.ebooks.oeb.base import OEBBook, DirWriter
+from calibre.ebooks.oeb.base import OEBBook
 from lxml import etree, html
 from lxml.etree import XPath
 

src/calibre/ebooks/epub/split.py

@@ -15,7 +15,7 @@ from lxml.cssselect import CSSSelector
 
 from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.epub import tostring, rules
-from calibre import CurrentDir, LoggingInterface
+from calibre import CurrentDir
 
 XPath = functools.partial(_XPath, namespaces={'re':'http://exslt.org/regular-expressions'})
 content = functools.partial(os.path.join, 'content')
@@ -32,10 +32,9 @@ class SplitError(ValueError):
 
     
 
-class Splitter(LoggingInterface):
+class Splitter(object):
     
     def __init__(self, path, opts, stylesheet_map, opf):
-        LoggingInterface.__init__(self, logging.getLogger('htmlsplit'))
         self.setup_cli_handler(opts.verbose)
         self.path = path
         self.always_remove = not opts.preserve_tag_structure or \

src/calibre/ebooks/html.py

@@ -19,11 +19,10 @@ from lxml.html import HtmlElementClassLookup, HTMLParser as _HTMLParser, \
 from lxml.etree import XPath
 get_text = XPath("//text()")
 
-from calibre import LoggingInterface, unicode_path, entity_to_unicode
+from calibre import unicode_path, entity_to_unicode
 from calibre.ebooks.chardet import xml_to_unicode, ENCODING_PATS
 from calibre.utils.config import Config, StringConfig
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.meta import get_metadata
 from calibre.ebooks.metadata.opf2 import OPF, OPFCreator
 from calibre.ptempfile import PersistentTemporaryDirectory, PersistentTemporaryFile
 from calibre.utils.zipfile import ZipFile
@@ -401,7 +400,7 @@ class PreProcessor(object):
             html = rule[0].sub(rule[1], html)
         return html
     
-class Parser(PreProcessor, LoggingInterface):
+class Parser(PreProcessor):
 #    SELF_CLOSING_TAGS = 'hr|br|link|img|meta|input|area|base|basefont'
 #    SELF_CLOSING_RULES = [re.compile(p[0]%SELF_CLOSING_TAGS, re.IGNORECASE) for p in 
 #                          [
@@ -412,7 +411,6 @@ class Parser(PreProcessor, LoggingInterface):
 #                          ]
     
     def __init__(self, htmlfile, opts, tdir, resource_map, htmlfiles, name='htmlparser'):
-        LoggingInterface.__init__(self, logging.getLogger(name))
         self.setup_cli_handler(opts.verbose)
         self.htmlfile = htmlfile
         self.opts = opts
@@ -859,7 +857,7 @@ class Processor(Parser):
             except ValueError:
                 setting = ''
             face = font.attrib.pop('face', None)
-            if face is not None:
+            if face:
                 faces = []
                 for face in face.split(','):
                     face = face.strip()
@@ -1038,6 +1036,7 @@ def merge_metadata(htmlfile, opf, opts):
     if opf:
         mi = MetaInformation(opf)
     elif htmlfile:
+        from calibre.ebooks.metadata.meta import get_metadata
         try:
             mi =  get_metadata(open(htmlfile, 'rb'), 'html')
         except:

src/calibre/ebooks/lrf/comic/convert_from.py

@@ -143,6 +143,7 @@ class PageProcessor(list):
                     MagickRotateImage(wand, pw, -90)
                     
                 # 25 percent fuzzy trim?
+                if not self.opts.disable_trim:
                     MagickTrimImage(wand, 25*65535/100)
                 MagickSetImagePage(wand, 0,0,0,0)   #Clear page after trim, like a "+repage"
                 # Do the Photoshop "Auto Levels" equivalent
@@ -303,6 +304,9 @@ def config(defaults=None,output_format='lrf'):
               help=_('Maintain picture aspect ratio. Default is to fill the screen.'))
     c.add_opt('dont_sharpen', ['-s', '--disable-sharpen'], default=False,
               help=_('Disable sharpening.'))
+    c.add_opt('disable_trim', ['--disable-trim'], default=False,
+              help=_('Disable trimming of comic pages. For some comics, '
+                     'trimming might remove content as well as borders.'))
     c.add_opt('landscape', ['-l', '--landscape'], default=False,
               help=_("Don't split landscape images into two portrait images"))
     c.add_opt('wide', ['-w', '--wide-aspect'], default=False,

src/calibre/ebooks/lrf/html/convert_from.py

@@ -31,7 +31,7 @@ from calibre.ebooks.lrf import option_parser as lrf_option_parser
 from calibre.ebooks import ConversionError
 from calibre.ebooks.lrf.html.table import Table 
 from calibre import filename_to_utf8,  setup_cli_handlers, __appname__, \
-                    fit_image, LoggingInterface, preferred_encoding
+                    fit_image, preferred_encoding
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.devices.interface import Device
 from calibre.ebooks.lrf.html.color_map import lrs_color
@@ -78,7 +78,7 @@ def tag_regex(tagname):
     return dict(open=r'(?:<\s*%(t)s\s+[^<>]*?>|<\s*%(t)s\s*>)'%dict(t=tagname), \
                 close=r'</\s*%(t)s\s*>'%dict(t=tagname))
 
-class HTMLConverter(object, LoggingInterface):
+class HTMLConverter(object):
     SELECTOR_PAT   = re.compile(r"([A-Za-z0-9\-\_\:\.]+[A-Za-z0-9\-\_\:\.\s\,]*)\s*\{([^\}]*)\}")
     PAGE_BREAK_PAT = re.compile(r'page-break-(?:after|before)\s*:\s*(\w+)', re.IGNORECASE)
     IGNORED_TAGS   = (Comment, Declaration, ProcessingInstruction)
@@ -99,6 +99,10 @@ class HTMLConverter(object, LoggingInterface):
                         # Replace common line break patterns with line breaks
                         (re.compile(r'<p>(&nbsp;|\s)*</p>', re.IGNORECASE), lambda m: '<br />'),
                         
+                        # Replace empty headers with line breaks
+                        (re.compile(r'<h[0-5]?>(&nbsp;|\s)*</h[0-5]?>', 
+                                    re.IGNORECASE), lambda m: '<br />'),
+                        
                         # Replace entities
                         (re.compile(ur'&(\S+?);'), partial(entity_to_unicode, 
                                                            exceptions=['lt', 'gt', 'amp'])),
@@ -209,7 +213,6 @@ class HTMLConverter(object, LoggingInterface):
         '''
         # Defaults for various formatting tags        
         object.__setattr__(self, 'options', options)
-        LoggingInterface.__init__(self, logger)
         self.fonts = fonts #: dict specifying font families to use
         # Memory 
         self.scaled_images    = {}    #: Temporary files with scaled version of images        

src/calibre/ebooks/lrf/lrs/convert_from.py

@@ -29,6 +29,7 @@ class LrsParser(object):
         self.logger = logger
         src = stream.read()
         self.soup = BeautifulStoneSoup(xml_to_unicode(src)[0],
+                       convertEntities=BeautifulStoneSoup.XML_ENTITIES, 
                        selfClosingTags=self.SELF_CLOSING_TAGS)
         self.objects = {}
         for obj in self.soup.findAll(objid=True):

src/calibre/ebooks/lrf/meta.py

@@ -530,7 +530,7 @@ class LRFMetaFile(object):
         """ See L{file.write} """
         self._file.write(val)
         
-    def objects(self):
+    def _objects(self):
         self._file.seek(self.object_index_offset)
         c = self.number_of_objects
         while c > 0:
@@ -543,7 +543,7 @@ class LRFMetaFile(object):
     def get_objects_by_type(self, type):
         from calibre.ebooks.lrf.tags import Tag
         objects = []
-        for id, offset, size in self.objects():
+        for id, offset, size in self._objects():
             self._file.seek(offset)
             tag = Tag(self._file)
             if tag.id == 0xF500:
@@ -554,7 +554,7 @@ class LRFMetaFile(object):
     
     def get_object_by_id(self, tid):
         from calibre.ebooks.lrf.tags import Tag
-        for id, offset, size in self.objects():
+        for id, offset, size in self._objects():
             self._file.seek(offset)
             tag = Tag(self._file)
             if tag.id == 0xF500:

src/calibre/ebooks/metadata/isbndb.py

@@ -112,7 +112,8 @@ key is the account key you generate after signing up for a free account from isb
                       default=None, help=_('The title of the book to search for.'))
     parser.add_option('-p', '--publisher', default=None, dest='publisher',
                       help=_('The publisher of the book to search for.'))
-    parser.add_option('--verbose', default=False, action='store_true', help=_('Verbose processing'))
+    parser.add_option('-v', '--verbose', default=False, 
+                      action='store_true', help=_('Verbose processing'))
     
     return parser
     

src/calibre/ebooks/metadata/lit.py

@@ -19,14 +19,22 @@ def get_metadata(stream):
     for item in opf.iterguide():
         if 'cover' not in item.get('type', '').lower():
             continue
+        ctype = item.get('type')
         href = item.get('href', '')
         candidates = [href, href.replace('&', '%26')]
         for item in litfile.manifest.values():
             if item.path in candidates:
-                covers.append(item.internal)
+                try:
+                    covers.append((litfile.get_file('/data/'+item.internal), 
+                                   ctype))
+                except:
+                    pass
                 break
-    covers = [litfile.get_file('/data/' + i) for i in covers]
-    covers.sort(cmp=lambda x, y:cmp(len(x), len(y)))
-    mi.cover_data = ('jpg', covers[-1])
+    covers.sort(cmp=lambda x, y:cmp(len(x[0]), len(y[0])), reverse=True)
+    idx = 0
+    if len(covers) > 1:
+        if covers[1][1] == covers[1][0]+'-standard':
+            idx = 1
+    mi.cover_data = ('jpg', covers[idx][0])
     return mi
 

src/calibre/ebooks/metadata/meta.py

@@ -15,7 +15,7 @@ _METADATA_PRIORITIES = [
                        'html', 'htm', 'xhtml', 'xhtm',
                        'rtf', 'fb2', 'pdf', 'prc', 'odt',
                        'epub', 'lit', 'lrx', 'lrf', 'mobi',
-                       'rb', 'imp'
+                       'rb', 'imp', 'azw'
                       ]
 
 # The priorities for loading metadata from different file types
@@ -41,7 +41,9 @@ def metadata_from_formats(formats):
     for path, ext in zip(formats, extensions):
         with open(path, 'rb') as stream:
             try:
-                mi.smart_update(get_metadata(stream, stream_type=ext, use_libprs_metadata=True))
+                newmi = get_metadata(stream, stream_type=ext, 
+                                     use_libprs_metadata=True)
+                mi.smart_update(newmi)
             except:
                 continue
             if getattr(mi, 'application_id', None) is not None:
@@ -58,7 +60,7 @@ def get_metadata(stream, stream_type='lrf', use_libprs_metadata=False):
     if stream_type: stream_type = stream_type.lower()
     if stream_type in ('html', 'html', 'xhtml', 'xhtm', 'xml'):
         stream_type = 'html'
-    if stream_type in ('mobi', 'prc'):
+    if stream_type in ('mobi', 'prc', 'azw'):
         stream_type = 'mobi'
     if stream_type in ('odt', 'ods', 'odp', 'odg', 'odf'):
         stream_type = 'odt'

src/calibre/ebooks/metadata/opf2.py

@@ -444,6 +444,7 @@ class OPF(object):
         if not hasattr(stream, 'read'):
             stream = open(stream, 'rb')
         self.basedir  = self.base_dir = basedir
+        self.path_to_html_toc = None
         raw, self.encoding = xml_to_unicode(stream.read(), strip_encoding_pats=True, resolve_entities=True)
         raw = raw[raw.find('<'):]
         self.root     = etree.fromstring(raw, self.PARSER)
@@ -495,6 +496,7 @@ class OPF(object):
                     if f:
                         self.toc.read_ncx_toc(f[0])
             else:
+                self.path_to_html_toc = toc
                 self.toc.read_html_toc(toc)
         except:
             pass

src/calibre/ebooks/metadata/topaz.py

@@ -0,0 +1,40 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+''' Read metadata from Amazon's topaz format '''
+
+def read_record(raw, name):
+    idx = raw.find(name)
+    if idx > -1:
+        length = ord(raw[idx+len(name)])
+        return raw[idx+len(name)+1:idx+len(name)+1+length]
+
+def get_metadata(stream):
+    raw = stream.read(8*1024)
+    if not raw.startswith('TPZ'):
+        raise ValueError('Not a Topaz file')
+    first = raw.find('metadata')
+    if first < 0:
+        raise ValueError('Invalid Topaz file')
+    second = raw.find('metadata', first+10)
+    if second < 0:
+        raise ValueError('Invalid Topaz file')
+    raw = raw[second:second+1000]
+    authors = read_record(raw, 'Authors')
+    if authors:
+        authors = authors.decode('utf-8', 'replace').split(';')
+    else:
+        authors = [_('Unknown')]
+    title = read_record(raw, 'Title')
+    if title:
+        title = title.decode('utf-8', 'replace')
+    else:
+        raise ValueError('No metadata in file')
+    from calibre.ebooks.metadata import MetaInformation
+    return MetaInformation(title, authors)
+
+if __name__ == '__main__':
+    import sys
+    print get_metadata(open(sys.argv[1], 'rb'))

src/calibre/ebooks/mobi/input.py

@@ -3,8 +3,6 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os
-
 from calibre.customize.conversion import InputFormatPlugin
 
 class MOBIInput(InputFormatPlugin):
@@ -14,16 +12,19 @@ class MOBIInput(InputFormatPlugin):
     description = 'Convert MOBI files (.mobi, .prc, .azw) to HTML'
     file_types  = set(['mobi', 'prc', 'azw'])
     
-    def convert(self, stream, options, file_ext, parse_cache, log):
+    def convert(self, stream, options, file_ext, parse_cache, log, 
+                accelerators):
         from calibre.ebooks.mobi.reader import MobiReader
         mr = MobiReader(stream, log, options.input_encoding, 
                         options.debug_input)
-        mr.extract_content(output_dir=os.getcwdu(), parse_cache)
+        mr.extract_content('.', parse_cache)
         raw = parse_cache.get('calibre_raw_mobi_markup', False)
         if raw:
             if isinstance(raw, unicode):
                 raw = raw.encode('utf-8')
             open('debug-raw.html', 'wb').write(raw)
-            
+        for f, root in parse_cache.items():
+            if '.' in f:
+                accelerators[f] = {'pagebreaks':root.xpath(
+                                            '//div[@class="mbp_pagebreak"]')}
         return mr.created_opf_path
-        

src/calibre/ebooks/mobi/reader.py

@@ -92,6 +92,8 @@ class BookHeader(object):
             self.sublanguage = 'NEUTRAL'
             self.exth_flag, self.exth = 0, None
             self.ancient = True
+            self.first_image_index = -1
+            self.mobi_version = 1
         else:
             self.ancient = False
             self.doctype = raw[16:20]
@@ -312,7 +314,7 @@ class MobiReader(object):
         mobi_version = self.book_header.mobi_version
         for i, tag in enumerate(root.iter(etree.Element)):
             if tag.tag in ('country-region', 'place', 'placetype', 'placename',
-                           'state', 'city'):
+                           'state', 'city', 'street', 'address'):
                 tag.tag = 'span'
                 for key in tag.attrib.keys():
                     tag.attrib.pop(key)
@@ -389,6 +391,12 @@ class MobiReader(object):
             opf.cover = 'images/%05d.jpg'%(self.book_header.exth.cover_offset+1)
         elif mi.cover is not None:
             opf.cover = mi.cover
+        else:
+            opf.cover = 'images/%05d.jpg'%1
+            if not os.path.exists(os.path.join(os.path.dirname(htmlfile),
+                                               *opf.cover.split('/'))):
+                opf.cover = None
+
         manifest = [(htmlfile, 'text/x-oeb1-document'),
                     (os.path.abspath('styles.css'), 'text/css')]
         bp = os.path.dirname(htmlfile)
@@ -531,7 +539,7 @@ class MobiReader(object):
             os.makedirs(output_dir)
         image_index = 0
         self.image_names = []
-        start = self.book_header.first_image_index
+        start = getattr(self.book_header, 'first_image_index', -1)
         if start > self.num_sections or start < 0:
             # BAEN PRC files have bad headers 
             start=0

src/calibre/ebooks/mobi/writer.py

@@ -9,7 +9,6 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.cam>'
 import sys
 import os
 from struct import pack
-import functools
 import time
 import random
 from cStringIO import StringIO
@@ -18,13 +17,12 @@ from itertools import izip, count
 from collections import defaultdict
 from urlparse import urldefrag
 import logging
-from lxml import etree
 from PIL import Image
 from calibre.ebooks.oeb.base import XML_NS, XHTML, XHTML_NS, OEB_DOCS, \
     OEB_RASTER_IMAGES
-from calibre.ebooks.oeb.base import xpath, barename, namespace, prefixname
+from calibre.ebooks.oeb.base import namespace, prefixname
 from calibre.ebooks.oeb.base import urlnormalize
-from calibre.ebooks.oeb.base import Logger, OEBBook
+from calibre.ebooks.oeb.base import OEBBook
 from calibre.ebooks.oeb.profile import Context
 from calibre.ebooks.oeb.transforms.flatcss import CSSFlattener
 from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer
@@ -33,7 +31,7 @@ from calibre.ebooks.oeb.transforms.htmltoc import HTMLTOCAdder
 from calibre.ebooks.oeb.transforms.manglecase import CaseMangler
 from calibre.ebooks.mobi.palmdoc import compress_doc
 from calibre.ebooks.mobi.langcodes import iana2mobi
-from calibre.ebooks.mobi.mobiml import MBP_NS, MBP, MobiMLizer
+from calibre.ebooks.mobi.mobiml import MBP_NS, MobiMLizer
 from calibre.customize.ui import run_plugins_on_postprocess
 from calibre.utils.config import Config, StringConfig
 
@@ -162,7 +160,7 @@ class Serializer(object):
         hrefs = self.oeb.manifest.hrefs
         buffer.write('<guide>')
         for ref in self.oeb.guide.values():
-            path, frag = urldefrag(ref.href)
+            path = urldefrag(ref.href)[0]
             if hrefs[path].media_type not in OEB_DOCS:
                 continue
             buffer.write('<reference type="')
@@ -457,8 +455,6 @@ class MobiWriter(object):
         self._oeb.logger.info('Serializing images...')
         images = [(index, href) for href, index in self._images.items()]
         images.sort()
-        metadata = self._oeb.metadata
-        coverid = metadata.cover[0] if metadata.cover else None
         for _, href in images:
             item = self._oeb.manifest.hrefs[href]
             try:

src/calibre/ebooks/oeb/base.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
-import os, sys, re, uuid
+import os, re, uuid
 from mimetypes import types_map
 from collections import defaultdict
 from itertools import count
@@ -15,7 +15,6 @@ from urlparse import urldefrag, urlparse, urlunparse
 from urllib import unquote as urlunquote
 from lxml import etree, html
 import calibre
-from calibre import LoggingInterface
 from calibre.translations.dynamic import translate
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
@@ -204,22 +203,6 @@ class OEBError(Exception):
     """Generic OEB-processing error."""
     pass
 
-
-class FauxLogger(object):
-    """Fake logging interface."""
-    def __getattr__(self, name):
-        return self
-    def __call__(self, message):
-        print message
-
-class Logger(LoggingInterface, object):
-    """A logging object which provides both the standard `logging.Logger` and
-    calibre-specific interfaces.
-    """
-    def __getattr__(self, name):
-        return object.__getattribute__(self, 'log_' + name)
-
-
 class NullContainer(object):
     """An empty container.
 
@@ -1233,16 +1216,20 @@ class PageList(object):
 class OEBBook(object):
     """Representation of a book in the IDPF OEB data model."""
     
-    def __init__(self, encoding=None, pretty_print=False, logger=FauxLogger()):
+    def __init__(self, logger, parse_cache={}, encoding='utf-8', 
+                 pretty_print=False):
         """Create empty book.  Optional arguments:
         
+        :param parse_cache: A cache of parsed XHTML/CSS. Keys are absolute
+            paths to te cached files and values are lxml root objects and
+            cssutils stylesheets.
         :param:`encoding`: Default encoding for textual content read
             from an external container.
         :param:`pretty_print`: Whether or not the canonical string form
             of XML markup is pretty-printed.
-        :prama:`logger`: A Logger object to use for logging all messages
+        :param:`logger`: A Log object to use for logging all messages
             related to the processing of this book.  It is accessible
-            via the instance data member :attr:`logger`.
+            via the instance data members :attr:`logger,log`.
         
         It provides the following public instance data members for
         accessing various parts of the OEB data model:
@@ -1260,7 +1247,7 @@ class OEBBook(object):
         """
         self.encoding = encoding
         self.pretty_print = pretty_print
-        self.logger = logger
+        self.logger = self.log = logger
         self.version = '2.0'
         self.container = NullContainer()
         self.metadata = Metadata(self)

src/calibre/ebooks/oeb/output.py

@@ -0,0 +1,17 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+from calibre.customize.conversion import OutputFormatPlugin
+
+class OEBOutput(OutputFormatPlugin):
+    
+    name = 'OEB Output'
+    author = 'Kovid Goyal'
+    file_type = 'oeb'
+    
+    
+    def convert(self, oeb_book, input_plugin, options, parse_cache, log):
+        pass
+    

src/calibre/ebooks/oeb/reader.py

@@ -19,9 +19,9 @@ from calibre.ebooks.oeb.base import OEB_DOCS, OEB_STYLES, OEB_IMAGES, \
     PAGE_MAP_MIME, JPEG_MIME, NCX_MIME, SVG_MIME
 from calibre.ebooks.oeb.base import XMLDECL_RE, COLLAPSE_RE, CSSURL_RE, \
     ENTITY_RE, LINK_SELECTORS, MS_COVER_TYPE
-from calibre.ebooks.oeb.base import namespace, barename, qname, XPath, xpath
-from calibre.ebooks.oeb.base import urlnormalize, xml2str
-from calibre.ebooks.oeb.base import OEBError, OEBBook, DirContainer
+from calibre.ebooks.oeb.base import namespace, barename, qname, XPath, xpath, \
+                                    urlnormalize, BINARY_MIME, \
+                                    OEBError, OEBBook, DirContainer
 from calibre.ebooks.oeb.writer import OEBWriter
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
 from calibre.ebooks.metadata.epub import CoverRenderer
@@ -45,9 +45,6 @@ class OEBReader(object):
     TRANSFORMS = []
     """List of transforms to apply to content read with this Reader."""
 
-    def __init__(self):
-        return
-    
     @classmethod
     def config(cls, cfg):
         """Add any book-reading options to the :class:`Config` object
@@ -65,7 +62,7 @@ class OEBReader(object):
         :param:`oeb`.
         """
         self.oeb = oeb
-        self.logger = oeb.logger
+        self.logger = self.log = oeb.logger
         oeb.container = self.Container(path)
         opf = self._read_opf()
         self._all_from_opf(opf)

src/calibre/ebooks/oeb/stylizer.py

@@ -19,7 +19,7 @@ from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \
     CSSValueList, cssproperties
 from cssutils.profiles import profiles as cssprofiles
 from lxml import etree
-from lxml.cssselect import css_to_xpath, ExpressionError
+from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
 from calibre.ebooks.oeb.base import XPNSMAP, xpath, urlnormalize
 from calibre.ebooks.oeb.profile import PROFILES
@@ -159,7 +159,9 @@ class Stylizer(object):
         for _, _, cssdict, text, _ in rules:
             try:
                 selector = CSSSelector(text)
-            except ExpressionError:
+            except (AssertionError, ExpressionError, etree.XPathSyntaxError,\
+                NameError, # gets thrown on OS X instead of SelectorSyntaxError
+                SelectorSyntaxError):
                 continue
             for elem in selector(tree):
                 self.style(elem)._update_cssdict(cssdict)

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -6,18 +6,14 @@ from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys
-import os
 import re
 import operator
 import math
-from itertools import chain
 from collections import defaultdict
 from lxml import etree
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS
 from calibre.ebooks.oeb.base import CSS_MIME, OEB_STYLES
 from calibre.ebooks.oeb.base import namespace, barename
-from calibre.ebooks.oeb.base import OEBBook
 from calibre.ebooks.oeb.stylizer import Stylizer
 
 COLLAPSE = re.compile(r'[ \t\r\n\v]+')

src/calibre/ebooks/oeb/transforms/htmltoc.py

@@ -6,9 +6,6 @@ from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys
-import os
-from lxml import etree
 from calibre.ebooks.oeb.base import XML, XHTML, XHTML_NS
 from calibre.ebooks.oeb.base import XHTML_MIME, CSS_MIME
 from calibre.ebooks.oeb.base import element
@@ -66,6 +63,8 @@ class HTMLTOCAdder(object):
     def __call__(self, oeb, context):
         if 'toc' in oeb.guide:
             return
+        if not getattr(getattr(oeb, 'toc', False), 'nodes', False):
+            return
         oeb.logger.info('Generating in-line TOC...')
         title = self.title or oeb.translate(DEFAULT_TITLE)
         style = self.style

src/calibre/ebooks/oeb/transforms/manglecase.py

@@ -6,13 +6,6 @@ from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys
-import os
-import re
-import operator
-import math
-from itertools import chain
-from collections import defaultdict
 from lxml import etree
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS
 from calibre.ebooks.oeb.base import CSS_MIME

src/calibre/ebooks/oeb/transforms/rasterize.py

@@ -6,7 +6,6 @@ from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys
 import os
 from urlparse import urldefrag
 import base64
@@ -20,9 +19,9 @@ from PyQt4.QtGui import QImage
 from PyQt4.QtGui import QPainter
 from PyQt4.QtSvg import QSvgRenderer
 from PyQt4.QtGui import QApplication
-from calibre.ebooks.oeb.base import XHTML_NS, XHTML, SVG_NS, SVG, XLINK
-from calibre.ebooks.oeb.base import SVG_MIME, PNG_MIME, JPEG_MIME
-from calibre.ebooks.oeb.base import xml2str, xpath, namespace, barename
+from calibre.ebooks.oeb.base import XHTML, XLINK
+from calibre.ebooks.oeb.base import SVG_MIME, PNG_MIME
+from calibre.ebooks.oeb.base import xml2str, xpath
 from calibre.ebooks.oeb.base import urlnormalize
 from calibre.ebooks.oeb.stylizer import Stylizer
 
@@ -88,7 +87,7 @@ class SVGRasterizer(object):
         hrefs = self.oeb.manifest.hrefs
         for elem in xpath(svg, '//svg:*[@xl:href]'):
             href = urlnormalize(elem.attrib[XLINK('href')])
-            path, frag = urldefrag(href)
+            path = urldefrag(href)[0]
             if not path:
                 continue
             abshref = item.abshref(path)

src/calibre/gui2/add.py

@@ -224,7 +224,10 @@ class AddRecursive(Add):
             files = _('<p>Books with the same title as the following already '
                       'exist in the database. Add them anyway?<ul>')
             for mi in self.duplicates:
-                files += '<li>'+mi[0].title+'</li>\n'
+                title = mi[0].title
+                if not isinstance(title, unicode):
+                    title = title.decode(preferred_encoding, 'replace')
+                files += '<li>'+title+'</li>\n'
             d = WarningDialog (_('Duplicates found!'), 
                               _('Duplicates found!'), 
                               files+'</ul></p>', parent=self._parent)

src/calibre/gui2/dialogs/comicconf.ui

@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <ui version="4.0">
  <class>Dialog</class>
  <widget class="QDialog" name="Dialog">
@@ -6,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>646</width>
-    <height>468</height>
+    <height>503</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -100,21 +101,21 @@
      </property>
     </widget>
    </item>
-   <item row="8" column="0" >
+   <item row="9" column="0">
     <widget class="QCheckBox" name="opt_landscape">
      <property name="text">
       <string>&amp;Landscape</string>
      </property>
     </widget>
    </item>
-   <item row="10" column="0" >
+   <item row="11" column="0">
     <widget class="QCheckBox" name="opt_no_sort">
      <property name="text">
       <string>Don't so&amp;rt</string>
      </property>
     </widget>
    </item>
-   <item row="12" column="1" >
+   <item row="13" column="1">
     <widget class="QDialogButtonBox" name="buttonBox">
      <property name="orientation">
       <enum>Qt::Horizontal</enum>
@@ -124,27 +125,34 @@
      </property>
     </widget>
    </item>
-   <item row="9" column="0" >
+   <item row="10" column="0">
     <widget class="QCheckBox" name="opt_right2left">
      <property name="text">
       <string>&amp;Right to left</string>
      </property>
     </widget>
    </item>
-   <item row="11" column="0" >
+   <item row="12" column="0">
     <widget class="QCheckBox" name="opt_despeckle">
      <property name="text">
       <string>De&amp;speckle</string>
      </property>
     </widget>
    </item>
-   <item row="7" column="0" >
+   <item row="8" column="0">
     <widget class="QCheckBox" name="opt_wide">
      <property name="text">
       <string>&amp;Wide</string>
      </property>
     </widget>
    </item>
+   <item row="7" column="0">
+    <widget class="QCheckBox" name="opt_disable_trim">
+     <property name="text">
+      <string>Disable &amp;Trimming</string>
+     </property>
+    </widget>
+   </item>
   </layout>
  </widget>
  <resources>

src/calibre/gui2/dialogs/config.py

@@ -196,7 +196,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
             self.language.addItem(language_codes[lang], QVariant(lang))
         else:
             lang = 'en'
-            self.language.addItem('English', 'en')
+            self.language.addItem('English', QVariant('en'))
         items = [(l, language_codes[l]) for l in translations.keys() \
                  if l != lang]
         if lang != 'en':

src/calibre/gui2/dialogs/metadata_single.py

@@ -346,19 +346,22 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                 pix = QPixmap()
                 pix.loadFromData(cover_data)
                 if pix.isNull():
-                    error_dialog(self.window, "The cover is not a valid picture").exec_()
+                    error_dialog(self.window, _('Bad cover'), 
+                                 _('The cover is not a valid picture')).exec_()
                 else:
                     self.cover.setPixmap(pix)
                     self.cover_changed = True
                     self.cpixmap = pix
             except LibraryThingError, err:
-                error_dialog(self, _('Could not fetch cover'), _('<b>Could not fetch cover.</b><br/>')+repr(err)).exec_()
+                error_dialog(self, _('Cannot fetch cover'), 
+                    _('<b>Could not fetch cover.</b><br/>')+repr(err)).exec_()
             finally:
                 self.fetch_cover_button.setEnabled(True)
                 self.unsetCursor()
                 
         else:
-            error_dialog(self, _('Cannot fetch cover'), _('You must specify the ISBN identifier for this book.')).exec_()
+            error_dialog(self, _('Cannot fetch cover'), 
+                _('You must specify the ISBN identifier for this book.')).exec_()
                 
     
     def fetch_metadata(self):

src/calibre/gui2/dialogs/metadata_single.ui

@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <ui version="4.0">
  <class>MetadataSingleDialog</class>
  <widget class="QDialog" name="MetadataSingleDialog">
@@ -10,7 +11,7 @@
    </rect>
   </property>
   <property name="sizePolicy">
-   <sizepolicy vsizetype="MinimumExpanding" hsizetype="MinimumExpanding" >
+   <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
     <horstretch>0</horstretch>
     <verstretch>0</verstretch>
    </sizepolicy>
@@ -51,7 +52,7 @@
         <number>0</number>
        </property>
        <item>
-        <widget class="QWidget" native="1" name="central_widget" >
+        <widget class="QWidget" name="central_widget" native="true">
          <property name="minimumSize">
           <size>
            <width>800</width>
@@ -92,7 +93,7 @@
                    </property>
                   </widget>
                  </item>
-                 <item rowspan="2" row="0" column="2" >
+                 <item row="0" column="2" rowspan="2">
                   <widget class="QToolButton" name="swap_button">
                    <property name="toolTip">
                     <string>Swap the author and title</string>
@@ -226,7 +227,7 @@
                    <item>
                     <widget class="QLineEdit" name="tags">
                      <property name="toolTip">
-                      <string>Tags categorize the book. This is particularly useful while searching. &lt;br>&lt;br>They can be any words or phrases, separated by commas.</string>
+                      <string>Tags categorize the book. This is particularly useful while searching. &lt;br&gt;&lt;br&gt;They can be any words or phrases, separated by commas.</string>
                      </property>
                     </widget>
                    </item>
@@ -270,7 +271,7 @@
                    <item>
                     <widget class="QComboBox" name="series">
                      <property name="sizePolicy">
-                      <sizepolicy vsizetype="Fixed" hsizetype="MinimumExpanding" >
+                      <sizepolicy hsizetype="MinimumExpanding" vsizetype="Fixed">
                        <horstretch>0</horstretch>
                        <verstretch>0</verstretch>
                       </sizepolicy>
@@ -374,7 +375,7 @@
               <item>
                <widget class="QPushButton" name="fetch_metadata_button">
                 <property name="text">
-                 <string>Fetch metadata from server</string>
+                 <string>&amp;Fetch metadata from server</string>
                 </property>
                </widget>
               </item>
@@ -385,7 +386,7 @@
               <item>
                <widget class="QGroupBox" name="af_group_box">
                 <property name="sizePolicy">
-                 <sizepolicy vsizetype="Minimum" hsizetype="Preferred" >
+                 <sizepolicy hsizetype="Preferred" vsizetype="Minimum">
                   <horstretch>0</horstretch>
                   <verstretch>0</verstretch>
                  </sizepolicy>
@@ -396,10 +397,10 @@
                 <layout class="QVBoxLayout" name="verticalLayout">
                  <item>
                   <layout class="QGridLayout" name="gridLayout">
-                   <item rowspan="3" row="0" column="0" >
+                   <item row="0" column="0" rowspan="3">
                     <widget class="QListWidget" name="formats">
                      <property name="sizePolicy">
-                      <sizepolicy vsizetype="Minimum" hsizetype="Minimum" >
+                      <sizepolicy hsizetype="Minimum" vsizetype="Minimum">
                        <horstretch>0</horstretch>
                        <verstretch>0</verstretch>
                       </sizepolicy>
@@ -487,7 +488,7 @@
               <item>
                <widget class="QGroupBox" name="bc_box">
                 <property name="sizePolicy">
-                 <sizepolicy vsizetype="Expanding" hsizetype="Preferred" >
+                 <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
                   <horstretch>0</horstretch>
                   <verstretch>10</verstretch>
                  </sizepolicy>
@@ -499,7 +500,7 @@
                  <item>
                   <widget class="ImageView" name="cover">
                    <property name="sizePolicy">
-                    <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
+                    <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
                      <horstretch>0</horstretch>
                      <verstretch>0</verstretch>
                     </sizepolicy>
@@ -588,7 +589,7 @@
                    <item>
                     <widget class="QPushButton" name="fetch_cover_button">
                      <property name="text">
-                      <string>Fetch cover image from server</string>
+                      <string>Fetch &amp;cover image from server</string>
                      </property>
                     </widget>
                    </item>
@@ -598,7 +599,7 @@
                       <string>Change the username and/or password for your account at LibraryThing.com</string>
                      </property>
                      <property name="text">
-                      <string>Change password</string>
+                      <string>Change &amp;password</string>
                      </property>
                     </widget>
                    </item>

src/calibre/gui2/library.py

@@ -94,6 +94,7 @@ class DateDelegate(QStyledItemDelegate):
     def createEditor(self, parent, option, index):
         qde = QStyledItemDelegate.createEditor(self, parent, option, index)
         qde.setDisplayFormat('MM/dd/yyyy')
+        qde.setMinimumDate(QDate(100,1,1))
         return qde
 
 class BooksModel(QAbstractTableModel):
@@ -420,6 +421,7 @@ class BooksModel(QAbstractTableModel):
     def get_preferred_formats(self, rows, formats, paths=False, 
                               set_metadata=False, specific_format=None):
         ans = []
+        need_auto = []
         if specific_format is not None:
             formats = [specific_format.lower()]
         for row in (row.row() for row in rows):
@@ -444,8 +446,9 @@ class BooksModel(QAbstractTableModel):
                 pt.close() if paths else pt.seek(0)
                 ans.append(pt)
             else:
+                need_auto.append(row)
                 ans.append(None)
-        return ans
+        return ans, need_auto
 
     def id(self, row):
         return self.db.id(getattr(row, 'row', lambda:row)())
@@ -1069,3 +1072,4 @@ class SearchBox(QLineEdit):
         self.emit(SIGNAL('search(PyQt_PyObject, PyQt_PyObject)'), txt, False)
         self.end(False)
         self.initial_state = False
+

src/calibre/gui2/main.py

@@ -38,7 +38,8 @@ from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
 from calibre.gui2.dialogs.jobs import JobsDialog
 from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog
 from calibre.gui2.tools import convert_single_ebook, convert_bulk_ebooks, \
-                                set_conversion_defaults, fetch_scheduled_recipe
+                                set_conversion_defaults, fetch_scheduled_recipe, \
+                                auto_convert_ebook
 from calibre.gui2.dialogs.config import ConfigDialog
 from calibre.gui2.dialogs.search import SearchDialog
 from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
@@ -237,6 +238,7 @@ class Main(MainWindow, Ui_MainWindow):
 
         QObject.connect(self.config_button, SIGNAL('clicked(bool)'), self.do_config)
         self.connect(self.preferences_action, SIGNAL('triggered(bool)'), self.do_config)
+        self.connect(self.action_preferences, SIGNAL('triggered(bool)'), self.do_config)
         QObject.connect(self.advanced_search_button, SIGNAL('clicked(bool)'), self.do_advanced_search)
         
         ####################### Library view ########################
@@ -818,7 +820,8 @@ class Main(MainWindow, Ui_MainWindow):
         rows = self.library_view.selectionModel().selectedRows()
         previous = self.library_view.currentIndex()
         if not rows or len(rows) == 0:
-            d = error_dialog(self, _('Cannot edit metadata'), _('No books selected'))
+            d = error_dialog(self, _('Cannot edit metadata'), 
+                             _('No books selected'))
             d.exec_()
             return
 
@@ -904,9 +907,8 @@ class Main(MainWindow, Ui_MainWindow):
         on_card = config['send_to_storage_card_by_default']
         self.sync_to_device(on_card, False, specific_format=fmt)
         
-    
-    def sync_to_device(self, on_card, delete_from_library, specific_format=None):
-        rows = self.library_view.selectionModel().selectedRows()
+    def sync_to_device(self, on_card, delete_from_library, specific_format=None, send_rows=None, auto_convert=True):
+        rows = self.library_view.selectionModel().selectedRows() if send_rows is None else send_rows
         if not self.device_manager or not rows or len(rows) == 0:
             return
         ids = iter(self.library_view.model().id(r) for r in rows)
@@ -917,7 +919,7 @@ class Main(MainWindow, Ui_MainWindow):
             if cdata:
                 mi['cover'] = self.cover_to_thumbnail(cdata)
         metadata, full_metadata = iter(metadata), iter(full_metadata)
-        _files   = self.library_view.model().get_preferred_formats(rows,
+        _files, _auto_rows   = self.library_view.model().get_preferred_formats(rows,
                                     self.device_manager.device_class.FORMATS, 
                                     paths=True, set_metadata=True,
                                     specific_format=specific_format)
@@ -952,8 +954,27 @@ class Main(MainWindow, Ui_MainWindow):
         remove = remove_ids if delete_from_library else []
         self.upload_books(gf, names, good, on_card, memory=(_files, remove))
         self.status_bar.showMessage(_('Sending books to device.'), 5000)
+        
         if bad:
+            if specific_format is None:
+                if 'epub' in self.device_manager.device_class.FORMATS:
+                    format = 'epub'
+                elif 'mobi' in self.device_manager.device_class.FORMATS or 'prc' in self.device_manager.device_class.FORMATS:
+                    format = 'mobi'
+                elif 'lrf' in self.device_manager.device_class.FORMATS:
+                    format = 'lrf'
+            else:
+                format = specific_format
+            
+            if format not in ('epub', 'mobi'):
+                auto_convert = False
+
             bad = '\n'.join('<li>%s</li>'%(i,) for i in bad)
+            if auto_convert:
+                d = info_dialog(self, _('No suitable formats'),
+                    _('Auto converting the following books before uploading to the device:<br><ul>%s</ul>')%(bad,))
+                self.auto_convert(_auto_rows, on_card, format)
+            else:
                 d = warning_dialog(self, _('No suitable formats'),
                     _('Could not upload the following books to the device, as no suitable formats were found:<br><ul>%s</ul>')%(bad,)) 
             d.exec_()
@@ -1048,6 +1069,32 @@ class Main(MainWindow, Ui_MainWindow):
 
     ############################### Convert ####################################
     
+    def auto_convert(self, rows, on_card, format):
+        previous = self.library_view.currentIndex()
+     
+        comics, others = [], []
+        db = self.library_view.model().db
+        for r in rows:
+            formats = db.formats(r)
+            if not formats: continue
+            formats = formats.lower().split(',')
+            if 'cbr' in formats or 'cbz' in formats:
+                comics.append(r)
+            else:
+                others.append(r)
+        
+        jobs, changed, bad_rows = auto_convert_ebook(format, self, self.library_view.model().db, comics, others)
+        for func, args, desc, fmt, id, temp_files in jobs:
+            if id not in bad_rows:
+                job = self.job_manager.run_job(Dispatcher(self.book_auto_converted), 
+                                            func, args=args, description=desc)
+                self.conversion_jobs[job] = (temp_files, fmt, id, on_card)
+            
+        if changed:
+            self.library_view.model().refresh_rows(rows)
+            current = self.library_view.currentIndex()
+            self.library_view.model().current_changed(current, previous)
+    
     def get_books_for_conversion(self):
         rows = [r.row() for r in self.library_view.selectionModel().selectedRows()]
         if not rows or len(rows) == 0:
@@ -1109,6 +1156,31 @@ class Main(MainWindow, Ui_MainWindow):
             current = self.library_view.currentIndex()
             self.library_view.model().current_changed(current, previous)
                   
+    def book_auto_converted(self, job):
+        temp_files, fmt, book_id, on_card = self.conversion_jobs.pop(job)
+        try:
+            if job.exception is not None:
+                self.job_exception(job)
+                return
+            data = open(temp_files[-1].name, 'rb')
+            self.library_view.model().db.add_format(book_id, fmt, data, index_is_id=True)
+            data.close()
+            self.status_bar.showMessage(job.description + (' completed'), 2000)
+        finally:
+            for f in temp_files:
+                try:
+                    if os.path.exists(f.name):
+                        os.remove(f.name)
+                except:
+                    pass
+        self.tags_view.recount()
+        if self.current_view() is self.library_view:
+            current = self.library_view.currentIndex()
+            self.library_view.model().current_changed(current, QModelIndex())
+
+        r = self.library_view.model().index(self.library_view.model().db.row(book_id), 0)
+        self.sync_to_device(on_card, False, specific_format=fmt, send_rows=[r], auto_convert=False)
+    
     def book_converted(self, job):
         temp_files, fmt, book_id = self.conversion_jobs.pop(job)
         try:
@@ -1149,10 +1221,14 @@ class Main(MainWindow, Ui_MainWindow):
             if ext in config['internally_viewed_formats']:
                 if ext == 'LRF':
                     args = ['lrfviewer', name]
-                    self.job_manager.server.run_free_job('lrfviewer', kwdargs=dict(args=args))
+                    self.job_manager.server.run_free_job('lrfviewer', 
+                                                        kwdargs=dict(args=args))
                 else:
                     args = ['ebook-viewer', name]
-                    self.job_manager.server.run_free_job('ebook-viewer', kwdargs=dict(args=args))
+                    if isosx:
+                        args.append('--raise-window')
+                    self.job_manager.server.run_free_job('ebook-viewer', 
+                                                        kwdargs=dict(args=args))
             else:
                 QDesktopServices.openUrl(QUrl('file:'+name))#launch(name)
             time.sleep(5) # User feedback
@@ -1247,7 +1323,7 @@ class Main(MainWindow, Ui_MainWindow):
 
     ############################### Do config ##################################
 
-    def do_config(self):
+    def do_config(self, *args):
         if self.job_manager.has_jobs():
             d = error_dialog(self, _('Cannot configure'), _('Cannot configure while there are running jobs.'))
             d.exec_()
@@ -1406,6 +1482,14 @@ class Main(MainWindow, Ui_MainWindow):
                 dir = os.path.expanduser('~/Library')
             self.library_path = os.path.abspath(dir)
         if not os.path.exists(self.library_path):
+            try:
+                os.makedirs(self.library_path)
+            except:
+                self.library_path = os.path.expanduser('~/Library')
+                error_dialog(self, _('Invalid library location'), 
+                     _('Could not access %s. Using %s as the library.')%
+                     (repr(self.library_path), repr(self.library_path)) 
+                             ).exec_()
                 os.makedirs(self.library_path)
 
 
@@ -1610,3 +1694,4 @@ if __name__ == '__main__':
             log = open(logfile).read().decode('utf-8', 'ignore')
             d = QErrorMessage('<b>Error:</b>%s<br><b>Traceback:</b><br>%s<b>Log:</b><br>'%(unicode(err), unicode(tb), log))
             d.exec_()
+

src/calibre/gui2/main.ui

@@ -1,3 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
 <ui version="4.0">
  <author>Kovid Goyal</author>
  <class>MainWindow</class>
@@ -6,12 +7,12 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>865</width>
+    <width>1012</width>
     <height>822</height>
    </rect>
   </property>
   <property name="sizePolicy">
-   <sizepolicy vsizetype="Preferred" hsizetype="Preferred" >
+   <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
     <horstretch>0</horstretch>
     <verstretch>0</verstretch>
    </sizepolicy>
@@ -33,7 +34,7 @@
       <item>
        <widget class="LocationView" name="location_view">
         <property name="sizePolicy">
-         <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
+         <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
           <horstretch>0</horstretch>
           <verstretch>0</verstretch>
          </sizepolicy>
@@ -110,7 +111,7 @@
         <item>
          <widget class="QLabel" name="vanity">
           <property name="sizePolicy">
-           <sizepolicy vsizetype="Preferred" hsizetype="Preferred" >
+           <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
             <horstretch>0</horstretch>
             <verstretch>0</verstretch>
            </sizepolicy>
@@ -195,7 +196,7 @@
          <bool>true</bool>
         </property>
         <property name="sizePolicy">
-         <sizepolicy vsizetype="Fixed" hsizetype="Expanding" >
+         <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
           <horstretch>1</horstretch>
           <verstretch>0</verstretch>
          </sizepolicy>
@@ -204,10 +205,10 @@
          <bool>false</bool>
         </property>
         <property name="toolTip">
-         <string>Search the list of books by title or author&lt;br>&lt;br>Words separated by spaces are ANDed</string>
+         <string>Search the list of books by title or author&lt;br&gt;&lt;br&gt;Words separated by spaces are ANDed</string>
         </property>
         <property name="whatsThis">
-         <string>Search the list of books by title, author, publisher, tags and comments&lt;br>&lt;br>Words separated by spaces are ANDed</string>
+         <string>Search the list of books by title, author, publisher, tags and comments&lt;br&gt;&lt;br&gt;Words separated by spaces are ANDed</string>
         </property>
         <property name="autoFillBackground">
          <bool>false</bool>
@@ -273,7 +274,7 @@
     <item row="2" column="0">
      <widget class="QStackedWidget" name="stack">
       <property name="sizePolicy">
-       <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
+       <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
         <horstretch>100</horstretch>
         <verstretch>100</verstretch>
        </sizepolicy>
@@ -335,7 +336,7 @@
           <item>
            <widget class="BooksView" name="library_view">
             <property name="sizePolicy">
-             <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
+             <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
               <horstretch>100</horstretch>
               <verstretch>10</verstretch>
              </sizepolicy>
@@ -375,7 +376,7 @@
         <item row="0" column="0">
          <widget class="DeviceBooksView" name="memory_view">
           <property name="sizePolicy">
-           <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
+           <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
             <horstretch>100</horstretch>
             <verstretch>10</verstretch>
            </sizepolicy>
@@ -413,7 +414,7 @@
         <item row="0" column="0">
          <widget class="DeviceBooksView" name="card_view">
           <property name="sizePolicy">
-           <sizepolicy vsizetype="Expanding" hsizetype="Preferred" >
+           <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
             <horstretch>10</horstretch>
             <verstretch>10</verstretch>
            </sizepolicy>
@@ -482,15 +483,16 @@
     <bool>false</bool>
    </attribute>
    <addaction name="action_add"/>
-   <addaction name="action_del" />
    <addaction name="action_edit"/>
+   <addaction name="action_convert"/>
+   <addaction name="action_view"/>
+   <addaction name="action_news"/>
    <addaction name="separator"/>
    <addaction name="action_sync"/>
    <addaction name="action_save"/>
+   <addaction name="action_del"/>
    <addaction name="separator"/>
-   <addaction name="action_news" />
-   <addaction name="action_convert" />
-   <addaction name="action_view" />
+   <addaction name="action_preferences"/>
   </widget>
   <widget class="QStatusBar" name="statusBar">
    <property name="mouseTracking">
@@ -665,6 +667,21 @@
     <string>Send specific format to device</string>
    </property>
   </action>
+  <action name="action_preferences">
+   <property name="icon">
+    <iconset resource="images.qrc">
+     <normaloff>:/images/config.svg</normaloff>:/images/config.svg</iconset>
+   </property>
+   <property name="text">
+    <string>Preferences</string>
+   </property>
+   <property name="toolTip">
+    <string>Configure calibre</string>
+   </property>
+   <property name="shortcut">
+    <string>Ctrl+P</string>
+   </property>
+  </action>
  </widget>
  <customwidgets>
   <customwidget>

src/calibre/gui2/tools.py

@@ -18,7 +18,9 @@ from calibre.gui2 import warning_dialog
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.ebooks.lrf import preferred_source_formats as LRF_PREFERRED_SOURCE_FORMATS
 from calibre.ebooks.metadata.opf import OPFCreator
-from calibre.ebooks.epub.from_any import SOURCE_FORMATS as EPUB_PREFERRED_SOURCE_FORMATS
+from calibre.ebooks.epub.from_any import SOURCE_FORMATS as EPUB_PREFERRED_SOURCE_FORMATS, config as epubconfig
+from calibre.ebooks.mobi.from_any import config as mobiconfig
+from calibre.ebooks.lrf.comic.convert_from import config as comicconfig
 
 def get_dialog(fmt):
     return {
@@ -26,6 +28,122 @@ def get_dialog(fmt):
               'mobi':MOBIConvert,
               }[fmt]
 
+def get_config(fmt):
+    return {
+              'epub':epubconfig,
+              'mobi':mobiconfig,
+           }[fmt]
+
+def auto_convert(fmt, parent, db, comics, others):
+    changed = False
+    jobs = []
+    
+    total = sum(map(len, (others, comics)))
+    if total == 0:
+        return
+    parent.status_bar.showMessage(_('Starting auto conversion of %d books')%total, 2000)
+    
+    i = 0
+    bad_rows = []
+    
+    for i, row in enumerate(others+comics):
+        row_id = db.id(row)
+        
+        if row in others:
+            temp_files = []
+            
+            data = None
+            for _fmt in EPUB_PREFERRED_SOURCE_FORMATS:
+                try:
+                    data = db.format(row, _fmt.upper())
+                    if data is not None:
+                        break
+                except:
+                    continue
+            if data is None:
+                bad_rows.append(row)
+                continue
+        
+            defaults = db.conversion_options(db.id(row), fmt)
+            defaults = defaults if defaults else ''
+            options = get_config(fmt)(defaults=defaults).parse()
+            
+            mi = db.get_metadata(row)
+            opf = OPFCreator(os.getcwdu(), mi)
+            opf_file = PersistentTemporaryFile('.opf')
+            opf.render(opf_file)
+            opf_file.close()
+            pt = PersistentTemporaryFile('.'+_fmt.lower())
+            pt.write(data)
+            pt.close()
+            of = PersistentTemporaryFile('.'+fmt)
+            of.close()
+            cover = db.cover(row)
+            cf = None
+            if cover:
+                cf = PersistentTemporaryFile('.jpeg')
+                cf.write(cover)
+                cf.close()
+                options.cover = cf.name
+            options.output = of.name
+            options.from_opf = opf_file.name
+            args = [options, pt.name]
+            desc = _('Auto convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
+            temp_files = [cf] if cf is not None else []
+            temp_files.extend([opf_file, pt, of])
+            jobs.append(('any2'+fmt, args, desc, fmt.upper(), row_id, temp_files))
+
+            changed = True
+        else:
+            defaults = db.conversion_options(db.id(row), fmt)
+            defaults = defaults if defaults else ''
+            options = comicconfig(defaults=defaults).parse()
+            
+            mi = db.get_metadata(row)
+            if mi.title:
+                options.title = mi.title
+            if mi.authors:
+                options.author =  ','.join(mi.authors)
+            data = None
+            for _fmt in ['cbz', 'cbr']:
+                try:
+                    data = db.format(row, _fmt.upper())
+                    if data is not None:
+                        break                    
+                except:
+                    continue
+            
+            if data is None:
+                bad_rows.append(row)
+                continue
+            
+            pt = PersistentTemporaryFile('.'+_fmt.lower())
+            pt.write(data)
+            pt.close()
+            of = PersistentTemporaryFile('.'+fmt)
+            of.close()
+            setattr(options, 'output', of.name)
+            options.verbose = 1
+            args = [pt.name, options]
+            desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
+            jobs.append(('comic2'+fmt, args, desc, fmt.upper(), row_id, [pt, of]))
+                
+            changed = True
+
+    if bad_rows:
+        res = []
+        for row in bad_rows:
+            title = db.title(row)
+            res.append('<li>%s</li>'%title)
+        
+        msg = _('<p>Could not convert %d of %d books, because no suitable source format was found.<ul>%s</ul>')%(len(res), total, '\n'.join(res))
+        warning_dialog(parent, _('Could not convert some books'), msg).exec_()
+        
+    return jobs, changed, bad_rows
+
+def auto_convert_lrf(fmt, parent, db, comics, others):
+    pass
+
 def convert_single(fmt, parent, db, comics, others):
     changed = False
     jobs = []
@@ -386,6 +504,12 @@ def fetch_scheduled_recipe(recipe, script):
     args.append(script)
     return 'feeds2'+fmt, [args], _('Fetch news from ')+recipe.title, fmt.upper(), [pt]
             
+def auto_convert_ebook(*args):
+    fmt = args[0] if args[0] else 'epub'
+    if fmt == 'lrf':
+        return auto_convert_lrf()
+    elif fmt in ('epub', 'mobi'):
+        return auto_convert(*args)
 
 def convert_single_ebook(*args):
     fmt = prefs['output_format'].lower()
@@ -411,3 +535,4 @@ def set_conversion_defaults(comic, parent, db):
 def fetch_news(data):
     fmt = prefs['output_format'].lower()
     return _fetch_news(data, fmt)
+

src/calibre/gui2/viewer/documentview.py

@@ -450,7 +450,9 @@ class DocumentView(QWebView):
                 self.manager.scrolled(self.scroll_fraction)
             
     def wheel_event(self, down=True):
-        QWebView.wheelEvent(self, QWheelEvent(QPoint(100, 100), (-120 if down else 120), Qt.NoButton, Qt.NoModifier))
+        QWebView.wheelEvent(self, 
+            QWheelEvent(QPoint(100, 100), (-120 if down else 120), 
+                        Qt.NoButton, Qt.NoModifier))
     
     def next_page(self):
         if self.document.at_bottom:
@@ -538,6 +540,26 @@ class DocumentView(QWebView):
             self.next_page()
         elif key in [Qt.Key_PageUp, Qt.Key_Backspace, Qt.Key_Up]:
             self.previous_page()
+        elif key in [Qt.Key_Home]:
+            if event.modifiers() & Qt.ControlModifier:
+                if self.manager is not None:
+                    self.manager.goto_start()
+            else:
+                self.scroll_to(0)
+        elif key in [Qt.Key_End]:
+            if event.modifiers() & Qt.ControlModifier:
+                if self.manager is not None:
+                    self.manager.goto_end()
+            else:
+                self.scroll_to(1)
+        elif key in [Qt.Key_J]:
+            self.wheel_event()
+        elif key in [Qt.Key_K]:
+            self.wheel_event(down=False)
+        elif key in [Qt.Key_H]:
+            self.scroll_by(x=-15)
+        elif key in [Qt.Key_L]:
+            self.scroll_by(x=15)
         else:
             return QWebView.keyPressEvent(self, event)
         

src/calibre/gui2/viewer/main.py

@@ -342,6 +342,12 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         if pos is not None:
             self.goto_page(pos)
    
+    def goto_start(self):
+        self.goto_page(1)
+        
+    def goto_end(self):
+        self.goto_page(self.pos.maximum())
+    
     def goto_page(self, new_page):
         if self.current_page is not None:
             for page in self.iterator.spine:
@@ -604,6 +610,10 @@ def config(defaults=None):
         c = Config('viewer', desc)
     else:
         c = StringConfig(defaults, desc)
+        
+    c.add_opt('raise_window', ['--raise-window'], default=False, 
+              help=_('If specified, viewer window will try to come to the '
+                     'front when started.'))
     return c
 
 def option_parser():
@@ -617,7 +627,7 @@ View an ebook.
 
 def main(args=sys.argv):
     parser = option_parser()
-    args = parser.parse_args(args)[-1]
+    opts, args = parser.parse_args(args)
     pid = os.fork() if False and islinux else -1
     if pid <= 0:
         app = Application(args)
@@ -627,6 +637,8 @@ def main(args=sys.argv):
         main = EbookViewer(args[1] if len(args) > 1 else None)
         sys.excepthook = main.unhandled_exception
         main.show()
+        if opts.raise_window:
+            main.raise_()
         with main:
             return app.exec_()       
     return 0

src/calibre/library/cli.py

@@ -244,7 +244,10 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
             if os.path.isdir(path):
                 dirs.append(path)
             else:
+                if os.path.exists(path):
                     files.append(path)
+                else:
+                    print path, 'not found'
 
         formats, metadata = [], []
         for book in files:
@@ -262,12 +265,14 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
             formats.append(format)
             metadata.append(mi)
         
-        file_duplicates = db.add_books(files, formats, metadata, add_duplicates=add_duplicates)
-        if not file_duplicates[0]:
         file_duplicates = []
-        else:
+        if files:
+            file_duplicates = db.add_books(files, formats, metadata, 
+                                           add_duplicates=add_duplicates)
+            if file_duplicates:
                 file_duplicates = file_duplicates[0]
                 
+
         dir_dups = []
         for dir in dirs:
             if recurse:
@@ -286,7 +291,9 @@ def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
                 db.import_book(mi, formats)
         else:
             if dir_dups or file_duplicates:
-                print >>sys.stderr, _('The following books were not added as they already exist in the database (see --duplicates option):')
+                print >>sys.stderr, _('The following books were not added as '
+                                      'they already exist in the database '
+                                      '(see --duplicates option):')
             for mi, formats in dir_dups:
                 title = mi.title
                 if isinstance(title, unicode):

src/calibre/library/database2.py

@@ -15,7 +15,7 @@ from PyQt4.QtCore import QCoreApplication, QThread, QReadWriteLock
 from PyQt4.QtGui import QApplication, QImage
 __app = None
 
-from calibre.library import title_sort
+from calibre.ebooks.metadata import title_sort
 from calibre.library.database import LibraryDatabase
 from calibre.library.sqlite import connect, IntegrityError
 from calibre.utils.search_query_parser import SearchQueryParser
@@ -1567,3 +1567,4 @@ books_series_link      feeds
                     break
 
         return duplicates
+

src/calibre/library/server.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 HTTP server for remote access to the calibre database.
 '''
 
-import sys, textwrap, mimetypes, operator, os, re, logging
+import sys, textwrap, operator, os, re, logging
 from itertools import repeat
 from logging.handlers import RotatingFileHandler
 from datetime import datetime
@@ -18,11 +18,13 @@ from PyQt4.Qt import QImage, QApplication, QByteArray, Qt, QBuffer
 
 from calibre.constants import __version__, __appname__
 from calibre.utils.genshi.template import MarkupTemplate
-from calibre import fit_image
+from calibre import fit_image, guess_type
 from calibre.resources import jquery, server_resources, build_time
 from calibre.library import server_config as config
 from calibre.library.database2 import LibraryDatabase2, FIELD_MAP
 from calibre.utils.config import config_dir
+from calibre.utils.mdns import publish as publish_zeroconf, \
+                               stop_server as stop_zeroconf
 
 build_time = datetime.strptime(build_time, '%d %m %Y %H%M%S')
 server_resources['jquery.js'] = jquery
@@ -77,7 +79,7 @@ class LibraryServer(object):
         <id>urn:calibre:${record[FM['id']]}</id>
         <author><name>${authors}</name></author>
         <updated>${record[FM['timestamp']].strftime('%Y-%m-%dT%H:%M:%S+00:00')}</updated>
-        <link type="application/epub+zip" href="/get/epub/${record[FM['id']]}" />
+        <link type="${mimetype}" href="/get/${fmt}/${record[FM['id']]}" />
         <link rel="x-stanza-cover-image" type="image/jpeg" href="/get/cover/${record[FM['id']]}" />
         <link rel="x-stanza-cover-image-thumbnail" type="image/jpeg" href="/get/thumb/${record[FM['id']]}" />
         <content type="xhtml">
@@ -171,11 +173,14 @@ class LibraryServer(object):
         try:
             cherrypy.engine.start()
             self.is_running = True
+            publish_zeroconf('Books in calibre', '_stanza._tcp', 
+                             self.opts.port, {'path':'/stanza'})
             cherrypy.engine.block()
         except Exception, e:
             self.exception = e
         finally:
             self.is_running = False
+            stop_zeroconf()
         
     def exit(self):
         cherrypy.engine.exit()
@@ -218,7 +223,7 @@ class LibraryServer(object):
         fmt = self.db.format(id, format, index_is_id=True, as_file=True, mode='rb')
         if fmt is None:
             raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format))
-        mt = mimetypes.guess_type('dummy.'+format.lower())[0]
+        mt = guess_type('dummy.'+format.lower())[0]
         if mt is None:
             mt = 'application/octet-stream'
         cherrypy.response.headers['Content-Type'] = mt
@@ -258,8 +263,9 @@ class LibraryServer(object):
         for record in iter(self.db):
             r = record[FIELD_MAP['formats']]
             r = r.upper() if r else ''
-            if 'EPUB' in r:
-                authors = ' & '.join([i.replace('|', ',') for i in record[FIELD_MAP['authors']].split(',')])
+            if 'EPUB' in r or 'PDB' in r:
+                authors = ' & '.join([i.replace('|', ',') for i in 
+                                      record[FIELD_MAP['authors']].split(',')])
                 extra = []
                 rating = record[FIELD_MAP['rating']]
                 if rating > 0:
@@ -270,11 +276,17 @@ class LibraryServer(object):
                     extra.append('TAGS: %s<br />'%', '.join(tags.split(',')))
                 series = record[FIELD_MAP['series']]
                 if series:
-                    extra.append('SERIES: %s [%d]<br />'%(series, record[FIELD_MAP['series_index']]))
-                books.append(self.STANZA_ENTRY.generate(authors=authors,
+                    extra.append('SERIES: %s [%d]<br />'%(series, 
+                                            record[FIELD_MAP['series_index']]))
+                fmt = 'epub' if 'EPUB' in r else 'pdb'
+                mimetype = guess_type('dummy.'+fmt)[0]
+                books.append(self.STANZA_ENTRY.generate(
+                                                authors=authors,
                                                 record=record, FM=FIELD_MAP,
                                                 port=self.opts.port,
                                                 extra = ''.join(extra),
+                                                mimetype=mimetype,
+                                                fmt=fmt,
                                                 ).render('xml').decode('utf8'))
         
         updated = self.db.last_modified()
@@ -325,7 +337,11 @@ class LibraryServer(object):
     @expose
     def index(self, **kwargs):
         'The / URL'
+        stanza = cherrypy.request.headers.get('Stanza-Device-Name', 919)
+        if stanza == 919:
             return self.static('index.html')
+        return self.stanza()
+        
     
     @expose
     def get(self, what, id):

src/calibre/linux.py

@@ -1,9 +1,8 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 ''' Post installation script for linux '''
-import sys, os, re, shutil
+import sys, os, shutil
 from subprocess import check_call, call
-from tempfile import NamedTemporaryFile
 
 from calibre import __version__, __appname__
 from calibre.devices import devices
@@ -18,15 +17,8 @@ entry_points = {
         'console_scripts': [ \
              'ebook-device       = calibre.devices.prs500.cli.main:main',
              'ebook-meta         = calibre.ebooks.metadata.cli:main',
-             'txt2lrf            = calibre.ebooks.lrf.txt.convert_from:main',
-             'html2lrf           = calibre.ebooks.lrf.html.convert_from:main',
-             'html2oeb           = calibre.ebooks.html:main',
-             'html2epub          = calibre.ebooks.epub.from_html:main',
-             'odt2oeb            = calibre.ebooks.odt.to_oeb:main',
+             'ebook-convert      = calibre.ebooks.conversion.cli:main',
              'markdown-calibre   = calibre.ebooks.markdown.markdown:main',
-             'lit2lrf            = calibre.ebooks.lrf.lit.convert_from:main',
-             'epub2lrf           = calibre.ebooks.lrf.epub.convert_from:main',
-             'rtf2lrf            = calibre.ebooks.lrf.rtf.convert_from:main',
              'web2disk           = calibre.web.fetch.simple:main',
              'feeds2disk         = calibre.web.feeds.main:main',
              'calibre-server     = calibre.library.server:main',
@@ -34,22 +26,10 @@ entry_points = {
              'feeds2epub         = calibre.ebooks.epub.from_feeds:main',
              'feeds2mobi         = calibre.ebooks.mobi.from_feeds:main',
              'web2lrf            = calibre.ebooks.lrf.web.convert_from:main',
-             'pdf2lrf            = calibre.ebooks.lrf.pdf.convert_from:main',
-             'mobi2lrf           = calibre.ebooks.lrf.mobi.convert_from:main',
-             'fb22lrf            = calibre.ebooks.lrf.fb2.convert_from:main',
-             'any2lrf            = calibre.ebooks.lrf.any.convert_from:main',
-             'any2epub           = calibre.ebooks.epub.from_any:main',
-             'any2lit            = calibre.ebooks.lit.from_any:main',
-             'any2mobi           = calibre.ebooks.mobi.from_any:main',
              'lrf2lrs            = calibre.ebooks.lrf.lrfparser:main',
              'lrs2lrf            = calibre.ebooks.lrf.lrs.convert_from:main',
-             'pdfreflow          = calibre.ebooks.lrf.pdf.reflow:main',
              'isbndb             = calibre.ebooks.metadata.isbndb:main',
              'librarything       = calibre.ebooks.metadata.library_thing:main',
-             'mobi2oeb           = calibre.ebooks.mobi.reader:main',
-             'oeb2mobi           = calibre.ebooks.mobi.writer:main',
-             'lit2oeb            = calibre.ebooks.lit.reader:main',
-             'oeb2lit            = calibre.ebooks.lit.writer:main',
              'comic2lrf          = calibre.ebooks.lrf.comic.convert_from:main',
              'comic2epub         = calibre.ebooks.epub.from_comic:main',
              'comic2mobi         = calibre.ebooks.mobi.from_comic:main',
@@ -60,7 +40,6 @@ entry_points = {
              'calibre-parallel   = calibre.parallel:main',
              'calibre-customize  = calibre.customize.ui:main',
              'pdftrim            = calibre.ebooks.pdf.pdftrim:main' ,
-             'any2pdf  = calibre.ebooks.pdf.from_any:main',
         ],
         'gui_scripts'    : [
             __appname__+' = calibre.gui2.main:main',
@@ -171,22 +150,13 @@ def setup_completion(fatal_errors):
         from calibre.ebooks.lrf.lrfparser import option_parser as lrf2lrsop
         from calibre.gui2.lrf_renderer.main import option_parser as lrfviewerop
         from calibre.ebooks.lrf.pdf.reflow import option_parser as pdfhtmlop
-        from calibre.ebooks.mobi.reader import option_parser as mobioeb
-        from calibre.ebooks.lit.reader import option_parser as lit2oeb
         from calibre.web.feeds.main import option_parser as feeds2disk
         from calibre.web.feeds.recipes import titles as feed_titles
         from calibre.ebooks.lrf.feeds.convert_from import option_parser as feeds2lrf
         from calibre.ebooks.lrf.comic.convert_from import option_parser as comicop
-        from calibre.ebooks.epub.from_html import option_parser as html2epub
-        from calibre.ebooks.html import option_parser as html2oeb
-        from calibre.ebooks.odt.to_oeb import option_parser as odt2oeb
         from calibre.ebooks.epub.from_feeds import option_parser as feeds2epub
         from calibre.ebooks.mobi.from_feeds import option_parser as feeds2mobi
-        from calibre.ebooks.epub.from_any import option_parser as any2epub
-        from calibre.ebooks.lit.from_any import option_parser as any2lit
         from calibre.ebooks.epub.from_comic import option_parser as comic2epub
-        from calibre.ebooks.mobi.from_any import option_parser as any2mobi
-        from calibre.ebooks.mobi.writer import option_parser as oeb2mobi
         from calibre.gui2.main import option_parser as guiop
         any_formats = ['epub', 'htm', 'html', 'xhtml', 'xhtm', 'rar', 'zip',
              'txt', 'lit', 'rtf', 'pdf', 'prc', 'mobi', 'fb2', 'odt']
@@ -209,16 +179,10 @@ def setup_completion(fatal_errors):
         f.write(opts_and_exts('pdf2lrf', htmlop, ['pdf']))
         f.write(opts_and_exts('any2lrf', htmlop, any_formats))
         f.write(opts_and_exts('calibre', guiop, any_formats))
-        f.write(opts_and_exts('any2epub', any2epub, any_formats))
-        f.write(opts_and_exts('any2lit', any2lit, any_formats))
-        f.write(opts_and_exts('any2mobi', any2mobi, any_formats))
-        f.write(opts_and_exts('oeb2mobi', oeb2mobi, ['opf']))
         f.write(opts_and_exts('lrf2lrs', lrf2lrsop, ['lrf']))
         f.write(opts_and_exts('ebook-meta', metaop, list(meta_filetypes())))
         f.write(opts_and_exts('lrfviewer', lrfviewerop, ['lrf']))
         f.write(opts_and_exts('pdfrelow', pdfhtmlop, ['pdf']))
-        f.write(opts_and_exts('mobi2oeb', mobioeb, ['mobi', 'prc']))
-        f.write(opts_and_exts('lit2oeb', lit2oeb, ['lit']))
         f.write(opts_and_exts('comic2lrf', comicop, ['cbz', 'cbr']))
         f.write(opts_and_exts('comic2epub', comic2epub, ['cbz', 'cbr']))
         f.write(opts_and_exts('comic2mobi', comic2epub, ['cbz', 'cbr']))
@@ -227,9 +191,6 @@ def setup_completion(fatal_errors):
         f.write(opts_and_words('feeds2lrf', feeds2lrf, feed_titles))
         f.write(opts_and_words('feeds2epub', feeds2epub, feed_titles))
         f.write(opts_and_words('feeds2mobi', feeds2mobi, feed_titles))
-        f.write(opts_and_exts('html2epub', html2epub, ['html', 'htm', 'xhtm', 'xhtml', 'opf']))
-        f.write(opts_and_exts('html2oeb', html2oeb, ['html', 'htm', 'xhtm', 'xhtml']))
-        f.write(opts_and_exts('odt2oeb', odt2oeb, ['odt']))
         f.write('''
 _prs500_ls()
 {
@@ -392,43 +353,27 @@ def option_parser():
                       help='Save a manifest of all installed files to the specified location')
     return parser
 
-def install_man_pages(fatal_errors):
-    from bz2 import compress
-    import subprocess
+def install_man_pages(fatal_errors, use_destdir=False):
+    from calibre.utils.help2man import create_man_page
+    prefix = os.environ.get('DESTDIR', '/') if use_destdir else '/'
+    manpath = os.path.join(prefix, 'usr/share/man/man1')
+    if not os.path.exists(manpath):
+        os.makedirs(manpath)
     print 'Installing MAN pages...'
-    manpath = '/usr/share/man/man1'
-    f = NamedTemporaryFile()
-    f.write('[see also]\nhttp://%s.kovidgoyal.net\n'%__appname__)
-    f.flush()
     manifest = []
-    os.environ['PATH'] += ':'+os.path.expanduser('~/bin')
     for src in entry_points['console_scripts']:
-        prog = src[:src.index('=')].strip()
-        if prog in ('ebook-device', 'markdown-calibre', 
-                    'calibre-fontconfig', 'calibre-parallel'):
+        prog, right = src.split('=')
+        prog = prog.strip()
+        module = __import__(right.split(':')[0].strip(), fromlist=['a'])
+        parser = getattr(module, 'option_parser', None)
+        if parser is None:
             continue
-
-        help2man = ('help2man', prog, '--name', 'part of %s'%__appname__,
-                    '--section', '1', '--no-info', '--include',
-                    f.name, '--manual', __appname__)
+        parser = parser()
+        raw = create_man_page(prog, parser)
         manfile = os.path.join(manpath, prog+'.1'+__appname__+'.bz2')
         print '\tInstalling MAN page for', prog
-        try:
-            p = subprocess.Popen(help2man, stdout=subprocess.PIPE)
-        except OSError, err:
-            import errno
-            if err.errno != errno.ENOENT:
-                raise
-            print 'Failed to install MAN pages as help2man is missing from your system'
-            break
-        o = p.stdout.read()
-        raw = re.compile(r'^\.IP\s*^([A-Z :]+)$', re.MULTILINE).sub(r'.SS\n\1', o)
-        if not raw.strip():
-            print 'Unable to create MAN page for', prog
-            continue
-        f2 = open_file(manfile)
-        manifest.append(f2.name)
-        f2.write(compress(raw))
+        open(manfile, 'wb').write(raw)
+        manifest.append(manfile)
     return manifest
 
 def post_install():
@@ -440,9 +385,9 @@ def post_install():
     manifest = []
     setup_desktop_integration(opts.fatal_errors)
     if opts.no_root or os.geteuid() == 0:
+        manifest += install_man_pages(opts.fatal_errors, use_destdir)
         manifest += setup_udev_rules(opts.group_file, not opts.dont_reload, opts.fatal_errors)
         manifest += setup_completion(opts.fatal_errors)
-        manifest += install_man_pages(opts.fatal_errors)
     else:
         print "Skipping udev, completion, and man-page install for non-root user."
 

src/calibre/manual/faq.rst

@@ -34,6 +34,8 @@ What formats does |app| support conversion to/from?
 |                   |        |                  |                       |                       |
 |                   |  ODT   |       ✔          |          ✔            |          ✔            |
 |                   |        |                  |                       |                       |
+|                   |  FB2   |       ✔          |          ✔            |          ✔            |
+|                   |        |                  |                       |                       |
 |                   |  HTML  |       ✔          |          ✔            |          ✔            |
 |                   |        |                  |                       |                       |
 | **Input formats** |  CBR   |       ✔          |          ✔            |          ✔            |
@@ -121,13 +123,12 @@ turned into a collection on the reader. Note that the PRS-500 does not support c
 How do I use |app| with my iPhone?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 First install the Stanza reader on your iPhone from http://www.lexcycle.com . Then,
-  * Set the output format for calibre to EPUB (this can be done in the configuration dialog accessed by the little hammer icon next to the search bar)
+
+  * Set the output format for calibre to EPUB (The output format can be set next to the big red heart) 
   * Convert the books you want to read on your iPhone to EPUB format by selecting them and clicking the Convert button.
-  * Turn on the Content Server in the configurations dialog and leave |app| running.
-  * In the Stanza reader on your iPhone, add a new catalog. The URL of the catalog is of the form 
-    ``http://10.34.56.89:8080/stanza``,  where you should replace the IP address ``10.34.56.89`` 
-    with the IP address of your computer. Stanza will the use the |app| content server to access all the
-    EPUB books in your |app| database.
+  * Turn on the Content Server in |app|'s preferences and leave |app| running.
+
+Now you should be able to access your books on your iPhone. 
 
 Library Management
 ------------------

src/calibre/parallel.py

@@ -227,7 +227,8 @@ class WorkerMother(object):
         return env
 
     def spawn_free_spirit_osx(self, arg, type='free_spirit'):
-        script = 'from calibre.parallel import main; main(args=["calibre-parallel", %s]);'%repr(arg)
+        script = ('from calibre.parallel import main; '
+                 'main(args=["calibre-parallel", %s]);')%repr(arg)
         exe = self.gui_executable if type == 'free_spirit' else self.executable
         cmdline = [exe, '-c', self.prefix+script]
         child = WorkerStatus(subprocess.Popen(cmdline, env=self.get_env()))

src/calibre/trac/donations/server.py

@@ -196,7 +196,7 @@ class Server(object):
     
     def calculate_month_trend(self, days=31):
         stats = self.get_slice(date.today()-timedelta(days=days-1), date.today())
-        fig = plt.figure(2, (12, 4), 96)#, facecolor, edgecolor, frameon, FigureClass)
+        fig = plt.figure(2, (10, 4), 96)#, facecolor, edgecolor, frameon, FigureClass)
         fig.clear()
         ax = fig.add_subplot(111)
         x = list(range(days-1, -1, -1))
@@ -216,7 +216,7 @@ Donors per day: %(dpd).2f
                  ad=stats.average_deviation,
                  dpd=len(stats.totals)/float(stats.period.days),
              )
-        text = ax.annotate(text, (0.6, 0.65), textcoords='axes fraction')
+        text = ax.annotate(text, (0.5, 0.65), textcoords='axes fraction')
         fig.savefig(self.MONTH_TRENDS)
     
     def calculate_trend(self):

src/calibre/trac/plugins/download.py

@@ -18,7 +18,6 @@ DEPENDENCIES = [
             ('lxml', '2.1.5', 'lxml', 'python-lxml', 'python-lxml'),
             ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil'),
             ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'),
-            ('help2man', '1.36.4', 'help2man', 'help2man', 'help2man'),
             ]
         
 
@@ -34,9 +33,10 @@ def get_linux_data(version='1.0.0'):
     data['title'] = 'Download calibre for linux'
     data['supported'] = []
     for name, title in [
-                        ('ubuntu', 'Ubuntu Jaunty Jackalope'),
                         ('debian', 'Debian Sid'),
                         ('exherbo', 'Exherbo'),
+                        ('foresight', 'Foresight 2.1'),
+                        ('ubuntu', 'Ubuntu Jaunty Jackalope'),
                         ]:
         data['supported'].append(CoolDistro(name, title, 
                                         prefix='http://calibre.kovidgoyal.net'))
@@ -177,11 +177,11 @@ else:
                 compatibility='%s works on OS X Tiger and above.'%(__appname__,),
                 path=MOBILEREAD+file, app=__appname__,
                 note=Markup(\
-    '''
+    u'''
     <ol>
     <li>Before trying to use the command line tools, you must run the app at least once. This will ask you for you password and then setup the symbolic links for the command line tools.</li>
     <li>The app cannot be run from within the dmg. You must drag it to a folder on your filesystem (The Desktop, Applications, wherever).</li> 
-    <li>In order for localization of the user interface in your language, select your language in the configuration dialog (by clicking the hammer icon next to the search bar) and select your language.</li>
+    <li>In order for localization of the user interface in your language, select your language in the preferences (by pressing u\2318+P) and select your language.</li>
     </ol>
     '''))
             return 'binary.html', data, None

src/calibre/trac/plugins/templates/linux.html

@@ -88,7 +88,7 @@ sudo python -c "import urllib2; exec urllib2.urlopen('http://calibre.kovidgoyal.
                                     be ignored.
                                 </li>
                                 <li>
-                                    You must have help2man and xdg-utils installed 
+                                    You must have xdg-utils installed 
                                     on your system before running the installer.
                                 </li>
                                 <li>

src/calibre/utils/help2man.py

@@ -0,0 +1,59 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import time, bz2
+
+from calibre.constants import __version__, __appname__, __author__
+
+
+def create_man_page(prog, parser):
+    usage  = parser.usage.splitlines()
+    for i, line in enumerate(list(usage)):
+        if not line.strip():
+            usage[i] = '.PP'
+        else:
+            usage[i] = line.replace('%prog', prog)
+    lines = [
+             '.TH ' + prog.upper() + ' "1" ' + time.strftime('"%B %Y"') + 
+             ' "%s (%s %s)" "%s"'%(prog, __appname__, __version__, __appname__),
+             '.SH NAME',
+             prog + r' \- part of '+__appname__,
+             '.SH SYNOPSIS',
+             '.B "%s"'%prog + r'\fR '+' '.join(usage[0].split()[1:]),
+             '.SH DESCRIPTION',
+             ]
+    lines += usage[1:]
+    
+    lines += [
+              '.SH OPTIONS'
+              ]
+    def format_option(opt):
+        ans = ['.TP']
+        opts = []
+        opts += opt._short_opts
+        opts.append(opt.get_opt_string())
+        opts = [r'\fB'+x.replace('-', r'\-')+r'\fR' for x in opts]
+        ans.append(', '.join(opts))
+        help = opt.help if opt.help else ''
+        ans.append(help.replace('%prog', prog).replace('%default', str(opt.default)))
+        return ans
+    
+    for opt in parser.option_list:
+        lines.extend(format_option(opt))
+    for group in parser.option_groups:
+        lines.append('.SS '+group.title)
+        if group.description:
+            lines.extend(['.PP', group.description])
+        for opt in group.option_list:
+            lines.extend(format_option(opt))
+    
+    lines += ['.SH SEE ALSO', 
+              'The User Manual is available at '
+              'http://calibre.kovidgoyal.net/user_manual',
+              '.PP', '.B Created by '+__author__]
+    
+    return  bz2.compress('\n'.join(lines))
+
+

src/calibre/utils/logging.py

@@ -13,13 +13,25 @@ ERROR = 3
 import sys, traceback
 from functools import partial
 
-from calibre import prints
-from calibre.utils.terminfo import TerminalController
 
-class ANSIStream:
+
+
+class Stream(object):
+    
+    def __init__(self, stream):
+        from calibre import prints
+        self._prints = prints
+        self.stream = stream
+        
+    def flush(self):
+        self.stream.flush()
+
+
+class ANSIStream(Stream):
     
     def __init__(self, stream=sys.stdout):
-        self.stream = stream
+        Stream.__init__(self, stream)
+        from calibre.utils.terminfo import TerminalController
         tc = TerminalController(stream)
         self.color = {
                       DEBUG: tc.GREEN,
@@ -32,16 +44,16 @@ class ANSIStream:
     def prints(self, level, *args, **kwargs):
         self.stream.write(self.color[level])
         kwargs['file'] = self.stream
-        prints(*args, **kwargs)
+        self._prints(*args, **kwargs)
         self.stream.write(self.normal)
         
     def flush(self):
         self.stream.flush()
         
-class HTMLStream:
+class HTMLStream(Stream):
     
     def __init__(self, stream=sys.stdout):
-        self.stream = stream
+        Stream.__init__(self, stream)
         self.color = {
                       DEBUG: '<span style="color:green">',
                       INFO:'<span>',
@@ -53,7 +65,7 @@ class HTMLStream:
     def prints(self, level, *args, **kwargs):
         self.stream.write(self.color[level])
         kwargs['file'] = self.stream
-        prints(*args, **kwargs)
+        self._prints(*args, **kwargs)
         self.stream.write(self.normal)
         
     def flush(self):

src/calibre/utils/mdns.py

@@ -0,0 +1,60 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import socket
+
+_server = None
+
+def get_external_ip():
+    'Get IP address of interface used to connect to the outside world'
+    try:
+        ipaddr = socket.gethostbyname(socket.gethostname())
+    except:
+        ipaddr = '127.0.0.1'
+    if ipaddr == '127.0.0.1':
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            s.connect(('google.com', 0))
+            ipaddr = s.getsockname()[0]
+        except:
+            pass
+    return ipaddr
+
+def start_server():
+    global _server
+    if _server is None:
+        from calibre.utils.Zeroconf import Zeroconf
+        _server = Zeroconf()
+    return _server
+
+def publish(desc, type, port, properties=None, add_hostname=True):
+    '''
+    Publish a service.
+    
+    :param desc: Description of service
+    :param type: Name and type of service. For example _stanza._tcp
+    :param port: Port the service listens on
+    :param properties: An optional dictionary whose keys and values will be put 
+                       into the TXT record. 
+    '''
+    port = int(port)
+    server = start_server()
+    hostname = socket.gethostname().partition('.')[0]
+    if add_hostname:
+        desc += ' (on %s)'%hostname
+    local_ip = get_external_ip()
+    type = type+'.local.'
+    from calibre.utils.Zeroconf import ServiceInfo
+    service = ServiceInfo(type, desc+'.'+type,
+                          address=socket.inet_aton(local_ip),
+                          port=port,
+                          properties=properties,
+                          server=hostname+'.local.')
+    server.registerService(service)
+    
+def stop_server():
+    global _server
+    if _server is not None:
+        _server.close()

src/calibre/utils/rss_gen.py

@@ -0,0 +1,443 @@
+"""PyRSS2Gen - A Python library for generating RSS 2.0 feeds."""
+
+__name__ = "PyRSS2Gen"
+__version__ = (1, 0, 0)
+__author__ = "Andrew Dalke <dalke@dalkescientific.com>"
+
+_generator_name = __name__ + "-" + ".".join(map(str, __version__))
+
+import datetime
+
+# Could make this the base class; will need to add 'publish'
+class WriteXmlMixin:
+    def write_xml(self, outfile, encoding = "iso-8859-1"):
+        from xml.sax import saxutils
+        handler = saxutils.XMLGenerator(outfile, encoding)
+        handler.startDocument()
+        self.publish(handler)
+        handler.endDocument()
+
+    def to_xml(self, encoding = "iso-8859-1"):
+        try:
+            import cStringIO as StringIO
+        except ImportError:
+            import StringIO
+        f = StringIO.StringIO()
+        self.write_xml(f, encoding)
+        return f.getvalue()
+
+
+def _element(handler, name, obj, d = {}):
+    if isinstance(obj, basestring) or obj is None:
+        # special-case handling to make the API easier
+        # to use for the common case.
+        handler.startElement(name, d)
+        if obj is not None:
+            handler.characters(obj)
+        handler.endElement(name)
+    else:
+        # It better know how to emit the correct XML.
+        obj.publish(handler)
+
+def _opt_element(handler, name, obj):
+    if obj is None:
+        return
+    _element(handler, name, obj)
+
+
+def _format_date(dt):
+    """convert a datetime into an RFC 822 formatted date
+
+    Input date must be in GMT.
+    """
+    # Looks like:
+    #   Sat, 07 Sep 2002 00:00:01 GMT
+    # Can't use strftime because that's locale dependent
+    #
+    # Isn't there a standard way to do this for Python?  The
+    # rfc822 and email.Utils modules assume a timestamp.  The
+    # following is based on the rfc822 module.
+    return "%s, %02d %s %04d %02d:%02d:%02d GMT" % (
+            ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"][dt.weekday()],
+            dt.day,
+            ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
+             "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"][dt.month-1],
+            dt.year, dt.hour, dt.minute, dt.second)
+
+        
+##
+# A couple simple wrapper objects for the fields which
+# take a simple value other than a string.
+class IntElement:
+    """implements the 'publish' API for integers
+
+    Takes the tag name and the integer value to publish.
+    
+    (Could be used for anything which uses str() to be published
+    to text for XML.)
+    """
+    element_attrs = {}
+    def __init__(self, name, val):
+        self.name = name
+        self.val = val
+    def publish(self, handler):
+        handler.startElement(self.name, self.element_attrs)
+        handler.characters(str(self.val))
+        handler.endElement(self.name)
+
+class DateElement:
+    """implements the 'publish' API for a datetime.datetime
+
+    Takes the tag name and the datetime to publish.
+
+    Converts the datetime to RFC 2822 timestamp (4-digit year).
+    """
+    def __init__(self, name, dt):
+        self.name = name
+        self.dt = dt
+    def publish(self, handler):
+        _element(handler, self.name, _format_date(self.dt))
+####
+
+class Category:
+    """Publish a category element"""
+    def __init__(self, category, domain = None):
+        self.category = category
+        self.domain = domain
+    def publish(self, handler):
+        d = {}
+        if self.domain is not None:
+            d["domain"] = self.domain
+        _element(handler, "category", self.category, d)
+
+class Cloud:
+    """Publish a cloud"""
+    def __init__(self, domain, port, path,
+                 registerProcedure, protocol):
+        self.domain = domain
+        self.port = port
+        self.path = path
+        self.registerProcedure = registerProcedure
+        self.protocol = protocol
+    def publish(self, handler):
+        _element(handler, "cloud", None, {
+            "domain": self.domain,
+            "port": str(self.port),
+            "path": self.path,
+            "registerProcedure": self.registerProcedure,
+            "protocol": self.protocol})
+
+class Image:
+    """Publish a channel Image"""
+    element_attrs = {}
+    def __init__(self, url, title, link,
+                 width = None, height = None, description = None):
+        self.url = url
+        self.title = title
+        self.link = link
+        self.width = width
+        self.height = height
+        self.description = description
+        
+    def publish(self, handler):
+        handler.startElement("image", self.element_attrs)
+
+        _element(handler, "url", self.url)
+        _element(handler, "title", self.title)
+        _element(handler, "link", self.link)
+
+        width = self.width
+        if isinstance(width, int):
+            width = IntElement("width", width)
+        _opt_element(handler, "width", width)
+        
+        height = self.height
+        if isinstance(height, int):
+            height = IntElement("height", height)
+        _opt_element(handler, "height", height)
+
+        _opt_element(handler, "description", self.description)
+
+        handler.endElement("image")
+
+class Guid:
+    """Publish a guid
+
+    Defaults to being a permalink, which is the assumption if it's
+    omitted.  Hence strings are always permalinks.
+    """
+    def __init__(self, guid, isPermaLink = 1):
+        self.guid = guid
+        self.isPermaLink = isPermaLink
+    def publish(self, handler):
+        d = {}
+        if self.isPermaLink:
+            d["isPermaLink"] = "true"
+        else:
+            d["isPermaLink"] = "false"
+        _element(handler, "guid", self.guid, d)
+
+class TextInput:
+    """Publish a textInput
+
+    Apparently this is rarely used.
+    """
+    element_attrs = {}
+    def __init__(self, title, description, name, link):
+        self.title = title
+        self.description = description
+        self.name = name
+        self.link = link
+
+    def publish(self, handler):
+        handler.startElement("textInput", self.element_attrs)
+        _element(handler, "title", self.title)
+        _element(handler, "description", self.description)
+        _element(handler, "name", self.name)
+        _element(handler, "link", self.link)
+        handler.endElement("textInput")
+        
+
+class Enclosure:
+    """Publish an enclosure"""
+    def __init__(self, url, length, type):
+        self.url = url
+        self.length = length
+        self.type = type
+    def publish(self, handler):
+        _element(handler, "enclosure", None,
+                 {"url": self.url,
+                  "length": str(self.length),
+                  "type": self.type,
+                  })
+
+class Source:
+    """Publish the item's original source, used by aggregators"""
+    def __init__(self, name, url):
+        self.name = name
+        self.url = url
+    def publish(self, handler):
+        _element(handler, "source", self.name, {"url": self.url})
+
+class SkipHours:
+    """Publish the skipHours
+
+    This takes a list of hours, as integers.
+    """
+    element_attrs = {}
+    def __init__(self, hours):
+        self.hours = hours
+    def publish(self, handler):
+        if self.hours:
+            handler.startElement("skipHours", self.element_attrs)
+            for hour in self.hours:
+                _element(handler, "hour", str(hour))
+            handler.endElement("skipHours")
+
+class SkipDays:
+    """Publish the skipDays
+
+    This takes a list of days as strings.
+    """
+    element_attrs = {}
+    def __init__(self, days):
+        self.days = days
+    def publish(self, handler):
+        if self.days:
+            handler.startElement("skipDays", self.element_attrs)
+            for day in self.days:
+                _element(handler, "day", day)
+            handler.endElement("skipDays")
+
+class RSS2(WriteXmlMixin):
+    """The main RSS class.
+
+    Stores the channel attributes, with the "category" elements under
+    ".categories" and the RSS items under ".items".
+    """
+    
+    rss_attrs = {"version": "2.0"}
+    element_attrs = {}
+    def __init__(self,
+                 title,
+                 link,
+                 description,
+
+                 language = None,
+                 copyright = None,
+                 managingEditor = None,
+                 webMaster = None,
+                 pubDate = None,  # a datetime, *in* *GMT*
+                 lastBuildDate = None, # a datetime
+                 
+                 categories = None, # list of strings or Category
+                 generator = _generator_name,
+                 docs = "http://blogs.law.harvard.edu/tech/rss",
+                 cloud = None,    # a Cloud
+                 ttl = None,      # integer number of minutes
+
+                 image = None,     # an Image
+                 rating = None,    # a string; I don't know how it's used
+                 textInput = None, # a TextInput
+                 skipHours = None, # a SkipHours with a list of integers
+                 skipDays = None,  # a SkipDays with a list of strings
+
+                 items = None,     # list of RSSItems
+                 ):
+        self.title = title
+        self.link = link
+        self.description = description
+        self.language = language
+        self.copyright = copyright
+        self.managingEditor = managingEditor
+
+        self.webMaster = webMaster
+        self.pubDate = pubDate
+        self.lastBuildDate = lastBuildDate
+        
+        if categories is None:
+            categories = []
+        self.categories = categories
+        self.generator = generator
+        self.docs = docs
+        self.cloud = cloud
+        self.ttl = ttl
+        self.image = image
+        self.rating = rating
+        self.textInput = textInput
+        self.skipHours = skipHours
+        self.skipDays = skipDays
+
+        if items is None:
+            items = []
+        self.items = items
+
+    def publish(self, handler):
+        handler.startElement("rss", self.rss_attrs)
+        handler.startElement("channel", self.element_attrs)
+        _element(handler, "title", self.title)
+        _element(handler, "link", self.link)
+        _element(handler, "description", self.description)
+
+        self.publish_extensions(handler)
+        
+        _opt_element(handler, "language", self.language)
+        _opt_element(handler, "copyright", self.copyright)
+        _opt_element(handler, "managingEditor", self.managingEditor)
+        _opt_element(handler, "webMaster", self.webMaster)
+
+        pubDate = self.pubDate
+        if isinstance(pubDate, datetime.datetime):
+            pubDate = DateElement("pubDate", pubDate)
+        _opt_element(handler, "pubDate", pubDate)
+
+        lastBuildDate = self.lastBuildDate
+        if isinstance(lastBuildDate, datetime.datetime):
+            lastBuildDate = DateElement("lastBuildDate", lastBuildDate)
+        _opt_element(handler, "lastBuildDate", lastBuildDate)
+
+        for category in self.categories:
+            if isinstance(category, basestring):
+                category = Category(category)
+            category.publish(handler)
+
+        _opt_element(handler, "generator", self.generator)
+        _opt_element(handler, "docs", self.docs)
+
+        if self.cloud is not None:
+            self.cloud.publish(handler)
+
+        ttl = self.ttl
+        if isinstance(self.ttl, int):
+            ttl = IntElement("ttl", ttl)
+        _opt_element(handler, "tt", ttl)
+
+        if self.image is not None:
+            self.image.publish(handler)
+
+        _opt_element(handler, "rating", self.rating)
+        if self.textInput is not None:
+            self.textInput.publish(handler)
+        if self.skipHours is not None:
+            self.skipHours.publish(handler)
+        if self.skipDays is not None:
+            self.skipDays.publish(handler)
+
+        for item in self.items:
+            item.publish(handler)
+
+        handler.endElement("channel")
+        handler.endElement("rss")
+
+    def publish_extensions(self, handler):
+        # Derived classes can hook into this to insert
+        # output after the three required fields.
+        pass
+
+    
+    
+class RSSItem(WriteXmlMixin):
+    """Publish an RSS Item"""
+    element_attrs = {}
+    def __init__(self,
+                 title = None,  # string
+                 link = None,   # url as string
+                 description = None, # string
+                 author = None,      # email address as string
+                 categories = None,  # list of string or Category
+                 comments = None,  # url as string
+                 enclosure = None, # an Enclosure
+                 guid = None,    # a unique string
+                 pubDate = None, # a datetime
+                 source = None,  # a Source
+                 ):
+        
+        if title is None and description is None:
+            raise TypeError(
+                "must define at least one of 'title' or 'description'")
+        self.title = title
+        self.link = link
+        self.description = description
+        self.author = author
+        if categories is None:
+            categories = []
+        self.categories = categories
+        self.comments = comments
+        self.enclosure = enclosure
+        self.guid = guid
+        self.pubDate = pubDate
+        self.source = source
+        # It sure does get tedious typing these names three times...
+
+    def publish(self, handler):
+        handler.startElement("item", self.element_attrs)
+        _opt_element(handler, "title", self.title)
+        _opt_element(handler, "link", self.link)
+        self.publish_extensions(handler)
+        _opt_element(handler, "description", self.description)
+        _opt_element(handler, "author", self.author)
+
+        for category in self.categories:
+            if isinstance(category, basestring):
+                category = Category(category)
+            category.publish(handler)
+        
+        _opt_element(handler, "comments", self.comments)
+        if self.enclosure is not None:
+            self.enclosure.publish(handler)
+        _opt_element(handler, "guid", self.guid)
+
+        pubDate = self.pubDate
+        if isinstance(pubDate, datetime.datetime):
+            pubDate = DateElement("pubDate", pubDate)
+        _opt_element(handler, "pubDate", pubDate)
+
+        if self.source is not None:
+            self.source.publish(handler)
+        
+        handler.endElement("item")
+
+    def publish_extensions(self, handler):
+        # Derived classes can hook into this to insert
+        # output after the title and link elements
+        pass

src/calibre/web/feeds/__init__.py

@@ -156,7 +156,6 @@ class Feed(object):
             content = None
         if not link and not content:
             return
-        
         article = Article(id, title, link, description, published, content)
         delta = datetime.utcnow() - article.utctime
         if delta.days*24*3600 + delta.seconds <= 24*3600*self.oldest_article:

src/calibre/web/feeds/news.py

@@ -17,7 +17,7 @@ from PyQt4.Qt import QApplication, QFile, Qt, QPalette, QSize, QImage, QPainter,
 from PyQt4.QtWebKit import QWebPage
 
 
-from calibre import browser, __appname__, iswindows, LoggingInterface, \
+from calibre import browser, __appname__, iswindows, \
                     strftime, __version__, preferred_encoding
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
 from calibre.ebooks.metadata.opf2 import OPFCreator
@@ -32,7 +32,7 @@ from calibre.ptempfile import PersistentTemporaryFile
 from calibre.gui2 import images_rc # Needed for default cover
 
 
-class BasicNewsRecipe(object, LoggingInterface):
+class BasicNewsRecipe(object):
     '''
     Abstract base class that contains logic needed in all feed fetchers.
     '''
@@ -444,7 +444,6 @@ class BasicNewsRecipe(object, LoggingInterface):
         :param parser:  Command line option parser. Used to intelligently merge options.
         :param progress_reporter: A Callable that takes two arguments: progress (a number between 0 and 1) and a string message. The message should be optional.
         '''
-        LoggingInterface.__init__(self, logging.getLogger('feeds2disk'))
         if not isinstance(self.title, unicode):
             self.title = unicode(self.title, 'utf-8', 'replace')
         
@@ -1013,6 +1012,7 @@ class BasicNewsRecipe(object, LoggingInterface):
                 parsed_feeds.append(feed)
                 self.log_exception(msg)
         
+        
         return parsed_feeds
     
     @classmethod

src/calibre/web/feeds/recipes/__init__.py

@@ -5,8 +5,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Builtin recipes.
 '''
 recipe_modules = ['recipe_' + r for r in (
-           'newsweek', 'atlantic', 'economist', 'portfolio',
-           'nytimes', 'usatoday', 'outlook_india', 'bbc', 'greader', 'wsj',
+           'newsweek', 'atlantic', 'economist', 'portfolio', 'the_register',
+           'usatoday', 'outlook_india', 'bbc', 'greader', 'wsj',
            'wired', 'globe_and_mail', 'smh', 'espn', 'business_week', 'miami_herald',
            'ars_technica', 'upi', 'new_yorker', 'irish_times', 'iht', 'lanacion',
            'discover_magazine', 'scientific_american', 'new_york_review_of_books',
@@ -33,7 +33,8 @@ recipe_modules = ['recipe_' + r for r in (
            'la_republica', 'physics_today', 'chicago_tribune', 'e_novine',
            'al_jazeera', 'winsupersite', 'borba', 'courrierinternational',
            'lamujerdemivida', 'soldiers', 'theonion', 'news_times',
-           'el_universal', 'mediapart',
+           'el_universal', 'mediapart', 'wikinews_en', 'ecogeek', 'daily_mail',
+           'new_york_review_of_books_no_sub', 'politico',
           )]
 
 import re, imp, inspect, time, os
@@ -86,11 +87,15 @@ def compile_recipe(src):
         match = re.search(r'coding[:=]\s*([-\w.]+)', src[:200])
         enc = match.group(1) if match else 'utf-8'
         src = src.decode(enc)
+    src = re.sub(r'from __future__.*', '', src)
     f = open(temp, 'wb')
     src = 'from %s.web.feeds.news import BasicNewsRecipe, AutomaticNewsRecipe\n'%__appname__ + src
     src = 'from %s.ebooks.lrf.web.profiles import DefaultProfile, FullContentProfile\n'%__appname__ + src
     src = '# coding: utf-8\n' + src
-    f.write(src.replace('from libprs500', 'from calibre').encode('utf-8'))
+    src = 'from __future__ import with_statement\n' + src
+    
+    src = src.replace('from libprs500', 'from calibre').encode('utf-8')
+    f.write(src)
     f.close()
     module = imp.find_module(temp.namebase, [temp.dirname()])
     module = imp.load_module(temp.namebase, *module)

src/calibre/web/feeds/recipes/recipe_daily_mail.py

@@ -0,0 +1,33 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class TheDailyMail(BasicNewsRecipe):
+    title          = u'The Daily Mail'
+    oldest_article = 2
+    language = _('English')
+    author = 'RufusA'
+    simultaneous_downloads= 1
+    max_articles_per_feed = 50
+
+    extra_css = 'h1 {text-align: left;}'
+
+    remove_tags = [ dict(name='ul', attrs={'class':'article-icons-links'}) ]
+    remove_tags_after  = dict(name='h3', attrs={'class':'social-links-title'})
+    remove_tags_before  = dict(name='div', attrs={'id':'content'})
+    no_stylesheets = True
+    
+    feeds          = [
+	(u'Home', u'http://www.dailymail.co.uk/home/index.rss'),
+	(u'News', u'http://www.dailymail.co.uk/news/index.rss'),
+	(u'Sport', u'http://www.dailymail.co.uk/sport/index.rss'),
+	(u'TV and Showbiz', u'http://www.dailymail.co.uk/tvshowbiz/index.rss'),
+	(u'Femail', u'http://www.dailymail.co.uk/femail/index.rss'),
+	(u'Health', u'http://www.dailymail.co.uk/health/index.rss'),
+	(u'Science and Technology', u'http://www.dailymail.co.uk/sciencetech/index.rss'),
+	(u'Money', u'http://www.dailymail.co.uk/money/index.rss'),
+	(u'Property', u'http://www.dailymail.co.uk/property/index.rss'),
+	(u'Motoring', u'http://www.dailymail.co.uk/motoring/index.rss'),
+	(u'Travel', u'http://www.dailymail.co.uk/travel/index.rss')]
+
+    def print_version(self, url):
+        main = url.partition('?')[0]
+        return main + '?printingPage=true'

src/calibre/web/feeds/recipes/recipe_ecogeek.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+EcoGeek.org
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class EcoGeek(BasicNewsRecipe):
+    title                 = 'EcoGeek'
+    __author__            = 'Darko Miletic'
+    description           = 'EcoGeek - Technology for the Environment Blog Feed'    
+    publisher             = 'EcoGeek'
+    language              = _('English')
+    category              = 'news, ecology, blog'
+    oldest_article        = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = True
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+
+    feeds = [(u'Posts', u'http://feeds2.feedburner.com/EcoGeek')]

src/calibre/web/feeds/recipes/recipe_iht.py

@@ -3,6 +3,7 @@ __copyright__ = '2008, Derry FitzGerald'
 '''
 iht.com
 '''
+import re
 
 from calibre.web.feeds.news import BasicNewsRecipe
 from calibre.ptempfile import PersistentTemporaryFile
@@ -16,7 +17,12 @@ class InternationalHeraldTribune(BasicNewsRecipe):
     max_articles_per_feed = 10
     no_stylesheets = True
 
-    remove_tags    = [dict(name='div', attrs={'class':'footer'})]
+    remove_tags    = [dict(name='div', attrs={'class':'footer'}),
+                      dict(name=['form'])]
+    preprocess_regexps = [
+            (re.compile(r'<!-- webtrends.*', re.DOTALL), 
+             lambda m:'</body></html>')
+                          ]
     extra_css      = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt  }' 
 
     feeds          = [

src/calibre/web/feeds/recipes/recipe_new_scientist.py

@@ -1,14 +1,7 @@
 #!/usr/bin/env  python
 
 __license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
-'''
-newscientist.com
-'''
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>, AprilHare'
+__copyright__ = '2008-2009, AprilHare, Darko Miletic <darko.miletic at gmail.com>'
 '''
 newscientist.com
 '''
@@ -16,23 +9,34 @@ newscientist.com
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class NewScientist(BasicNewsRecipe):
-    title                 = u'New Scientist - Online News'
+    title                 = 'New Scientist - Online News'
     __author__            = 'Darko Miletic'
-    description           = 'News from Science'
+    description           = 'Science news and science articles from New Scientist.'
     language              = _('English')
+    publisher             = 'New Scientist'
+    category              = 'science news, science articles, science jobs, drugs, cancer, depression, computer software, sex'    
+    delay                 = 3    
     oldest_article        = 7
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False
+    remove_javascript     = True
+    encoding              = 'utf-8'
     
-    keep_only_tags = [
-                        dict(name='div'  , attrs={'id':'pgtop'   })
-                       ,dict(name='div'  , attrs={'id':'maincol' })
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
                         ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    
+    keep_only_tags = [dict(name='div', attrs={'id':['pgtop','maincol']})]
+
     remove_tags = [
-                     dict(name='div'  , attrs={'class':'hldBd' })
-                    ,dict(name='div'  , attrs={'id':'compnl' })
-                    ,dict(name='div'  , attrs={'id':'artIssueInfo' })
+                     dict(name='div', attrs={'class':['hldBd','adline','pnl','infotext' ]})
+                    ,dict(name='div', attrs={'id'   :['compnl','artIssueInfo','artTools']})
+                    ,dict(name='p'  , attrs={'class':['marker','infotext'               ]})
                   ]
 
     feeds          = [
@@ -46,3 +50,20 @@ class NewScientist(BasicNewsRecipe):
                        ,(u'Science in Society'      , u'http://www.newscientist.com/feed/view?id=5&type=channel' )
                        ,(u'Tech'                    , u'http://www.newscientist.com/feed/view?id=7&type=channel' )
                      ]
+
+    def get_article_url(self, article):
+        url = article.get('link',  None)
+        raw = article.get('description',  None)
+        rsoup = self.index_to_soup(raw)
+        atags = rsoup.findAll('a',href=True)
+        for atag in atags:
+            if atag['href'].startswith('http://res.feedsportal.com/viral/sendemail2.html?'):
+               st, sep, rest = atag['href'].partition('&link=')
+               real_url, sep2, drest = rest.partition('" target=')
+               return real_url
+        return url
+
+    def print_version(self, url):
+        rawurl, sep, params = url.partition('?')
+        return rawurl + '?full=true&print=true'
+                     

src/calibre/web/feeds/recipes/recipe_new_york_review_of_books_no_sub.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+nybooks.com
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+from lxml import html
+from calibre.constants import preferred_encoding
+
+class NewYorkReviewOfBooks(BasicNewsRecipe):
+    
+    title = u'New York Review of Books (no subscription)'
+    description = u'Book reviews'
+    language = _('English')
+    __author__ = 'Kovid Goyal' 
+    remove_tags_before = {'id':'container'}
+    remove_tags = [{'class':['noprint', 'ad', 'footer']}, {'id':'right-content'}]
+
+    def parse_index(self):
+        root = html.fromstring(self.browser.open('http://www.nybooks.com/current-issue').read())
+        date = root.xpath('//h4[@class = "date"]')[0]
+        self.timefmt = ' ['+date.text.encode(preferred_encoding)+']'
+        articles = []
+        for tag in date.itersiblings():
+            if tag.tag == 'h4': break
+            if tag.tag == 'p':
+                if tag.get('class') == 'indented':
+                    articles[-1]['description'] += html.tostring(tag)
+                else:
+                    href = tag.xpath('descendant::a[@href]')[0].get('href')
+                    article = {
+                               'title': u''.join(tag.xpath('descendant::text()')),
+                               'date' : '',
+                               'url'  : 'http://www.nybooks.com'+href,
+                               'description': '',
+                               }
+                    articles.append(article)
+                    
+        return [('Current Issue', articles)]
+