Sync to trunk

.pydevproject

@@ -2,8 +2,9 @@
 <?eclipse-pydev version="1.0"?>
 
 <pydev_project>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.5</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/calibre/src</path>
 </pydev_pathproperty>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
 </pydev_project>

setup.py

@@ -121,7 +121,7 @@ if __name__ == '__main__':
                 buf = cStringIO.StringIO()
                 print 'Creating translations template'
                 tempdir = tempfile.mkdtemp()
-                pygettext(buf, ['-p', tempdir]+files)
+                pygettext(buf, ['-k', '__', '-p', tempdir]+files)
                 src = buf.getvalue()
                 pot = os.path.join(tempdir, 'calibre.pot')
                 f = open(pot, 'wb')

src/calibre/__init__.py

@@ -20,6 +20,7 @@ import mechanize
 mimetypes.add_type('application/epub+zip', '.epub')
 mimetypes.add_type('text/x-sony-bbeb+xml', '.lrs')
 mimetypes.add_type('application/x-sony-bbeb', '.lrf')
+mimetypes.add_type('application/x-dtbncx+xml', '.ncx')
 
 def to_unicode(raw, encoding='utf-8', errors='strict'):
     if isinstance(raw, unicode):

src/calibre/devices/cybookg3/driver.py

@@ -10,6 +10,7 @@ from itertools import cycle
 from calibre.devices.errors import FreeSpaceError
 from calibre.devices.usbms.driver import USBMS
 import calibre.devices.cybookg3.t2b as t2b
+from calibre.devices.errors import FreeSpaceError
 
 class CYBOOKG3(USBMS):
     # Ordered list of supported formats

src/calibre/devices/usbms/device.py

@@ -3,10 +3,10 @@ __copyright__ = '2009, John Schember <john at nachtimwald.com>'
 '''
 Generic device driver. This is not a complete stand alone driver. It is
 intended to be subclassed with the relevant parts implemented for a particular
-device. This class handles devive detection.
+device. This class handles device detection.
 '''
 
-import os, re, subprocess, time
+import os, subprocess, time, re
 
 from calibre.devices.interface import Device as _Device
 from calibre.devices.errors import DeviceError
@@ -18,21 +18,21 @@ class Device(_Device):
     as USB Mass Storage devices. If you are writing such a driver, inherit from this
     class.
     '''
-    
+
     VENDOR_ID   = 0x0
     PRODUCT_ID  = 0x0
     BCD         = None
-    
+
     VENDOR_NAME = None
     WINDOWS_MAIN_MEM = None
     WINDOWS_CARD_MEM = None
-    
+
     OSX_MAIN_MEM = None
     OSX_CARD_MEM = None
-    
+
     MAIN_MEMORY_VOLUME_LABEL  = ''
     STORAGE_CARD_VOLUME_LABEL = ''
-        
+
     FDI_TEMPLATE = \
 '''
   <device>
@@ -65,15 +65,15 @@ class Device(_Device):
   </device>
 '''
     FDI_BCD_TEMPLATE = '<match key="@info.parent:@info.parent:@info.parent:@info.parent:usb.device_revision_bcd" int="%(bcd)s">'
-    
+
-    
+
     def __init__(self, key='-1', log_packets=False, report_progress=None) :
         self._main_prefix = self._card_prefix = None
-    
+
     @classmethod
     def get_fdi(cls):
         fdi = ''
-        
+
         fdi_base_values = dict(
                                app=__appname__,
                                deviceclass=cls.__name__,
@@ -92,12 +92,12 @@ class Device(_Device):
                 fdi_bcd_values['BCD_start'] = cls.FDI_BCD_TEMPLATE % dict(bcd=hex(bcd))
                 fdi_bcd_values['BCD_end'] = '</match>'
                 fdi += cls.FDI_TEMPLATE % fdi_bcd_values
-                
+
         return fdi
-    
+
     def set_progress_reporter(self, report_progress):
         self.report_progress = report_progress
-    
+
     def card_prefix(self, end_session=True):
         return self._card_prefix
 
@@ -117,7 +117,7 @@ class Device(_Device):
             else: raise
         mult = sectors_per_cluster * bytes_per_sector
         return total_clusters * mult, free_clusters * mult
-    
+
     def total_space(self, end_session=True):
         msz = csz = 0
         print self._main_prefix
@@ -131,9 +131,9 @@ class Device(_Device):
         else:
             msz = self._windows_space(self._main_prefix)[0]
             csz = self._windows_space(self._card_prefix)[0]
-                
+
         return (msz, 0, csz)
-    
+
     def free_space(self, end_session=True):
         msz = csz = 0
         if not iswindows:
@@ -146,15 +146,15 @@ class Device(_Device):
         else:
             msz = self._windows_space(self._main_prefix)[1]
             csz = self._windows_space(self._card_prefix)[1]
-                
+
         return (msz, 0, csz)
 
     def windows_match_device(self, pnp_id, device_id):
         pnp_id = pnp_id.upper()
-        
+
         if device_id and pnp_id is not None:
             device_id = device_id.upper()
-            
+
             if 'VEN_' + self.VENDOR_NAME in pnp_id and 'PROD_' + device_id in pnp_id:
                 return True
 
@@ -162,44 +162,45 @@ class Device(_Device):
 
     def windows_get_drive_prefix(self, drive):
         prefix = None
-        
+
         try:
             partition = drive.associators("Win32_DiskDriveToDiskPartition")[0]
             logical_disk = partition.associators('Win32_LogicalDiskToPartition')[0]
             prefix = logical_disk.DeviceID + os.sep
         except IndexError:
             pass
-            
+
         return prefix
 
     def open_windows(self):
         drives = {}
-        wmi = __import__('wmi', globals(), locals(), [], -1) 
+        wmi = __import__('wmi', globals(), locals(), [], -1)
         c = wmi.WMI()
         for drive in c.Win32_DiskDrive():
             if self.windows_match_device(str(drive.PNPDeviceID), self.WINDOWS_MAIN_MEM):
                 drives['main'] = self.windows_get_drive_prefix(drive)
             elif self.windows_match_device(str(drive.PNPDeviceID), self.WINDOWS_CARD_MEM):
                 drives['card'] = self.windows_get_drive_prefix(drive)
-                
+
             if 'main' and 'card' in drives.keys():
                 break
-                
+
         if not drives:
             raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.') % self.__class__.__name__)
-            
+
-        self._main_prefix = drives['main'] if 'main' in drives.keys() else None
+        self._main_prefix = drives.get('main', None)
-        self._card_prefix = drives['card'] if 'card' in drives.keys() else None
+        self._card_prefix = drives.get('card', None)
 
     def get_osx_mountpoints(self, raw=None):
         if raw is None:
             ioreg = '/usr/sbin/ioreg'
             if not os.access(ioreg, os.X_OK):
                 ioreg = 'ioreg'
-            raw = subprocess.Popen((ioreg+' -w 0 -S -c IOMedia').split(), stdout=subprocess.PIPE).stdout.read()
+            raw = subprocess.Popen((ioreg+' -w 0 -S -c IOMedia').split(),
+                                   stdout=subprocess.PIPE).stdout.read()
         lines = raw.splitlines()
         names = {}
-        
+
         def get_dev_node(lines, loc):
             for line in lines:
                 line = line.strip()
@@ -209,7 +210,7 @@ class Device(_Device):
                 if match is not None:
                     names[loc] = match.group(1)
                     break
-                    
+
         for i, line in enumerate(lines):
             if self.OSX_MAIN_MEM is not None and line.strip().endswith('<class IOMedia>') and self.OSX_MAIN_MEM in line:
                 get_dev_node(lines[i+1:], 'main')
@@ -218,7 +219,7 @@ class Device(_Device):
             if len(names.keys()) == 2:
                 break
         return names
-    
+
     def open_osx(self):
         mount = subprocess.Popen('mount', shell=True,  stdout=subprocess.PIPE).stdout.read()
         names = self.get_osx_mountpoints()
@@ -231,12 +232,12 @@ class Device(_Device):
         if card_pat is not None:
             card_pat = dev_pat % card_pat
             self._card_prefix = re.search(card_pat, mount).group(2) + os.sep
-            
+
     def open_linux(self):
         import dbus
-        bus = dbus.SystemBus() 
+        bus = dbus.SystemBus()
         hm  = dbus.Interface(bus.get_object("org.freedesktop.Hal", "/org/freedesktop/Hal/Manager"), "org.freedesktop.Hal.Manager")
-        
+
         def conditional_mount(dev):
             mmo = bus.get_object("org.freedesktop.Hal", dev)
             label = mmo.GetPropertyString('volume.label', dbus_interface='org.freedesktop.Hal.Device')
@@ -245,10 +246,10 @@ class Device(_Device):
             fstype = mmo.GetPropertyString('volume.fstype', dbus_interface='org.freedesktop.Hal.Device')
             if is_mounted:
                 return str(mount_point)
-            mmo.Mount(label, fstype, ['umask=077', 'uid='+str(os.getuid()), 'sync'], 
+            mmo.Mount(label, fstype, ['umask=077', 'uid='+str(os.getuid()), 'sync'],
                           dbus_interface='org.freedesktop.Hal.Device.Volume')
             return os.path.normpath('/media/'+label)+'/'
-        
+
         mm = hm.FindDeviceStringMatch(__appname__+'.mainvolume', self.__class__.__name__)
         if not mm:
             raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%(self.__class__.__name__,))
@@ -259,13 +260,13 @@ class Device(_Device):
                 break
             except dbus.exceptions.DBusException:
                 continue
-                
+
         if not self._main_prefix:
             raise DeviceError('Could not open device for reading. Try a reboot.')
-            
+
         self._card_prefix = None
         cards = hm.FindDeviceStringMatch(__appname__+'.cardvolume', self.__class__.__name__)
-        
+
         for dev in cards:
             try:
                 self._card_prefix = conditional_mount(dev)+os.sep

src/calibre/devices/usbms/driver.py

@@ -34,24 +34,25 @@ class USBMS(Device):
     SUPPORTS_SUB_DIRS = False
 
     def __init__(self, key='-1', log_packets=False, report_progress=None):
-        Device.__init__(self, key, log_packets, report_progress)
+        Device.__init__(self, key=key, log_packets=log_packets, 
-            
+                        report_progress=report_progress)
+
     def get_device_information(self, end_session=True):
-        """ 
+        """
-        Ask device for device information. See L{DeviceInfoQuery}. 
+        Ask device for device information. See L{DeviceInfoQuery}.
         @return: (device name, device version, software version on device, mime type)
         """
         return (self.__class__.__name__, '', '', '')
-    
+
     def books(self, oncard=False, end_session=True):
         bl = BookList()
-        
+
-        if oncard and self._card_prefix is None:
+        if oncard and self._card_prefix is None:
-            return bl
+            return bl
 
         prefix = self._card_prefix if oncard else self._main_prefix
         ebook_dir = self.EBOOK_DIR_CARD if oncard else self.EBOOK_DIR_MAIN
-        
+
         # Get all books in all directories under the root ebook_dir directory
         for path, dirs, files in os.walk(os.path.join(prefix, ebook_dir)):
             # Filter out anything that isn't in the list of supported ebook
@@ -59,15 +60,15 @@ class USBMS(Device):
             for book_type in self.FORMATS:
                 for filename in fnmatch.filter(files, '*.%s' % (book_type)):
                     title, author, mime = self.__class__.extract_book_metadata_by_filename(filename)
-                    
+
-                    bl.append(Book(os.path.join(path, filename), title, author, mime))
+                    bl.append(Book(os.path.join(path, filename), title, author, mime))
         return bl
-    
+
-    def upload_books(self, files, names, on_card=False, end_session=True, 
+    def upload_books(self, files, names, on_card=False, end_session=True,
                      metadata=None):
         if on_card and not self._card_prefix:
             raise ValueError(_('The reader has no storage card connected.'))
-            
+
         if not on_card:
             path = os.path.join(self._main_prefix, self.EBOOK_DIR_MAIN)
         else:
@@ -84,21 +85,21 @@ class USBMS(Device):
         sizes = [get_size(f) for f in files]
         size = sum(sizes)
 
-        if on_card and size > self.free_space()[2] - 1024*1024: 
+        if on_card and size > self.free_space()[2] - 1024*1024:
             raise FreeSpaceError(_("There is insufficient free space on the storage card"))
-        if not on_card and size > self.free_space()[0] - 2*1024*1024: 
+        if not on_card and size > self.free_space()[0] - 2*1024*1024:
             raise FreeSpaceError(_("There is insufficient free space in main memory"))
 
         paths = []
         names = iter(names)
         metadata = iter(metadata)
-        
+
         for infile in files:
             newpath = path
-            
+
             if self.SUPPORTS_SUB_DIRS:
                 mdata = metadata.next()
-                
+
                 if 'tags' in mdata.keys():
                     for tag in mdata['tags']:
                         if tag.startswith('/'):
@@ -108,35 +109,36 @@ class USBMS(Device):
 
             if not os.path.exists(newpath):
                 os.makedirs(newpath)
-            
+
             filepath = os.path.join(newpath, names.next())
             paths.append(filepath)
-            
+
             if hasattr(infile, 'read'):
                 infile.seek(0)
-                
+
                 dest = open(filepath, 'wb')
                 shutil.copyfileobj(infile, dest, 10*1024*1024)
 
-                dest.flush()                
+                dest.flush()
                 dest.close()
             else:
                 shutil.copy2(infile, filepath)
-    
+
         return zip(paths, cycle([on_card]))
-    
+
     @classmethod
-    def add_books_to_metadata(cls, locations, metadata, booklists):    
+    def add_books_to_metadata(cls, locations, metadata, booklists):
         for location in locations:
             path = location[0]
             on_card = 1 if location[1] else 0
-            
+
             title, author, mime = cls.extract_book_metadata_by_filename(os.path.basename(path))
             book = Book(path, title, author, mime)
-            
+
             if not book in booklists[on_card]:
                 booklists[on_card].append(book)
-    
+
+
     def delete_books(self, paths, end_session=True):
         for path in paths:
             if os.path.exists(path):
@@ -147,7 +149,7 @@ class USBMS(Device):
                         os.removedirs(os.path.dirname(path))
                     except:
                         pass
-    
+
     @classmethod
     def remove_books_from_metadata(cls, paths, booklists):
         for path in paths:
@@ -155,14 +157,14 @@ class USBMS(Device):
                 for book in bl:
                     if path.endswith(book.path):
                         bl.remove(book)
-        
+
     def sync_booklists(self, booklists, end_session=True):
         # There is no meta data on the device to update. The device is treated
         # as a mass storage device and does not use a meta data xml file like
         # the Sony Readers.
         pass
-    
+
-    def get_file(self, path, outfile, end_session=True): 
+    def get_file(self, path, outfile, end_session=True):
         path = self.munge_path(path)
         src = open(path, 'rb')
         shutil.copyfileobj(src, outfile, 10*1024*1024)
@@ -232,7 +234,7 @@ class USBMS(Device):
         # the filename without the extension
         else:
             book_title = os.path.splitext(filename)[0].replace('_', ' ')
-           
+
         fileext = os.path.splitext(filename)[1][1:]
 
         if fileext in cls.FORMATS:

src/calibre/ebooks/epub/from_html.py

@@ -160,7 +160,11 @@ class HTMLProcessor(Processor, Rationalizer):
                 br.text = u'\u00a0'
                 
         if self.opts.profile.remove_object_tags:
-            for tag in self.root.xpath('//object|//embed'):
+            for tag in self.root.xpath('//embed'):
+                tag.getparent().remove(tag)
+            for tag in self.root.xpath('//object'):
+                if tag.get('type', '').lower().strip() in ('image/svg+xml',):
+                    continue
                 tag.getparent().remove(tag)
     
     def save(self):

src/calibre/ebooks/mobi/from_any.py

@@ -14,18 +14,21 @@ import sys, os, glob, logging
 from calibre.ebooks.epub.from_any import any2epub, formats, USAGE
 from calibre.ebooks.epub import config as common_config
 from calibre.ptempfile import TemporaryDirectory
-from calibre.ebooks.mobi.writer import oeb2mobi, add_mobi_options
+from calibre.ebooks.mobi.writer import oeb2mobi, config as mobi_config
 
 def config(defaults=None):
-    return common_config(defaults=defaults, name='mobi')
+    c = common_config(defaults=defaults, name='mobi')
+    c.remove_opt('profile')
+    mobic = mobi_config(defaults=defaults)
+    c.update(mobic)
+    return c 
 
 def option_parser(usage=USAGE):
     usage = usage % ('Mobipocket', formats())
     parser = config().option_parser(usage=usage)
-    add_mobi_options(parser)
     return parser
 
-def any2mobi(opts, path):
+def any2mobi(opts, path, notification=None):
     ext = os.path.splitext(path)[1]
     if not ext:
         raise ValueError('Unknown file type: '+path)

src/calibre/ebooks/mobi/from_feeds.py

@@ -0,0 +1,74 @@
+from __future__ import with_statement
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Convert feeds to MOBI ebook
+'''
+
+import sys, glob, os
+from calibre.web.feeds.main import config as feeds2disk_config, USAGE, run_recipe
+from calibre.ebooks.mobi.writer import config as oeb2mobi_config, oeb2mobi
+from calibre.ptempfile import TemporaryDirectory
+from calibre import strftime, sanitize_file_name
+
+def config(defaults=None):
+    c = feeds2disk_config(defaults=defaults)
+    c.remove('lrf')
+    c.remove('epub')
+    c.remove('mobi')
+    c.remove('output_dir')
+    c.update(oeb2mobi_config(defaults=defaults))
+    c.remove('encoding')
+    c.remove('source_profile')
+    c.add_opt('output', ['-o', '--output'], default=None,
+              help=_('Output file. Default is derived from input filename.'))
+    return c
+
+def option_parser():
+    c = config()
+    return c.option_parser(usage=USAGE)
+
+def convert(opts, recipe_arg, notification=None):
+    opts.lrf  = False
+    opts.epub = False
+    opts.mobi = True
+    if opts.debug:
+        opts.verbose = 2
+    parser = option_parser()
+    with TemporaryDirectory('_feeds2mobi') as tdir:
+        opts.output_dir = tdir
+        recipe = run_recipe(opts, recipe_arg, parser, notification=notification)
+        c = config()
+        recipe_opts = c.parse_string(recipe.oeb2mobi_options)
+        c.smart_update(recipe_opts, opts)
+        opts = recipe_opts
+        opf = glob.glob(os.path.join(tdir, '*.opf'))
+        if not opf:
+            raise Exception('Downloading of recipe: %s failed'%recipe_arg)
+        opf = opf[0]
+        
+        if opts.output is None:
+            fname = recipe.title + strftime(recipe.timefmt) + '.mobi'
+            opts.output = os.path.join(os.getcwd(), sanitize_file_name(fname))
+        
+        print 'Generating MOBI...'
+        opts.encoding = 'utf-8'
+        opts.source_profile = 'Browser'
+        oeb2mobi(opts, opf)
+    
+
+def main(args=sys.argv, notification=None, handler=None):
+    parser = option_parser()
+    opts, args = parser.parse_args(args)
+    if len(args) != 2 and opts.feeds is None:
+        parser.print_help()
+        return 1
+    recipe_arg = args[1] if len(args) > 1 else None
+    convert(opts, recipe_arg, notification=notification)
+        
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/ebooks/mobi/writer.py

@@ -34,8 +34,7 @@ from calibre.ebooks.mobi.palmdoc import compress_doc
 from calibre.ebooks.mobi.langcodes import iana2mobi
 from calibre.ebooks.mobi.mobiml import MBP_NS, MBP, MobiMLizer
 from calibre.customize.ui import run_plugins_on_postprocess
-from calibre.utils.config import OptionParser
+from calibre.utils.config import Config, StringConfig
-from optparse import OptionGroup
 
 # TODO:
 # - Allow override CSS (?)
@@ -502,44 +501,45 @@ class MobiWriter(object):
             self._write(record)
 
 
-def add_mobi_options(parser):
+def config(defaults=None):
-    profiles = Context.PROFILES.keys()
+    desc = _('Options to control the conversion to MOBI')
-    profiles.sort()
+    _profiles = list(sorted(Context.PROFILES.keys()))
-    profiles = ', '.join(profiles)
+    if defaults is None:
-    group = OptionGroup(parser, _('Mobipocket'),
+        c = Config('mobi', desc)
-        _('Mobipocket-specific options.'))
+    else:
-    group.add_option(
+        c = StringConfig(defaults, desc)
-        '-c', '--compress', default=False, action='store_true',
+        
-        help=_('Compress file text using PalmDOC compression. '
+    mobi = c.add_group('mobipocket', _('Mobipocket-specific options.'))
+    mobi('compress', ['--compress'], default=False,
+         help=_('Compress file text using PalmDOC compression. '
                'Results in smaller files, but takes a long time to run.'))
-    group.add_option(
+    mobi('rescale_images', ['--rescale-images'], default=False, 
-        '-r', '--rescale-images', default=False, action='store_true',
         help=_('Modify images to meet Palm device size limitations.'))
-    group.add_option(
+    mobi('toc_title', ['--toc-title'], default=None, 
-        '--toc-title', default=None, action='store',
+         help=_('Title for any generated in-line table of contents.'))
-        help=_('Title for any generated in-line table of contents.'))
+    profiles = c.add_group('profiles', _('Device renderer profiles. '
-    parser.add_option_group(group)
+        'Affects conversion of font sizes, image rescaling and rasterization '
-    group = OptionGroup(parser, _('Profiles'), _('Device renderer profiles. '
+        'of tables. Valid profiles are: %s.') % ', '.join(_profiles))
-        'Affects conversion of default font sizes and rasterization '
+    profiles('source_profile', ['--source-profile'],
-        'resolution.  Valid profiles are: %s.') % profiles)
+             default='Browser', choices=_profiles,
-    group.add_option(
+             help=_("Source renderer profile. Default is %default."))
-        '--source-profile', default='Browser', metavar='PROFILE',
+    profiles('dest_profile', ['--dest-profile'],
-        help=_("Source renderer profile. Default is 'Browser'."))
+             default='CybookG3', choices=_profiles,
-    group.add_option(
+             help=_("Destination renderer profile. Default is %default."))
-        '--dest-profile', default='CybookG3', metavar='PROFILE',
+    c.add_opt('encoding', ['--encoding'], default=None,
-        help=_("Destination renderer profile. Default is 'CybookG3'."))
+              help=_('Character encoding for HTML files. Default is to auto detect.'))
-    parser.add_option_group(group)
+    return c
-    return
+    
-            
+
 def option_parser():
-    parser = OptionParser(usage=_('%prog [options] OPFFILE'))
+    c = config()
+    parser = c.option_parser(usage='%prog '+_('[options]')+' file.opf')
     parser.add_option(
         '-o', '--output', default=None, 
         help=_('Output file. Default is derived from input filename.'))
     parser.add_option(
         '-v', '--verbose', default=0, action='count',
         help=_('Useful for debugging.'))
-    add_mobi_options(parser)
     return parser
 
 def oeb2mobi(opts, inpath):
@@ -560,7 +560,7 @@ def oeb2mobi(opts, inpath):
     compression = PALMDOC if opts.compress else UNCOMPRESSED
     imagemax = PALM_MAX_IMAGE_SIZE if opts.rescale_images else None
     context = Context(source, dest)
-    oeb = OEBBook(inpath, logger=logger)
+    oeb = OEBBook(inpath, logger=logger, encoding=opts.encoding)
     tocadder = HTMLTOCAdder(title=opts.toc_title)
     tocadder.transform(oeb, context)
     mangler = CaseMangler()

src/calibre/ebooks/oeb/base.py

@@ -90,6 +90,9 @@ def prefixname(name, nsrmap):
         return barename(name)
     return ':'.join((prefix, barename(name)))
 
+def XPath(expr):
+    return etree.XPath(expr, namespaces=XPNSMAP)
+
 def xpath(elem, expr):
     return elem.xpath(expr, namespaces=XPNSMAP)
 
@@ -292,15 +295,19 @@ class Metadata(object):
 class Manifest(object):
     class Item(object):
         NUM_RE = re.compile('^(.*)([0-9][0-9.]*)(?=[.]|$)')
+        META_XP = XPath('/h:html/h:head/h:meta[@http-equiv="Content-Type"]')
     
-        def __init__(self, id, href, media_type,
+        def __init__(self, oeb, id, href, media_type,
                      fallback=None, loader=str, data=None):
+            self.oeb = oeb
             self.id = id
             self.href = self.path = urlnormalize(href)
             self.media_type = media_type
             self.fallback = fallback
             self.spine_position = None
             self.linear = True
+            if loader is None and data is None:
+                loader = oeb.container.read
             self._loader = loader
             self._data = data
 
@@ -309,16 +316,20 @@ class Manifest(object):
                 % (self.id, self.href, self.media_type)
 
         def _force_xhtml(self, data):
+            if self.oeb.encoding is not None:
+                data = data.decode(self.oeb.encoding, 'replace')
             try:
                 data = etree.fromstring(data, parser=XML_PARSER)
             except etree.XMLSyntaxError:
-                data = html.fromstring(data, parser=XML_PARSER)
+                data = html.fromstring(data)
                 data = etree.tostring(data, encoding=unicode)
                 data = etree.fromstring(data, parser=XML_PARSER)
             if namespace(data.tag) != XHTML_NS:
                 data.attrib['xmlns'] = XHTML_NS
-                data = etree.tostring(data)
+                data = etree.tostring(data, encoding=unicode)
                 data = etree.fromstring(data, parser=XML_PARSER)
+            for meta in self.META_XP(data):
+                meta.getparent().remove(meta)
             return data
         
         def data():
@@ -395,9 +406,8 @@ class Manifest(object):
         self.hrefs = {}
 
     def add(self, id, href, media_type, fallback=None, loader=None, data=None):
-        loader = loader or self.oeb.container.read
         item = self.Item(
-            id, href, media_type, fallback, loader, data)
+            self.oeb, id, href, media_type, fallback, loader, data)
         self.ids[item.id] = item
         self.hrefs[item.href] = item
         return item
@@ -535,27 +545,36 @@ class Spine(object):
 
 class Guide(object):
     class Reference(object):
-        _TYPES_TITLES = [('cover', 'Cover'), ('title-page', 'Title Page'),
+        _TYPES_TITLES = [('cover', __('Cover')),
-            ('toc', 'Table of Contents'), ('index', 'Index'),
+                         ('title-page', __('Title Page')),
-            ('glossary', 'Glossary'), ('acknowledgements', 'Acknowledgements'),
+                         ('toc', __('Table of Contents')),
-            ('bibliography', 'Bibliography'), ('colophon', 'Colophon'),
+                         ('index', __('Index')),
-            ('copyright-page', 'Copyright'), ('dedication', 'Dedication'),
+                         ('glossary', __('Glossary')),
-            ('epigraph', 'Epigraph'), ('foreword', 'Foreword'),
+                         ('acknowledgements', __('Acknowledgements')),
-            ('loi', 'List of Illustrations'), ('lot', 'List of Tables'),
+                         ('bibliography', __('Bibliography')),
-            ('notes', 'Notes'), ('preface', 'Preface'),
+                         ('colophon', __('Colophon')),
-            ('text', 'Main Text')]
+                         ('copyright-page', __('Copyright')),
+                         ('dedication', __('Dedication')),
+                         ('epigraph', __('Epigraph')),
+                         ('foreword', __('Foreword')),
+                         ('loi', __('List of Illustrations')),
+                         ('lot', __('List of Tables')),
+                         ('notes', __('Notes')),
+                         ('preface', __('Preface')),
+                         ('text', __('Main Text'))]
         TYPES = set(t for t, _ in _TYPES_TITLES)
         TITLES = dict(_TYPES_TITLES)
         ORDER = dict((t, i) for (t, _), i in izip(_TYPES_TITLES, count(0)))
         
-        def __init__(self, type, title, href):
+        def __init__(self, oeb, type, title, href):
+            self.oeb = oeb
             if type.lower() in self.TYPES:
                 type = type.lower()
             elif type not in self.TYPES and \
                  not type.startswith('other.'):
                 type = 'other.' + type
-            if not title:
+            if not title and type in self.TITLES:
-                title = self.TITLES.get(type, None)
+                title = oeb.translate(self.TITLES[type])
             self.type = type
             self.title = title
             self.href = urlnormalize(href)
@@ -574,13 +593,21 @@ class Guide(object):
             if not isinstance(other, Guide.Reference):
                 return NotImplemented
             return cmp(self._order, other._order)
+        
+        def item():
+            def fget(self):
+                path, frag = urldefrag(self.href)
+                hrefs = self.oeb.manifest.hrefs
+                return hrefs.get(path, None)
+            return property(fget=fget)
+        item = item()
     
     def __init__(self, oeb):
         self.oeb = oeb
         self.refs = {}
     
     def add(self, type, title, href):
-        ref = self.Reference(type, title, href)
+        ref = self.Reference(self.oeb, type, title, href)
         self.refs[type] = ref
         return ref
     
@@ -590,9 +617,7 @@ class Guide(object):
     __iter__ = iterkeys
     
     def values(self):
-        values = list(self.refs.values())
+        return sorted(self.refs.values())
-        values.sort()
-        return values
     
     def items(self):
         for type, ref in self.refs.items():
@@ -696,11 +721,13 @@ class TOC(object):
 
     
 class OEBBook(object):
-    def __init__(self, opfpath=None, container=None, logger=FauxLogger()):
+    def __init__(self, opfpath=None, container=None, encoding=None,
+                 logger=FauxLogger()):
         if opfpath and not container:
             container = DirContainer(os.path.dirname(opfpath))
             opfpath = os.path.basename(opfpath)
         self.container = container
+        self.encoding = encoding
         self.logger = logger
         if opfpath or container:
             opf = self._read_opf(opfpath)

src/calibre/ebooks/oeb/stylizer.py

@@ -223,8 +223,11 @@ class Stylizer(object):
             for key in composition:
                 style[key] = 'inherit'
         else:
-            primitives = [v.cssText for v in cssvalue]
+            try:
-            primitites.reverse()
+                primitives = [v.cssText for v in cssvalue]
+            except TypeError:
+                primitives = [cssvalue.cssText]
+            primitives.reverse()
             value = primitives.pop()
             for key in composition:
                 if cssproperties.cssvalues[key](value):

src/calibre/ebooks/oeb/transforms/htmltoc.py

@@ -13,6 +13,10 @@ from calibre.ebooks.oeb.base import XML, XHTML, XHTML_NS
 from calibre.ebooks.oeb.base import XHTML_MIME, CSS_MIME
 from calibre.ebooks.oeb.base import element
 
+__all__ = ['HTMLTOCAdder']
+
+DEFAULT_TITLE = __('Table of Contents')
+
 STYLE_CSS = {
     'nested': """
 .calibre_toc_header {
@@ -52,7 +56,7 @@ class HTMLTOCAdder(object):
         if 'toc' in oeb.guide:
             return
         oeb.logger.info('Generating in-line TOC...')
-        title = self.title or oeb.translate('Table of Contents')
+        title = self.title or oeb.translate(DEFAULT_TITLE)
         style = self.style
         if style not in STYLE_CSS:
             oeb.logger.error('Unknown TOC style %r' % style)

src/calibre/gui2/dialogs/epub.py

@@ -15,7 +15,7 @@ from lxml.etree import XPath
 from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 from calibre.gui2.dialogs.epub_ui import Ui_Dialog 
 from calibre.gui2 import error_dialog, choose_images, pixmap_to_data
-from calibre.ebooks.epub.from_any import SOURCE_FORMATS, config
+from calibre.ebooks.epub.from_any import SOURCE_FORMATS, config as epubconfig
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.ebooks.metadata.opf import OPFCreator
@@ -24,9 +24,12 @@ from calibre.ebooks.metadata import authors_to_string, string_to_authors
 
 class Config(QDialog, Ui_Dialog):
     
-    def __init__(self, parent, db, row=None):
+    OUTPUT = 'EPUB'
+        
+    def __init__(self, parent, db, row=None, config=epubconfig):
         QDialog.__init__(self, parent)
         self.setupUi(self)
+        self.hide_controls()
         self.connect(self.category_list, SIGNAL('itemEntered(QListWidgetItem *)'),
                         self.show_category_help)
         self.connect(self.cover_button, SIGNAL("clicked()"), self.select_cover)
@@ -38,7 +41,7 @@ class Config(QDialog, Ui_Dialog):
         if row is not None:
             self.id = db.id(row)
             base = config().as_string() + '\n\n'
-            defaults = self.db.conversion_options(self.id, 'epub')
+            defaults = self.db.conversion_options(self.id, self.OUTPUT.lower())
             defaults = base + (defaults if defaults else '')
             self.config = config(defaults=defaults)
         else:
@@ -47,9 +50,18 @@ class Config(QDialog, Ui_Dialog):
         self.get_source_format()
         self.category_list.setCurrentRow(0)
         if self.row is None:
-            self.setWindowTitle(_('Bulk convert to EPUB'))
+            self.setWindowTitle(_('Bulk convert to ')+self.OUTPUT)
         else:
-            self.setWindowTitle(_(u'Convert %s to EPUB')%unicode(self.title.text()))
+            self.setWindowTitle((_(u'Convert %s to ')%unicode(self.title.text()))+self.OUTPUT)
+    
+    def hide_controls(self):
+        self.source_profile_label.setVisible(False)
+        self.opt_source_profile.setVisible(False)
+        self.dest_profile_label.setVisible(False)
+        self.opt_dest_profile.setVisible(False)
+        self.opt_toc_title.setVisible(False)
+        self.toc_title_label.setVisible(False)
+        self.opt_rescale_images.setVisible(False)
         
     def initialize(self):
         self.__w = []
@@ -81,8 +93,8 @@ class Config(QDialog, Ui_Dialog):
     def show_category_help(self, item):
         text = unicode(item.text())
         help = {
-                _('Metadata')          : _('Specify metadata such as title and author for the book.\n\nMetadata will be updated in the database as well as the generated EPUB file.'),
+                _('Metadata')          : _('Specify metadata such as title and author for the book.\n\nMetadata will be updated in the database as well as the generated %s file.')%self.OUTPUT,
-                _('Look & Feel')       : _('Adjust the look of the generated EPUB file by specifying things like font sizes.'),
+                _('Look & Feel')       : _('Adjust the look of the generated ebook by specifying things like font sizes.'),
                 _('Page Setup')        : _('Specify the page layout settings like margins.'),
                 _('Chapter Detection') : _('Fine tune the detection of chapter and section headings.'),                  
                 }
@@ -195,7 +207,7 @@ class Config(QDialog, Ui_Dialog):
                 elif isinstance(g, QCheckBox):
                     self.config.set(pref.name, bool(g.isChecked()))
         if self.row is not None:
-            self.db.set_conversion_options(self.id, 'epub', self.config.src)        
+            self.db.set_conversion_options(self.id, self.OUTPUT.lower(), self.config.src)        
         
     
     def initialize_options(self):
@@ -235,7 +247,7 @@ class Config(QDialog, Ui_Dialog):
             elif len(choices) == 1:
                 self.source_format = choices[0]
             else:
-                d = ChooseFormatDialog(self.parent(), _('Choose the format to convert to EPUB'), choices)
+                d = ChooseFormatDialog(self.parent(), _('Choose the format to convert to ')+self.OUTPUT, choices)
                 if d.exec_() == QDialog.Accepted:
                     self.source_format = d.format()
                 

src/calibre/gui2/dialogs/epub.ui

@@ -89,36 +89,6 @@
               <string>Book Cover</string>
              </property>
              <layout class="QGridLayout" name="_2" >
-              <item row="0" column="0" >
-               <layout class="QHBoxLayout" name="_3" >
-                <item>
-                 <widget class="ImageView" name="cover" >
-                  <property name="text" >
-                   <string/>
-                  </property>
-                  <property name="pixmap" >
-                   <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
-                  </property>
-                  <property name="scaledContents" >
-                   <bool>true</bool>
-                  </property>
-                  <property name="alignment" >
-                   <set>Qt::AlignCenter</set>
-                  </property>
-                 </widget>
-                </item>
-               </layout>
-              </item>
-              <item row="2" column="0" >
-               <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
-                <property name="text" >
-                 <string>Use cover from &amp;source file</string>
-                </property>
-                <property name="checked" >
-                 <bool>true</bool>
-                </property>
-               </widget>
-              </item>
               <item row="1" column="0" >
                <layout class="QVBoxLayout" name="_4" >
                 <property name="spacing" >
@@ -170,6 +140,36 @@
                 </item>
                </layout>
               </item>
+              <item row="2" column="0" >
+               <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
+                <property name="text" >
+                 <string>Use cover from &amp;source file</string>
+                </property>
+                <property name="checked" >
+                 <bool>true</bool>
+                </property>
+               </widget>
+              </item>
+              <item row="0" column="0" >
+               <layout class="QHBoxLayout" name="_3" >
+                <item>
+                 <widget class="ImageView" name="cover" >
+                  <property name="text" >
+                   <string/>
+                  </property>
+                  <property name="pixmap" >
+                   <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
+                  </property>
+                  <property name="scaledContents" >
+                   <bool>true</bool>
+                  </property>
+                  <property name="alignment" >
+                   <set>Qt::AlignCenter</set>
+                  </property>
+                 </widget>
+                </item>
+               </layout>
+              </item>
              </layout>
              <zorder>opt_prefer_metadata_cover</zorder>
              <zorder></zorder>
@@ -456,6 +456,13 @@
              </property>
             </widget>
            </item>
+           <item row="4" column="0" >
+            <widget class="QCheckBox" name="opt_rescale_images" >
+             <property name="text" >
+              <string>&amp;Rescale images</string>
+             </property>
+            </widget>
+           </item>
           </layout>
          </item>
          <item>
@@ -475,7 +482,7 @@
        <widget class="QWidget" name="pagesetup_page" >
         <layout class="QGridLayout" name="_13" >
          <item row="0" column="0" >
-          <widget class="QLabel" name="label_11" >
+          <widget class="QLabel" name="profile_label" >
            <property name="text" >
             <string>&amp;Profile:</string>
            </property>
@@ -494,7 +501,7 @@
            </property>
           </widget>
          </item>
-         <item row="1" column="0" >
+         <item row="3" column="0" >
           <widget class="QLabel" name="label_12" >
            <property name="text" >
             <string>&amp;Left Margin:</string>
@@ -504,7 +511,7 @@
            </property>
           </widget>
          </item>
-         <item row="1" column="1" >
+         <item row="3" column="1" >
           <widget class="QSpinBox" name="opt_margin_left" >
            <property name="suffix" >
             <string> pt</string>
@@ -517,7 +524,7 @@
            </property>
           </widget>
          </item>
-         <item row="2" column="0" >
+         <item row="4" column="0" >
           <widget class="QLabel" name="label_13" >
            <property name="text" >
             <string>&amp;Right Margin:</string>
@@ -527,7 +534,7 @@
            </property>
           </widget>
          </item>
-         <item row="2" column="1" >
+         <item row="4" column="1" >
           <widget class="QSpinBox" name="opt_margin_right" >
            <property name="suffix" >
             <string> pt</string>
@@ -540,7 +547,7 @@
            </property>
           </widget>
          </item>
-         <item row="3" column="0" >
+         <item row="5" column="0" >
           <widget class="QLabel" name="label_14" >
            <property name="text" >
             <string>&amp;Top Margin:</string>
@@ -550,7 +557,7 @@
            </property>
           </widget>
          </item>
-         <item row="3" column="1" >
+         <item row="5" column="1" >
           <widget class="QSpinBox" name="opt_margin_top" >
            <property name="suffix" >
             <string> pt</string>
@@ -563,7 +570,7 @@
            </property>
           </widget>
          </item>
-         <item row="4" column="0" >
+         <item row="6" column="0" >
           <widget class="QLabel" name="label_15" >
            <property name="text" >
             <string>&amp;Bottom Margin:</string>
@@ -573,7 +580,7 @@
            </property>
           </widget>
          </item>
-         <item row="4" column="1" >
+         <item row="6" column="1" >
           <widget class="QSpinBox" name="opt_margin_bottom" >
            <property name="suffix" >
             <string> pt</string>
@@ -586,13 +593,39 @@
            </property>
           </widget>
          </item>
-         <item row="5" column="0" >
+         <item row="7" column="0" >
           <widget class="QCheckBox" name="opt_dont_split_on_page_breaks" >
            <property name="text" >
             <string>Do not &amp;split on page breaks</string>
            </property>
           </widget>
          </item>
+         <item row="1" column="0" >
+          <widget class="QLabel" name="source_profile_label" >
+           <property name="text" >
+            <string>&amp;Source profile:</string>
+           </property>
+           <property name="buddy" >
+            <cstring>opt_source_profile</cstring>
+           </property>
+          </widget>
+         </item>
+         <item row="1" column="1" >
+          <widget class="QComboBox" name="opt_source_profile" />
+         </item>
+         <item row="2" column="0" >
+          <widget class="QLabel" name="dest_profile_label" >
+           <property name="text" >
+            <string>&amp;Destination profile:</string>
+           </property>
+           <property name="buddy" >
+            <cstring>opt_dest_profile</cstring>
+           </property>
+          </widget>
+         </item>
+         <item row="2" column="1" >
+          <widget class="QComboBox" name="opt_dest_profile" />
+         </item>
         </layout>
        </widget>
        <widget class="QWidget" name="chapterdetection_page" >
@@ -721,6 +754,19 @@ p, li { white-space: pre-wrap; }
             <item row="5" column="1" >
              <widget class="QLineEdit" name="opt_level2_toc" />
             </item>
+            <item row="6" column="1" >
+             <widget class="QLineEdit" name="opt_toc_title" />
+            </item>
+            <item row="6" column="0" >
+             <widget class="QLabel" name="toc_title_label" >
+              <property name="text" >
+               <string>&amp;Title for generated TOC</string>
+              </property>
+              <property name="buddy" >
+               <cstring>opt_toc_title</cstring>
+              </property>
+             </widget>
+            </item>
            </layout>
           </widget>
          </item>

src/calibre/gui2/dialogs/mobi.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+from calibre.gui2.dialogs.epub import Config as _Config
+from calibre.ebooks.mobi.from_any import config as mobiconfig
+
+class Config(_Config):
+    
+    OUTPUT = 'MOBI'
+    
+    def __init__(self, parent, db, row=None):
+        _Config.__init__(self, parent, db, row=row, config=mobiconfig)
+        
+    def hide_controls(self):
+        self.profile_label.setVisible(False)
+        self.opt_profile.setVisible(False)
+        self.opt_dont_split_on_page_breaks.setVisible(False)
+        self.opt_preserve_tag_structure.setVisible(False)

src/calibre/gui2/main.py

@@ -25,7 +25,6 @@ from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            max_available_height, config, info_dialog, \
                            available_width
 from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
-from calibre.library.database import LibraryDatabase
 from calibre.gui2.dialogs.scheduler import Scheduler
 from calibre.gui2.update import CheckForUpdates
 from calibre.gui2.dialogs.progress import ProgressDialog
@@ -131,14 +130,14 @@ class Main(MainWindow, Ui_MainWindow):
         QObject.connect(self.stack, SIGNAL('currentChanged(int)'),
                         self.location_view.location_changed)
         
-        self.output_formats = sorted(['EPUB', 'LRF'])
+        self.output_formats = sorted(['EPUB', 'MOBI', 'LRF'])
         for f in self.output_formats:
             self.output_format.addItem(f)
         self.output_format.setCurrentIndex(self.output_formats.index(prefs['output_format']))
         def change_output_format(x):
             of = unicode(x).strip()
             if of != prefs['output_format']:
-                if of in ('EPUB', 'LIT'):
+                if of not in ('LRF',):
                     warning_dialog(self, 'Warning', 
                                    '<p>%s support is still in beta. If you find bugs, please report them by opening a <a href="http://calibre.kovidgoyal.net">ticket</a>.'%of).exec_()
                 prefs.set('output_format', of)

src/calibre/gui2/tools.py

@@ -12,6 +12,7 @@ from PyQt4.Qt import QDialog
 from calibre.utils.config import prefs
 from calibre.gui2.dialogs.lrf_single import LRFSingleDialog, LRFBulkDialog
 from calibre.gui2.dialogs.epub import Config as EPUBConvert
+from calibre.gui2.dialogs.mobi import Config as MOBIConvert
 import calibre.gui2.dialogs.comicconf as ComicConf
 from calibre.gui2 import warning_dialog
 from calibre.ptempfile import PersistentTemporaryFile
@@ -19,14 +20,20 @@ from calibre.ebooks.lrf import preferred_source_formats as LRF_PREFERRED_SOURCE_
 from calibre.ebooks.metadata.opf import OPFCreator
 from calibre.ebooks.epub.from_any import SOURCE_FORMATS as EPUB_PREFERRED_SOURCE_FORMATS
 
-def convert_single_epub(parent, db, comics, others):
+def get_dialog(fmt):
+    return {
+              'epub':EPUBConvert,
+              'mobi':MOBIConvert,
+              }[fmt]
+
+def convert_single(fmt, parent, db, comics, others):
     changed = False
     jobs = []
     others_ids = [db.id(row) for row in others]
     comics_ids = [db.id(row) for row in comics]
     for row, row_id in zip(others, others_ids):
         temp_files = []
-        d = EPUBConvert(parent, db, row)
+        d = get_dialog(fmt)(parent, db, row)
         if d.source_format is not None:
             d.exec_()
             if d.result() == QDialog.Accepted:
@@ -35,7 +42,7 @@ def convert_single_epub(parent, db, comics, others):
                 pt = PersistentTemporaryFile('.'+d.source_format.lower())
                 pt.write(data)
                 pt.close()
-                of = PersistentTemporaryFile('.epub')
+                of = PersistentTemporaryFile('.'+fmt)
                 of.close()
                 opts.output = of.name
                 opts.from_opf = d.opf_file.name
@@ -45,8 +52,8 @@ def convert_single_epub(parent, db, comics, others):
                     temp_files.append(d.cover_file)
                     opts.cover = d.cover_file.name
                 temp_files.extend([d.opf_file, pt, of])
-                jobs.append(('any2epub', args, _('Convert book: ')+d.mi.title, 
+                jobs.append(('any2'+fmt, args, _('Convert book: ')+d.mi.title, 
-                             'EPUB', row_id, temp_files))
+                             fmt.upper(), row_id, temp_files))
                 changed = True
                 
     for row, row_id in zip(comics, comics_ids):
@@ -61,24 +68,24 @@ def convert_single_epub(parent, db, comics, others):
         if defaults is not None:
             db.set_conversion_options(db.id(row), 'comic', defaults)
         if opts is None: continue
-        for fmt in ['cbz', 'cbr']:
+        for _fmt in ['cbz', 'cbr']:
             try:
-                data = db.format(row, fmt.upper())
+                data = db.format(row, _fmt.upper())
                 if data is not None:
                     break                    
             except:
                 continue
-        pt = PersistentTemporaryFile('.'+fmt)
+        pt = PersistentTemporaryFile('.'+_fmt)
         pt.write(data)
         pt.close()
-        of = PersistentTemporaryFile('.epub')
+        of = PersistentTemporaryFile('.'+fmt)
         of.close()
         opts.output = of.name
         opts.verbose = 2
         args = [pt.name, opts]
         changed = True
-        jobs.append(('comic2epub', args, _('Convert comic: ')+opts.title, 
+        jobs.append(('comic2'+fmt, args, _('Convert comic: ')+opts.title, 
-                     'EPUB', row_id, [pt, of]))
+                     fmt.upper(), row_id, [pt, of]))
         
     return jobs, changed
     
@@ -146,9 +153,9 @@ def convert_single_lrf(parent, db, comics, others):
         
     return jobs, changed
 
-def convert_bulk_epub(parent, db, comics, others):
+def convert_bulk(fmt, parent, db, comics, others):
     if others:
-        d = EPUBConvert(parent, db)
+        d = get_dialog(fmt)(parent, db)
         if d.exec_() != QDialog.Accepted:
             others = []
         else:
@@ -169,9 +176,9 @@ def convert_bulk_epub(parent, db, comics, others):
         row_id = db.id(row)
         if row in others:
             data = None
-            for fmt in EPUB_PREFERRED_SOURCE_FORMATS:
+            for _fmt in EPUB_PREFERRED_SOURCE_FORMATS:
                 try:
-                    data = db.format(row, fmt.upper())
+                    data = db.format(row, _fmt.upper())
                     if data is not None:
                         break
                 except:
@@ -185,10 +192,10 @@ def convert_bulk_epub(parent, db, comics, others):
             opf_file = PersistentTemporaryFile('.opf')
             opf.render(opf_file)
             opf_file.close()
-            pt = PersistentTemporaryFile('.'+fmt.lower())
+            pt = PersistentTemporaryFile('.'+_fmt.lower())
             pt.write(data)
             pt.close()
-            of = PersistentTemporaryFile('.epub')
+            of = PersistentTemporaryFile('.'+fmt)
             of.close()
             cover = db.cover(row)
             cf = None
@@ -203,7 +210,7 @@ def convert_bulk_epub(parent, db, comics, others):
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
             temp_files = [cf] if cf is not None else []
             temp_files.extend([opf_file, pt, of])
-            jobs.append(('any2epub', args, desc, 'EPUB', row_id, temp_files))
+            jobs.append(('any2'+fmt, args, desc, fmt.upper(), row_id, temp_files))
         else:
             options = comic_opts.copy()
             mi = db.get_metadata(row)
@@ -212,24 +219,24 @@ def convert_bulk_epub(parent, db, comics, others):
             if mi.authors:
                 options.author =  ','.join(mi.authors)
             data = None
-            for fmt in ['cbz', 'cbr']:
+            for _fmt in ['cbz', 'cbr']:
                 try:
-                    data = db.format(row, fmt.upper())
+                    data = db.format(row, _fmt.upper())
                     if data is not None:
                         break                    
                 except:
                     continue
             
-            pt = PersistentTemporaryFile('.'+fmt.lower())
+            pt = PersistentTemporaryFile('.'+_fmt.lower())
             pt.write(data)
             pt.close()
-            of = PersistentTemporaryFile('.epub')
+            of = PersistentTemporaryFile('.'+fmt)
             of.close()
             setattr(options, 'output', of.name)
             options.verbose = 1
             args = [pt.name, options]
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
-            jobs.append(('comic2epub', args, desc, 'EPUB', row_id, [pt, of]))        
+            jobs.append(('comic2'+fmt, args, desc, fmt.upper(), row_id, [pt, of]))        
         
     if bad_rows:
         res = []
@@ -345,15 +352,14 @@ def set_conversion_defaults_lrf(comic, parent, db):
     else:
         LRFSingleDialog(parent, None, None).exec_()
 
-def set_conversion_defaults_epub(comic, parent, db):
+def _set_conversion_defaults(dialog, comic, parent, db):
     if comic:
         ComicConf.set_conversion_defaults(parent)
     else:
-        d = EPUBConvert(parent, db)
+        d = dialog(parent, db)
         d.setWindowTitle(_('Set conversion defaults'))
         d.exec_()
 
-
 def _fetch_news(data, fmt):
     pt = PersistentTemporaryFile(suffix='_feeds2%s.%s'%(fmt.lower(), fmt.lower()))
     pt.close()
@@ -385,22 +391,22 @@ def convert_single_ebook(*args):
     fmt = prefs['output_format'].lower()
     if fmt == 'lrf':
         return convert_single_lrf(*args)
-    elif fmt == 'epub':
+    elif fmt in ('epub', 'mobi'):
-        return convert_single_epub(*args)
+        return convert_single(fmt, *args)
     
 def convert_bulk_ebooks(*args):
     fmt = prefs['output_format'].lower()
     if fmt == 'lrf':
         return convert_bulk_lrf(*args)
-    elif fmt == 'epub':
+    elif fmt in ('epub', 'mobi'):
-        return convert_bulk_epub(*args)
+        return convert_bulk(fmt, *args)
     
 def set_conversion_defaults(comic, parent, db):
     fmt = prefs['output_format'].lower()
     if fmt == 'lrf':
         return set_conversion_defaults_lrf(comic, parent, db)
-    elif fmt == 'epub':
+    elif fmt in ('epub', 'mobi'):
-        return set_conversion_defaults_epub(comic, parent, db)
+        return _set_conversion_defaults(get_dialog(fmt), comic, parent, db)
 
 def fetch_news(data):
     fmt = prefs['output_format'].lower()

src/calibre/library/database2.py

@@ -224,9 +224,17 @@ class ResultCache(SearchQueryParser):
         return False
     
     def refresh_ids(self, conn, ids):
+        '''
+        Refresh the data in the cache for books identified by ids.
+        Returns a list of affected rows or None if the rows are filtered.
+        '''
         for id in ids:
             self._data[id] = conn.get('SELECT * from meta WHERE id=?', (id,))[0]
-        return map(self.row, ids)
+        try:
+            return map(self.row, ids)
+        except ValueError:
+            pass
+        return None
     
     def books_added(self, ids, conn):
         if not ids:

src/calibre/linux.py

@@ -40,6 +40,7 @@ entry_points = {
                              'calibre-server = calibre.library.server:main',
                              'feeds2lrf  = calibre.ebooks.lrf.feeds.convert_from:main',
                              'feeds2epub = calibre.ebooks.epub.from_feeds:main',
+                             'feeds2mobi = calibre.ebooks.mobi.from_feeds:main',
                              'web2lrf   = calibre.ebooks.lrf.web.convert_from:main',
                              'pdf2lrf   = calibre.ebooks.lrf.pdf.convert_from:main',
                              'mobi2lrf  = calibre.ebooks.lrf.mobi.convert_from:main',
@@ -189,6 +190,7 @@ def setup_completion(fatal_errors):
         from calibre.ebooks.html import option_parser as html2oeb
         from calibre.ebooks.odt.to_oeb import option_parser as odt2oeb
         from calibre.ebooks.epub.from_feeds import option_parser as feeds2epub
+        from calibre.ebooks.mobi.from_feeds import option_parser as feeds2mobi
         from calibre.ebooks.epub.from_any import option_parser as any2epub
         from calibre.ebooks.lit.from_any import option_parser as any2lit
         from calibre.ebooks.epub.from_comic import option_parser as comic2epub
@@ -219,7 +221,7 @@ def setup_completion(fatal_errors):
         f.write(opts_and_exts('any2epub', any2epub, any_formats))
         f.write(opts_and_exts('any2lit', any2lit, any_formats))
         f.write(opts_and_exts('any2mobi', any2mobi, any_formats))
-        f.write(opts_and_exts('oeb2mobi', oeb2mobi, ['mobi', 'prc']))
+        f.write(opts_and_exts('oeb2mobi', oeb2mobi, ['opf']))
         f.write(opts_and_exts('lrf2lrs', lrf2lrsop, ['lrf']))
         f.write(opts_and_exts('lrf-meta', metaop, ['lrf']))
         f.write(opts_and_exts('rtf-meta', metaop, ['rtf']))
@@ -239,7 +241,8 @@ def setup_completion(fatal_errors):
         f.write(opts_and_exts('comic2pdf', comic2epub, ['cbz', 'cbr']))
         f.write(opts_and_words('feeds2disk', feeds2disk, feed_titles))
         f.write(opts_and_words('feeds2lrf', feeds2lrf, feed_titles))
-        f.write(opts_and_words('feeds2lrf', feeds2epub, feed_titles))
+        f.write(opts_and_words('feeds2epub', feeds2epub, feed_titles))
+        f.write(opts_and_words('feeds2mobi', feeds2mobi, feed_titles))
         f.write(opts_and_exts('html2epub', html2epub, ['html', 'htm', 'xhtm', 'xhtml', 'opf']))
         f.write(opts_and_exts('html2oeb', html2oeb, ['html', 'htm', 'xhtm', 'xhtml']))
         f.write(opts_and_exts('odt2oeb', odt2oeb, ['odt']))

src/calibre/parallel.py

@@ -67,7 +67,15 @@ PARALLEL_FUNCS = {
         
       'comic2epub'    :
         ('calibre.ebooks.epub.from_comic', 'convert', {}, 'notification'),
-
+        
+      'any2mobi'     :
+        ('calibre.ebooks.mobi.from_any', 'any2mobi', {}, None),
+        
+      'feeds2mobi'   :
+        ('calibre.ebooks.mobi.from_feeds', 'main', {}, 'notification'),
+        
+      'comic2mobi'    :
+        ('calibre.ebooks.mobi.from_comic', 'convert', {}, 'notification'),
 }
 
 

src/calibre/startup.py

@@ -13,6 +13,10 @@ from gettext import GNUTranslations
 import __builtin__
 __builtin__.__dict__['_'] = lambda s: s
 
+# For strings which belong in the translation tables, but which shouldn't be
+# immediately translated to the environment language
+__builtin__.__dict__['__'] = lambda s: s
+
 from calibre.constants import iswindows, preferred_encoding, plugins
 from calibre.utils.config import prefs
 from calibre.translations.msgfmt import make

src/calibre/web/feeds/main.py

@@ -45,6 +45,8 @@ If you specify this option, any argument to %prog is ignored and a default recip
                  help='Optimize fetching for subsequent conversion to LRF.')
     c.add_opt('epub', ['--epub'], default=False, action='store_true', 
                  help='Optimize fetching for subsequent conversion to EPUB.')
+    c.add_opt('mobi', ['--mobi'], default=False, action='store_true', 
+                 help='Optimize fetching for subsequent conversion to MOBI.')
     c.add_opt('recursions', ['--recursions'], default=0,
                  help=_('Number of levels of links to follow on webpages that are linked to from feeds. Defaul %default'))
     c.add_opt('output_dir', ['--output-dir'], default='.', 

src/calibre/web/feeds/news.py

@@ -20,7 +20,7 @@ from PyQt4.QtWebKit import QWebPage
 from calibre import browser, __appname__, iswindows, LoggingInterface, \
                     strftime, __version__, preferred_encoding
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
-from calibre.ebooks.metadata.opf import OPFCreator
+from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.ebooks.lrf import entity_to_unicode
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import MetaInformation
@@ -152,6 +152,8 @@ class BasicNewsRecipe(object, LoggingInterface):
     
     #: Options to pass to html2epub to customize generation of EPUB ebooks.
     html2epub_options     = ''
+    #: Options to pass to oeb2mobi to customize generation of MOBI ebooks.
+    oeb2mobi_options     = ''
     
     #: List of tags to be removed. Specified tags are removed from downloaded HTML.
     #: A tag is specified as a dictionary of the form::
@@ -876,6 +878,7 @@ class BasicNewsRecipe(object, LoggingInterface):
         
         manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))]
         manifest.append(os.path.join(dir, 'index.html'))
+        manifest.append(os.path.join(dir, 'index.ncx'))
         cpath = getattr(self, 'cover_path', None)
         if cpath is None:
             pf = PersistentTemporaryFile('_recipe_cover.jpg')
@@ -885,6 +888,9 @@ class BasicNewsRecipe(object, LoggingInterface):
             opf.cover = cpath
             manifest.append(cpath)
         opf.create_manifest_from_files_in(manifest)
+        for mani in opf.manifest:
+            if mani.path.endswith('.ncx'):
+                mani.id = 'ncx'
         
         entries = ['index.html']
         toc = TOC(base_path=dir)

src/calibre/web/feeds/recipes/recipe_newsweek.py

@@ -19,9 +19,16 @@ class Newsweek(BasicNewsRecipe):
 
     remove_tags = [
         dict(name=['script',  'noscript']),
-        dict(name='div',  attrs={'class':['ad', 'SocialLinks', 'SocialLinksDiv', 'channel', 'bot', 'nav', 'top', 'EmailArticleBlock']}),
+        dict(name='div',  attrs={'class':['ad', 'SocialLinks', 'SocialLinksDiv',
+                                          'channel', 'bot', 'nav', 'top', 
+                                          'EmailArticleBlock', 
+                                          'comments-and-social-links-wrapper',
+                                          'inline-social-links-wrapper',
+                                          'inline-social-links',
+                                          ]}),
         dict(name='div',  attrs={'class':re.compile('box')}),
-        dict(id=['ToolBox', 'EmailMain', 'EmailArticle', ])
+        dict(id=['ToolBox', 'EmailMain', 'EmailArticle', 'comment-box',
+                 'nw-comments'])
     ]
     
     recursions = 1

src/calibre/web/feeds/recipes/recipe_xkcd.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Fetch xkcd.
 '''
 
-import time
+import time, re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class XkcdCom(BasicNewsRecipe):
@@ -17,6 +17,11 @@ class XkcdCom(BasicNewsRecipe):
     keep_only_tags = [dict(id='middleContent')]
     remove_tags = [dict(name='ul'), dict(name='h3'), dict(name='br')]
     no_stylesheets = True
+    # turn image bubblehelp into a paragraph
+    preprocess_regexps = [
+        (re.compile(r'(<img.*title=")([^"]+)(".*>)'),
+         lambda m: '%s%s<p>%s</p>' % (m.group(1), m.group(3), m.group(2)))
+    ]
     
     def parse_index(self):
         INDEX = 'http://xkcd.com/archive/'