Sync to trunk

.pydevproject

@@ -2,8 +2,9 @@
 <?eclipse-pydev version="1.0"?>
 
 <pydev_project>
-<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.5</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.6</pydev_property>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
 <path>/calibre/src</path>
 </pydev_pathproperty>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
 </pydev_project>

installer/windows/build_installer.py

@@ -8,16 +8,18 @@ __docformat__ = 'restructuredtext en'
 import sys, time, subprocess, os, re
 from calibre import __appname__, __version__
 
+INSTALLJAMMER = '/home/kovid/installjammer/installjammer' 
+
 sv = re.sub(r'[a-z]\d+', '', __version__)
 
 cmdline = [
-    '/usr/local/installjammer/installjammer',
+    INSTALLJAMMER,
     '--build-dir', '/tmp/calibre-installjammer',
     '-DAppName', __appname__,
     '-DShortAppName', __appname__,
     '-DApplicationURL', 'http://%s.kovidgoyal.net'%__appname__,
     '-DCopyright', time.strftime('%Y Kovid Goyal'),
-    '-DPackageDescription', '%s is an e-book library manager. It can view, convert and catalog e-books in most of the major e-book formats. It can also talk to a few e-book reader devices. It can go out to the internet and fetch metadata for your books. It can download newspapers and convert them into e-books for convenient reading.'%__appname__,
+    '-DPackageDescription', '%s is an e-book library manager. It can view, convert and catalog e-books in most of the major e-book formats. It can also talk to e-book reader devices. It can go out to the internet and fetch metadata for your books. It can download newspapers and convert them into e-books for convenient reading.'%__appname__,
     '-DPackageSummary', '%s: E-book library management'%__appname__,
     '-DVersion', __version__,
     '-DInstallVersion', sv + '.0',

installer/windows/calibre/calibre.mpi

@@ -138,7 +138,7 @@ ProjectID
 DA98A0C6-9102-73EC-2516-B147E972D3F7
 
 ProjectVersion
-1.2.7.0
+1.2.12.0
 
 SaveOnlyToplevelDirs
 No
@@ -211,7 +211,8 @@ File ::8E5D85A4-7608-47A1-CF7C-309060D5FF40 -filemethod {Always overwrite files}
 Component ::F6829AB7-9F66-4CEE-CA0E-21F54C6D3609 -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows} -name Main -parent Components
 SetupType ::D9ADE41C-B744-690C-2CED-CF826BF03D2E -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows} -name Typical -parent SetupTypes
 
-InstallComponent 3EA07B17-04D8-6508-B535-96CC7173B49A -setup Install -type pane -title {Welcome Screen} -component Welcome -active Yes -parent StandardInstall
+InstallComponent 3EA07B17-04D8-6508-B535-96CC7173B49A -setup Install -type pane -conditions D7F585DB-0DEC-A94E-DAB0-94D558D82764 -title {Welcome Screen} -component Welcome -command reorder -active Yes -parent StandardInstall
+Condition D7F585DB-0DEC-A94E-DAB0-94D558D82764 -active Yes -parent 3EA07B17-04D8-6508-B535-96CC7173B49A -title {Execute Script Condition} -component ExecuteScriptCondition -TreeObject::id D7F585DB-0DEC-A94E-DAB0-94D558D82764
 InstallComponent 7CCDA4BB-861C-C21E-3011-E93DB58F07D6 -setup Install -type action -title {Check for Previous Install} -component CheckForPreviousInstall -active Yes -parent 3EA07B17-04D8-6508-B535-96CC7173B49A
 InstallComponent 580ACF2C-517F-5E48-9DEF-7DAEFBA59FDD -setup Install -type action -conditions 6DE3B369-9D6B-6BC1-4EA0-2C54ECE159EB -title {Set Virtual Text} -component SetVirtualText -command insert -active Yes -parent 3EA07B17-04D8-6508-B535-96CC7173B49A
 Condition 6DE3B369-9D6B-6BC1-4EA0-2C54ECE159EB -active Yes -parent 580ACF2C-517F-5E48-9DEF-7DAEFBA59FDD -title {String Is Condition} -component StringIsCondition -TreeObject::id 6DE3B369-9D6B-6BC1-4EA0-2C54ECE159EB
@@ -513,7 +514,7 @@ false
 1
 
 3EA07B17-04D8-6508-B535-96CC7173B49A,Conditions
-{0 conditions}
+{1 condition}
 
 3EA07B17-04D8-6508-B535-96CC7173B49A,Message,subst
 1
@@ -1100,6 +1101,9 @@ AAFE58A0-2DFB-CA20-1F6E-D3815F885996,Alias
 AIX-ppc,Active
 No
 
+AIX-ppc,BuildSeparateArchives
+No
+
 AIX-ppc,DefaultDirectoryPermission
 0755
 
@@ -1286,6 +1290,26 @@ CFBE4459-450B-1FAB-3422-609544334AA2,String
 D79DC0D2-38BC-9D9F-2DF4-3C76D89BF933,ExitType
 Finish
 
+D7F585DB-0DEC-A94E-DAB0-94D558D82764,CheckCondition
+{Before Next Pane is Displayed}
+
+D7F585DB-0DEC-A94E-DAB0-94D558D82764,Comment
+{Check if calibre.exe is still running}
+
+D7F585DB-0DEC-A94E-DAB0-94D558D82764,FailureMessage
+{calibre is still running. Please shut it down before proceeding. You can quit calibre by right clicking on the calibre system tray icon.}
+
+D7F585DB-0DEC-A94E-DAB0-94D558D82764,ResultVirtualText
+CalibreRunning
+
+D7F585DB-0DEC-A94E-DAB0-94D558D82764,Script
+{set pid [::InstallAPI::FindProcesses -name calibre.exe]
+if {$pid eq ""} {
+    ## myapp.exe is not running
+    return 1
+}
+return 0}
+
 D86BBA5C-4903-33BA-59F8-4266A3D45896,Conditions
 {2 conditions}
 
@@ -1475,6 +1499,9 @@ FBA33088-C809-DD6B-D337-EADBF1CEE966,String
 FreeBSD-4-x86,Active
 No
 
+FreeBSD-4-x86,BuildSeparateArchives
+No
+
 FreeBSD-4-x86,DefaultDirectoryPermission
 0755
 
@@ -1526,6 +1553,9 @@ FreeBSD-4-x86,RootInstallDir
 FreeBSD-x86,Active
 No
 
+FreeBSD-x86,BuildSeparateArchives
+No
+
 FreeBSD-x86,DefaultDirectoryPermission
 0755
 
@@ -1577,6 +1607,9 @@ FreeBSD-x86,RootInstallDir
 HPUX-hppa,Active
 No
 
+HPUX-hppa,BuildSeparateArchives
+No
+
 HPUX-hppa,DefaultDirectoryPermission
 0755
 
@@ -1628,6 +1661,9 @@ HPUX-hppa,RootInstallDir
 Linux-x86,Active
 No
 
+Linux-x86,BuildSeparateArchives
+No
+
 Linux-x86,DefaultDirectoryPermission
 0755
 
@@ -1679,6 +1715,9 @@ Linux-x86,RootInstallDir
 Solaris-sparc,Active
 No
 
+Solaris-sparc,BuildSeparateArchives
+No
+
 Solaris-sparc,DefaultDirectoryPermission
 0755
 
@@ -1730,6 +1769,9 @@ Solaris-sparc,RootInstallDir
 TarArchive,Active
 No
 
+TarArchive,BuildSeparateArchives
+No
+
 TarArchive,CompressionLevel
 6
 
@@ -1790,9 +1832,15 @@ TarArchive,VirtualTextMap
 Windows,Active
 Yes
 
+Windows,BuildSeparateArchives
+No
+
 Windows,Executable
 <%AppName%>-<%Version%><%Ext%>
 
+Windows,FileDescription
+{<%AppName%> <%Version%> Setup}
+
 Windows,IncludeTWAPI
 Yes
 
@@ -1829,6 +1877,9 @@ Windows,WindowsIcon
 ZipArchive,Active
 No
 
+ZipArchive,BuildSeparateArchives
+No
+
 ZipArchive,CompressionLevel
 6
 

setup.py

@@ -121,7 +121,7 @@ if __name__ == '__main__':
                 buf = cStringIO.StringIO()
                 print 'Creating translations template'
                 tempdir = tempfile.mkdtemp()
-                pygettext(buf, ['-p', tempdir]+files)
+                pygettext(buf, ['-k', '__', '-p', tempdir]+files)
                 src = buf.getvalue()
                 pot = os.path.join(tempdir, 'calibre.pot')
                 f = open(pot, 'wb')

src/calibre/__init__.py

@@ -20,6 +20,7 @@ import mechanize
 mimetypes.add_type('application/epub+zip', '.epub')
 mimetypes.add_type('text/x-sony-bbeb+xml', '.lrs')
 mimetypes.add_type('application/x-sony-bbeb', '.lrf')
+mimetypes.add_type('application/x-dtbncx+xml', '.ncx')
 
 def to_unicode(raw, encoding='utf-8', errors='strict'):
     if isinstance(raw, unicode):

src/calibre/devices/cybookg3/driver.py

@@ -7,8 +7,10 @@ Device driver for Bookeen's Cybook Gen 3
 import os, shutil
 from itertools import cycle
 
+from calibre.devices.errors import FreeSpaceError
 from calibre.devices.usbms.driver import USBMS
 import calibre.devices.cybookg3.t2b as t2b
+from calibre.devices.errors import FreeSpaceError
 
 class CYBOOKG3(USBMS):
     # Ordered list of supported formats
@@ -50,10 +52,10 @@ class CYBOOKG3(USBMS):
                 return size
             return os.path.getsize(obj)
 
-        sizes = map(get_size, files)
+        sizes = [get_size(f) for f in files]
         size = sum(sizes)
 
-        if on_card and size > self.free_space()[2] - 1024*1024: 
+        if on_card and size > self.free_space()[2] - 1024*1024:
             raise FreeSpaceError(_("There is insufficient free space on the storage card"))
         if not on_card and size > self.free_space()[0] - 2*1024*1024: 
             raise FreeSpaceError(_("There is insufficient free space in main memory"))

src/calibre/devices/prs505/driver.py

@@ -146,36 +146,7 @@ class PRS505(Device):
             self._card_prefix = re.search(card_pat, mount).group(2) + os.sep
             
     
-    def open_windows_nowmi(self):
-        from calibre import plugins
-        winutil = plugins['winutil'][0]
-        volumes = winutil.get_mounted_volumes_for_usb_device(self.VENDOR_ID, self.PRODUCT_ID)
-        main = None
-        for device_id in volumes.keys():
-            if 'PRS-505/UC&' in device_id:
-                main = volumes[device_id]+':\\'
-        if not main:
-            raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%self.__class__.__name__)
-        self._main_prefix = main
-        card = self._card_prefix = None
-        win32api = __import__('win32api')
-        for device_id in volumes.keys():
-            if 'PRS-505/UC:' in device_id:
-                card = volumes[device_id]+':\\'
-                try:
-                    win32api.GetVolumeInformation(card)
-                    self._card_prefix = card
-                    break
-                except:
-                    continue
-            
-    
     def open_windows(self):
-        try:
-            self.open_windows_nowmi()
-            return
-        except:
-            pass
         drives = []
         wmi = __import__('wmi', globals(), locals(), [], -1) 
         c = wmi.WMI()

src/calibre/devices/usbms/books.py

@@ -18,6 +18,9 @@ class Book(object):
         self.thumbnail = None
         self.tags = []
         
+    def __eq__(self, other):
+        return self.path == other.path
+        
     @apply
     def title_sorter():
         doc = '''String to sort the title. If absent, title is returned'''

src/calibre/devices/usbms/device.py

@@ -3,10 +3,10 @@ __copyright__ = '2009, John Schember <john at nachtimwald.com>'
 '''
 Generic device driver. This is not a complete stand alone driver. It is
 intended to be subclassed with the relevant parts implemented for a particular
-device. This class handles devive detection.
+device. This class handles device detection.
 '''
 
-import os, subprocess, time
+import os, subprocess, time, re
 
 from calibre.devices.interface import Device as _Device
 from calibre.devices.errors import DeviceError
@@ -18,21 +18,21 @@ class Device(_Device):
     as USB Mass Storage devices. If you are writing such a driver, inherit from this
     class.
     '''
-    
+
     VENDOR_ID   = 0x0
     PRODUCT_ID  = 0x0
     BCD         = None
-    
+
     VENDOR_NAME = None
     WINDOWS_MAIN_MEM = None
     WINDOWS_CARD_MEM = None
-    
+
     OSX_MAIN_MEM = None
     OSX_CARD_MEM = None
-    
+
     MAIN_MEMORY_VOLUME_LABEL  = ''
     STORAGE_CARD_VOLUME_LABEL = ''
-        
+
     FDI_TEMPLATE = \
 '''
   <device>
@@ -65,15 +65,15 @@ class Device(_Device):
   </device>
 '''
     FDI_BCD_TEMPLATE = '<match key="@info.parent:@info.parent:@info.parent:@info.parent:usb.device_revision_bcd" int="%(bcd)s">'
-    
+
-    
+
     def __init__(self, key='-1', log_packets=False, report_progress=None) :
         self._main_prefix = self._card_prefix = None
-    
+
     @classmethod
     def get_fdi(cls):
         fdi = ''
-        
+
         fdi_base_values = dict(
                                app=__appname__,
                                deviceclass=cls.__name__,
@@ -92,12 +92,12 @@ class Device(_Device):
                 fdi_bcd_values['BCD_start'] = cls.FDI_BCD_TEMPLATE % dict(bcd=hex(bcd))
                 fdi_bcd_values['BCD_end'] = '</match>'
                 fdi += cls.FDI_TEMPLATE % fdi_bcd_values
-                
+
         return fdi
-    
+
     def set_progress_reporter(self, report_progress):
         self.report_progress = report_progress
-    
+
     def card_prefix(self, end_session=True):
         return self._card_prefix
 
@@ -117,7 +117,7 @@ class Device(_Device):
             else: raise
         mult = sectors_per_cluster * bytes_per_sector
         return total_clusters * mult, free_clusters * mult
-    
+
     def total_space(self, end_session=True):
         msz = csz = 0
         print self._main_prefix
@@ -131,9 +131,9 @@ class Device(_Device):
         else:
             msz = self._windows_space(self._main_prefix)[0]
             csz = self._windows_space(self._card_prefix)[0]
-                
+
         return (msz, 0, csz)
-    
+
     def free_space(self, end_session=True):
         msz = csz = 0
         if not iswindows:
@@ -146,15 +146,15 @@ class Device(_Device):
         else:
             msz = self._windows_space(self._main_prefix)[1]
             csz = self._windows_space(self._card_prefix)[1]
-                
+
         return (msz, 0, csz)
 
     def windows_match_device(self, pnp_id, device_id):
         pnp_id = pnp_id.upper()
-        
+
         if device_id and pnp_id is not None:
             device_id = device_id.upper()
-            
+
             if 'VEN_' + self.VENDOR_NAME in pnp_id and 'PROD_' + device_id in pnp_id:
                 return True
 
@@ -162,45 +162,45 @@ class Device(_Device):
 
     def windows_get_drive_prefix(self, drive):
         prefix = None
-        
+
         try:
             partition = drive.associators("Win32_DiskDriveToDiskPartition")[0]
             logical_disk = partition.associators('Win32_LogicalDiskToPartition')[0]
             prefix = logical_disk.DeviceID + os.sep
         except IndexError:
             pass
-            
+
         return prefix
 
     def open_windows(self):
         drives = {}
-        wmi = __import__('wmi', globals(), locals(), [], -1) 
+        wmi = __import__('wmi', globals(), locals(), [], -1)
         c = wmi.WMI()
         for drive in c.Win32_DiskDrive():
-            if self.windows_match_device(str(drive.PNPDeviceID), WINDOWS_MAIN_MEM):
+            if self.windows_match_device(str(drive.PNPDeviceID), self.WINDOWS_MAIN_MEM):
                 drives['main'] = self.windows_get_drive_prefix(drive)
-            elif self.windows_match_device(str(drive.PNPDeviceID), WINDOWS_CARD_MEM):
+            elif self.windows_match_device(str(drive.PNPDeviceID), self.WINDOWS_CARD_MEM):
                 drives['card'] = self.windows_get_drive_prefix(drive)
-                
+
             if 'main' and 'card' in drives.keys():
                 break
-                
+
         if not drives:
             raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.') % self.__class__.__name__)
-            
-        self._main_prefix = drives['main'] if 'main' in names.keys() else None
-        self._card_prefix = drives['card'] if 'card' in names.keys() else None
 
-    @classmethod
+        self._main_prefix = drives.get('main', None)
+        self._card_prefix = drives.get('card', None)
+
     def get_osx_mountpoints(self, raw=None):
         if raw is None:
             ioreg = '/usr/sbin/ioreg'
             if not os.access(ioreg, os.X_OK):
                 ioreg = 'ioreg'
-            raw = subprocess.Popen((ioreg+' -w 0 -S -c IOMedia').split(), stdout=subprocess.PIPE).stdout.read()
+            raw = subprocess.Popen((ioreg+' -w 0 -S -c IOMedia').split(),
+                                   stdout=subprocess.PIPE).stdout.read()
         lines = raw.splitlines()
         names = {}
-        
+
         def get_dev_node(lines, loc):
             for line in lines:
                 line = line.strip()
@@ -210,7 +210,7 @@ class Device(_Device):
                 if match is not None:
                     names[loc] = match.group(1)
                     break
-                    
+
         for i, line in enumerate(lines):
             if self.OSX_MAIN_MEM is not None and line.strip().endswith('<class IOMedia>') and self.OSX_MAIN_MEM in line:
                 get_dev_node(lines[i+1:], 'main')
@@ -219,25 +219,25 @@ class Device(_Device):
             if len(names.keys()) == 2:
                 break
         return names
-    
+
     def open_osx(self):
         mount = subprocess.Popen('mount', shell=True,  stdout=subprocess.PIPE).stdout.read()
         names = self.get_osx_mountpoints()
         dev_pat = r'/dev/%s(\w*)\s+on\s+([^\(]+)\s+'
         if 'main' not in names.keys():
             raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%self.__class__.__name__)
-        main_pat = dev_pat%names['main']
+        main_pat = dev_pat % names['main']
         self._main_prefix = re.search(main_pat, mount).group(2) + os.sep
         card_pat = names['card'] if 'card' in names.keys() else None
         if card_pat is not None:
-            card_pat = dev_pat%card_pat
+            card_pat = dev_pat % card_pat
             self._card_prefix = re.search(card_pat, mount).group(2) + os.sep
-            
+
     def open_linux(self):
         import dbus
-        bus = dbus.SystemBus() 
+        bus = dbus.SystemBus()
         hm  = dbus.Interface(bus.get_object("org.freedesktop.Hal", "/org/freedesktop/Hal/Manager"), "org.freedesktop.Hal.Manager")
-        
+
         def conditional_mount(dev):
             mmo = bus.get_object("org.freedesktop.Hal", dev)
             label = mmo.GetPropertyString('volume.label', dbus_interface='org.freedesktop.Hal.Device')
@@ -246,10 +246,10 @@ class Device(_Device):
             fstype = mmo.GetPropertyString('volume.fstype', dbus_interface='org.freedesktop.Hal.Device')
             if is_mounted:
                 return str(mount_point)
-            mmo.Mount(label, fstype, ['umask=077', 'uid='+str(os.getuid()), 'sync'], 
+            mmo.Mount(label, fstype, ['umask=077', 'uid='+str(os.getuid()), 'sync'],
                           dbus_interface='org.freedesktop.Hal.Device.Volume')
             return os.path.normpath('/media/'+label)+'/'
-        
+
         mm = hm.FindDeviceStringMatch(__appname__+'.mainvolume', self.__class__.__name__)
         if not mm:
             raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%(self.__class__.__name__,))
@@ -260,13 +260,13 @@ class Device(_Device):
                 break
             except dbus.exceptions.DBusException:
                 continue
-                
+
         if not self._main_prefix:
             raise DeviceError('Could not open device for reading. Try a reboot.')
-            
+
         self._card_prefix = None
         cards = hm.FindDeviceStringMatch(__appname__+'.cardvolume', self.__class__.__name__)
-        
+
         for dev in cards:
             try:
                 self._card_prefix = conditional_mount(dev)+os.sep

src/calibre/devices/usbms/driver.py

@@ -11,9 +11,22 @@ from itertools import cycle
 
 from calibre.devices.usbms.device import Device
 from calibre.devices.usbms.books import BookList, Book
-from calibre.devices.errors import FreeSpaceError
+from calibre.devices.errors import FreeSpaceError, PathError
 from calibre.devices.mime import MIME_MAP
 
+class File(object):
+    def __init__(self, path):
+        stats = os.stat(path)
+        self.is_dir = os.path.isdir(path)
+        self.is_readonly = not os.access(path, os.W_OK)
+        self.ctime = stats.st_ctime
+        self.wtime = stats.st_mtime
+        self.size  = stats.st_size
+        if path.endswith(os.sep):
+            path = path[:-1]
+        self.path = path
+        self.name = os.path.basename(path)
+
 class USBMS(Device):
     FORMATS = []
     EBOOK_DIR_MAIN = ''
@@ -21,39 +34,41 @@ class USBMS(Device):
     SUPPORTS_SUB_DIRS = False
 
     def __init__(self, key='-1', log_packets=False, report_progress=None):
-        pass
+        Device.__init__(self, key=key, log_packets=log_packets, 
-            
+                        report_progress=report_progress)
+
     def get_device_information(self, end_session=True):
-        """ 
+        """
-        Ask device for device information. See L{DeviceInfoQuery}. 
+        Ask device for device information. See L{DeviceInfoQuery}.
         @return: (device name, device version, software version on device, mime type)
         """
         return (self.__class__.__name__, '', '', '')
-    
+
     def books(self, oncard=False, end_session=True):
         bl = BookList()
-        
+
-        if oncard and self._card_prefix is None:
+        if oncard and self._card_prefix is None:
-            return bl
+            return bl
 
         prefix = self._card_prefix if oncard else self._main_prefix
         ebook_dir = self.EBOOK_DIR_CARD if oncard else self.EBOOK_DIR_MAIN
-        
+
         # Get all books in all directories under the root ebook_dir directory
         for path, dirs, files in os.walk(os.path.join(prefix, ebook_dir)):
-            # Filter out anything that isn't in the list of supported ebook types
+            # Filter out anything that isn't in the list of supported ebook
+            # types
             for book_type in self.FORMATS:
                 for filename in fnmatch.filter(files, '*.%s' % (book_type)):
                     title, author, mime = self.__class__.extract_book_metadata_by_filename(filename)
-                    
+
-                    bl.append(Book(os.path.join(path, filename), title, author, mime))
+                    bl.append(Book(os.path.join(path, filename), title, author, mime))
         return bl
-    
+
-    def upload_books(self, files, names, on_card=False, end_session=True, 
+    def upload_books(self, files, names, on_card=False, end_session=True,
                      metadata=None):
         if on_card and not self._card_prefix:
             raise ValueError(_('The reader has no storage card connected.'))
-            
+
         if not on_card:
             path = os.path.join(self._main_prefix, self.EBOOK_DIR_MAIN)
         else:
@@ -67,24 +82,24 @@ class USBMS(Device):
                 return size
             return os.path.getsize(obj)
 
-        sizes = map(get_size, files)
+        sizes = [get_size(f) for f in files]
         size = sum(sizes)
 
-        if on_card and size > self.free_space()[2] - 1024*1024: 
+        if on_card and size > self.free_space()[2] - 1024*1024:
             raise FreeSpaceError(_("There is insufficient free space on the storage card"))
-        if not on_card and size > self.free_space()[0] - 2*1024*1024: 
+        if not on_card and size > self.free_space()[0] - 2*1024*1024:
             raise FreeSpaceError(_("There is insufficient free space in main memory"))
 
         paths = []
         names = iter(names)
         metadata = iter(metadata)
-        
+
         for infile in files:
             newpath = path
-            
+
             if self.SUPPORTS_SUB_DIRS:
                 mdata = metadata.next()
-                
+
                 if 'tags' in mdata.keys():
                     for tag in mdata['tags']:
                         if tag.startswith('/'):
@@ -94,32 +109,36 @@ class USBMS(Device):
 
             if not os.path.exists(newpath):
                 os.makedirs(newpath)
-            
+
-            filepath = os.path.join(newpath, names.next())                
+            filepath = os.path.join(newpath, names.next())
             paths.append(filepath)
-            
+
             if hasattr(infile, 'read'):
                 infile.seek(0)
-                
+
                 dest = open(filepath, 'wb')
                 shutil.copyfileobj(infile, dest, 10*1024*1024)
 
-                dest.flush()                
+                dest.flush()
                 dest.close()
             else:
                 shutil.copy2(infile, filepath)
-    
+
         return zip(paths, cycle([on_card]))
-    
+
     @classmethod
-    def add_books_to_metadata(cls, locations, metadata, booklists):    
+    def add_books_to_metadata(cls, locations, metadata, booklists):
         for location in locations:
             path = location[0]
             on_card = 1 if location[1] else 0
-            
+
             title, author, mime = cls.extract_book_metadata_by_filename(os.path.basename(path))
-            booklists[on_card].append(Book(path, title, author, mime))
+            book = Book(path, title, author, mime)
-    
+
+            if not book in booklists[on_card]:
+                booklists[on_card].append(book)
+
+
     def delete_books(self, paths, end_session=True):
         for path in paths:
             if os.path.exists(path):
@@ -130,7 +149,7 @@ class USBMS(Device):
                         os.removedirs(os.path.dirname(path))
                     except:
                         pass
-    
+
     @classmethod
     def remove_books_from_metadata(cls, paths, booklists):
         for path in paths:
@@ -138,14 +157,14 @@ class USBMS(Device):
                 for book in bl:
                     if path.endswith(book.path):
                         bl.remove(book)
-        
+
     def sync_booklists(self, booklists, end_session=True):
         # There is no meta data on the device to update. The device is treated
         # as a mass storage device and does not use a meta data xml file like
         # the Sony Readers.
         pass
-    
+
-    def get_file(self, path, outfile, end_session=True): 
+    def get_file(self, path, outfile, end_session=True):
         path = self.munge_path(path)
         src = open(path, 'rb')
         shutil.copyfileobj(src, outfile, 10*1024*1024)
@@ -215,7 +234,7 @@ class USBMS(Device):
         # the filename without the extension
         else:
             book_title = os.path.splitext(filename)[0].replace('_', ' ')
-           
+
         fileext = os.path.splitext(filename)[1][1:]
 
         if fileext in cls.FORMATS:

src/calibre/ebooks/epub/__init__.py

@@ -15,15 +15,17 @@ from lxml import etree
 
 class DefaultProfile(object):
     
-    flow_size   = sys.maxint
+    flow_size            = sys.maxint
-    screen_size = None
+    screen_size          = None
     remove_special_chars = False
+    remove_object_tags   = False
     
 class PRS505(DefaultProfile):
     
-    flow_size   = 270000
+    flow_size            = 270000
-    screen_size = (590, 765)
+    screen_size          = (590, 765)
     remove_special_chars = re.compile(u'[\u200b\u00ad]')
+    remove_object_tags   = True
         
 
 PROFILES = {
@@ -156,7 +158,7 @@ to auto-generate a Table of Contents.
            help=_('Set the right margin in pts. Default is %default'))
     layout('base_font_size2', ['--base-font-size'], default=12.0,
            help=_('The base font size in pts. Default is %defaultpt. Set to 0 to disable rescaling of fonts.'))
-    layout('remove_paragraph_spacing', ['--remove-paragraph-spacing'], default=True,
+    layout('remove_paragraph_spacing', ['--remove-paragraph-spacing'], default=False,
            help=_('Remove spacing between paragraphs. Will not work if the source file forces inter-paragraph spacing.'))
     layout('preserve_tag_structure', ['--preserve-tag-structure'], default=False,
            help=_('Preserve the HTML tag structure while splitting large HTML files. This is only neccessary if the HTML files contain CSS that uses sibling selectors. Enabling this greatly slows down processing of large HTML files.'))

src/calibre/ebooks/epub/from_any.py

@@ -16,7 +16,7 @@ from calibre.ebooks.epub import config as common_config, process_encryption
 from calibre.ebooks.epub.from_html import convert as html2epub, find_html_index
 from calibre.ptempfile import TemporaryDirectory
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.opf2 import OPFCreator
+from calibre.ebooks.metadata.opf2 import OPFCreator, OPF
 from calibre.utils.zipfile import ZipFile
 from calibre.customize.ui import run_plugins_on_preprocess
 
@@ -25,9 +25,36 @@ def lit2opf(path, tdir, opts):
     print 'Exploding LIT file:', path
     reader = LitReader(path)
     reader.extract_content(tdir, False)
-    for f in walk(tdir):
+    opf = None
-        if f.lower().endswith('.opf'):
+    for opf in walk(tdir):
-            return f
+        if opf.lower().endswith('.opf'):
+            break
+    if not opf.endswith('.opf'):
+        opf = None
+    if opf is not None: # Check for url-quoted filenames
+        _opf = OPF(opf, os.path.dirname(opf))
+        replacements = []
+        for item in _opf.itermanifest():
+            href = item.get('href', '')
+            path = os.path.join(os.path.dirname(opf), *(href.split('/')))
+            if not os.path.exists(path) and os.path.exists(path.replace('&', '%26')):
+                npath = path
+                path = path.replace('&', '%26')
+                replacements.append((path, npath))
+        if replacements:
+            print 'Fixing quoted filenames...'
+            for path, npath in replacements:
+                if os.path.exists(path):
+                    os.rename(path, npath)
+            for f in walk(tdir):
+                with open(f, 'r+b') as f:
+                    raw = f.read()
+                    for path, npath in replacements:
+                        raw = raw.replace(os.path.basename(path), os.path.basename(npath))
+                        f.seek(0)
+                        f.truncate()
+                        f.write(raw)
+    return opf
 
 def mobi2opf(path, tdir, opts):
     from calibre.ebooks.mobi.reader import MobiReader

src/calibre/ebooks/epub/from_feeds.py

@@ -52,6 +52,7 @@ def convert(opts, recipe_arg, notification=None):
         
         print 'Generating epub...'
         opts.encoding = 'utf-8'
+        opts.remove_paragraph_spacing = True
         html2epub(opf, opts, notification=notification)
     
 

src/calibre/ebooks/epub/from_html.py

@@ -128,6 +128,8 @@ class HTMLProcessor(Processor, Rationalizer):
         if hasattr(self.body, 'xpath'):
             for script in list(self.body.xpath('descendant::script')):
                 script.getparent().remove(script)
+                
+        self.fix_markup()
             
     def convert_image(self, img):
         rpath = img.get('src', '')
@@ -145,6 +147,25 @@ class HTMLProcessor(Processor, Rationalizer):
                     if val == rpath:
                         self.resource_map[key] = rpath+'_calibre_converted.jpg'
         img.set('src', rpath+'_calibre_converted.jpg')
+        
+    def fix_markup(self):
+        '''
+        Perform various markup transforms to get the output to render correctly 
+        in the quirky ADE.
+        '''
+        # Replace <br> that are children of <body> with <p>&nbsp;</p>
+        if hasattr(self.body, 'xpath'):
+            for br in self.body.xpath('./br'):
+                br.tag = 'p'
+                br.text = u'\u00a0'
+                
+        if self.opts.profile.remove_object_tags:
+            for tag in self.root.xpath('//embed'):
+                tag.getparent().remove(tag)
+            for tag in self.root.xpath('//object'):
+                if tag.get('type', '').lower().strip() in ('image/svg+xml',):
+                    continue
+                tag.getparent().remove(tag)
     
     def save(self):
         for meta in list(self.root.xpath('//meta')):

src/calibre/ebooks/epub/iterator.py

@@ -95,7 +95,7 @@ class EbookIterator(object):
                 for match in re.compile(r'@font-face\s*{([^}]+)}').finditer(css):
                     block  = match.group(1)
                     family = re.compile(r'font-family\s*:\s*([^;]+)').search(block)
-                    url    = re.compile(r'url\s*\((.+?)\)', re.DOTALL).search(block)
+                    url    = re.compile(r'url\s*\([\'"]*(.+?)[\'"]*\)', re.DOTALL).search(block)
                     if url:
                         path = url.group(1).split('/')
                         path = os.path.join(os.path.dirname(item.path), *path) 

src/calibre/ebooks/html.py

@@ -848,7 +848,7 @@ class Processor(Parser):
         # Workaround for anchor rendering bug in ADE
         css += '\n\na { color: inherit; text-decoration: inherit; cursor: default; }\na[href] { color: blue; text-decoration: underline; cursor:pointer; }'
         if self.opts.remove_paragraph_spacing:
-            css += '\n\np {text-indent: 2em; margin-top:0pt; margin-bottom:0pt; padding:0pt; border:0pt;}'
+            css += '\n\np {text-indent: 1.5em; margin-top:0pt; margin-bottom:0pt; padding:0pt; border:0pt;}'
         if self.opts.override_css:
             css += '\n\n' + self.opts.override_css
         self.override_css = self.css_parser.parseString(self.preprocess_css(css))

src/calibre/ebooks/metadata/lit.py

@@ -6,33 +6,28 @@ Support for reading the metadata from a LIT file.
 
 import sys, cStringIO, os
 
-from calibre import relpath
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks.metadata.opf import OPFReader
+from calibre.ebooks.metadata.opf2 import OPF
 from calibre.ebooks.lit.reader import LitReader
 
 def get_metadata(stream):
-    try:
+    litfile = LitReader(stream)
-        litfile = LitReader(stream)
+    src = litfile.meta.encode('utf-8')
-        src = litfile.meta.encode('utf-8')
+    opf = OPF(cStringIO.StringIO(src), os.getcwd())
-        mi = OPFReader(cStringIO.StringIO(src), dir=os.getcwd())
+    mi = MetaInformation(opf)
-        cover_url, cover_item = mi.cover, None
+    covers = []
-        if cover_url:
+    for item in opf.iterguide():
-            cover_url = relpath(cover_url, os.getcwd())
+        if 'cover' not in item.get('type', '').lower():
-            for item in litfile.manifest.values():
+            continue
-                if item.path == cover_url:
+        href = item.get('href', '')
-                    cover_item = item.internal
+        candidates = [href, href.replace('&', '%26')]
-        if cover_item is not None:
+        for item in litfile.manifest.values():
-            ext = cover_url.rpartition('.')[-1]
+            if item.path in candidates:
-            if not ext:
+                covers.append(item.internal)
-                ext = 'jpg'
+                break
-            else:
+    covers = [litfile.get_file('/data/' + i) for i in covers]
-                ext = ext.lower()
+    covers.sort(cmp=lambda x, y:cmp(len(x), len(y)))
-            cd = litfile.get_file('/data/' + cover_item)
+    mi.cover_data = ('jpg', covers[-1])
-            mi.cover_data = (ext, cd) if cd else (None, None)
-    except:
-        title = stream.name if hasattr(stream, 'name') and stream.name else 'Unknown'
-        mi = MetaInformation(title, ['Unknown'])
     return mi
 
 def main(args=sys.argv):

src/calibre/ebooks/metadata/mobi.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+'''
+
+import sys, os
+
+from calibre.ebooks.mobi.reader import get_metadata
+
+def main(args=sys.argv):
+    if len(args) != 2:
+        print >>sys.stderr, 'Usage: %s file.mobi' % args[0]
+        return 1
+    fname = args[1]
+    mi = get_metadata(open(fname, 'rb'))
+    print unicode(mi)
+    if mi.cover_data[1]:
+        cover = os.path.abspath(
+            '.'.join((os.path.splitext(os.path.basename(fname))[0],
+                      mi.cover_data[0].lower())))
+        open(cover, 'wb').write(mi.cover_data[1])
+        print _('Cover saved to'), cover
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/ebooks/metadata/opf2.py

@@ -435,7 +435,7 @@ class OPF(object):
     rating          = MetadataField('rating', is_dc=False, formatter=int)
     
     
-    def __init__(self, stream, basedir=os.getcwdu()):
+    def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True):
         if not hasattr(stream, 'read'):
             stream = open(stream, 'rb')
         self.basedir  = self.base_dir = basedir
@@ -446,7 +446,8 @@ class OPF(object):
         if not self.metadata:
             raise ValueError('Malformed OPF file: No <metadata> element')
         self.metadata      = self.metadata[0]
-        self.unquote_urls()
+        if unquote_urls:
+            self.unquote_urls()
         self.manifest = Manifest()
         m = self.manifest_path(self.root)
         if m:

src/calibre/ebooks/mobi/from_any.py

@@ -14,18 +14,21 @@ import sys, os, glob, logging
 from calibre.ebooks.epub.from_any import any2epub, formats, USAGE
 from calibre.ebooks.epub import config as common_config
 from calibre.ptempfile import TemporaryDirectory
-from calibre.ebooks.mobi.writer import oeb2mobi, add_mobi_options
+from calibre.ebooks.mobi.writer import oeb2mobi, config as mobi_config
 
 def config(defaults=None):
-    return common_config(defaults=defaults, name='mobi')
+    c = common_config(defaults=defaults, name='mobi')
+    c.remove_opt('profile')
+    mobic = mobi_config(defaults=defaults)
+    c.update(mobic)
+    return c 
 
 def option_parser(usage=USAGE):
     usage = usage % ('Mobipocket', formats())
     parser = config().option_parser(usage=usage)
-    add_mobi_options(parser)
     return parser
 
-def any2mobi(opts, path):
+def any2mobi(opts, path, notification=None):
     ext = os.path.splitext(path)[1]
     if not ext:
         raise ValueError('Unknown file type: '+path)

src/calibre/ebooks/mobi/from_feeds.py

@@ -0,0 +1,74 @@
+from __future__ import with_statement
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Convert feeds to MOBI ebook
+'''
+
+import sys, glob, os
+from calibre.web.feeds.main import config as feeds2disk_config, USAGE, run_recipe
+from calibre.ebooks.mobi.writer import config as oeb2mobi_config, oeb2mobi
+from calibre.ptempfile import TemporaryDirectory
+from calibre import strftime, sanitize_file_name
+
+def config(defaults=None):
+    c = feeds2disk_config(defaults=defaults)
+    c.remove('lrf')
+    c.remove('epub')
+    c.remove('mobi')
+    c.remove('output_dir')
+    c.update(oeb2mobi_config(defaults=defaults))
+    c.remove('encoding')
+    c.remove('source_profile')
+    c.add_opt('output', ['-o', '--output'], default=None,
+              help=_('Output file. Default is derived from input filename.'))
+    return c
+
+def option_parser():
+    c = config()
+    return c.option_parser(usage=USAGE)
+
+def convert(opts, recipe_arg, notification=None):
+    opts.lrf  = False
+    opts.epub = False
+    opts.mobi = True
+    if opts.debug:
+        opts.verbose = 2
+    parser = option_parser()
+    with TemporaryDirectory('_feeds2mobi') as tdir:
+        opts.output_dir = tdir
+        recipe = run_recipe(opts, recipe_arg, parser, notification=notification)
+        c = config()
+        recipe_opts = c.parse_string(recipe.oeb2mobi_options)
+        c.smart_update(recipe_opts, opts)
+        opts = recipe_opts
+        opf = glob.glob(os.path.join(tdir, '*.opf'))
+        if not opf:
+            raise Exception('Downloading of recipe: %s failed'%recipe_arg)
+        opf = opf[0]
+        
+        if opts.output is None:
+            fname = recipe.title + strftime(recipe.timefmt) + '.mobi'
+            opts.output = os.path.join(os.getcwd(), sanitize_file_name(fname))
+        
+        print 'Generating MOBI...'
+        opts.encoding = 'utf-8'
+        opts.source_profile = 'Browser'
+        oeb2mobi(opts, opf)
+    
+
+def main(args=sys.argv, notification=None, handler=None):
+    parser = option_parser()
+    opts, args = parser.parse_args(args)
+    if len(args) != 2 and opts.feeds is None:
+        parser.print_help()
+        return 1
+    recipe_arg = args[1] if len(args) > 1 else None
+    convert(opts, recipe_arg, notification=notification)
+        
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/ebooks/mobi/mobiml.py

@@ -12,7 +12,7 @@ import copy
 import re
 from lxml import etree
 from calibre.ebooks.oeb.base import namespace, barename
-from calibre.ebooks.oeb.base import XHTML, XHTML_NS
+from calibre.ebooks.oeb.base import XHTML, XHTML_NS, OEB_DOCS
 from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.oeb.transforms.flatcss import KeyMapper
 
@@ -96,8 +96,11 @@ class MobiMLizer(object):
         href = oeb.guide['cover'].href
         del oeb.guide['cover']
         item = oeb.manifest.hrefs[href]
-        oeb.manifest.remove(item)
+        if item.spine_position is not None:
-
+            oeb.spine.remove(item)                
+            if item.media_type in OEB_DOCS:
+                self.oeb.manifest.remove(item)
+    
     def mobimlize_spine(self):
         for item in self.oeb.spine:
             stylizer = Stylizer(item.data, item.href, self.oeb, self.profile)
@@ -137,7 +140,7 @@ class MobiMLizer(object):
         para = bstate.para
         if tag in SPECIAL_TAGS and not text:
             para = para if para is not None else bstate.body
-        elif para is None:
+        elif para is None or tag in ('td', 'th'):
             body = bstate.body
             if bstate.pbreak:
                 etree.SubElement(body, MBP('pagebreak'))
@@ -157,7 +160,8 @@ class MobiMLizer(object):
             elif indent != 0 and abs(indent) < self.profile.fbase:
                 indent = (indent / abs(indent)) * self.profile.fbase
             if tag in NESTABLE_TAGS:
-                para = wrapper = etree.SubElement(parent, XHTML(tag))
+                para = wrapper = etree.SubElement(
+                    parent, XHTML(tag), attrib=istate.attrib)
                 bstate.nested.append(para)
                 if tag == 'li' and len(istates) > 1:
                     istates[-2].list_num += 1
@@ -337,6 +341,10 @@ class MobiMLizer(object):
             tag = 'tr'
         elif display == 'table-cell':
             tag = 'td'
+        if tag in TABLE_TAGS:
+            for attr in ('rowspan', 'colspan'):
+                if attr in elem.attrib:
+                    istate.attrib[attr] = elem.attrib[attr]
         text = None
         if elem.text:
             if istate.preserve:
@@ -374,6 +382,6 @@ class MobiMLizer(object):
                 bstate.vpadding += bstate.vmargin
                 bstate.vmargin = 0
                 bstate.vpadding += vpadding
-        if tag in NESTABLE_TAGS and bstate.nested:
+        if bstate.nested and bstate.nested[-1].tag == elem.tag:
             bstate.nested.pop()
         istates.pop()

src/calibre/ebooks/mobi/reader.py

@@ -124,6 +124,7 @@ class BookHeader(object):
             sublangid = (langcode >> 10) & 0xFF
             self.language = main_language.get(langid, 'ENGLISH')
             self.sublanguage = sub_language.get(sublangid, 'NEUTRAL')
+            self.first_image_index = struct.unpack('>L', raw[0x6c:0x6c+4])[0]
             
             self.exth_flag, = struct.unpack('>L', raw[0x80:0x84])
             self.exth = None
@@ -310,7 +311,7 @@ class MobiReader(object):
             opf.cover = 'images/%05d.jpg'%(self.book_header.exth.cover_offset+1)
         manifest = [(htmlfile, 'text/x-oeb1-document')]
         bp = os.path.dirname(htmlfile)
-        for i in self.image_names:
+        for i in getattr(self, 'image_names', []):
             manifest.append((os.path.join(bp, 'images/', i), 'image/jpg'))
         
         opf.create_manifest(manifest)
@@ -441,17 +442,18 @@ class MobiReader(object):
             os.makedirs(output_dir)
         image_index = 0
         self.image_names = []
-        for i in range(self.num_sections):
+        for i in range(self.book_header.first_image_index, self.num_sections):
             if i in processed_records:
                 continue
             processed_records.append(i)
             data  = self.sections[i][0]
             buf = cStringIO.StringIO(data)
+            image_index += 1
             try:
                 im = PILImage.open(buf)                
             except IOError:
                 continue
-            image_index += 1 
+             
             path = os.path.join(output_dir, '%05d.jpg'%image_index)
             self.image_names.append(os.path.basename(path))
             im.convert('RGB').save(open(path, 'wb'), format='JPEG')
@@ -474,31 +476,23 @@ def get_metadata(stream):
     if mr.book_header.exth is None:
         mi = MetaInformation(mr.name, [_('Unknown')])
     else:
-        tdir = tempfile.mkdtemp('_mobi_meta', __appname__+'_')
-        atexit.register(shutil.rmtree, tdir)
-        mr.extract_images([], tdir)
         mi = mr.create_opf('dummy.html')
-        if mi.cover:
+        try:
-            cover =  os.path.join(tdir, mi.cover)
+            if hasattr(mr.book_header.exth, 'cover_offset'):
-            if not os.access(cover, os.R_OK):
+                cover_index = mr.book_header.first_image_index + mr.book_header.exth.cover_offset
-                fname = os.path.basename(cover)
+                data  = mr.sections[cover_index][0]
-                match = re.match(r'(\d+)(.+)', fname)
+            else:
-                if match:
+                data  = mr.sections[mr.book_header.first_image_index][0]
-                    num, ext = int(match.group(1), 10), match.group(2)
+            buf = cStringIO.StringIO(data)
-                    while num > 0:
+            im = PILImage.open(buf)
-                        num -= 1
+            obuf = cStringIO.StringIO()
-                        candidate = os.path.join(os.path.dirname(cover), '%05d%s'%(num, ext))
+            im.convert('RGBA').save(obuf, format='JPEG')
-                        if os.access(candidate, os.R_OK):
+            mi.cover_data = ('jpg', obuf.getvalue())
-                            cover = candidate
+        except:
-                            break
+            import traceback
-                    
+            traceback.print_exc()
-            if os.access(cover, os.R_OK):
+        return mi
-                mi.cover_data = ('JPEG', open(os.path.join(tdir, cover), 'rb').read())
+        
-        else:
-            path = os.path.join(tdir, 'images', '00001.jpg')
-            if os.access(path, os.R_OK):
-                mi.cover_data = ('JPEG', open(path, 'rb').read())
-    return mi
         
 def option_parser():
     from calibre.utils.config import OptionParser

src/calibre/ebooks/mobi/writer.py

@@ -34,8 +34,7 @@ from calibre.ebooks.mobi.palmdoc import compress_doc
 from calibre.ebooks.mobi.langcodes import iana2mobi
 from calibre.ebooks.mobi.mobiml import MBP_NS, MBP, MobiMLizer
 from calibre.customize.ui import run_plugins_on_postprocess
-from calibre.utils.config import OptionParser
+from calibre.utils.config import Config, StringConfig
-from optparse import OptionGroup
 
 # TODO:
 # - Allow override CSS (?)
@@ -95,6 +94,7 @@ class Serializer(object):
     def __init__(self, oeb, images):
         self.oeb = oeb
         self.images = images
+        self.logger = oeb.logger
         self.id_offsets = {}
         self.href_offsets = defaultdict(list)
         self.breaks = []
@@ -144,8 +144,8 @@ class Serializer(object):
         item = hrefs[path] if path else None
         if item and item.spine_position is None:
             return False
-        id =  item.id if item else base.id
+        path =  item.href if item else base.href
-        href = '#'.join((id, frag)) if frag else id
+        href = '#'.join((path, frag)) if frag else path
         buffer.write('filepos=')
         self.href_offsets[href].append(buffer.tell())
         buffer.write('0000000000')
@@ -170,7 +170,7 @@ class Serializer(object):
         buffer = self.buffer
         if not item.linear:
             self.breaks.append(buffer.tell() - 1)
-        self.id_offsets[item.id] = buffer.tell()
+        self.id_offsets[item.href] = buffer.tell()
         for elem in item.data.find(XHTML('body')):
             self.serialize_elem(elem, item)
         buffer.write('<mbp:pagebreak/>')
@@ -180,12 +180,11 @@ class Serializer(object):
         if not isinstance(elem.tag, basestring) \
            or namespace(elem.tag) not in nsrmap:
             return
-        hrefs = self.oeb.manifest.hrefs
         tag = prefixname(elem.tag, nsrmap)
         for attr in ('name', 'id'):
             if attr in elem.attrib:
-                id = '#'.join((item.id, elem.attrib[attr]))
+                href = '#'.join((item.href, elem.attrib[attr]))
-                self.id_offsets[id] = buffer.tell()
+                self.id_offsets[href] = buffer.tell()
                 del elem.attrib[attr]
         if tag == 'a' and not elem.attrib \
            and not len(elem) and not elem.text:
@@ -203,7 +202,7 @@ class Serializer(object):
                         continue
                 elif attr == 'src':
                     href = item.abshref(val)
-                    if href in hrefs:
+                    if href in self.images:
                         index = self.images[href]
                         buffer.write('recindex="%05d"' % index)
                         continue
@@ -233,8 +232,12 @@ class Serializer(object):
 
     def fixup_links(self):
         buffer = self.buffer
-        for id, hoffs in self.href_offsets.items():
+        id_offsets = self.id_offsets
-            ioff = self.id_offsets[id]
+        for href, hoffs in self.href_offsets.items():
+            if href not in id_offsets:
+                self.logger.warn('Hyperlink target %r not found' % href)
+                href, _ = urldefrag(href)
+            ioff = self.id_offsets[href]
             for hoff in hoffs:
                 buffer.seek(hoff)
                 buffer.write('%010d' % ioff)
@@ -360,7 +363,11 @@ class MobiWriter(object):
         if image.format not in ('JPEG', 'GIF'):
             width, height = image.size
             area = width * height
-            format = 'GIF' if area <= 40000 else 'JPEG'
+            if area <= 40000:
+                format = 'GIF'
+            else:
+                image = image.convert('RGBA')
+                format = 'JPEG'
             changed = True
         if dimen is not None:
             image.thumbnail(dimen, Image.ANTIALIAS)
@@ -494,41 +501,45 @@ class MobiWriter(object):
             self._write(record)
 
 
-def add_mobi_options(parser):
+def config(defaults=None):
-    profiles = Context.PROFILES.keys()
+    desc = _('Options to control the conversion to MOBI')
-    profiles.sort()
+    _profiles = list(sorted(Context.PROFILES.keys()))
-    profiles = ', '.join(profiles)
+    if defaults is None:
-    group = OptionGroup(parser, _('Mobipocket'),
+        c = Config('mobi', desc)
-        _('Mobipocket-specific options.'))
+    else:
-    group.add_option(
+        c = StringConfig(defaults, desc)
-        '-c', '--compress', default=False, action='store_true',
+        
-        help=_('Compress file text using PalmDOC compression. '
+    mobi = c.add_group('mobipocket', _('Mobipocket-specific options.'))
+    mobi('compress', ['--compress'], default=False,
+         help=_('Compress file text using PalmDOC compression. '
                'Results in smaller files, but takes a long time to run.'))
-    group.add_option(
+    mobi('rescale_images', ['--rescale-images'], default=False, 
-        '-r', '--rescale-images', default=False, action='store_true',
         help=_('Modify images to meet Palm device size limitations.'))
-    parser.add_option_group(group)
+    mobi('toc_title', ['--toc-title'], default=None, 
-    group = OptionGroup(parser, _('Profiles'), _('Device renderer profiles. '
+         help=_('Title for any generated in-line table of contents.'))
-        'Affects conversion of default font sizes and rasterization '
+    profiles = c.add_group('profiles', _('Device renderer profiles. '
-        'resolution.  Valid profiles are: %s.') % profiles)
+        'Affects conversion of font sizes, image rescaling and rasterization '
-    group.add_option(
+        'of tables. Valid profiles are: %s.') % ', '.join(_profiles))
-        '--source-profile', default='Browser', metavar='PROFILE',
+    profiles('source_profile', ['--source-profile'],
-        help=_("Source renderer profile. Default is 'Browser'."))
+             default='Browser', choices=_profiles,
-    group.add_option(
+             help=_("Source renderer profile. Default is %default."))
-        '--dest-profile', default='CybookG3', metavar='PROFILE',
+    profiles('dest_profile', ['--dest-profile'],
-        help=_("Destination renderer profile. Default is 'CybookG3'."))
+             default='CybookG3', choices=_profiles,
-    parser.add_option_group(group)
+             help=_("Destination renderer profile. Default is %default."))
-    return
+    c.add_opt('encoding', ['--encoding'], default=None,
-            
+              help=_('Character encoding for HTML files. Default is to auto detect.'))
+    return c
+    
+
 def option_parser():
-    parser = OptionParser(usage=_('%prog [options] OPFFILE'))
+    c = config()
+    parser = c.option_parser(usage='%prog '+_('[options]')+' file.opf')
     parser.add_option(
         '-o', '--output', default=None, 
         help=_('Output file. Default is derived from input filename.'))
     parser.add_option(
         '-v', '--verbose', default=0, action='count',
         help=_('Useful for debugging.'))
-    add_mobi_options(parser)
     return parser
 
 def oeb2mobi(opts, inpath):
@@ -549,8 +560,8 @@ def oeb2mobi(opts, inpath):
     compression = PALMDOC if opts.compress else UNCOMPRESSED
     imagemax = PALM_MAX_IMAGE_SIZE if opts.rescale_images else None
     context = Context(source, dest)
-    oeb = OEBBook(inpath, logger=logger)
+    oeb = OEBBook(inpath, logger=logger, encoding=opts.encoding)
-    tocadder = HTMLTOCAdder()
+    tocadder = HTMLTOCAdder(title=opts.toc_title)
     tocadder.transform(oeb, context)
     mangler = CaseMangler()
     mangler.transform(oeb, context)

src/calibre/ebooks/oeb/base.py

@@ -15,11 +15,12 @@ from urlparse import urldefrag, urlparse, urlunparse
 from urllib import unquote as urlunquote
 import logging
 import re
-import htmlentitydefs
 import uuid
 import copy
 from lxml import etree
+from lxml import html
 from calibre import LoggingInterface
+from calibre.translations.dynamic import translate
 
 XML_PARSER = etree.XMLParser(recover=True)
 XML_NS = 'http://www.w3.org/XML/1998/namespace'
@@ -67,14 +68,6 @@ OEB_IMAGES = set([GIF_MIME, JPEG_MIME, PNG_MIME, SVG_MIME])
 
 MS_COVER_TYPE = 'other.ms-coverimage-standard'
 
-recode = lambda s: s.decode('iso-8859-1').encode('ascii', 'xmlcharrefreplace')
-ENTITYDEFS = dict((k, recode(v)) for k, v in htmlentitydefs.entitydefs.items())
-del ENTITYDEFS['lt']
-del ENTITYDEFS['gt']
-del ENTITYDEFS['quot']
-del ENTITYDEFS['amp']
-del recode
-
 
 def element(parent, *args, **kwargs):
     if parent is not None:
@@ -97,6 +90,9 @@ def prefixname(name, nsrmap):
         return barename(name)
     return ':'.join((prefix, barename(name)))
 
+def XPath(expr):
+    return etree.XPath(expr, namespaces=XPNSMAP)
+
 def xpath(elem, expr):
     return elem.xpath(expr, namespaces=XPNSMAP)
 
@@ -298,17 +294,20 @@ class Metadata(object):
 
 class Manifest(object):
     class Item(object):
-        ENTITY_RE = re.compile(r'&([a-zA-Z_:][a-zA-Z0-9.-_:]+);')
         NUM_RE = re.compile('^(.*)([0-9][0-9.]*)(?=[.]|$)')
+        META_XP = XPath('/h:html/h:head/h:meta[@http-equiv="Content-Type"]')
     
-        def __init__(self, id, href, media_type,
+        def __init__(self, oeb, id, href, media_type,
                      fallback=None, loader=str, data=None):
+            self.oeb = oeb
             self.id = id
             self.href = self.path = urlnormalize(href)
             self.media_type = media_type
             self.fallback = fallback
             self.spine_position = None
             self.linear = True
+            if loader is None and data is None:
+                loader = oeb.container.read
             self._loader = loader
             self._data = data
 
@@ -317,13 +316,20 @@ class Manifest(object):
                 % (self.id, self.href, self.media_type)
 
         def _force_xhtml(self, data):
-            repl = lambda m: ENTITYDEFS.get(m.group(1), m.group(0))
+            if self.oeb.encoding is not None:
-            data = self.ENTITY_RE.sub(repl, data)
+                data = data.decode(self.oeb.encoding, 'replace')
-            data = etree.fromstring(data, parser=XML_PARSER)
+            try:
+                data = etree.fromstring(data, parser=XML_PARSER)
+            except etree.XMLSyntaxError:
+                data = html.fromstring(data)
+                data = etree.tostring(data, encoding=unicode)
+                data = etree.fromstring(data, parser=XML_PARSER)
             if namespace(data.tag) != XHTML_NS:
                 data.attrib['xmlns'] = XHTML_NS
-                data = etree.tostring(data)
+                data = etree.tostring(data, encoding=unicode)
                 data = etree.fromstring(data, parser=XML_PARSER)
+            for meta in self.META_XP(data):
+                meta.getparent().remove(meta)
             return data
         
         def data():
@@ -400,9 +406,8 @@ class Manifest(object):
         self.hrefs = {}
 
     def add(self, id, href, media_type, fallback=None, loader=None, data=None):
-        loader = loader or self.oeb.container.read
         item = self.Item(
-            id, href, media_type, fallback, loader, data)
+            self.oeb, id, href, media_type, fallback, loader, data)
         self.ids[item.id] = item
         self.hrefs[item.href] = item
         return item
@@ -506,6 +511,7 @@ class Spine(object):
         self.items.pop(index)
         for i in xrange(index, len(self.items)):
             self.items[i].spine_position = i
+        item.spine_position = None
     
     def __iter__(self):
         for item in self.items:
@@ -539,27 +545,36 @@ class Spine(object):
 
 class Guide(object):
     class Reference(object):
-        _TYPES_TITLES = [('cover', 'Cover'), ('title-page', 'Title Page'),
+        _TYPES_TITLES = [('cover', __('Cover')),
-            ('toc', 'Table of Contents'), ('index', 'Index'),
+                         ('title-page', __('Title Page')),
-            ('glossary', 'Glossary'), ('acknowledgements', 'Acknowledgements'),
+                         ('toc', __('Table of Contents')),
-            ('bibliography', 'Bibliography'), ('colophon', 'Colophon'),
+                         ('index', __('Index')),
-            ('copyright-page', 'Copyright'), ('dedication', 'Dedication'),
+                         ('glossary', __('Glossary')),
-            ('epigraph', 'Epigraph'), ('foreword', 'Foreword'),
+                         ('acknowledgements', __('Acknowledgements')),
-            ('loi', 'List of Illustrations'), ('lot', 'List of Tables'),
+                         ('bibliography', __('Bibliography')),
-            ('notes', 'Notes'), ('preface', 'Preface'),
+                         ('colophon', __('Colophon')),
-            ('text', 'Main Text')]
+                         ('copyright-page', __('Copyright')),
+                         ('dedication', __('Dedication')),
+                         ('epigraph', __('Epigraph')),
+                         ('foreword', __('Foreword')),
+                         ('loi', __('List of Illustrations')),
+                         ('lot', __('List of Tables')),
+                         ('notes', __('Notes')),
+                         ('preface', __('Preface')),
+                         ('text', __('Main Text'))]
         TYPES = set(t for t, _ in _TYPES_TITLES)
         TITLES = dict(_TYPES_TITLES)
         ORDER = dict((t, i) for (t, _), i in izip(_TYPES_TITLES, count(0)))
         
-        def __init__(self, type, title, href):
+        def __init__(self, oeb, type, title, href):
+            self.oeb = oeb
             if type.lower() in self.TYPES:
                 type = type.lower()
             elif type not in self.TYPES and \
                  not type.startswith('other.'):
                 type = 'other.' + type
-            if not title:
+            if not title and type in self.TITLES:
-                title = self.TITLES.get(type, None)
+                title = oeb.translate(self.TITLES[type])
             self.type = type
             self.title = title
             self.href = urlnormalize(href)
@@ -578,13 +593,21 @@ class Guide(object):
             if not isinstance(other, Guide.Reference):
                 return NotImplemented
             return cmp(self._order, other._order)
+        
+        def item():
+            def fget(self):
+                path, frag = urldefrag(self.href)
+                hrefs = self.oeb.manifest.hrefs
+                return hrefs.get(path, None)
+            return property(fget=fget)
+        item = item()
     
     def __init__(self, oeb):
         self.oeb = oeb
         self.refs = {}
     
     def add(self, type, title, href):
-        ref = self.Reference(type, title, href)
+        ref = self.Reference(self.oeb, type, title, href)
         self.refs[type] = ref
         return ref
     
@@ -594,9 +617,7 @@ class Guide(object):
     __iter__ = iterkeys
     
     def values(self):
-        values = list(self.refs.values())
+        return sorted(self.refs.values())
-        values.sort()
-        return values
     
     def items(self):
         for type, ref in self.refs.items():
@@ -680,31 +701,33 @@ class TOC(object):
             node.to_opf1(tour)
         return tour
     
-    def to_ncx(self, parent, playorder=None, depth=1):
+    def to_ncx(self, parent, order=None, depth=1):
-        if not playorder: playorder = [0]
+        if not order: order = [0]
         for node in self.nodes:
-            playorder[0] += 1
+            order[0] += 1
+            playOrder = str(order[0])
+            id = self.id or 'np' + playOrder
             point = etree.SubElement(parent,
-                NCX('navPoint'), attrib={'playOrder': str(playorder[0])})
+                NCX('navPoint'), id=id, playOrder=playOrder)
             if self.klass:
                 point.attrib['class'] = node.klass
-            if self.id:
-                point.attrib['id'] = node.id
             label = etree.SubElement(point, NCX('navLabel'))
             etree.SubElement(label, NCX('text')).text = node.title
             href = node.href if depth > 1 else urldefrag(node.href)[0]
             child = etree.SubElement(point,
                 NCX('content'), attrib={'src': href})
-            node.to_ncx(point, playorder, depth+1)
+            node.to_ncx(point, order, depth+1)
         return parent
 
     
 class OEBBook(object):
-    def __init__(self, opfpath=None, container=None, logger=FauxLogger()):
+    def __init__(self, opfpath=None, container=None, encoding=None,
+                 logger=FauxLogger()):
         if opfpath and not container:
             container = DirContainer(os.path.dirname(opfpath))
             opfpath = os.path.basename(opfpath)
         self.container = container
+        self.encoding = encoding
         self.logger = logger
         if opfpath or container:
             opf = self._read_opf(opfpath)
@@ -802,12 +825,20 @@ class OEBBook(object):
     def _manifest_from_opf(self, opf):
         self.manifest = manifest = Manifest(self)
         for elem in xpath(opf, '/o2:package/o2:manifest/o2:item'):
+            id = elem.get('id')
             href = elem.get('href')
+            media_type = elem.get('media-type')
+            fallback = elem.get('fallback')
+            if href in manifest.hrefs:
+                self.logger.warn(u'Duplicate manifest entry for %r.' % href)
+                continue
             if not self.container.exists(href):
                 self.logger.warn(u'Manifest item %r not found.' % href)
                 continue
-            manifest.add(elem.get('id'), href, elem.get('media-type'),
+            if id in manifest.ids:
-                         elem.get('fallback'))
+                self.logger.warn(u'Duplicate manifest id %r.' % id)
+                id, href = manifest.generate(id, href)
+            manifest.add(id, href, media_type, fallback)
     
     def _spine_from_opf(self, opf):
         self.spine = spine = Spine(self)
@@ -970,6 +1001,11 @@ class OEBBook(object):
         self._toc_from_opf(opf)
         self._ensure_cover_image()
 
+    def translate(self, text):
+        lang = str(self.metadata.language[0])
+        lang = lang.split('-', 1)[0].lower()
+        return translate(lang, text)
+    
     def to_opf1(self):
         package = etree.Element('package',
             attrib={'unique-identifier': self.uid.id})
@@ -983,22 +1019,11 @@ class OEBBook(object):
         guide = self.guide.to_opf1(package)
         return {OPF_MIME: ('content.opf', package)}
 
-    def _generate_ncx_item(self):
-        id = 'ncx'
-        index = 0
-        while id in self.manifest:
-            id = 'ncx' + str(index)
-            index = index + 1
-        href = 'toc'
-        index = 0
-        while (href + '.ncx') in self.manifest.hrefs:
-            href = 'toc' + str(index)
-        href += '.ncx'
-        return (id, href)
-        
     def _to_ncx(self):
-        ncx = etree.Element(NCX('ncx'), attrib={'version': '2005-1'},
+        lang = unicode(self.metadata.language[0])
-                            nsmap={None: NCX_NS})
+        ncx = etree.Element(NCX('ncx'),
+            attrib={'version': '2005-1', XML('lang'): lang},
+            nsmap={None: NCX_NS})
         head = etree.SubElement(ncx, NCX('head'))
         etree.SubElement(head, NCX('meta'),
             attrib={'name': 'dtb:uid', 'content': unicode(self.uid)})
@@ -1021,7 +1046,7 @@ class OEBBook(object):
             nsmap={None: OPF2_NS})
         metadata = self.metadata.to_opf2(package)
         manifest = self.manifest.to_opf2(package)
-        id, href = self._generate_ncx_item()
+        id, href = self.manifest.generate('ncx', 'toc.ncx')
         etree.SubElement(manifest, OPF('item'),
             attrib={'id': id, 'href': href, 'media-type': NCX_MIME})
         spine = self.spine.to_opf2(package)

src/calibre/ebooks/oeb/stylizer.py

@@ -223,8 +223,11 @@ class Stylizer(object):
             for key in composition:
                 style[key] = 'inherit'
         else:
-            primitives = [v.cssText for v in cssvalue]
+            try:
-            primitites.reverse()
+                primitives = [v.cssText for v in cssvalue]
+            except TypeError:
+                primitives = [cssvalue.cssText]
+            primitives.reverse()
             value = primitives.pop()
             for key in composition:
                 if cssproperties.cssvalues[key](value):

src/calibre/ebooks/oeb/transforms/htmltoc.py

@@ -13,6 +13,10 @@ from calibre.ebooks.oeb.base import XML, XHTML, XHTML_NS
 from calibre.ebooks.oeb.base import XHTML_MIME, CSS_MIME
 from calibre.ebooks.oeb.base import element
 
+__all__ = ['HTMLTOCAdder']
+
+DEFAULT_TITLE = __('Table of Contents')
+
 STYLE_CSS = {
     'nested': """
 .calibre_toc_header {
@@ -44,13 +48,15 @@ body > .calibre_toc_block {
     }
 
 class HTMLTOCAdder(object):
-    def __init__(self, style='nested'):
+    def __init__(self, title=None, style='nested'):
+        self.title = title
         self.style = style
     
     def transform(self, oeb, context):
         if 'toc' in oeb.guide:
             return
         oeb.logger.info('Generating in-line TOC...')
+        title = self.title or oeb.translate(DEFAULT_TITLE)
         style = self.style
         if style not in STYLE_CSS:
             oeb.logger.error('Unknown TOC style %r' % style)
@@ -61,15 +67,15 @@ class HTMLTOCAdder(object):
         contents = element(None, XHTML('html'), nsmap={None: XHTML_NS},
                            attrib={XML('lang'): language})
         head = element(contents, XHTML('head'))
-        title = element(head, XHTML('title'))
+        htitle = element(head, XHTML('title'))
-        title.text = 'Table of Contents'
+        htitle.text = title
         element(head, XHTML('link'), rel='stylesheet', type=CSS_MIME,
                 href=css_href)
         body = element(contents, XHTML('body'),
                        attrib={'class': 'calibre_toc'})
         h1 = element(body, XHTML('h1'),
                      attrib={'class': 'calibre_toc_header'})
-        h1.text = 'Table of Contents'
+        h1.text = title
         self.add_toc_level(body, oeb.toc)
         id, href = oeb.manifest.generate('contents', 'contents.xhtml')
         item = oeb.manifest.add(id, href, XHTML_MIME, data=contents)

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -41,8 +41,9 @@ class ManifestTrimmer(object):
         while unchecked:
             new = set()
             for item in unchecked:
-                if item.media_type in OEB_DOCS or \
+                if (item.media_type in OEB_DOCS or 
-                   item.media_type[-4:] in ('/xml', '+xml'):
+                    item.media_type[-4:] in ('/xml', '+xml')) and \
+                   item.data is not None:
                     hrefs = [sel(item.data) for sel in LINK_SELECTORS]
                     for href in chain(*hrefs):
                         href = item.abshref(href)

src/calibre/gui2/dialogs/epub.py

@@ -15,7 +15,7 @@ from lxml.etree import XPath
 from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 from calibre.gui2.dialogs.epub_ui import Ui_Dialog 
 from calibre.gui2 import error_dialog, choose_images, pixmap_to_data
-from calibre.ebooks.epub.from_any import SOURCE_FORMATS, config
+from calibre.ebooks.epub.from_any import SOURCE_FORMATS, config as epubconfig
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.ebooks.metadata.opf import OPFCreator
@@ -24,9 +24,12 @@ from calibre.ebooks.metadata import authors_to_string, string_to_authors
 
 class Config(QDialog, Ui_Dialog):
     
-    def __init__(self, parent, db, row=None):
+    OUTPUT = 'EPUB'
+        
+    def __init__(self, parent, db, row=None, config=epubconfig):
         QDialog.__init__(self, parent)
         self.setupUi(self)
+        self.hide_controls()
         self.connect(self.category_list, SIGNAL('itemEntered(QListWidgetItem *)'),
                         self.show_category_help)
         self.connect(self.cover_button, SIGNAL("clicked()"), self.select_cover)
@@ -38,7 +41,7 @@ class Config(QDialog, Ui_Dialog):
         if row is not None:
             self.id = db.id(row)
             base = config().as_string() + '\n\n'
-            defaults = self.db.conversion_options(self.id, 'epub')
+            defaults = self.db.conversion_options(self.id, self.OUTPUT.lower())
             defaults = base + (defaults if defaults else '')
             self.config = config(defaults=defaults)
         else:
@@ -47,9 +50,18 @@ class Config(QDialog, Ui_Dialog):
         self.get_source_format()
         self.category_list.setCurrentRow(0)
         if self.row is None:
-            self.setWindowTitle(_('Bulk convert to EPUB'))
+            self.setWindowTitle(_('Bulk convert to ')+self.OUTPUT)
         else:
-            self.setWindowTitle(_(u'Convert %s to EPUB')%unicode(self.title.text()))
+            self.setWindowTitle((_(u'Convert %s to ')%unicode(self.title.text()))+self.OUTPUT)
+    
+    def hide_controls(self):
+        self.source_profile_label.setVisible(False)
+        self.opt_source_profile.setVisible(False)
+        self.dest_profile_label.setVisible(False)
+        self.opt_dest_profile.setVisible(False)
+        self.opt_toc_title.setVisible(False)
+        self.toc_title_label.setVisible(False)
+        self.opt_rescale_images.setVisible(False)
         
     def initialize(self):
         self.__w = []
@@ -81,8 +93,8 @@ class Config(QDialog, Ui_Dialog):
     def show_category_help(self, item):
         text = unicode(item.text())
         help = {
-                _('Metadata')          : _('Specify metadata such as title and author for the book.\n\nMetadata will be updated in the database as well as the generated EPUB file.'),
+                _('Metadata')          : _('Specify metadata such as title and author for the book.\n\nMetadata will be updated in the database as well as the generated %s file.')%self.OUTPUT,
-                _('Look & Feel')       : _('Adjust the look of the generated EPUB file by specifying things like font sizes.'),
+                _('Look & Feel')       : _('Adjust the look of the generated ebook by specifying things like font sizes.'),
                 _('Page Setup')        : _('Specify the page layout settings like margins.'),
                 _('Chapter Detection') : _('Fine tune the detection of chapter and section headings.'),                  
                 }
@@ -195,7 +207,7 @@ class Config(QDialog, Ui_Dialog):
                 elif isinstance(g, QCheckBox):
                     self.config.set(pref.name, bool(g.isChecked()))
         if self.row is not None:
-            self.db.set_conversion_options(self.id, 'epub', self.config.src)        
+            self.db.set_conversion_options(self.id, self.OUTPUT.lower(), self.config.src)        
         
     
     def initialize_options(self):
@@ -235,7 +247,7 @@ class Config(QDialog, Ui_Dialog):
             elif len(choices) == 1:
                 self.source_format = choices[0]
             else:
-                d = ChooseFormatDialog(self.parent(), _('Choose the format to convert to EPUB'), choices)
+                d = ChooseFormatDialog(self.parent(), _('Choose the format to convert to ')+self.OUTPUT, choices)
                 if d.exec_() == QDialog.Accepted:
                     self.source_format = d.format()
                 

src/calibre/gui2/dialogs/epub.ui

@@ -89,36 +89,6 @@
               <string>Book Cover</string>
              </property>
              <layout class="QGridLayout" name="_2" >
-              <item row="0" column="0" >
-               <layout class="QHBoxLayout" name="_3" >
-                <item>
-                 <widget class="ImageView" name="cover" >
-                  <property name="text" >
-                   <string/>
-                  </property>
-                  <property name="pixmap" >
-                   <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
-                  </property>
-                  <property name="scaledContents" >
-                   <bool>true</bool>
-                  </property>
-                  <property name="alignment" >
-                   <set>Qt::AlignCenter</set>
-                  </property>
-                 </widget>
-                </item>
-               </layout>
-              </item>
-              <item row="2" column="0" >
-               <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
-                <property name="text" >
-                 <string>Use cover from &amp;source file</string>
-                </property>
-                <property name="checked" >
-                 <bool>true</bool>
-                </property>
-               </widget>
-              </item>
               <item row="1" column="0" >
                <layout class="QVBoxLayout" name="_4" >
                 <property name="spacing" >
@@ -170,6 +140,36 @@
                 </item>
                </layout>
               </item>
+              <item row="2" column="0" >
+               <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
+                <property name="text" >
+                 <string>Use cover from &amp;source file</string>
+                </property>
+                <property name="checked" >
+                 <bool>true</bool>
+                </property>
+               </widget>
+              </item>
+              <item row="0" column="0" >
+               <layout class="QHBoxLayout" name="_3" >
+                <item>
+                 <widget class="ImageView" name="cover" >
+                  <property name="text" >
+                   <string/>
+                  </property>
+                  <property name="pixmap" >
+                   <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
+                  </property>
+                  <property name="scaledContents" >
+                   <bool>true</bool>
+                  </property>
+                  <property name="alignment" >
+                   <set>Qt::AlignCenter</set>
+                  </property>
+                 </widget>
+                </item>
+               </layout>
+              </item>
              </layout>
              <zorder>opt_prefer_metadata_cover</zorder>
              <zorder></zorder>
@@ -456,6 +456,13 @@
              </property>
             </widget>
            </item>
+           <item row="4" column="0" >
+            <widget class="QCheckBox" name="opt_rescale_images" >
+             <property name="text" >
+              <string>&amp;Rescale images</string>
+             </property>
+            </widget>
+           </item>
           </layout>
          </item>
          <item>
@@ -475,7 +482,7 @@
        <widget class="QWidget" name="pagesetup_page" >
         <layout class="QGridLayout" name="_13" >
          <item row="0" column="0" >
-          <widget class="QLabel" name="label_11" >
+          <widget class="QLabel" name="profile_label" >
            <property name="text" >
             <string>&amp;Profile:</string>
            </property>
@@ -494,7 +501,7 @@
            </property>
           </widget>
          </item>
-         <item row="1" column="0" >
+         <item row="3" column="0" >
           <widget class="QLabel" name="label_12" >
            <property name="text" >
             <string>&amp;Left Margin:</string>
@@ -504,7 +511,7 @@
            </property>
           </widget>
          </item>
-         <item row="1" column="1" >
+         <item row="3" column="1" >
           <widget class="QSpinBox" name="opt_margin_left" >
            <property name="suffix" >
             <string> pt</string>
@@ -517,7 +524,7 @@
            </property>
           </widget>
          </item>
-         <item row="2" column="0" >
+         <item row="4" column="0" >
           <widget class="QLabel" name="label_13" >
            <property name="text" >
             <string>&amp;Right Margin:</string>
@@ -527,7 +534,7 @@
            </property>
           </widget>
          </item>
-         <item row="2" column="1" >
+         <item row="4" column="1" >
           <widget class="QSpinBox" name="opt_margin_right" >
            <property name="suffix" >
             <string> pt</string>
@@ -540,7 +547,7 @@
            </property>
           </widget>
          </item>
-         <item row="3" column="0" >
+         <item row="5" column="0" >
           <widget class="QLabel" name="label_14" >
            <property name="text" >
             <string>&amp;Top Margin:</string>
@@ -550,7 +557,7 @@
            </property>
           </widget>
          </item>
-         <item row="3" column="1" >
+         <item row="5" column="1" >
           <widget class="QSpinBox" name="opt_margin_top" >
            <property name="suffix" >
             <string> pt</string>
@@ -563,7 +570,7 @@
            </property>
           </widget>
          </item>
-         <item row="4" column="0" >
+         <item row="6" column="0" >
           <widget class="QLabel" name="label_15" >
            <property name="text" >
             <string>&amp;Bottom Margin:</string>
@@ -573,7 +580,7 @@
            </property>
           </widget>
          </item>
-         <item row="4" column="1" >
+         <item row="6" column="1" >
           <widget class="QSpinBox" name="opt_margin_bottom" >
            <property name="suffix" >
             <string> pt</string>
@@ -586,13 +593,39 @@
            </property>
           </widget>
          </item>
-         <item row="5" column="0" >
+         <item row="7" column="0" >
           <widget class="QCheckBox" name="opt_dont_split_on_page_breaks" >
            <property name="text" >
             <string>Do not &amp;split on page breaks</string>
            </property>
           </widget>
          </item>
+         <item row="1" column="0" >
+          <widget class="QLabel" name="source_profile_label" >
+           <property name="text" >
+            <string>&amp;Source profile:</string>
+           </property>
+           <property name="buddy" >
+            <cstring>opt_source_profile</cstring>
+           </property>
+          </widget>
+         </item>
+         <item row="1" column="1" >
+          <widget class="QComboBox" name="opt_source_profile" />
+         </item>
+         <item row="2" column="0" >
+          <widget class="QLabel" name="dest_profile_label" >
+           <property name="text" >
+            <string>&amp;Destination profile:</string>
+           </property>
+           <property name="buddy" >
+            <cstring>opt_dest_profile</cstring>
+           </property>
+          </widget>
+         </item>
+         <item row="2" column="1" >
+          <widget class="QComboBox" name="opt_dest_profile" />
+         </item>
         </layout>
        </widget>
        <widget class="QWidget" name="chapterdetection_page" >
@@ -721,6 +754,19 @@ p, li { white-space: pre-wrap; }
             <item row="5" column="1" >
              <widget class="QLineEdit" name="opt_level2_toc" />
             </item>
+            <item row="6" column="1" >
+             <widget class="QLineEdit" name="opt_toc_title" />
+            </item>
+            <item row="6" column="0" >
+             <widget class="QLabel" name="toc_title_label" >
+              <property name="text" >
+               <string>&amp;Title for generated TOC</string>
+              </property>
+              <property name="buddy" >
+               <cstring>opt_toc_title</cstring>
+              </property>
+             </widget>
+            </item>
            </layout>
           </widget>
          </item>

src/calibre/gui2/dialogs/mobi.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+from calibre.gui2.dialogs.epub import Config as _Config
+from calibre.ebooks.mobi.from_any import config as mobiconfig
+
+class Config(_Config):
+    
+    OUTPUT = 'MOBI'
+    
+    def __init__(self, parent, db, row=None):
+        _Config.__init__(self, parent, db, row=row, config=mobiconfig)
+        
+    def hide_controls(self):
+        self.profile_label.setVisible(False)
+        self.opt_profile.setVisible(False)
+        self.opt_dont_split_on_page_breaks.setVisible(False)
+        self.opt_preserve_tag_structure.setVisible(False)

src/calibre/gui2/library.py

@@ -162,7 +162,8 @@ class BooksModel(QAbstractTableModel):
 
     def refresh_ids(self, ids, current_row=-1):
         rows = self.db.refresh_ids(ids)
-        self.refresh_rows(rows, current_row=current_row)
+        if rows:
+            self.refresh_rows(rows, current_row=current_row)
             
     def refresh_rows(self, rows, current_row=-1):
         for row in rows:
@@ -261,7 +262,17 @@ class BooksModel(QAbstractTableModel):
             self.reset()
         self.sorted_on = (self.column_map[col], order)
         
-
+    
+    def refresh(self, reset=True):
+        try:
+            col = self.column_map.index(self.sorted_on[0])
+        except:
+            col = 0
+        self.db.refresh(field=self.column_map[col], 
+                        ascending=self.sorted_on[1]==Qt.AscendingOrder)
+        if reset:
+            self.reset()
+    
     def resort(self, reset=True):
         try:
             col = self.column_map.index(self.sorted_on[0])

src/calibre/gui2/main.py

@@ -25,7 +25,6 @@ from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            max_available_height, config, info_dialog, \
                            available_width
 from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
-from calibre.library.database import LibraryDatabase
 from calibre.gui2.dialogs.scheduler import Scheduler
 from calibre.gui2.update import CheckForUpdates
 from calibre.gui2.dialogs.progress import ProgressDialog
@@ -131,14 +130,14 @@ class Main(MainWindow, Ui_MainWindow):
         QObject.connect(self.stack, SIGNAL('currentChanged(int)'),
                         self.location_view.location_changed)
         
-        self.output_formats = sorted(['EPUB', 'LRF'])
+        self.output_formats = sorted(['EPUB', 'MOBI', 'LRF'])
         for f in self.output_formats:
             self.output_format.addItem(f)
         self.output_format.setCurrentIndex(self.output_formats.index(prefs['output_format']))
         def change_output_format(x):
             of = unicode(x).strip()
             if of != prefs['output_format']:
-                if of in ('EPUB', 'LIT'):
+                if of not in ('LRF',):
                     warning_dialog(self, 'Warning', 
                                    '<p>%s support is still in beta. If you find bugs, please report them by opening a <a href="http://calibre.kovidgoyal.net">ticket</a>.'%of).exec_()
                 prefs.set('output_format', of)
@@ -296,6 +295,8 @@ class Main(MainWindow, Ui_MainWindow):
         self.card_view.connect_dirtied_signal(self.upload_booklists)
 
         self.show()
+        if self.system_tray_icon.isVisible() and opts.start_in_tray:
+            self.hide()
         self.stack.setCurrentIndex(0)
         try:
             db = LibraryDatabase2(self.library_path)
@@ -309,18 +310,7 @@ class Main(MainWindow, Ui_MainWindow):
                 self.library_path = dir
                 db = LibraryDatabase2(self.library_path)
         self.library_view.set_database(db)
-        if self.olddb is not None:
+        prefs['library_path'] = self.library_path
-            pd = QProgressDialog('', '', 0, 100, self)
-            pd.setWindowModality(Qt.ApplicationModal)
-            pd.setCancelButton(None)
-            pd.setWindowTitle(_('Migrating database'))
-            pd.show()
-            number_of_books = db.migrate_old(self.olddb, pd)
-            self.olddb.close()
-            if number_of_books == 0:
-                os.remove(self.olddb.dbpath)
-            self.olddb = None
-            prefs['library_path'] = self.library_path
         self.library_view.sortByColumn(*dynamic.get('sort_column', ('timestamp', Qt.DescendingOrder)))
         if not self.library_view.restore_column_widths():
             self.library_view.resizeColumnsToContents()
@@ -488,7 +478,7 @@ class Main(MainWindow, Ui_MainWindow):
             self.raise_()
             self.activateWindow()
         elif msg.startswith('refreshdb:'):
-            self.library_view.model().resort()
+            self.library_view.model().refresh()
             self.library_view.model().research()
         else:
             print msg
@@ -1392,39 +1382,14 @@ class Main(MainWindow, Ui_MainWindow):
 
     def initialize_database(self):
         self.library_path = prefs['library_path']
-        self.olddb = None
         if self.library_path is None: # Need to migrate to new database layout
-            QMessageBox.information(self, 'Database format changed',
-                '''\
-<p>calibre's book storage format has changed. Instead of storing book files in a database, the
-files are now stored in a folder on your filesystem. You will now be asked to choose the folder 
-in which you want to store your books files. Any existing books will be automatically migrated.
-                ''')
-            self.database_path = prefs['database_path']
-            if not os.access(os.path.dirname(self.database_path), os.W_OK):
-                error_dialog(self, _('Database does not exist'), 
-                             _('The directory in which the database should be: %s no longer exists. Please choose a new database location.')%self.database_path).exec_()
-                self.database_path = choose_dir(self, 'database path dialog', 
-                                                _('Choose new location for database'))
-                if not self.database_path:
-                    self.database_path = os.path.expanduser('~').decode(sys.getfilesystemencoding())
-                if not os.path.exists(self.database_path):
-                    os.makedirs(self.database_path)
-                self.database_path = os.path.join(self.database_path, 'library1.db')
-                prefs['database_path'] = self.database_path
-            home = os.path.dirname(self.database_path)
-            if not os.path.exists(home):
-                home = os.getcwd()
             dir = unicode(QFileDialog.getExistingDirectory(self, 
-                            _('Choose a location for your ebook library.'), home))
+                            _('Choose a location for your ebook library.'), os.getcwd()))
             if not dir:
-                dir = os.path.dirname(self.database_path)
+                dir = os.path.expanduser('~/Library')
             self.library_path = os.path.abspath(dir)
-            try:
+        if not os.path.exists(self.library_path):
-                self.olddb = LibraryDatabase(self.database_path)
+            os.makedirs(self.library_path)
-            except:
-                traceback.print_exc()
-                self.olddb = None
 
 
     def read_settings(self):
@@ -1563,6 +1528,8 @@ path_to_ebook to the database.
 ''')
     parser.add_option('--with-library', default=None, action='store', 
                       help=_('Use the library located at the specified path.'))
+    parser.add_option('--start-in-tray', default=False, action='store_true',
+                      help=_('Start minimized to system tray.'))
     parser.add_option('-v', '--verbose', default=0, action='count',
                       help=_('Log debugging information to console'))
     return parser

src/calibre/gui2/tools.py

@@ -12,6 +12,7 @@ from PyQt4.Qt import QDialog
 from calibre.utils.config import prefs
 from calibre.gui2.dialogs.lrf_single import LRFSingleDialog, LRFBulkDialog
 from calibre.gui2.dialogs.epub import Config as EPUBConvert
+from calibre.gui2.dialogs.mobi import Config as MOBIConvert
 import calibre.gui2.dialogs.comicconf as ComicConf
 from calibre.gui2 import warning_dialog
 from calibre.ptempfile import PersistentTemporaryFile
@@ -19,14 +20,20 @@ from calibre.ebooks.lrf import preferred_source_formats as LRF_PREFERRED_SOURCE_
 from calibre.ebooks.metadata.opf import OPFCreator
 from calibre.ebooks.epub.from_any import SOURCE_FORMATS as EPUB_PREFERRED_SOURCE_FORMATS
 
-def convert_single_epub(parent, db, comics, others):
+def get_dialog(fmt):
+    return {
+              'epub':EPUBConvert,
+              'mobi':MOBIConvert,
+              }[fmt]
+
+def convert_single(fmt, parent, db, comics, others):
     changed = False
     jobs = []
     others_ids = [db.id(row) for row in others]
     comics_ids = [db.id(row) for row in comics]
     for row, row_id in zip(others, others_ids):
         temp_files = []
-        d = EPUBConvert(parent, db, row)
+        d = get_dialog(fmt)(parent, db, row)
         if d.source_format is not None:
             d.exec_()
             if d.result() == QDialog.Accepted:
@@ -35,7 +42,7 @@ def convert_single_epub(parent, db, comics, others):
                 pt = PersistentTemporaryFile('.'+d.source_format.lower())
                 pt.write(data)
                 pt.close()
-                of = PersistentTemporaryFile('.epub')
+                of = PersistentTemporaryFile('.'+fmt)
                 of.close()
                 opts.output = of.name
                 opts.from_opf = d.opf_file.name
@@ -45,8 +52,8 @@ def convert_single_epub(parent, db, comics, others):
                     temp_files.append(d.cover_file)
                     opts.cover = d.cover_file.name
                 temp_files.extend([d.opf_file, pt, of])
-                jobs.append(('any2epub', args, _('Convert book: ')+d.mi.title, 
+                jobs.append(('any2'+fmt, args, _('Convert book: ')+d.mi.title, 
-                             'EPUB', row_id, temp_files))
+                             fmt.upper(), row_id, temp_files))
                 changed = True
                 
     for row, row_id in zip(comics, comics_ids):
@@ -61,24 +68,24 @@ def convert_single_epub(parent, db, comics, others):
         if defaults is not None:
             db.set_conversion_options(db.id(row), 'comic', defaults)
         if opts is None: continue
-        for fmt in ['cbz', 'cbr']:
+        for _fmt in ['cbz', 'cbr']:
             try:
-                data = db.format(row, fmt.upper())
+                data = db.format(row, _fmt.upper())
                 if data is not None:
                     break                    
             except:
                 continue
-        pt = PersistentTemporaryFile('.'+fmt)
+        pt = PersistentTemporaryFile('.'+_fmt)
         pt.write(data)
         pt.close()
-        of = PersistentTemporaryFile('.epub')
+        of = PersistentTemporaryFile('.'+fmt)
         of.close()
         opts.output = of.name
         opts.verbose = 2
         args = [pt.name, opts]
         changed = True
-        jobs.append(('comic2epub', args, _('Convert comic: ')+opts.title, 
+        jobs.append(('comic2'+fmt, args, _('Convert comic: ')+opts.title, 
-                     'EPUB', row_id, [pt, of]))
+                     fmt.upper(), row_id, [pt, of]))
         
     return jobs, changed
     
@@ -146,9 +153,9 @@ def convert_single_lrf(parent, db, comics, others):
         
     return jobs, changed
 
-def convert_bulk_epub(parent, db, comics, others):
+def convert_bulk(fmt, parent, db, comics, others):
     if others:
-        d = EPUBConvert(parent, db)
+        d = get_dialog(fmt)(parent, db)
         if d.exec_() != QDialog.Accepted:
             others = []
         else:
@@ -169,9 +176,9 @@ def convert_bulk_epub(parent, db, comics, others):
         row_id = db.id(row)
         if row in others:
             data = None
-            for fmt in EPUB_PREFERRED_SOURCE_FORMATS:
+            for _fmt in EPUB_PREFERRED_SOURCE_FORMATS:
                 try:
-                    data = db.format(row, fmt.upper())
+                    data = db.format(row, _fmt.upper())
                     if data is not None:
                         break
                 except:
@@ -185,10 +192,10 @@ def convert_bulk_epub(parent, db, comics, others):
             opf_file = PersistentTemporaryFile('.opf')
             opf.render(opf_file)
             opf_file.close()
-            pt = PersistentTemporaryFile('.'+fmt.lower())
+            pt = PersistentTemporaryFile('.'+_fmt.lower())
             pt.write(data)
             pt.close()
-            of = PersistentTemporaryFile('.epub')
+            of = PersistentTemporaryFile('.'+fmt)
             of.close()
             cover = db.cover(row)
             cf = None
@@ -203,7 +210,7 @@ def convert_bulk_epub(parent, db, comics, others):
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
             temp_files = [cf] if cf is not None else []
             temp_files.extend([opf_file, pt, of])
-            jobs.append(('any2epub', args, desc, 'EPUB', row_id, temp_files))
+            jobs.append(('any2'+fmt, args, desc, fmt.upper(), row_id, temp_files))
         else:
             options = comic_opts.copy()
             mi = db.get_metadata(row)
@@ -212,24 +219,24 @@ def convert_bulk_epub(parent, db, comics, others):
             if mi.authors:
                 options.author =  ','.join(mi.authors)
             data = None
-            for fmt in ['cbz', 'cbr']:
+            for _fmt in ['cbz', 'cbr']:
                 try:
-                    data = db.format(row, fmt.upper())
+                    data = db.format(row, _fmt.upper())
                     if data is not None:
                         break                    
                 except:
                     continue
             
-            pt = PersistentTemporaryFile('.'+fmt.lower())
+            pt = PersistentTemporaryFile('.'+_fmt.lower())
             pt.write(data)
             pt.close()
-            of = PersistentTemporaryFile('.epub')
+            of = PersistentTemporaryFile('.'+fmt)
             of.close()
             setattr(options, 'output', of.name)
             options.verbose = 1
             args = [pt.name, options]
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
-            jobs.append(('comic2epub', args, desc, 'EPUB', row_id, [pt, of]))        
+            jobs.append(('comic2'+fmt, args, desc, fmt.upper(), row_id, [pt, of]))        
         
     if bad_rows:
         res = []
@@ -345,15 +352,14 @@ def set_conversion_defaults_lrf(comic, parent, db):
     else:
         LRFSingleDialog(parent, None, None).exec_()
 
-def set_conversion_defaults_epub(comic, parent, db):
+def _set_conversion_defaults(dialog, comic, parent, db):
     if comic:
         ComicConf.set_conversion_defaults(parent)
     else:
-        d = EPUBConvert(parent, db)
+        d = dialog(parent, db)
         d.setWindowTitle(_('Set conversion defaults'))
         d.exec_()
 
-
 def _fetch_news(data, fmt):
     pt = PersistentTemporaryFile(suffix='_feeds2%s.%s'%(fmt.lower(), fmt.lower()))
     pt.close()
@@ -385,22 +391,22 @@ def convert_single_ebook(*args):
     fmt = prefs['output_format'].lower()
     if fmt == 'lrf':
         return convert_single_lrf(*args)
-    elif fmt == 'epub':
+    elif fmt in ('epub', 'mobi'):
-        return convert_single_epub(*args)
+        return convert_single(fmt, *args)
     
 def convert_bulk_ebooks(*args):
     fmt = prefs['output_format'].lower()
     if fmt == 'lrf':
         return convert_bulk_lrf(*args)
-    elif fmt == 'epub':
+    elif fmt in ('epub', 'mobi'):
-        return convert_bulk_epub(*args)
+        return convert_bulk(fmt, *args)
     
 def set_conversion_defaults(comic, parent, db):
     fmt = prefs['output_format'].lower()
     if fmt == 'lrf':
         return set_conversion_defaults_lrf(comic, parent, db)
-    elif fmt == 'epub':
+    elif fmt in ('epub', 'mobi'):
-        return set_conversion_defaults_epub(comic, parent, db)
+        return _set_conversion_defaults(get_dialog(fmt), comic, parent, db)
 
 def fetch_news(data):
     fmt = prefs['output_format'].lower()

src/calibre/library/__init__.py

@@ -29,5 +29,5 @@ def server_config(defaults=None):
     c.add_opt('develop', ['--develop'], default=False,
               help='Development mode. Server automatically restarts on file changes and serves code files (html, css, js) from the file system instead of calibre\'s resource system.')
     c.add_opt('max_cover', ['--max-cover'], default='600x800', 
-              help=_('The maximum size for displayed covers'))
+              help=_('The maximum size for displayed covers. Default is %default.'))
     return c

src/calibre/library/database2.py

@@ -224,9 +224,17 @@ class ResultCache(SearchQueryParser):
         return False
     
     def refresh_ids(self, conn, ids):
+        '''
+        Refresh the data in the cache for books identified by ids.
+        Returns a list of affected rows or None if the rows are filtered.
+        '''
         for id in ids:
             self._data[id] = conn.get('SELECT * from meta WHERE id=?', (id,))[0]
-        return map(self.row, ids)
+        try:
+            return map(self.row, ids)
+        except ValueError:
+            pass
+        return None
     
     def books_added(self, ids, conn):
         if not ids:

src/calibre/library/static/gui.js

@@ -40,7 +40,7 @@ function create_table_headers() {
 
 
 function format_url(format, id, title) {
-    return 'get/'+format.toLowerCase() + '/'+title + '_' + id+'.'+format.toLowerCase();
+    return 'get/'+format.toLowerCase() + '/'+encodeURIComponent(title) + '_' + id+'.'+format.toLowerCase();
 }
 
 function render_book(book) {

src/calibre/linux.py

@@ -26,6 +26,7 @@ entry_points = {
                              'opf-meta  = calibre.ebooks.metadata.opf2:main',
                              'odt-meta  = calibre.ebooks.metadata.odt:main',
                              'epub-meta = calibre.ebooks.metadata.epub:main',
+                             'mobi-meta = calibre.ebooks.metadata.mobi:main',
                              'txt2lrf   = calibre.ebooks.lrf.txt.convert_from:main',
                              'html2lrf  = calibre.ebooks.lrf.html.convert_from:main',
                              'html2oeb  = calibre.ebooks.html:main',
@@ -40,6 +41,7 @@ entry_points = {
                              'calibre-server = calibre.library.server:main',
                              'feeds2lrf  = calibre.ebooks.lrf.feeds.convert_from:main',
                              'feeds2epub = calibre.ebooks.epub.from_feeds:main',
+                             'feeds2mobi = calibre.ebooks.mobi.from_feeds:main',
                              'web2lrf   = calibre.ebooks.lrf.web.convert_from:main',
                              'pdf2lrf   = calibre.ebooks.lrf.pdf.convert_from:main',
                              'mobi2lrf  = calibre.ebooks.lrf.mobi.convert_from:main',
@@ -189,6 +191,7 @@ def setup_completion(fatal_errors):
         from calibre.ebooks.html import option_parser as html2oeb
         from calibre.ebooks.odt.to_oeb import option_parser as odt2oeb
         from calibre.ebooks.epub.from_feeds import option_parser as feeds2epub
+        from calibre.ebooks.mobi.from_feeds import option_parser as feeds2mobi
         from calibre.ebooks.epub.from_any import option_parser as any2epub
         from calibre.ebooks.lit.from_any import option_parser as any2lit
         from calibre.ebooks.epub.from_comic import option_parser as comic2epub
@@ -219,7 +222,7 @@ def setup_completion(fatal_errors):
         f.write(opts_and_exts('any2epub', any2epub, any_formats))
         f.write(opts_and_exts('any2lit', any2lit, any_formats))
         f.write(opts_and_exts('any2mobi', any2mobi, any_formats))
-        f.write(opts_and_exts('oeb2mobi', oeb2mobi, ['mobi', 'prc']))
+        f.write(opts_and_exts('oeb2mobi', oeb2mobi, ['opf']))
         f.write(opts_and_exts('lrf2lrs', lrf2lrsop, ['lrf']))
         f.write(opts_and_exts('lrf-meta', metaop, ['lrf']))
         f.write(opts_and_exts('rtf-meta', metaop, ['rtf']))
@@ -239,7 +242,8 @@ def setup_completion(fatal_errors):
         f.write(opts_and_exts('comic2pdf', comic2epub, ['cbz', 'cbr']))
         f.write(opts_and_words('feeds2disk', feeds2disk, feed_titles))
         f.write(opts_and_words('feeds2lrf', feeds2lrf, feed_titles))
-        f.write(opts_and_words('feeds2lrf', feeds2epub, feed_titles))
+        f.write(opts_and_words('feeds2epub', feeds2epub, feed_titles))
+        f.write(opts_and_words('feeds2mobi', feeds2mobi, feed_titles))
         f.write(opts_and_exts('html2epub', html2epub, ['html', 'htm', 'xhtm', 'xhtml', 'opf']))
         f.write(opts_and_exts('html2oeb', html2oeb, ['html', 'htm', 'xhtm', 'xhtml']))
         f.write(opts_and_exts('odt2oeb', odt2oeb, ['odt']))
@@ -420,7 +424,7 @@ def install_man_pages(fatal_errors):
         if prog in ('prs500', 'pdf-meta', 'epub-meta', 'lit-meta',
                     'markdown-calibre', 'calibre-debug', 'fb2-meta',
                     'calibre-fontconfig', 'calibre-parallel', 'odt-meta',
-                    'rb-meta', 'imp-meta'):
+                    'rb-meta', 'imp-meta', 'mobi-meta'):
             continue
 
         help2man = ('help2man', prog, '--name', 'part of %s'%__appname__,

src/calibre/parallel.py

@@ -67,7 +67,15 @@ PARALLEL_FUNCS = {
         
       'comic2epub'    :
         ('calibre.ebooks.epub.from_comic', 'convert', {}, 'notification'),
-
+        
+      'any2mobi'     :
+        ('calibre.ebooks.mobi.from_any', 'any2mobi', {}, None),
+        
+      'feeds2mobi'   :
+        ('calibre.ebooks.mobi.from_feeds', 'main', {}, 'notification'),
+        
+      'comic2mobi'    :
+        ('calibre.ebooks.mobi.from_comic', 'convert', {}, 'notification'),
 }
 
 

src/calibre/startup.py

@@ -13,6 +13,10 @@ from gettext import GNUTranslations
 import __builtin__
 __builtin__.__dict__['_'] = lambda s: s
 
+# For strings which belong in the translation tables, but which shouldn't be
+# immediately translated to the environment language
+__builtin__.__dict__['__'] = lambda s: s
+
 from calibre.constants import iswindows, preferred_encoding, plugins
 from calibre.utils.config import prefs
 from calibre.translations.msgfmt import make

src/calibre/translations/dynamic.py

@@ -0,0 +1,26 @@
+'''
+Dynamic language lookup of translations for user-visible strings.
+'''
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
+
+from cStringIO import StringIO
+from gettext import GNUTranslations
+from calibre.translations.compiled import translations
+
+__all__ = ['translate']
+
+_CACHE = {}
+
+def translate(lang, text):
+    trans = None
+    if lang in _CACHE:
+        trans = _CACHE[lang]
+    elif lang in translations:
+        buf = StringIO(translations[lang])
+        trans = GNUTranslations(buf)
+        _CACHE[lang] = trans
+    if trans is None:
+        return getattr(__builtins__, '_', lambda x: x)(text)
+    return trans.ugettext(text)

src/calibre/web/feeds/main.py

@@ -45,6 +45,8 @@ If you specify this option, any argument to %prog is ignored and a default recip
                  help='Optimize fetching for subsequent conversion to LRF.')
     c.add_opt('epub', ['--epub'], default=False, action='store_true', 
                  help='Optimize fetching for subsequent conversion to EPUB.')
+    c.add_opt('mobi', ['--mobi'], default=False, action='store_true', 
+                 help='Optimize fetching for subsequent conversion to MOBI.')
     c.add_opt('recursions', ['--recursions'], default=0,
                  help=_('Number of levels of links to follow on webpages that are linked to from feeds. Defaul %default'))
     c.add_opt('output_dir', ['--output-dir'], default='.', 

src/calibre/web/feeds/news.py

@@ -20,7 +20,7 @@ from PyQt4.QtWebKit import QWebPage
 from calibre import browser, __appname__, iswindows, LoggingInterface, \
                     strftime, __version__, preferred_encoding
 from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString, CData, Tag
-from calibre.ebooks.metadata.opf import OPFCreator
+from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.ebooks.lrf import entity_to_unicode
 from calibre.ebooks.metadata.toc import TOC
 from calibre.ebooks.metadata import MetaInformation
@@ -152,6 +152,8 @@ class BasicNewsRecipe(object, LoggingInterface):
     
     #: Options to pass to html2epub to customize generation of EPUB ebooks.
     html2epub_options     = ''
+    #: Options to pass to oeb2mobi to customize generation of MOBI ebooks.
+    oeb2mobi_options     = ''
     
     #: List of tags to be removed. Specified tags are removed from downloaded HTML.
     #: A tag is specified as a dictionary of the form::
@@ -532,7 +534,9 @@ class BasicNewsRecipe(object, LoggingInterface):
             if body is not None:
                 templ = self.navbar.generate(False, f, a, feed_len, 
                                              not self.has_single_feed, 
-                                             url, __appname__, center=self.center_navbar)
+                                             url, __appname__, 
+                                             center=self.center_navbar,
+                                             extra_css=self.extra_css)
                 elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div')
                 body.insert(0, elem)
         if self.remove_javascript:
@@ -575,7 +579,8 @@ class BasicNewsRecipe(object, LoggingInterface):
     
     def feeds2index(self, feeds):
         templ = templates.IndexTemplate()
-        return templ.generate(self.title, self.timefmt, feeds).render(doctype='xhtml')
+        return templ.generate(self.title, self.timefmt, feeds,
+                              extra_css=self.extra_css).render(doctype='xhtml')
     
     @classmethod
     def description_limiter(cls, src):
@@ -626,7 +631,8 @@ class BasicNewsRecipe(object, LoggingInterface):
                 
                 
         templ = templates.FeedTemplate()
-        return templ.generate(feed, self.description_limiter).render(doctype='xhtml')
+        return templ.generate(feed, self.description_limiter, 
+                              extra_css=self.extra_css).render(doctype='xhtml')
         
     
     def create_logger(self, feed_number, article_number):
@@ -872,6 +878,7 @@ class BasicNewsRecipe(object, LoggingInterface):
         
         manifest = [os.path.join(dir, 'feed_%d'%i) for i in range(len(feeds))]
         manifest.append(os.path.join(dir, 'index.html'))
+        manifest.append(os.path.join(dir, 'index.ncx'))
         cpath = getattr(self, 'cover_path', None)
         if cpath is None:
             pf = PersistentTemporaryFile('_recipe_cover.jpg')
@@ -881,6 +888,9 @@ class BasicNewsRecipe(object, LoggingInterface):
             opf.cover = cpath
             manifest.append(cpath)
         opf.create_manifest_from_files_in(manifest)
+        for mani in opf.manifest:
+            if mani.path.endswith('.ncx'):
+                mani.id = 'ncx'
         
         entries = ['index.html']
         toc = TOC(base_path=dir)

src/calibre/web/feeds/recipes/__init__.py

@@ -22,7 +22,8 @@ recipe_modules = ['recipe_' + r for r in (
            'time_magazine', 'endgadget', 'fudzilla', 'nspm_int', 'nspm', 'pescanik',
            'spiegel_int', 'themarketticker', 'tomshardware', 'xkcd', 'ftd', 'zdnet',
            'joelonsoftware', 'telepolis', 'common_dreams', 'nin', 'tomshardware_de',
-           'pagina12', 'infobae', 'ambito', 'elargentino', 'sueddeutsche', 
+           'pagina12', 'infobae', 'ambito', 'elargentino', 'sueddeutsche', 'the_age',
+           'laprensa',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_laprensa.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
+'''
+laprensa.com.ar
+'''
+import urllib
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class LaPrensa(BasicNewsRecipe):
+    title                 = 'La Prensa'
+    __author__            = 'Darko Miletic'
+    description           = 'Informacion Libre las 24 horas'    
+    oldest_article        = 7
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    encoding              = 'cp1252'
+    cover_url             = 'http://www.laprensa.com.ar/imgs/logo.gif'
+
+    html2lrf_options = [
+                          '--comment'       , description
+                        , '--category'      , 'news, Argentina'
+                        , '--publisher'     , title
+                        ]
+                            
+    feeds = [ 
+              (u'Politica'    , u'http://www.laprensa.com.ar/Rss.aspx?Rss=4' )
+             ,(u'Economia'    , u'http://www.laprensa.com.ar/Rss.aspx?Rss=5' )
+             ,(u'Opinion'     , u'http://www.laprensa.com.ar/Rss.aspx?Rss=6' )
+             ,(u'El Mundo'    , u'http://www.laprensa.com.ar/Rss.aspx?Rss=7' )
+             ,(u'Actualidad'  , u'http://www.laprensa.com.ar/Rss.aspx?Rss=8' )
+             ,(u'Deportes'    , u'http://www.laprensa.com.ar/Rss.aspx?Rss=9' )
+             ,(u'Espectaculos', u'http://www.laprensa.com.ar/Rss.aspx?Rss=10')
+            ]
+
+    def print_version(self, url):
+        return url.replace('.note.aspx','.NotePrint.note.aspx')
+
+    def get_article_url(self, article):
+        raw = article.get('link',  None).encode('utf8')
+        final = urllib.quote(raw,':/') 
+        return final
+
+    def preprocess_html(self, soup):
+        del soup.body['onload']
+        return soup
+    

src/calibre/web/feeds/recipes/recipe_new_yorker.py

@@ -16,6 +16,14 @@ class NewYorker(BasicNewsRecipe):
     max_articles_per_feed = 100
     no_stylesheets        = False
     use_embedded_content  = False
+    extra_css = '''
+    .calibre_feed_list {font-size:xx-small}
+    .calibre_article_list {font-size:xx-small}
+    .calibre_feed_title {font-size:normal}
+    .calibre_recipe_title {font-size:normal}
+    .calibre_feed_description {font-size:xx-small}
+    '''
+                    
 
     keep_only_tags = [
                         dict(name='div'  , attrs={'id':'printbody'   })

src/calibre/web/feeds/recipes/recipe_newsweek.py

@@ -19,9 +19,16 @@ class Newsweek(BasicNewsRecipe):
 
     remove_tags = [
         dict(name=['script',  'noscript']),
-        dict(name='div',  attrs={'class':['ad', 'SocialLinks', 'SocialLinksDiv', 'channel', 'bot', 'nav', 'top', 'EmailArticleBlock']}),
+        dict(name='div',  attrs={'class':['ad', 'SocialLinks', 'SocialLinksDiv',
+                                          'channel', 'bot', 'nav', 'top', 
+                                          'EmailArticleBlock', 
+                                          'comments-and-social-links-wrapper',
+                                          'inline-social-links-wrapper',
+                                          'inline-social-links',
+                                          ]}),
         dict(name='div',  attrs={'class':re.compile('box')}),
-        dict(id=['ToolBox', 'EmailMain', 'EmailArticle', ])
+        dict(id=['ToolBox', 'EmailMain', 'EmailArticle', 'comment-box',
+                 'nw-comments'])
     ]
     
     recursions = 1

src/calibre/web/feeds/recipes/recipe_the_age.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2009, Matthew Briggs <hal.sulphur@gmail.com>'
+__docformat__ = 'restructuredtext en'
+
+'''
+theage.com.au
+'''
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+from calibre.ebooks.BeautifulSoup import BeautifulSoup
+
+
+class TheAge(BasicNewsRecipe):
+    
+    title = 'The Age'
+    description = 'Business News, World News and Breaking News in Melbourne, Australia'
+    __author__ = 'Matthew Briggs'
+    
+    def get_browser(self):
+        br = BasicNewsRecipe.get_browser()
+        br.set_handle_refresh(False)
+        return br
+    
+    def parse_index(self):
+        
+        soup = BeautifulSoup(self.browser.open('http://www.theage.com.au/text/').read())
+        
+        feeds, articles = [], []
+        feed = None
+        
+        
+        for tag in soup.findAll(['h3', 'a']):
+            if tag.name == 'h3':
+                if articles:
+                    feeds.append((feed, articles))
+                    articles = []
+                feed = self.tag_to_string(tag)
+            elif feed is not None and tag.has_key('href') and tag['href'].strip():
+                url = tag['href'].strip()
+                if url.startswith('/'):
+                    url   = 'http://www.theage.com.au' + url 
+                title = self.tag_to_string(tag)
+                articles.append({
+                                 'title': title,
+                                 'url'  : url,
+                                 'date' : strftime('%a, %d %b'),
+                                 'description' : '',
+                                 'content'     : '',
+                                 }) 
+                
+        return feeds
+                
+
+

src/calibre/web/feeds/recipes/recipe_xkcd.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Fetch xkcd.
 '''
 
-import time
+import time, re
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class XkcdCom(BasicNewsRecipe):
@@ -17,6 +17,11 @@ class XkcdCom(BasicNewsRecipe):
     keep_only_tags = [dict(id='middleContent')]
     remove_tags = [dict(name='ul'), dict(name='h3'), dict(name='br')]
     no_stylesheets = True
+    # turn image bubblehelp into a paragraph
+    preprocess_regexps = [
+        (re.compile(r'(<img.*title=")([^"]+)(".*>)'),
+         lambda m: '%s%s<p>%s</p>' % (m.group(1), m.group(3), m.group(2)))
+    ]
     
     def parse_index(self):
         INDEX = 'http://xkcd.com/archive/'

src/calibre/web/feeds/templates.py

@@ -32,6 +32,11 @@ class NavBarTemplate(Template):
       xmlns:py="http://genshi.edgewall.org/" 
        
 >
+    <head>
+        <style py:if="extra_css" type="text/css">
+        ${extra_css}
+        </style>
+    </head>
     <body>
         <div class="navbar" style="text-align:${'center' if center else 'left'};">
             <hr py:if="bottom" />
@@ -60,14 +65,15 @@ class NavBarTemplate(Template):
 ''')
 
     def generate(self, bottom, feed, art, number_of_articles_in_feed, 
-                 two_levels, url, __appname__, prefix='', center=True):
+                 two_levels, url, __appname__, prefix='', center=True,
+                 extra_css=None):
         if prefix and not prefix.endswith('/'):
             prefix += '/'
         return Template.generate(self, bottom=bottom, art=art, feed=feed,
                                  num=number_of_articles_in_feed, 
                                  two_levels=two_levels, url=url,
                                  __appname__=__appname__, prefix=prefix,
-                                 center=center)
+                                 center=center, extra_css=extra_css)
     
 
 class IndexTemplate(Template):
@@ -88,11 +94,14 @@ class IndexTemplate(Template):
         <style py:if="style" type="text/css">
             ${style}
         </style>
+        <style py:if="extra_css" type="text/css">
+            ${extra_css}
+        </style>
     </head>
     <body>
-        <h1>${title}</h1>
+        <h1 class="calibre_recipe_title">${title}</h1>
         <p style="text-align:right">${date}</p>
-        <ul>
+        <ul class="calibre_feed_list">
             <py:for each="i, feed in enumerate(feeds)">
             <li py:if="feed" id="feed_${str(i)}">
                 <a class="feed" href="${'feed_%d/index.html'%i}">${feed.title}</a>
@@ -103,11 +112,12 @@ class IndexTemplate(Template):
 </html>
 ''')
 
-    def generate(self, title, datefmt, feeds):
+    def generate(self, title, datefmt, feeds, extra_css=None):
         if isinstance(datefmt, unicode):
             datefmt = datefmt.encode(preferred_encoding)
         date = strftime(datefmt)
-        return Template.generate(self, title=title, date=date, feeds=feeds)
+        return Template.generate(self, title=title, date=date, feeds=feeds,
+                                 extra_css=extra_css)
     
     
 class FeedTemplate(Template):
@@ -128,18 +138,21 @@ class FeedTemplate(Template):
         <style py:if="style" type="text/css">
             ${style}
         </style>
+        <style py:if="extra_css" type="text/css">
+            ${extra_css}
+        </style>
     </head>
     <body style="page-break-before:always">
-        <h2 class="feed_title">${feed.title}</h2>
+        <h2 class="calibre_feed_title">${feed.title}</h2>
         <py:if test="getattr(feed, 'image', None)">
-        <div class="feed_image">
+        <div class="calibre_feed_image">
             <img alt="${feed.image_alt}" src="${feed.image_url}" />
         </div>
         </py:if>
-        <div py:if="getattr(feed, 'description', None)">
+        <div class="calibre_feed_description" py:if="getattr(feed, 'description', None)">
             ${feed.description}<br />
         </div>
-        <ul>
+        <ul class="calibre_article_list">
             <py:for each="i, article in enumerate(feed.articles)">
             <li id="${'article_%d'%i}" py:if="getattr(article, 'downloaded', False)" style="padding-bottom:0.5em">
                 <a class="article" href="${article.url}">${article.title}</a>
@@ -157,8 +170,9 @@ class FeedTemplate(Template):
 </html>
 ''')
         
-    def generate(self, feed, cutoff):
+    def generate(self, feed, cutoff, extra_css=None):
-        return Template.generate(self, feed=feed, cutoff=cutoff)
+        return Template.generate(self, feed=feed, cutoff=cutoff, 
+                                 extra_css=extra_css)
 
 class EmbeddedContent(Template):
     

src/calibre/web/fetch/simple.py

@@ -11,6 +11,8 @@ import sys, socket, os, urlparse, logging, re, time, copy, urllib2, threading, t
 from urllib import url2pathname
 from threading import RLock
 from httplib import responses
+from PIL import Image
+from cStringIO import StringIO
 
 from calibre import setup_cli_handlers, browser, sanitize_file_name, \
                     relpath, LoggingInterface
@@ -183,8 +185,9 @@ class RecursiveFetcher(object, LoggingInterface):
             except urllib2.URLError, err:
                 if hasattr(err, 'code') and responses.has_key(err.code):
                     raise FetchError, responses[err.code]
-                if getattr(err, 'reason', [0])[0] == 104: # Connection reset by peer
+                if getattr(err, 'reason', [0])[0] == 104 or \
-                    self.log_debug('Connection reset by peer retrying in 1 second.')
+                    getattr(getattr(err, 'args', [None])[0], 'errno', None) == -2: # Connection reset by peer or Name or service not know
+                    self.log_debug('Temporary error, retrying in 1 second')
                     time.sleep(1)
                     with closing(self.browser.open(url)) as f:
                         data = response(f.read()+f.read())
@@ -304,12 +307,17 @@ class RecursiveFetcher(object, LoggingInterface):
             fname = sanitize_file_name('img'+str(c)+ext)
             if isinstance(fname, unicode):
                 fname = fname.encode('ascii', 'replace')
-            imgpath = os.path.join(diskpath, fname)
+            imgpath = os.path.join(diskpath, fname+'.jpg')
-            with self.imagemap_lock:
+            try:
-                self.imagemap[iurl] = imgpath
+                im = Image.open(StringIO(data)).convert('RGBA')
-            with open(imgpath, 'wb') as x:
+                with self.imagemap_lock:
-                x.write(data)                
+                    self.imagemap[iurl] = imgpath
-            tag['src'] = imgpath
+                with open(imgpath, 'wb') as x:
+                    im.save(x, 'JPEG')
+                tag['src'] = imgpath
+            except:
+                traceback.print_exc()
+                continue
 
     def absurl(self, baseurl, tag, key, filter=True): 
         iurl = tag[key]
@@ -398,7 +406,7 @@ class RecursiveFetcher(object, LoggingInterface):
                     _fname = basename(iurl)
                     if not isinstance(_fname, unicode):
                         _fname.decode('latin1', 'replace')
-                    _fname.encode('ascii', 'replace').replace('%', '')
+                    _fname = _fname.encode('ascii', 'replace').replace('%', '').replace(os.sep, '')
                     res = os.path.join(linkdiskpath, _fname)
                     self.downloaded_paths.append(res)
                     self.filemap[nurl] = res