Pull from Trunk

Changelog.yaml

@@ -4,6 +4,13 @@
 # for important features/bug fixes.
 # Also, each release can have new and improved recipes.
 
+- version: 0.7.0
+  date: 2010-06-04
+
+  new features:
+    - title: "Go to http://calibre-ebook.com/new-in/seven to see what's new in 0.7.0"
+      type: major
+
 - version: 0.6.55
   date: 2010-05-28
 

resources/recipes/clarin.recipe

@@ -5,7 +5,6 @@ __copyright__ = '2008-2010, Darko Miletic <darko.miletic at gmail.com>'
 clarin.com
 '''
 
-from calibre import strftime
 from calibre.web.feeds.news import BasicNewsRecipe
 
 class Clarin(BasicNewsRecipe):
@@ -18,11 +17,12 @@ class Clarin(BasicNewsRecipe):
     max_articles_per_feed = 100
     use_embedded_content  = False
     no_stylesheets        = True
-    cover_url             = strftime('http://www.clarin.com/diario/%Y/%m/%d/portada.jpg')
+    encoding              = 'utf8'
-    encoding              = 'cp1252'
+    language              = 'es_AR'
-    language              = 'es'
+    publication_type      = 'newspaper'
-    masthead_url          = 'http://www.clarin.com/shared/v10/img/Hd/lg_Clarin.gif'
+    INDEX                 = 'http://www.clarin.com'
-    extra_css             = ' body{font-family: Arial,Helvetica,sans-serif} h2{font-family: Georgia,"Times New Roman",Times,serif; font-size: xx-large} .Volan,.Pie,.Autor{ font-size: x-small} .Copete,.Hora{font-size: large} '
+    masthead_url          = 'http://www.clarin.com/static/CLAClarin/images/logo-clarin-print.jpg'
+    extra_css             = ' body{font-family: Arial,Helvetica,sans-serif} h2{font-family: Georgia,serif; font-size: xx-large} .hora{font-weight:bold} .hd p{font-size: small} .nombre-autor{color: #0F325A} '
 
     conversion_options = {
                           'comment'  : description
@@ -31,27 +31,32 @@ class Clarin(BasicNewsRecipe):
                         , 'language' : language
                         }
 
-    remove_tags = [
+    keep_only_tags = [dict(attrs={'class':['hd','mt']})]
-                     dict(name='a'   , attrs={'class':'Imp'   })
-                    ,dict(name='div' , attrs={'class':'Perma' })
-                    ,dict(name='h1'  , text='Imprimir'         )
-                  ]
 
     feeds = [
-               (u'Ultimo Momento', u'http://www.clarin.com/diario/hoy/um/sumariorss.xml')
+               (u'Pagina principal', u'http://www.clarin.com/rss/'             )
-              ,(u'El Pais'       , u'http://www.clarin.com/diario/hoy/elpais.xml'       )
+              ,(u'Politica'        , u'http://www.clarin.com/rss/politica/'    )
-              ,(u'Opinion'       , u'http://www.clarin.com/diario/hoy/opinion.xml'      )
+              ,(u'Deportes'        , u'http://www.clarin.com/rss/deportes/'    )
-              ,(u'El Mundo'      , u'http://www.clarin.com/diario/hoy/elmundo.xml'      )
+              ,(u'Economia'        , u'http://www.clarin.com/economia/'        )
-              ,(u'Sociedad'      , u'http://www.clarin.com/diario/hoy/sociedad.xml'     )
+              ,(u'Mundo'           , u'http://www.clarin.com/rss/mundo/'       )
-              ,(u'La Ciudad'     , u'http://www.clarin.com/diario/hoy/laciudad.xml'     )
+              ,(u'Espectaculos'    , u'http://www.clarin.com/rss/espectaculos/')
-              ,(u'Policiales'    , u'http://www.clarin.com/diario/hoy/policiales.xml'   )
+              ,(u'Sociedad'        , u'http://www.clarin.com/rss/sociedad/'    )
-              ,(u'Deportes'      , u'http://www.clarin.com/diario/hoy/deportes.xml'     )
+              ,(u'Ciudades'        , u'http://www.clarin.com/rss/ciudades/'    )
+              ,(u'Policiales'      , u'http://www.clarin.com/rss/policiales/'  )
+              ,(u'Internet'        , u'http://www.clarin.com/rss/internet/'    )
+              ,(u'Ciudades'        , u'http://www.clarin.com/rss/ciudades/'    )
             ]
 
     def print_version(self, url):
-        rest  = url.partition('-0')[-1]
+        return url + '?print=1'
-        lmain = rest.partition('.')[0]
-        lurl = u'http://www.servicios.clarin.com/notas/jsp/clarin/v9/notas/imprimir.jsp?pagid=' + lmain
-        return lurl
 
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('div',attrs={'class':'bb-md bb-md-edicion_papel'})
+        if cover_item:
+           ap = cover_item.find('a',attrs={'href':'/edicion-impresa/'})
+           if ap:
+              cover_url = self.INDEX + ap.img['src']
+        return cover_url
 

resources/recipes/welt.recipe

@@ -21,12 +21,16 @@ class weltDe(BasicNewsRecipe):
     no_stylesheets = True
     remove_stylesheets = True
     remove_javascript = True
-    encoding = 'iso-8859-1'
+    encoding = 'utf-8'
-    BasicNewsRecipe.summary_length = 200
+    html2epub_options = 'linearize_tables = True\nbase_font_size2=10'
+    BasicNewsRecipe.summary_length = 100
 
 
     remove_tags = [dict(id='jumplinks'),
 		   dict(id='ad1'),
+		   dict(id='top'),
+		   dict(id='header'),
+		   dict(id='additionalNavWrapper'),
 		   dict(id='fullimage_index'),
 		   dict(id='additionalNav'),
 		   dict(id='printMenu'),
@@ -35,6 +39,8 @@ class weltDe(BasicNewsRecipe):
 		   dict(id='servicesBox'),
 		   dict(id='servicesNav'),
 		   dict(id='ad2'),
+		   dict(id='banner_1'),
+		   dict(id='ssoInfoTop'),
 		   dict(id='brandingWrapper'),
 		   dict(id='links-intern'),
 		   dict(id='navigation'),
@@ -53,10 +59,22 @@ class weltDe(BasicNewsRecipe):
 		   dict(id='xmsg_comment'),
 		   dict(id='additionalNavWrapper'),
 		   dict(id='imagebox'),
+		   dict(id='footerContainer'),
 		   #dict(id=''),
                    dict(name='span'),
 		   dict(name='div', attrs={'class':'printURL'}),
+		   dict(name='ul', attrs={'class':'clear mainNavigation inline'}),
+		   dict(name='ul', attrs={'class':'inline'}),
+		   dict(name='ul', attrs={'class':'ubar'}),
+		   dict(name='hr', attrs={'class':'ubar'}),
+		   dict(name='li', attrs={'class':'counter'}),
+		   dict(name='li', attrs={'class':'browseBack'}),
+		   dict(name='li', attrs={'class':'browseNext'}),
+		   dict(name='li', attrs={'class':'selected'}),
+		   dict(name='div', attrs={'class':'floatLeft'}),
 		   dict(name='div', attrs={'class':'ad'}),
+		   dict(name='div', attrs={'class':'ftBarLeft'}),
+		   dict(name='div', attrs={'class':'clear additionalNav'}),
 		   dict(name='div', attrs={'class':'inlineBox inlineFurtherLinks'}),
 		   dict(name='div', attrs={'class':'inlineBox videoInlineBox'}),
 		   dict(name='div', attrs={'class':'inlineGallery'}),
@@ -65,6 +83,23 @@ class weltDe(BasicNewsRecipe):
 		   dict(name='div', attrs={'class':'articleOptions clear'}),
 		   dict(name='div', attrs={'class':'noPrint galleryIndex'}),
 		   dict(name='div', attrs={'class':'inlineBox inlineTagCloud'}),
+		   dict(name='div', attrs={'class':'clear module writeComment bgColor1'}),
+		   dict(name='div', attrs={'class':'clear module textGallery bgColor1'}),
+		   dict(name='div', attrs={'class':'clear module socialMedia bgColor1'}),
+		   dict(name='div', attrs={'class':'clear module continuativeLinks'}),
+		   dict(name='div', attrs={'class':'moreArtH3'}),
+		   dict(name='div', attrs={'class':'jqmWindow'}),
+		   dict(name='div', attrs={'class':'clear gap4'}),
+		   dict(name='div', attrs={'class':'hidden'}),
+		   dict(name='div', attrs={'class':'advertising'}),
+		   dict(name='div', attrs={'class':'ad adMarginBottom'}),
+		   dict(name='div', attrs={'class':'ad'}),
+		   dict(name='div', attrs={'class':'topLine'}),
+		   dict(name='div', attrs={'class':'toplineH2'}),
+		   dict(name='div', attrs={'class':'headLineH3'}),
+		   dict(name='div', attrs={'class':'print'}),
+		   dict(name='div', attrs={'class':'clear menu'}),
+		   dict(name='div', attrs={'class':'clear galleryContent'}),
 		   dict(name='p', attrs={'class':'jump'}),
 		   dict(name='a', attrs={'class':'commentLink'}),
 		   dict(name='h2', attrs={'class':'jumpHeading'}),
@@ -75,7 +110,7 @@ class weltDe(BasicNewsRecipe):
 		   dict(name='table', attrs={'class':'textGallery'}),
 		   dict(name='li', attrs={'class':'active'})]
 
-    remove_tags_after = [dict(id='tw_link_widget')]
+    remove_tags_after = [dict(name='div', attrs={'class':'clear departmentLine'})]
 
     extra_css = '''
                     h2{font-family:Arial,Helvetica,sans-serif; font-size: x-small; color: #003399;}
@@ -87,7 +122,6 @@ class weltDe(BasicNewsRecipe):
                     .photo {font-family:Arial,Helvetica,sans-serif; font-size: x-small; color: #666666;}                 '''
 
     feeds =  [ ('Politik', 'http://welt.de/politik/?service=Rss'),
-	       ('Deutsche Dinge', 'http://www.welt.de/deutsche-dinge/?service=Rss'),
 	       ('Wirtschaft', 'http://welt.de/wirtschaft/?service=Rss'),
 	       ('Finanzen', 'http://welt.de/finanzen/?service=Rss'),
 	       ('Sport', 'http://welt.de/sport/?service=Rss'),
@@ -101,4 +135,5 @@ class weltDe(BasicNewsRecipe):
 
 
     def print_version(self, url):
-        return url.replace ('.html', '.html?print=yes')
+        return url.replace ('.html', '.html?print=true')
+

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.6.55'
+__version__   = '0.7.0'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 
 import re

src/calibre/devices/apple/driver.py

@@ -44,6 +44,12 @@ if iswindows:
                 ]
 
 class ITUNES(DevicePlugin):
+    '''
+            try:
+                pythoncom.CoInitialize()
+            finally:
+                pythoncom.CoUninitialize()
+    '''
 
     name = 'Apple device interface'
     gui_name = 'Apple device'
@@ -51,7 +57,8 @@ class ITUNES(DevicePlugin):
     description    = _('Communicate with iBooks through iTunes.')
     supported_platforms = ['osx','windows']
     author = 'GRiker'
-    driver_version = '0.2'
+    #: The version of this plugin as a 3-tuple (major, minor, revision)
+    version        = (1, 0, 0)
 
     OPEN_FEEDBACK_MESSAGE = _(
         'Apple device detected, launching iTunes, please wait ...')
@@ -68,6 +75,7 @@ class ITUNES(DevicePlugin):
     # Properties
     cached_books = {}
     cache_dir = os.path.join(config_dir, 'caches', 'itunes')
+    ejected = False
     iTunes= None
     log = Log()
     path_template = 'iTunes/%s - %s.epub'
@@ -99,16 +107,19 @@ class ITUNES(DevicePlugin):
             if isosx:
                 if DEBUG:
                     self.log.info( "ITUNES.add_books_to_metadata()")
+                    self._dump_update_list('add_books_to_metadata()')
                 for (j,p_book) in enumerate(self.update_list):
-                    #self.log.info("ITUNES.add_books_to_metadata(): looking for %s" % p_book['lib_book'])
+                    self.log.info("ITUNES.add_books_to_metadata(): looking for %s" %
+                        str(p_book['lib_book'])[-9:])
                     for i,bl_book in enumerate(booklists[0]):
-                        #self.log.info("ITUNES.add_books_to_metadata(): evaluating %s" % bl_book.library_id)
                         if bl_book.library_id == p_book['lib_book']:
                             booklists[0].pop(i)
-                            #self.log.info("ITUNES.add_books_to_metadata(): removing %s" % p_book['title'])
+                            self.log.info("ITUNES.add_books_to_metadata(): removing %s %s" %
+                                (p_book['title'], str(p_book['lib_book'])[-9:]))
                             break
                     else:
-                        self.log.error(" update_list item '%s' not found in booklists[0]" % p_book['title'])
+                        self.log.error(" update_list item '%s' by %s %s not found in booklists[0]" %
+                            (p_book['title'], p_book['author'],str(p_book['lib_book'])[-9:]))
 
                     if self.report_progress is not None:
                         self.report_progress(j+1/task_count, _('Updating device metadata listing...'))
@@ -136,7 +147,12 @@ class ITUNES(DevicePlugin):
 
         # Add new books to booklists[0]
         for new_book in locations[0]:
+            if DEBUG:
+                self.log.info(" adding '%s' by '%s' to booklists[0]" %
+                    (new_book.title, new_book.author))
             booklists[0].append(new_book)
+        if DEBUG:
+            self._dump_booklist(booklists[0],'after add_books_to_metadata()')
 
     def books(self, oncard=None, end_session=True):
         """
@@ -153,10 +169,10 @@ class ITUNES(DevicePlugin):
         list of device books.
 
         """
-        if DEBUG:
-            self.log.info("ITUNES:books(oncard=%s)" % oncard)
-
         if not oncard:
+            if DEBUG:
+                self.log.info("ITUNES:books(oncard=%s)" % oncard)
+
             # Fetch a list of books from iPod device connected to iTunes
 
             # Fetch Library|Books
@@ -187,7 +203,7 @@ class ITUNES(DevicePlugin):
 
                         cached_books[this_book.path] = {
                          'title':book.name(),
-                         'author':book.artist(),
+                         'author':[book.artist()],
                          'lib_book':library_books[this_book.path] if this_book.path in library_books else None
                          }
 
@@ -232,7 +248,8 @@ class ITUNES(DevicePlugin):
                     self.report_progress(1.0, _('finished'))
                 self.cached_books = cached_books
                 if DEBUG:
-                    self._dump_cached_books()
+                    self._dump_booklist(booklist, 'returning from books():')
+                    self._dump_cached_books('returning from books():')
                 return booklist
         else:
             return []
@@ -254,31 +271,49 @@ class ITUNES(DevicePlugin):
 
         if self.iTunes:
             # Check for connected book-capable device
-            try:
+            self.sources = self._get_sources()
-                '''
+            if 'iPod' in self.sources:
-                names = [s.name() for s in self.iTunes.sources()]
+                #if DEBUG:
-                kinds = [str(s.kind()).rpartition('.')[2] for s in self.iTunes.sources()]
+                    #sys.stdout.write('.')
-                self.sources = sources = dict(zip(kinds,names))
+                    #sys.stdout.flush()
-                '''
+                return True
-                self.sources = self._get_sources()
+            else:
-                if 'iPod' in self.sources:
+                if DEBUG:
-                    if DEBUG:
+                    sys.stdout.write('-')
-                        sys.stdout.write('.')
+                    sys.stdout.flush()
-                        sys.stdout.flush()
-                    return True
-                else:
-                    if DEBUG:
-                        self.log.info("ITUNES.can_handle(): device ejected")
-                    return False
-            except:
-                # iTunes connection failed, probably not running anymore
-                self.log.error("ITUNES.can_handle(): lost connection to iTunes")
                 return False
         else:
-            # can_handle() is called once before open(), so need to return True
+            # Called at entry
-            # to keep things going
+            # We need to know if iTunes sees the iPad
+            # It may have been ejected
             if DEBUG:
-                self.log.info("ITUNES:can_handle(): iTunes not yet instantiated")
+                self.log.info("ITUNES.can_handle()")
+
+            self._launch_iTunes()
+            self.sources = self._get_sources()
+            if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
+                attempts = 9
+                while attempts:
+                    # If iTunes was just launched, device may not be detected yet
+                    self.sources = self._get_sources()
+                    if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
+                        attempts -= 1
+                        time.sleep(0.5)
+                        if DEBUG:
+                            self.log.warning(" waiting for identified iPad, attempt #%d" % (10 - attempts))
+                    else:
+                        if DEBUG:
+                            self.log.info(' found connected iPad in iTunes')
+                        break
+                else:
+                    # iTunes running, but not connected iPad
+                    if DEBUG:
+                        self.log.info(' self.ejected = True')
+                    self.ejected = True
+                    return False
+            else:
+                self.log.info(' found connected iPad in sources')
+
             return True
 
     def can_handle_windows(self, device_id, debug=False):
@@ -292,35 +327,74 @@ class ITUNES(DevicePlugin):
 
         :param device_info: On windows a device ID string. On Unix a tuple of
         ``(vendor_id, product_id, bcd)``.
+
+        iPad implementation notes:
+        It is necessary to use this method to check for the presence of a connected
+        iPad, as we have to return True if we can handle device interaction, or False if not.
+
         '''
         if self.iTunes:
-            # Check for connected book-capable device
+            # We've previously run, so the user probably ejected the device
             try:
-                '''
+                pythoncom.CoInitialize()
-                names = [s.name() for s in self.iTunes.sources()]
-                kinds = [str(s.kind()).rpartition('.')[2] for s in self.iTunes.sources()]
-                self.sources = sources = dict(zip(kinds,names))
-                '''
                 self.sources = self._get_sources()
                 if 'iPod' in self.sources:
                     if DEBUG:
                         sys.stdout.write('.')
                         sys.stdout.flush()
+                    if DEBUG:
+                        self.log.info('ITUNES.can_handle_windows:\n confirming connected iPad')
+                    self.ejected = False
                     return True
                 else:
                     if DEBUG:
-                        self.log.info("ITUNES.can_handle(): device ejected")
+                        self.log.info("ITUNES.can_handle_windows():\n device ejected")
+                    self.ejected = True
                     return False
             except:
                 # iTunes connection failed, probably not running anymore
-                self.log.error("ITUNES.can_handle(): lost connection to iTunes")
+
+                self.log.error("ITUNES.can_handle_windows():\n lost connection to iTunes")
                 return False
+            finally:
+                pythoncom.CoUninitialize()
 
         else:
-            # can_handle_windows() is called once before open(), so need to return True
+            # This is called at entry
-            # to keep things going
+            # We need to know if iTunes sees the iPad
+            # It may have been ejected
             if DEBUG:
-                self.log.info("ITUNES:can_handle(): iTunes not yet instantiated")
+                self.log.info("ITUNES:can_handle_windows():\n Launching iTunes")
+
+            try:
+                pythoncom.CoInitialize()
+                self._launch_iTunes()
+                self.sources = self._get_sources()
+                if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
+                    attempts = 9
+                    while attempts:
+                        # If iTunes was just launched, device may not be detected yet
+                        self.sources = self._get_sources()
+                        if (not 'iPod' in self.sources) or (self.sources['iPod'] == ''):
+                            attempts -= 1
+                            time.sleep(0.5)
+                            if DEBUG:
+                                self.log.warning(" waiting for identified iPad, attempt #%d" % (10 - attempts))
+                        else:
+                            if DEBUG:
+                                self.log.info(' found connected iPad in iTunes')
+                            break
+                    else:
+                        # iTunes running, but not connected iPad
+                        if DEBUG:
+                            self.log.info(' self.ejected = True')
+                        self.ejected = True
+                        return False
+                else:
+                    self.log.info(' found connected iPad in sources')
+            finally:
+                pythoncom.CoUninitialize()
+
             return True
 
     def card_prefix(self, end_session=True):
@@ -333,8 +407,6 @@ class ITUNES(DevicePlugin):
         ('place', None)
         (None, None)
         '''
-        if DEBUG:
-            self.log.info("ITUNES:card_prefix()")
         return (None,None)
 
     def delete_books(self, paths, end_session=True):
@@ -399,6 +471,8 @@ class ITUNES(DevicePlugin):
 
         @return: A 3 element list with free space in bytes of (1, 2, 3). If a
         particular device doesn't have any of these locations it should return -1.
+
+        In Windows, a sync-in-progress blocks this call until sync is complete
         """
         if DEBUG:
             self.log.info("ITUNES:free_space()")
@@ -411,13 +485,19 @@ class ITUNES(DevicePlugin):
 
         elif iswindows:
             if 'iPod' in self.sources:
-                try:
+
-                    pythoncom.CoInitialize()
+                while True:
-                    self.iTunes = win32com.client.Dispatch("iTunes.Application")
+                    try:
-                    connected_device = self.sources['iPod']
+                        try:
-                    free_space = self.iTunes.sources.ItemByName(connected_device).FreeSpace
+                            pythoncom.CoInitialize()
-                finally:
+                            self.iTunes = win32com.client.Dispatch("iTunes.Application")
-                    pythoncom.CoUninitialize()
+                            connected_device = self.sources['iPod']
+                            free_space = self.iTunes.sources.ItemByName(connected_device).FreeSpace
+                        finally:
+                            pythoncom.CoUninitialize()
+                            break
+                    except:
+                        self.log.error(' waiting for free_space() call to go through')
 
         return (free_space,-1,-1)
 
@@ -450,61 +530,11 @@ class ITUNES(DevicePlugin):
         mounted. The base class within USBMS device.py has a implementation of
         this function that should serve as a good example for USB Mass storage
         devices.
+
+        Note that most of the initialization is necessarily performed in can_handle(), as
+        we need to talk to iTunes to discover if there's a connected iPod
         '''
 
-        if isosx:
-            # Launch iTunes if not already running
-            if DEBUG:
-                self.log.info("ITUNES:open(): Instantiating iTunes")
-
-            # Instantiate iTunes
-            running_apps = appscript.app('System Events')
-            if not 'iTunes' in running_apps.processes.name():
-                if DEBUG:
-                    self.log.info( "ITUNES:open(): Launching iTunes" )
-                self.iTunes = iTunes= appscript.app('iTunes', hide=True)
-                iTunes.run()
-                initial_status = 'launched'
-            else:
-                self.iTunes = appscript.app('iTunes')
-                initial_status = 'already running'
-
-            if DEBUG:
-                self.log.info( " %s - %s (%s), driver version %s" %
-                 (self.iTunes.name(), self.iTunes.version(), initial_status, self.driver_version))
-
-            # Init the iTunes source list
-            '''
-            names = [s.name() for s in self.iTunes.sources()]
-            kinds = [str(s.kind()).rpartition('.')[2] for s in self.iTunes.sources()]
-            self.sources = dict(zip(kinds,names))
-            '''
-            self.sources = self._get_sources()
-
-        elif iswindows:
-            # Launch iTunes if not already running
-            if DEBUG:
-                self.log.info("ITUNES:open(): Instantiating iTunes")
-
-            # Instantiate iTunes
-            try:
-                pythoncom.CoInitialize()
-                self.iTunes = win32com.client.Dispatch("iTunes.Application")
-                if not DEBUG:
-                    self.iTunes.Windows[0].Minimized = True
-                initial_status = 'launched'
-
-                if DEBUG:
-                    self.log.info( " %s - %s (%s), driver version %s" %
-                     (self.iTunes.Windows[0].name, self.iTunes.Version, initial_status, self.driver_version))
-
-                # Init the iTunes source list
-                self.sources = self._get_sources()
-
-            finally:
-                pythoncom.CoUninitialize()
-
-
         # Confirm/create thumbs archive
         archive_path = os.path.join(self.cache_dir, "thumbs.zip")
 
@@ -545,10 +575,10 @@ class ITUNES(DevicePlugin):
                     self.log.error("ITUNES.remove_books_from_metadata(): '%s' not found in self.cached_book" % path)
 
                 # Remove from cached_books
+                self.cached_books.pop(path)
                 if DEBUG:
                     self.log.info("ITUNES.remove_books_from_metadata(): Removing '%s' from self.cached_books" % path)
-                self.cached_books.pop(path)
+                    self._dump_cached_books('remove_books_from_metadata()')
-
             else:
                 self.log.warning("ITUNES.remove_books_from_metadata(): skipping purchased book, can't remove via automation interface")
 
@@ -573,8 +603,6 @@ class ITUNES(DevicePlugin):
                                 If it is called with -1 that means that the
                                 task does not have any progress information
         '''
-        if DEBUG:
-            self.log.info("ITUNES:set_progress_reporter()")
         self.report_progress = report_progress
 
     def settings(self):
@@ -582,8 +610,6 @@ class ITUNES(DevicePlugin):
         Should return an opts object. The opts object should have one attribute
         `format_map` which is an ordered list of formats for the device.
         '''
-        if DEBUG:
-            self.log.info("ITUNES.settings()")
         klass = self if isinstance(self, type) else self.__class__
         c = Config('device_drivers_%s' % klass.__name__, _('settings for device drivers'))
         c.add_opt('format_map', default=self.FORMATS,
@@ -600,24 +626,23 @@ class ITUNES(DevicePlugin):
         if DEBUG:
             self.log.info("ITUNES:sync_booklists():")
         if self.update_needed:
+            if DEBUG:
+                self.log.info(' calling _update_device')
             self._update_device(msg=self.update_msg)
             self.update_needed = False
 
         # Get actual size of updated books on device
         if self.update_list:
             if DEBUG:
-                self.log.info("ITUNES:sync_booklists()\n update_list:")
+                self._dump_update_list(header='sync_booklists()')
-                for ub in self.update_list:
-                    self.log.info("  '%s' by %s" % (ub['title'], ub['author']))
-
             if isosx:
                 for updated_book in self.update_list:
-                    size_on_device = self._get_device_book_size(updated_book['title'], updated_book['author'])
+                    size_on_device = self._get_device_book_size(updated_book['title'],
+                                                                updated_book['author'][0])
                     if size_on_device:
                         for book in booklists[0]:
                             if book.title == updated_book['title'] and \
-                               book.author[0] == updated_book['author']:
+                               book.author == updated_book['author']:
-                                book.size = size_on_device
                                 break
                         else:
                             self.log.error("ITUNES:sync_booklists(): could not update book size for '%s'" % updated_book['title'])
@@ -705,15 +730,24 @@ class ITUNES(DevicePlugin):
                                      "Click 'Show Details' for a list.")
 
         if isosx:
+            if DEBUG:
+                self.log.info("ITUNES.upload_books():")
+                self._dump_files(files, header='upload_books()')
+                self._dump_cached_books('upload_books()')
+                self._dump_update_list('upload_books()')
             for (i,file) in enumerate(files):
                 path = self.path_template % (metadata[i].title, metadata[i].author[0])
                 # Delete existing from Library|Books, add to self.update_list
                 # for deletion from booklist[0] during add_books_to_metadata
                 if path in self.cached_books:
+                    if DEBUG:
+                        self.log.info(" adding '%s' by %s to self.update_list" %
+                                      (self.cached_books[path]['title'],self.cached_books[path]['author']))
+
+                    # *** Second time a book is updated the author is a list ***
                     self.update_list.append(self.cached_books[path])
 
                     if DEBUG:
-                        self.log.info("ITUNES.upload_books():")
                         self.log.info( " deleting existing '%s'" % (path))
                     self._remove_from_iTunes(self.cached_books[path])
 
@@ -941,26 +975,44 @@ class ITUNES(DevicePlugin):
         return (new_booklist, [], [])
 
     # Private methods
-    def _dump_booklist(self,booklist, header="booklists[0]"):
+    def _dump_booklist(self, booklist, header=None):
         '''
         '''
-        self.log.info()
+        if header:
-        self.log.info(header)
+            msg = '\nbooklist, %s' % header
-        self.log.info( "%s" % ('-' * len(header)))
+            self.log.info(msg)
-        for i,book in enumerate(booklist):
+            self.log.info('%s' % ('-' * len(msg)))
-            self.log.info( "%2d %-25.25s %s" % (i,book.title, book.library_id))
+
+        for book in booklist:
+            if isosx:
+                self.log.info("%-40.40s %-30.30s %-10.10s" %
+                 (book.title, book.author, str(book.library_id)[-9:]))
+            elif iswindows:
+                self.log.info("%-40.40s %-30.30s" %
+                 (book.title, book.author))
+
+    def _dump_cached_books(self, header=None):
+        '''
+        '''
+        if header:
+            msg = '\nself.cached_books, %s' % header
+            self.log.info(msg)
+            self.log.info( "%s" % ('-' * len(msg)))
+        if isosx:
+            for cb in self.cached_books.keys():
+                self.log.info("%-40.40s %-30.30s %-10.10s" %
+                 (self.cached_books[cb]['title'],
+                  self.cached_books[cb]['author'],
+                  str(self.cached_books[cb]['lib_book'])[-9:]))
+        elif iswindows:
+            for cb in self.cached_books.keys():
+                self.log.info("%-40.40s %-30.30s" %
+                 (self.cached_books[cb]['title'],
+                  self.cached_books[cb]['author']))
+
         self.log.info()
 
-    def _dump_cached_books(self):
+    def _dump_hex(self, src, length=16):
-        '''
-        '''
-        self.log.info("\n%-40.40s %-12.12s" % ('Device Books','In Library'))
-        self.log.info("%-40.40s %-12.12s" % ('------------','----------'))
-        for cb in self.cached_books.keys():
-            self.log.info("%-40.40s %6.6s" % (self.cached_books[cb]['title'], 'yes' if self.cached_books[cb]['lib_book'] else ' no'))
-        self.log.info("\n")
-
-    def _hexdump(self, src, length=16):
         '''
         '''
         FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)])
@@ -973,6 +1025,34 @@ class ITUNES(DevicePlugin):
            N+=length
         print result
 
+    def _dump_files(self, files, header=None):
+        if header:
+            msg = '\nfiles passed to %s:' % header
+            self.log.info(msg)
+            self.log.info( "%s" % ('-' * len(msg)))
+        for file in files:
+            self.log.info(file)
+        self.log.info()
+
+    def _dump_update_list(self,header=None):
+        if header:
+            msg = '\nself.update_list called from %s' % header
+            self.log.info(msg)
+            self.log.info( "%s" % ('-' * len(msg)))
+
+        if isosx:
+            for ub in self.update_list:
+                self.log.info("%-40.40s %-30.30s %-10.10s" %
+                 (ub['title'],
+                  ub['author'],
+                  str(ub['lib_book'])[-9:]))
+        elif iswindows:
+            for ub in self.update_list:
+                self.log.info("%-40.40s %-30.30s" %
+                 (ub['title'],
+                  ub['author']))
+        self.log.info()
+
     def _find_device_book(self, cached_book):
         '''
         Windows-only method to get a handle to a device book in the current pythoncom session
@@ -1034,11 +1114,11 @@ class ITUNES(DevicePlugin):
         except:
             zfw = zipfile.ZipFile(archive_path, mode='a')
         else:
-            if DEBUG:
+#             if DEBUG:
-                if isosx:
+#                 if isosx:
-                    self.log.info("ITUNES._generate_thumbnail(): cached thumb found for '%s'" % book.name())
+#                     self.log.info("ITUNES._generate_thumbnail(): cached thumb found for '%s'" % book.name())
-                elif iswindows:
+#                 elif iswindows:
-                    self.log.info("ITUNES._generate_thumbnail(): cached thumb found for '%s'" % book.Name)
+#                     self.log.info("ITUNES._generate_thumbnail(): cached thumb found for '%s'" % book.Name)
 
             return thumb_data
 
@@ -1046,7 +1126,7 @@ class ITUNES(DevicePlugin):
             try:
                 # Resize the cover
                 data = book.artworks[1].raw_data().data
-                #self._hexdump(data[:256])
+                #self._dump_hex(data[:256])
                 im = PILImage.open(cStringIO.StringIO(data))
                 scaled, width, height = fit_image(im.size[0],im.size[1], 60, 80)
                 im = im.resize((int(width),int(height)), PILImage.ANTIALIAS)
@@ -1097,27 +1177,27 @@ class ITUNES(DevicePlugin):
     def _get_device_book_size(self, title, author):
         '''
         Fetch the size of a book stored on the device
+
+        Windows: If sync-in-progress, this call blocked until sync completes
         '''
         if DEBUG:
-            self.log.info("ITUNES._get_device_book_size():\n looking for title: '%s' author: %s" % (title,author))
+            self.log.info("ITUNES._get_device_book_size():\n looking for title: '%s' author: '%s'" %
+                (title,author))
 
         device_books = self._get_device_books()
 
         if isosx:
             for d_book in device_books:
-                if DEBUG:
-                    self.log.info(" evaluating title: '%s' author: '%s'" % (d_book.name(), d_book.artist()))
                 if d_book.name() == title and d_book.artist() == author:
+                    if DEBUG:
+                        self.log.info(' found it')
                     return d_book.size()
             else:
-                self.log.error("ITUNES._get_device_book_size(): could not find '%s' by '%s' in device_books" % (title,author))
+                self.log.error("ITUNES._get_device_book_size():"
+                               " could not find '%s' by '%s' in device_books" % (title,author))
                 return None
         elif iswindows:
             for d_book in device_books:
-                '''
-                if DEBUG:
-                    self.log.info(" evaluating title: '%s' author: '%s'" % (d_book.Name, d_book.Artist))
-                '''
                 if d_book.Name == title and d_book.Artist == author:
                     self.log.info(" found it")
                     return d_book.Size
@@ -1143,6 +1223,10 @@ class ITUNES(DevicePlugin):
                 dev_playlists = [pl.Name for pl in dev.Playlists]
                 if 'Books' in dev_playlists:
                     return self.iTunes.sources.ItemByName(connected_device).Playlists.ItemByName('Books').Tracks
+                else:
+                    return []
+            if DEBUG:
+                self.log.warning('ITUNES._get_device_book(): No iPod device connected')
             return []
 
     def _get_library_books(self):
@@ -1188,9 +1272,10 @@ class ITUNES(DevicePlugin):
                     return []
             elif iswindows:
                 dev = self.iTunes.sources.ItemByName(connected_device)
-                dev_playlists = [pl.Name for pl in dev.Playlists]
+                if dev.Playlists is not None:
-                if 'Purchased' in dev_playlists:
+                    dev_playlists = [pl.Name for pl in dev.Playlists]
-                    return self.iTunes.sources.ItemByName(connected_device).Playlists.ItemByName('Purchased').Tracks
+                    if 'Purchased' in dev_playlists:
+                        return self.iTunes.sources.ItemByName(connected_device).Playlists.ItemByName('Purchased').Tracks
                 else:
                     return []
 
@@ -1203,6 +1288,7 @@ class ITUNES(DevicePlugin):
             kinds = [str(s.kind()).rpartition('.')[2] for s in self.iTunes.sources()]
             return dict(zip(kinds,names))
         elif iswindows:
+            # Assumes a pythoncom wrapper
             it_sources = ['Unknown','Library','iPod','AudioCD','MP3CD','Device','RadioTuner','SharedLibrary']
             names = [s.name for s in self.iTunes.sources]
             kinds = [it_sources[s.kind] for s in self.iTunes.sources]
@@ -1216,16 +1302,79 @@ class ITUNES(DevicePlugin):
         else:
             return True
 
+    def _launch_iTunes(self):
+        '''
+        '''
+        if DEBUG:
+            self.log.info("ITUNES:_launch_iTunes():\n Instantiating iTunes")
+
+        if isosx:
+            '''
+            Launch iTunes if not already running
+            '''
+            # Instantiate iTunes
+            running_apps = appscript.app('System Events')
+            if not 'iTunes' in running_apps.processes.name():
+                if DEBUG:
+                    self.log.info( "ITUNES:open(): Launching iTunes" )
+                self.iTunes = iTunes= appscript.app('iTunes', hide=True)
+                iTunes.run()
+                initial_status = 'launched'
+            else:
+                self.iTunes = appscript.app('iTunes')
+                initial_status = 'already running'
+
+            if DEBUG:
+                self.log.info( " [%s - %s (%s), driver version %d.%d.%d]" %
+                 (self.iTunes.name(), self.iTunes.version(), initial_status,
+                  self.version[0],self.version[1],self.version[2]))
+
+        if iswindows:
+            '''
+            Launch iTunes if not already running
+            Assumes pythoncom wrapper
+            '''
+            # Instantiate iTunes
+            self.iTunes = win32com.client.Dispatch("iTunes.Application")
+            if not DEBUG:
+                self.iTunes.Windows[0].Minimized = True
+            initial_status = 'launched'
+
+            if DEBUG:
+                self.log.info( " [%s - %s (%s), driver version %d.%d.%d]" %
+                 (self.iTunes.Windows[0].name, self.iTunes.Version, initial_status,
+                  self.version[0],self.version[1],self.version[2]))
+
     def _remove_from_iTunes(self, cached_book):
         '''
         iTunes does not delete books from storage when removing from database
         '''
         if isosx:
             storage_path = os.path.split(cached_book['lib_book'].location().path)
+            title_storage_path = storage_path[0]
             if DEBUG:
                 self.log.info("ITUNES._remove_from_iTunes():")
-                self.log.info(" removing storage_path: %s" % storage_path[0])
+                self.log.info(" removing title_storage_path: %s" % title_storage_path)
-            shutil.rmtree(storage_path[0])
+            try:
+                shutil.rmtree(title_storage_path)
+            except:
+                self.log.info(" '%s' not empty" % title_storage_path)
+
+            # Clean up title/author directories
+            author_storage_path = os.path.split(title_storage_path)[0]
+            self.log.info(" author_storage_path: %s" % author_storage_path)
+            author_files = os.listdir(author_storage_path)
+            if '.DS_Store' in author_files:
+                author_files.pop(author_files.index('.DS_Store'))
+            if not author_files:
+                shutil.rmtree(author_storage_path)
+                if DEBUG:
+                    self.log.info(" removing empty author_storage_path")
+            else:
+                if DEBUG:
+                    self.log.info(" author_storage_path not empty (%d objects):" % len(author_files))
+                    self.log.info(" %s" % '\n'.join(author_files))
+
             self.iTunes.delete(cached_book['lib_book'])
 
         elif iswindows:
@@ -1280,7 +1429,7 @@ class ITUNES(DevicePlugin):
             try:
                 pythoncom.CoInitialize()
                 self.iTunes = win32com.client.Dispatch("iTunes.Application")
-                #result = self.iTunes.UpdateIPod()
+                self.iTunes.UpdateIPod()
                 if wait:
                     if DEBUG:
                         sys.stdout.write(" waiting for iPad sync to complete ...")
@@ -1291,6 +1440,7 @@ class ITUNES(DevicePlugin):
                         pb_count = len(self._get_purchased_book_ids())
                         if db_count != lb_count + pb_count:
                             if DEBUG:
+                                #sys.stdout.write(' %d != %d + %d\n' % (db_count,lb_count,pb_count))
                                 sys.stdout.write('.')
                                 sys.stdout.flush()
                             time.sleep(2)
@@ -1333,8 +1483,6 @@ class BookList(list):
         Add the book to the booklist. Intent is to maintain any device-internal
         metadata. Return True if booklists must be sync'ed
         '''
-        if DEBUG:
-            self.log.info("BookList.add_book():\n%s" % book)
         self.append(book)
 
     def remove_book(self, book):

src/calibre/ebooks/metadata/html.py

@@ -44,7 +44,8 @@ def get_metadata_(src, encoding=None):
         author = match.group(2).replace(',', ';')
 
     ent_pat = re.compile(r'&(\S+)?;')
-    title = ent_pat.sub(entity_to_unicode, title)
+    if title:
+        title = ent_pat.sub(entity_to_unicode, title)
     if author:
         author = ent_pat.sub(entity_to_unicode, author)
     mi = MetaInformation(title, [author] if author else None)

src/calibre/ebooks/mobi/writer.py

@@ -1334,7 +1334,7 @@ class MobiWriter(object):
             item = self._oeb.manifest.hrefs[href]
             try:
                 data = rescale_image(item.data, self._imagemax)
-            except IOError:
+            except:
                 self._oeb.logger.warn('Bad image file %r' % item.href)
                 continue
             self._records.append(data)

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -201,6 +201,11 @@ class CSSFlattener(object):
         tag = barename(node.tag)
         style = stylizer.style(node)
         cssdict = style.cssdict()
+        try:
+            font_size = style['font-size']
+        except:
+            font_size = self.sbase if self.sbase is not None else \
+                self.context.source.fbase
         if 'align' in node.attrib:
             cssdict['text-align'] = node.attrib['align']
             del node.attrib['align']
@@ -219,13 +224,16 @@ class CSSFlattener(object):
                             esize = 1
                         if esize > 7:
                             esize = 7
-                        cssdict['font-size'] = fnums[esize]
+                        font_size = fnums[esize]
                     else:
                         try:
-                            cssdict['font-size'] = fnums[force_int(size)]
+                            font_size = fnums[force_int(size)]
                         except:
-                            cssdict['font-size'] = fnums[3]
+                            font_size = fnums[3]
+                    cssdict['font-size'] = '%.1fpt'%font_size
                 del node.attrib['size']
+            if 'face' in node.attrib:
+                del node.attrib['face']
         if 'color' in node.attrib:
             cssdict['color'] = node.attrib['color']
             del node.attrib['color']
@@ -244,7 +252,7 @@ class CSSFlattener(object):
                 cssdict['font-size'] = '%0.5fem'%(fsize/psize)
                 psize = fsize
             elif 'font-size' in cssdict or tag == 'body':
-                fsize = self.fmap[style['font-size']]
+                fsize = self.fmap[font_size]
                 cssdict['font-size'] = "%0.5fem" % (fsize / psize)
                 psize = fsize
         if cssdict:

src/calibre/gui2/library/views.py

@@ -75,6 +75,9 @@ class BooksView(QTableView): # {{{
             h.setSectionHidden(idx, True)
         elif action == 'show':
             h.setSectionHidden(idx, False)
+            if h.sectionSize(idx) < 3:
+               sz = h.sectionSizeHint(idx)
+               h.resizeSection(idx, sz)
         elif action == 'ascending':
             self.sortByColumn(idx, Qt.AscendingOrder)
         elif action == 'descending':
@@ -257,6 +260,11 @@ class BooksView(QTableView): # {{{
         for col, alignment in state.get('column_alignment', {}).items():
             self._model.change_alignment(col, alignment)
 
+        for i in range(h.count()):
+            if not h.isSectionHidden(i) and h.sectionSize(i) < 3:
+                sz = h.sectionSizeHint(i)
+                h.resizeSection(i, sz)
+
     def get_default_state(self):
         old_state = {
                 'hidden_columns': [],

src/calibre/library/server/opds.py

@@ -25,7 +25,7 @@ BASE_HREFS = {
         1 : '/opds',
 }
 
-STANZA_FORMATS = frozenset(['epub', 'pdb'])
+STANZA_FORMATS = frozenset(['epub', 'pdb', 'pdf', 'cbr', 'cbz', 'djvu'])
 
 def url_for(name, version, **kwargs):
     if not name.endswith('_'):
@@ -121,7 +121,7 @@ def CATALOG_GROUP_ENTRY(item, category, base_href, version, updated):
             TITLE(item.text),
             ID(id_),
             UPDATED(updated),
-            E.content(_('%d books')%item.count, type='text'),
+            E.content(_('%d items')%item.count, type='text'),
             link
             )
 

src/calibre/manual/faq.rst

@@ -135,29 +135,18 @@ turned into a collection on the reader. Note that the PRS-500 does not support c
 How do I use |app| with my iPad/iPhone/iTouch?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can access your calibre library on a iPad/iPhone/iTouch over the air using the calibre content server. 
+The easiest way to browse your |app| collection on your Apple device (iPad/iPhone/iPod) is by using the *free* Stanza app, available from the Apple app store. You need at least Stanza version 3.0. Stanza allows you to access your |app| collection wirelessly, over the air.
 
 First perform the following steps in |app|
 
   * Set the Preferred Output Format in |app| to EPUB (The output format can be set under Preferences->General) 
+  * Set the output profile to iPad (this will work for iPhone/iPods as well), under Preferences->Conversion->Page Setup
   * Convert the books you want to read on your iPhone to EPUB format by selecting them and clicking the Convert button.
   * Turn on the Content Server in |app|'s preferences and leave |app| running.
 
-For an iPad:
+Install the free Stanza reader app on your iPad/iPhone/iTouch using iTunes.
 
-Install the ReadMe app on your iPad using iTunes. Open the Readme builtin browser and browse to::
+Now you should be able to access your books on your iPhone by opening Stanza. Go to "Get Books" and then click the "Shared" tab. Under Shared you will see an entry "Books in calibre". If you don't, make sure your iPad/iPhone is connected using the WiFi network in your house, not 3G. If the |app| catalog is still not detected in Stanza, you can add it manually in Stanza. To do this, click the "Shared" tab, then click the "Edit" button and then click "Add book source" to add a new book source. In the Add Book Source screen enter whatever name you like and in the URL field, enter the following::
-
-    http://192.168.1.2:8080/
-
-Replace ``192.168.1.2`` with the local IP address of the computer running |app|. If you have changed the port the |app| content server is running on, you will have to change ``8080`` as well to the new port. The local IP address is the IP address you computer is assigned on your home network. A quick Google search will tell you how to find out your local IP address.
-
-The books in your |app| library will be presented as a list, 25 entries at a time. Click the right arrow to go to the next 25. You can also type in the search box to find specific books. Just click on the EPUB link of the book you want and it will be downloaded into your ReadMe library.
-
-For an iPhone/iTouch:   
-
-Install the free Stanza reader app on your iPhone/iTouch using iTunes.
-
-Now you should be able to access your books on your iPhone by opening Stanza. Go to "Get Books" and then click the "Shared" tab. Under Shared you will see an entry "Books in calibre". If you don't, make sure your iPhone is connected using the WiFi network in your house, not 3G. If the |app| catalog is still not detected in Stanza, you can add it manually in Stanza. To do this, click the "Shared" tab, then click the "Edit" button and then click "Add book source" to add a new book source. In the Add Book Source screen enter whatever name you like and in the URL field, enter the following::
 
     http://192.168.1.2:8080/
 
@@ -165,7 +154,10 @@ Replace ``192.168.1.2`` with the local IP address of the computer running |app|.
 
 If you get timeout errors while browsing the calibre catalog in Stanza, try increasing the connection timeout value in the stanza settings. Go to Info->Settings and increase the value of Download Timeout.
 
-Note that neither the Stanza, nor the ReadMe apps are in anyway associated with |app|.
+Alternative for the iPad
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+As of |app| version 0.7.0, on windows and OS X you can plugin your iPad into the computer using its charging cable, and |app| will detect it and show you a list of books on the iPad. You can then use the Send to device button to send books directly to iBooks on the iPad.
 
 How do I use |app| with my Android phone?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

src/calibre/manual/gui.rst

@@ -9,7 +9,7 @@ The Graphical User Interface *(GUI)* provides access to all
 library management and ebook format conversion features. The basic workflow
 for using |app| is to first add books to the library from your hard disk.
 |app| will automatically try to read metadata from the books and add them
-to its internal database. Once they are in the database, you can performa various
+to its internal database. Once they are in the database, you can perform a various
 :ref:`actions` on them that include conversion from one format to another,
 transfer to the reading device, viewing on your computer, editing metadata, including covers, etc.
 
@@ -243,7 +243,7 @@ Now, you can access your saved search in the Tag Browser under "Searches". A sin
 
 Preferences
 ---------------
-The Preferences dialog allows you to set some global defaults used by all of |app|. To access it, click the |cbi|.
+The Preferences dialog allows you to change the way various aspects of |app| work. To access it, click the |cbi|.
 
 .. |cbi| image:: images/configuration.png
 
@@ -251,7 +251,7 @@ The Preferences dialog allows you to set some global defaults used by all of |ap
 
 Guessing metadata from file names
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-In the :guilabel:`Advanced` section of the configuration dialog, you can specify a regularexpression that |app| will use to try and guess metadata from the names of ebook files
+In the :guilabel:`Add/Save` section of the configuration dialog, you can specify a regular expression that |app| will use to try and guess metadata from the names of ebook files
 that you add to the library. The default regular expression is::
 
     title - author
@@ -265,18 +265,13 @@ will be interpreted to have the title: Foundation and Earth and author: Isaac As
 .. tip::
     If the filename does not contain the hyphen, the regular expression will fail.
 
-.. tip::
-   If you want to only use metadata guessed from filenames and not metadata read from the file itself, you can tell |app| to do this, via the configuration dialog, accessed by the button to the right
-   of the search box.
-
 .. _book_details:
 
 Book Details
 -------------
 .. image:: images/book_details.png
 
-The Book Details display shows you extra information and the cover for the currently selected book. THe comments section is truncated if the comments are too long. To see the full comments as well as
+The Book Details display shows you extra information and the cover for the currently selected book.
-a larger image of the cover, click anywhere in the Book Details area.
 
 .. _jobs:
 

src/calibre/manual/news_recipe.rst

@@ -111,6 +111,8 @@ Pre/post processing of downloaded HTML
 
 .. automember:: BasicNewsRecipe.remove_javascript
 
+.. automethod:: BasicNewsRecipe.prepreprocess_html
+
 .. automethod:: BasicNewsRecipe.preprocess_html
 
 .. automethod:: BasicNewsRecipe.postprocess_html

src/calibre/utils/magick_draw.py

@@ -161,6 +161,19 @@ def create_text_arc(text, font_size, font=None, bgcolor='white'):
     p.MagickTrimImage(canvas, 0)
     return canvas
 
+def add_borders_to_image(path_to_image, left=0, top=0, right=0, bottom=0,
+        border_color='white'):
+    with p.ImageMagick():
+        img = load_image(path_to_image)
+        lwidth = p.MagickGetImageWidth(img)
+        lheight = p.MagickGetImageHeight(img)
+        canvas = create_canvas(lwidth+left+right, lheight+top+bottom,
+                border_color)
+        compose_image(canvas, img, left, top)
+        p.DestroyMagickWand(img)
+        with open(path_to_image, 'wb') as f:
+            p.MagickWriteImage(canvas, f)
+        p.DestroyMagickWand(canvas)
 
 def create_cover_page(top_lines, logo_path, width=590, height=750,
         bgcolor='white', output_format='png'):

src/calibre/web/feeds/news.py

@@ -267,7 +267,7 @@ class BasicNewsRecipe(Recipe):
             }
 
             a.article {
-                font-weight: bold;
+                font-weight: bold; text-align:left;
             }
 
             a.feed {
@@ -403,10 +403,25 @@ class BasicNewsRecipe(Recipe):
                     return url
         return article.get('link',  None)
 
+    def prepreprocess_html(self, soup):
+        '''
+        This method is called with the source of each downloaded :term:`HTML` file, before
+        any of the cleanup attributes like remove_tags, keep_only_tags are
+        applied. Note that preprocess_regexps will have already been applied.
+        It can be used to do arbitrarily powerful pre-processing on the :term:`HTML`.
+        It should return `soup` after processing it.
+
+        `soup`: A `BeautifulSoup <http://www.crummy.com/software/BeautifulSoup/documentation.html>`_
+        instance containing the downloaded :term:`HTML`.
+        '''
+        return soup
+
+
     def preprocess_html(self, soup):
         '''
         This method is called with the source of each downloaded :term:`HTML` file, before
-        it is parsed for links and images.
+        it is parsed for links and images. It is called after the cleanup as
+        specified by remove_tags etc.
         It can be used to do arbitrarily powerful pre-processing on the :term:`HTML`.
         It should return `soup` after processing it.
 
@@ -523,8 +538,8 @@ class BasicNewsRecipe(Recipe):
         Intended to be used to get article metadata like author/summary/etc.
         from the parsed HTML (soup).
         :param article: A object of class :class:`calibre.web.feeds.Article`.
-                       If you chane the sumamry, remeber to also change the
+        If you change the summary, remember to also change the
-                       text_summary
+        text_summary
         :param soup: Parsed HTML belonging to this article
         :param first: True iff the parsed HTML is the first page of the article.
         '''
@@ -603,7 +618,7 @@ class BasicNewsRecipe(Recipe):
 
         self.web2disk_options = web2disk_option_parser().parse_args(web2disk_cmdline)[0]
         for extra in ('keep_only_tags', 'remove_tags', 'preprocess_regexps',
-                      'preprocess_html', 'remove_tags_after',
+                      'prepreprocess_html', 'preprocess_html', 'remove_tags_after',
                       'remove_tags_before', 'is_link_wanted'):
             setattr(self.web2disk_options, extra, getattr(self, extra))
         self.web2disk_options.postprocess_html = self._postprocess_html
@@ -758,15 +773,15 @@ class BasicNewsRecipe(Recipe):
         if self.touchscreen:
             touchscreen_css = u'''
                     .summary_headline {
-                        font-size:large; font-weight:bold; margin-top:0px; margin-bottom:0px;
+                        font-weight:bold; text-align:left;
                     }
 
                     .summary_byline {
-                        font-size:small; margin-top:0px; margin-bottom:0px;
+                        font-family:monospace;
                     }
 
                     .summary_text {
-                        margin-top:0px; margin-bottom:0px;
+                        text-align:left;
                     }
 
                     .feed {
@@ -782,12 +797,6 @@ class BasicNewsRecipe(Recipe):
                         border-width:thin;
                     }
 
-                    table.toc {
-                        font-size:large;
-                    }
-                    td.article_count {
-                        text-align:right;
-                    }
             '''
 
             templ = templates.TouchscreenFeedTemplate()
@@ -1120,8 +1129,11 @@ class BasicNewsRecipe(Recipe):
         mi.publisher = __appname__
         mi.author_sort = __appname__
         if self.output_profile.name == 'iPad':
-            mi.authors = [strftime('%A, %d %B %Y')]
+            date_as_author = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
-            mi.author_sort = strftime('%Y-%m-%d')
+            mi = MetaInformation(self.short_title(), [date_as_author])
+            mi.publisher = __appname__
+            sort_author =  re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip()
+            mi.author_sort = '%s %s' % (sort_author, strftime('%Y-%m-%d'))
         mi.publication_type = 'periodical:'+self.publication_type
         mi.timestamp = nowf()
         mi.comments = self.description
@@ -1245,7 +1257,6 @@ class BasicNewsRecipe(Recipe):
         with nested(open(opf_path, 'wb'), open(ncx_path, 'wb')) as (opf_file, ncx_file):
             opf.render(opf_file, ncx_file)
 
-
     def article_downloaded(self, request, result):
         index = os.path.join(os.path.dirname(result[0]), 'index.html')
         if index != result[0]:

src/calibre/web/feeds/templates.py

@@ -120,6 +120,7 @@ class TouchscreenNavBarTemplate(Template):
             href = '%s%s/%s/index.html'%(prefix, up, next)
             navbar.text = '| '
             navbar.append(A('Next', href=href))
+
         href = '%s../index.html#article_%d'%(prefix, art)
         navbar.iterchildren(reversed=True).next().tail = ' | '
         navbar.append(A('Section Menu', href=href))
@@ -130,6 +131,7 @@ class TouchscreenNavBarTemplate(Template):
             href = '%s../article_%d/index.html'%(prefix, art-1)
             navbar.iterchildren(reversed=True).next().tail = ' | '
             navbar.append(A('Previous', href=href))
+
         navbar.iterchildren(reversed=True).next().tail = ' | '
         if not bottom:
             navbar.append(HR())
@@ -165,8 +167,14 @@ class TouchscreenIndexTemplate(Template):
     def _generate(self, title, masthead, datefmt, feeds, extra_css=None, style=None):
         if isinstance(datefmt, unicode):
             datefmt = datefmt.encode(preferred_encoding)
-        date = strftime(datefmt)
+        date = '%s, %s %s, %s' % (strftime('%A'), strftime('%B'), strftime('%d').lstrip('0'), strftime('%Y'))
-        masthead_img = IMG(src=masthead,alt="masthead")
+        masthead_p = etree.Element("p")
+        masthead_p.set("style","text-align:center")
+        masthead_img = etree.Element("img")
+        masthead_img.set("src",masthead)
+        masthead_img.set("alt","masthead")
+        masthead_p.append(masthead_img)
+
         head = HEAD(TITLE(title))
         if style:
             head.append(STYLE(style, type='text/css'))
@@ -177,15 +185,13 @@ class TouchscreenIndexTemplate(Template):
         for i, feed in enumerate(feeds):
             if feed:
                 tr = TR()
-                tr.append(TD( CLASS('toc_item'), A(feed.title, href='feed_%d/index.html'%i)))
+                tr.append(TD( CLASS('calibre_rescale_120'), A(feed.title, href='feed_%d/index.html'%i)))
-                tr.append(TD( CLASS('article_count'),'%d' % len(feed.articles)))
+                tr.append(TD( '%s' % len(feed.articles), style="text-align:right"))
                 toc.append(tr)
-
         div = DIV(
-                PT(masthead_img,style='text-align:center'),
+                masthead_p,
                 PT(date, style='text-align:center'),
-                toc,
+                toc)
-                CLASS('calibre_rescale_100'))
         self.root = HTML(head, BODY(div))
 
 class FeedTemplate(Template):
@@ -271,12 +277,15 @@ class TouchscreenFeedTemplate(Template):
                 continue
             tr = TR()
             td = TD(
-                    A(article.title, CLASS('article calibre_rescale_100',
+                    A(article.title, CLASS('summary_headline','calibre_rescale_120',
                                     href=article.url))
                     )
+            if article.author:
+                td.append(DIV(article.author,
+                    CLASS('summary_byline', 'calibre_rescale_100')))
             if article.summary:
                 td.append(DIV(cutoff(article.text_summary),
-                    CLASS('article_description', 'calibre_rescale_80')))
+                    CLASS('summary_text', 'calibre_rescale_100')))
             tr.append(td)
             toc.append(tr)
         div.append(toc)

src/calibre/web/fetch/simple.py

@@ -136,6 +136,7 @@ class RecursiveFetcher(object):
         self.remove_tags_before  = getattr(options, 'remove_tags_before', None)
         self.keep_only_tags      = getattr(options, 'keep_only_tags', [])
         self.preprocess_html_ext = getattr(options, 'preprocess_html', lambda soup: soup)
+        self.prepreprocess_html_ext = getattr(options, 'prepreprocess_html', lambda soup: soup)
         self.postprocess_html_ext= getattr(options, 'postprocess_html', None)
         self._is_link_wanted     = getattr(options, 'is_link_wanted',
                 default_is_link_wanted)
@@ -153,6 +154,8 @@ class RecursiveFetcher(object):
         nmassage.append((re.compile(r'<!--.*?-->', re.DOTALL), lambda m: ''))
         soup = BeautifulSoup(xml_to_unicode(src, self.verbose, strip_encoding_pats=True)[0], markupMassage=nmassage)
 
+        soup = self.prepreprocess_html_ext(soup)
+
         if self.keep_only_tags:
             body = Tag(soup, 'body')
             try: